1892 lines · cpp
1//===- AArch64ExpandPseudoInsts.cpp - Expand pseudo instructions ----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains a pass that expands pseudo instructions into target10// instructions to allow proper scheduling and other late optimizations. This11// pass should be run after register allocation but before the post-regalloc12// scheduling pass.13//14//===----------------------------------------------------------------------===//15 16#include "AArch64ExpandImm.h"17#include "AArch64InstrInfo.h"18#include "AArch64MachineFunctionInfo.h"19#include "AArch64Subtarget.h"20#include "MCTargetDesc/AArch64AddressingModes.h"21#include "Utils/AArch64BaseInfo.h"22#include "llvm/CodeGen/LivePhysRegs.h"23#include "llvm/CodeGen/MachineBasicBlock.h"24#include "llvm/CodeGen/MachineConstantPool.h"25#include "llvm/CodeGen/MachineFunction.h"26#include "llvm/CodeGen/MachineFunctionPass.h"27#include "llvm/CodeGen/MachineInstr.h"28#include "llvm/CodeGen/MachineInstrBuilder.h"29#include "llvm/CodeGen/MachineOperand.h"30#include "llvm/CodeGen/TargetSubtargetInfo.h"31#include "llvm/IR/DebugLoc.h"32#include "llvm/MC/MCInstrDesc.h"33#include "llvm/Pass.h"34#include "llvm/Support/CodeGen.h"35#include "llvm/Target/TargetMachine.h"36#include "llvm/TargetParser/Triple.h"37#include <cassert>38#include <cstdint>39#include <iterator>40 41using namespace llvm;42 43#define AARCH64_EXPAND_PSEUDO_NAME "AArch64 pseudo instruction expansion pass"44 45namespace {46 47class AArch64ExpandPseudo : public MachineFunctionPass {48public:49 const AArch64InstrInfo *TII;50 51 static char ID;52 53 AArch64ExpandPseudo() : MachineFunctionPass(ID) {}54 55 bool runOnMachineFunction(MachineFunction &Fn) override;56 57 StringRef getPassName() const override { return AARCH64_EXPAND_PSEUDO_NAME; }58 59private:60 bool expandMBB(MachineBasicBlock &MBB);61 bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,62 MachineBasicBlock::iterator &NextMBBI);63 bool expandMultiVecPseudo(MachineBasicBlock &MBB,64 MachineBasicBlock::iterator MBBI,65 TargetRegisterClass ContiguousClass,66 TargetRegisterClass StridedClass,67 unsigned ContiguousOpc, unsigned StridedOpc);68 bool expandFormTuplePseudo(MachineBasicBlock &MBB,69 MachineBasicBlock::iterator MBBI,70 MachineBasicBlock::iterator &NextMBBI,71 unsigned Size);72 bool expandMOVImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,73 unsigned BitSize);74 75 bool expand_DestructiveOp(MachineInstr &MI, MachineBasicBlock &MBB,76 MachineBasicBlock::iterator MBBI);77 bool expandCMP_SWAP(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,78 unsigned LdarOp, unsigned StlrOp, unsigned CmpOp,79 unsigned ExtendImm, unsigned ZeroReg,80 MachineBasicBlock::iterator &NextMBBI);81 bool expandCMP_SWAP_128(MachineBasicBlock &MBB,82 MachineBasicBlock::iterator MBBI,83 MachineBasicBlock::iterator &NextMBBI);84 bool expandSetTagLoop(MachineBasicBlock &MBB,85 MachineBasicBlock::iterator MBBI,86 MachineBasicBlock::iterator &NextMBBI);87 bool expandSVESpillFill(MachineBasicBlock &MBB,88 MachineBasicBlock::iterator MBBI, unsigned Opc,89 unsigned N);90 bool expandCALL_RVMARKER(MachineBasicBlock &MBB,91 MachineBasicBlock::iterator MBBI);92 bool expandCALL_BTI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);93 bool expandStoreSwiftAsyncContext(MachineBasicBlock &MBB,94 MachineBasicBlock::iterator MBBI);95 struct ConditionalBlocks {96 MachineBasicBlock &CondBB;97 MachineBasicBlock &EndBB;98 };99 ConditionalBlocks expandConditionalPseudo(MachineBasicBlock &MBB,100 MachineBasicBlock::iterator MBBI,101 DebugLoc DL,102 MachineInstrBuilder &Branch);103 MachineBasicBlock *expandRestoreZASave(MachineBasicBlock &MBB,104 MachineBasicBlock::iterator MBBI);105 MachineBasicBlock *expandCommitZASave(MachineBasicBlock &MBB,106 MachineBasicBlock::iterator MBBI);107 MachineBasicBlock *expandCondSMToggle(MachineBasicBlock &MBB,108 MachineBasicBlock::iterator MBBI);109};110 111} // end anonymous namespace112 113char AArch64ExpandPseudo::ID = 0;114 115INITIALIZE_PASS(AArch64ExpandPseudo, "aarch64-expand-pseudo",116 AARCH64_EXPAND_PSEUDO_NAME, false, false)117 118/// Transfer implicit operands on the pseudo instruction to the119/// instructions created from the expansion.120static void transferImpOps(MachineInstr &OldMI, MachineInstrBuilder &UseMI,121 MachineInstrBuilder &DefMI) {122 const MCInstrDesc &Desc = OldMI.getDesc();123 for (const MachineOperand &MO :124 llvm::drop_begin(OldMI.operands(), Desc.getNumOperands())) {125 assert(MO.isReg() && MO.getReg());126 if (MO.isUse())127 UseMI.add(MO);128 else129 DefMI.add(MO);130 }131}132 133/// Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more134/// real move-immediate instructions to synthesize the immediate.135bool AArch64ExpandPseudo::expandMOVImm(MachineBasicBlock &MBB,136 MachineBasicBlock::iterator MBBI,137 unsigned BitSize) {138 MachineInstr &MI = *MBBI;139 Register DstReg = MI.getOperand(0).getReg();140 uint64_t RenamableState =141 MI.getOperand(0).isRenamable() ? RegState::Renamable : 0;142 uint64_t Imm = MI.getOperand(1).getImm();143 144 if (DstReg == AArch64::XZR || DstReg == AArch64::WZR) {145 // Useless def, and we don't want to risk creating an invalid ORR (which146 // would really write to sp).147 MI.eraseFromParent();148 return true;149 }150 151 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;152 AArch64_IMM::expandMOVImm(Imm, BitSize, Insn);153 assert(Insn.size() != 0);154 155 SmallVector<MachineInstrBuilder, 4> MIBS;156 for (auto I = Insn.begin(), E = Insn.end(); I != E; ++I) {157 bool LastItem = std::next(I) == E;158 switch (I->Opcode)159 {160 default: llvm_unreachable("unhandled!"); break;161 162 case AArch64::ORRWri:163 case AArch64::ORRXri:164 if (I->Op1 == 0) {165 MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))166 .add(MI.getOperand(0))167 .addReg(BitSize == 32 ? AArch64::WZR : AArch64::XZR)168 .addImm(I->Op2));169 } else {170 Register DstReg = MI.getOperand(0).getReg();171 bool DstIsDead = MI.getOperand(0).isDead();172 MIBS.push_back(173 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))174 .addReg(DstReg, RegState::Define |175 getDeadRegState(DstIsDead && LastItem) |176 RenamableState)177 .addReg(DstReg)178 .addImm(I->Op2));179 }180 break;181 case AArch64::ORRWrs:182 case AArch64::ORRXrs: {183 Register DstReg = MI.getOperand(0).getReg();184 bool DstIsDead = MI.getOperand(0).isDead();185 MIBS.push_back(186 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))187 .addReg(DstReg, RegState::Define |188 getDeadRegState(DstIsDead && LastItem) |189 RenamableState)190 .addReg(DstReg)191 .addReg(DstReg)192 .addImm(I->Op2));193 } break;194 case AArch64::ANDXri:195 case AArch64::EORXri:196 if (I->Op1 == 0) {197 MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))198 .add(MI.getOperand(0))199 .addReg(BitSize == 32 ? AArch64::WZR : AArch64::XZR)200 .addImm(I->Op2));201 } else {202 Register DstReg = MI.getOperand(0).getReg();203 bool DstIsDead = MI.getOperand(0).isDead();204 MIBS.push_back(205 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))206 .addReg(DstReg, RegState::Define |207 getDeadRegState(DstIsDead && LastItem) |208 RenamableState)209 .addReg(DstReg)210 .addImm(I->Op2));211 }212 break;213 case AArch64::MOVNWi:214 case AArch64::MOVNXi:215 case AArch64::MOVZWi:216 case AArch64::MOVZXi: {217 bool DstIsDead = MI.getOperand(0).isDead();218 MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))219 .addReg(DstReg, RegState::Define |220 getDeadRegState(DstIsDead && LastItem) |221 RenamableState)222 .addImm(I->Op1)223 .addImm(I->Op2));224 } break;225 case AArch64::MOVKWi:226 case AArch64::MOVKXi: {227 Register DstReg = MI.getOperand(0).getReg();228 bool DstIsDead = MI.getOperand(0).isDead();229 MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))230 .addReg(DstReg,231 RegState::Define |232 getDeadRegState(DstIsDead && LastItem) |233 RenamableState)234 .addReg(DstReg)235 .addImm(I->Op1)236 .addImm(I->Op2));237 } break;238 }239 }240 transferImpOps(MI, MIBS.front(), MIBS.back());241 MI.eraseFromParent();242 return true;243}244 245bool AArch64ExpandPseudo::expandCMP_SWAP(246 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned LdarOp,247 unsigned StlrOp, unsigned CmpOp, unsigned ExtendImm, unsigned ZeroReg,248 MachineBasicBlock::iterator &NextMBBI) {249 MachineInstr &MI = *MBBI;250 MIMetadata MIMD(MI);251 const MachineOperand &Dest = MI.getOperand(0);252 Register StatusReg = MI.getOperand(1).getReg();253 bool StatusDead = MI.getOperand(1).isDead();254 // Duplicating undef operands into 2 instructions does not guarantee the same255 // value on both; However undef should be replaced by xzr anyway.256 assert(!MI.getOperand(2).isUndef() && "cannot handle undef");257 Register AddrReg = MI.getOperand(2).getReg();258 Register DesiredReg = MI.getOperand(3).getReg();259 Register NewReg = MI.getOperand(4).getReg();260 261 MachineFunction *MF = MBB.getParent();262 auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());263 auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());264 auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());265 266 MF->insert(++MBB.getIterator(), LoadCmpBB);267 MF->insert(++LoadCmpBB->getIterator(), StoreBB);268 MF->insert(++StoreBB->getIterator(), DoneBB);269 270 // .Lloadcmp:271 // mov wStatus, 0272 // ldaxr xDest, [xAddr]273 // cmp xDest, xDesired274 // b.ne .Ldone275 if (!StatusDead)276 BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::MOVZWi), StatusReg)277 .addImm(0).addImm(0);278 BuildMI(LoadCmpBB, MIMD, TII->get(LdarOp), Dest.getReg())279 .addReg(AddrReg);280 BuildMI(LoadCmpBB, MIMD, TII->get(CmpOp), ZeroReg)281 .addReg(Dest.getReg(), getKillRegState(Dest.isDead()))282 .addReg(DesiredReg)283 .addImm(ExtendImm);284 BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::Bcc))285 .addImm(AArch64CC::NE)286 .addMBB(DoneBB)287 .addReg(AArch64::NZCV, RegState::Implicit | RegState::Kill);288 LoadCmpBB->addSuccessor(DoneBB);289 LoadCmpBB->addSuccessor(StoreBB);290 291 // .Lstore:292 // stlxr wStatus, xNew, [xAddr]293 // cbnz wStatus, .Lloadcmp294 BuildMI(StoreBB, MIMD, TII->get(StlrOp), StatusReg)295 .addReg(NewReg)296 .addReg(AddrReg);297 BuildMI(StoreBB, MIMD, TII->get(AArch64::CBNZW))298 .addReg(StatusReg, getKillRegState(StatusDead))299 .addMBB(LoadCmpBB);300 StoreBB->addSuccessor(LoadCmpBB);301 StoreBB->addSuccessor(DoneBB);302 303 DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());304 DoneBB->transferSuccessors(&MBB);305 306 MBB.addSuccessor(LoadCmpBB);307 308 NextMBBI = MBB.end();309 MI.eraseFromParent();310 311 // Recompute livein lists.312 LivePhysRegs LiveRegs;313 computeAndAddLiveIns(LiveRegs, *DoneBB);314 computeAndAddLiveIns(LiveRegs, *StoreBB);315 computeAndAddLiveIns(LiveRegs, *LoadCmpBB);316 // Do an extra pass around the loop to get loop carried registers right.317 StoreBB->clearLiveIns();318 computeAndAddLiveIns(LiveRegs, *StoreBB);319 LoadCmpBB->clearLiveIns();320 computeAndAddLiveIns(LiveRegs, *LoadCmpBB);321 322 return true;323}324 325bool AArch64ExpandPseudo::expandCMP_SWAP_128(326 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,327 MachineBasicBlock::iterator &NextMBBI) {328 MachineInstr &MI = *MBBI;329 MIMetadata MIMD(MI);330 MachineOperand &DestLo = MI.getOperand(0);331 MachineOperand &DestHi = MI.getOperand(1);332 Register StatusReg = MI.getOperand(2).getReg();333 bool StatusDead = MI.getOperand(2).isDead();334 // Duplicating undef operands into 2 instructions does not guarantee the same335 // value on both; However undef should be replaced by xzr anyway.336 assert(!MI.getOperand(3).isUndef() && "cannot handle undef");337 Register AddrReg = MI.getOperand(3).getReg();338 Register DesiredLoReg = MI.getOperand(4).getReg();339 Register DesiredHiReg = MI.getOperand(5).getReg();340 Register NewLoReg = MI.getOperand(6).getReg();341 Register NewHiReg = MI.getOperand(7).getReg();342 343 unsigned LdxpOp, StxpOp;344 345 switch (MI.getOpcode()) {346 case AArch64::CMP_SWAP_128_MONOTONIC:347 LdxpOp = AArch64::LDXPX;348 StxpOp = AArch64::STXPX;349 break;350 case AArch64::CMP_SWAP_128_RELEASE:351 LdxpOp = AArch64::LDXPX;352 StxpOp = AArch64::STLXPX;353 break;354 case AArch64::CMP_SWAP_128_ACQUIRE:355 LdxpOp = AArch64::LDAXPX;356 StxpOp = AArch64::STXPX;357 break;358 case AArch64::CMP_SWAP_128:359 LdxpOp = AArch64::LDAXPX;360 StxpOp = AArch64::STLXPX;361 break;362 default:363 llvm_unreachable("Unexpected opcode");364 }365 366 MachineFunction *MF = MBB.getParent();367 auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());368 auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());369 auto FailBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());370 auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());371 372 MF->insert(++MBB.getIterator(), LoadCmpBB);373 MF->insert(++LoadCmpBB->getIterator(), StoreBB);374 MF->insert(++StoreBB->getIterator(), FailBB);375 MF->insert(++FailBB->getIterator(), DoneBB);376 377 // .Lloadcmp:378 // ldaxp xDestLo, xDestHi, [xAddr]379 // cmp xDestLo, xDesiredLo380 // sbcs xDestHi, xDesiredHi381 // b.ne .Ldone382 BuildMI(LoadCmpBB, MIMD, TII->get(LdxpOp))383 .addReg(DestLo.getReg(), RegState::Define)384 .addReg(DestHi.getReg(), RegState::Define)385 .addReg(AddrReg);386 BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::SUBSXrs), AArch64::XZR)387 .addReg(DestLo.getReg(), getKillRegState(DestLo.isDead()))388 .addReg(DesiredLoReg)389 .addImm(0);390 BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::CSINCWr), StatusReg)391 .addUse(AArch64::WZR)392 .addUse(AArch64::WZR)393 .addImm(AArch64CC::EQ);394 BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::SUBSXrs), AArch64::XZR)395 .addReg(DestHi.getReg(), getKillRegState(DestHi.isDead()))396 .addReg(DesiredHiReg)397 .addImm(0);398 BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::CSINCWr), StatusReg)399 .addUse(StatusReg, RegState::Kill)400 .addUse(StatusReg, RegState::Kill)401 .addImm(AArch64CC::EQ);402 BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::CBNZW))403 .addUse(StatusReg, getKillRegState(StatusDead))404 .addMBB(FailBB);405 LoadCmpBB->addSuccessor(FailBB);406 LoadCmpBB->addSuccessor(StoreBB);407 408 // .Lstore:409 // stlxp wStatus, xNewLo, xNewHi, [xAddr]410 // cbnz wStatus, .Lloadcmp411 BuildMI(StoreBB, MIMD, TII->get(StxpOp), StatusReg)412 .addReg(NewLoReg)413 .addReg(NewHiReg)414 .addReg(AddrReg);415 BuildMI(StoreBB, MIMD, TII->get(AArch64::CBNZW))416 .addReg(StatusReg, getKillRegState(StatusDead))417 .addMBB(LoadCmpBB);418 BuildMI(StoreBB, MIMD, TII->get(AArch64::B)).addMBB(DoneBB);419 StoreBB->addSuccessor(LoadCmpBB);420 StoreBB->addSuccessor(DoneBB);421 422 // .Lfail:423 // stlxp wStatus, xDestLo, xDestHi, [xAddr]424 // cbnz wStatus, .Lloadcmp425 BuildMI(FailBB, MIMD, TII->get(StxpOp), StatusReg)426 .addReg(DestLo.getReg())427 .addReg(DestHi.getReg())428 .addReg(AddrReg);429 BuildMI(FailBB, MIMD, TII->get(AArch64::CBNZW))430 .addReg(StatusReg, getKillRegState(StatusDead))431 .addMBB(LoadCmpBB);432 FailBB->addSuccessor(LoadCmpBB);433 FailBB->addSuccessor(DoneBB);434 435 DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());436 DoneBB->transferSuccessors(&MBB);437 438 MBB.addSuccessor(LoadCmpBB);439 440 NextMBBI = MBB.end();441 MI.eraseFromParent();442 443 // Recompute liveness bottom up.444 LivePhysRegs LiveRegs;445 computeAndAddLiveIns(LiveRegs, *DoneBB);446 computeAndAddLiveIns(LiveRegs, *FailBB);447 computeAndAddLiveIns(LiveRegs, *StoreBB);448 computeAndAddLiveIns(LiveRegs, *LoadCmpBB);449 450 // Do an extra pass in the loop to get the loop carried dependencies right.451 FailBB->clearLiveIns();452 computeAndAddLiveIns(LiveRegs, *FailBB);453 StoreBB->clearLiveIns();454 computeAndAddLiveIns(LiveRegs, *StoreBB);455 LoadCmpBB->clearLiveIns();456 computeAndAddLiveIns(LiveRegs, *LoadCmpBB);457 458 return true;459}460 461/// \brief Expand Pseudos to Instructions with destructive operands.462///463/// This mechanism uses MOVPRFX instructions for zeroing the false lanes464/// or for fixing relaxed register allocation conditions to comply with465/// the instructions register constraints. The latter case may be cheaper466/// than setting the register constraints in the register allocator,467/// since that will insert regular MOV instructions rather than MOVPRFX.468///469/// Example (after register allocation):470///471/// FSUB_ZPZZ_ZERO_B Z0, Pg, Z1, Z0472///473/// * The Pseudo FSUB_ZPZZ_ZERO_B maps to FSUB_ZPmZ_B.474/// * We cannot map directly to FSUB_ZPmZ_B because the register475/// constraints of the instruction are not met.476/// * Also the _ZERO specifies the false lanes need to be zeroed.477///478/// We first try to see if the destructive operand == result operand,479/// if not, we try to swap the operands, e.g.480///481/// FSUB_ZPmZ_B Z0, Pg/m, Z0, Z1482///483/// But because FSUB_ZPmZ is not commutative, this is semantically484/// different, so we need a reverse instruction:485///486/// FSUBR_ZPmZ_B Z0, Pg/m, Z0, Z1487///488/// Then we implement the zeroing of the false lanes of Z0 by adding489/// a zeroing MOVPRFX instruction:490///491/// MOVPRFX_ZPzZ_B Z0, Pg/z, Z0492/// FSUBR_ZPmZ_B Z0, Pg/m, Z0, Z1493///494/// Note that this can only be done for _ZERO or _UNDEF variants where495/// we can guarantee the false lanes to be zeroed (by implementing this)496/// or that they are undef (don't care / not used), otherwise the497/// swapping of operands is illegal because the operation is not498/// (or cannot be emulated to be) fully commutative.499bool AArch64ExpandPseudo::expand_DestructiveOp(500 MachineInstr &MI,501 MachineBasicBlock &MBB,502 MachineBasicBlock::iterator MBBI) {503 unsigned Opcode = AArch64::getSVEPseudoMap(MI.getOpcode());504 uint64_t DType = TII->get(Opcode).TSFlags & AArch64::DestructiveInstTypeMask;505 uint64_t FalseLanes = MI.getDesc().TSFlags & AArch64::FalseLanesMask;506 bool FalseZero = FalseLanes == AArch64::FalseLanesZero;507 Register DstReg = MI.getOperand(0).getReg();508 bool DstIsDead = MI.getOperand(0).isDead();509 bool UseRev = false;510 unsigned PredIdx, DOPIdx, SrcIdx, Src2Idx;511 512 switch (DType) {513 case AArch64::DestructiveBinaryComm:514 case AArch64::DestructiveBinaryCommWithRev:515 if (DstReg == MI.getOperand(3).getReg()) {516 // FSUB Zd, Pg, Zs1, Zd ==> FSUBR Zd, Pg/m, Zd, Zs1517 std::tie(PredIdx, DOPIdx, SrcIdx) = std::make_tuple(1, 3, 2);518 UseRev = true;519 break;520 }521 [[fallthrough]];522 case AArch64::DestructiveBinary:523 case AArch64::DestructiveBinaryImm:524 std::tie(PredIdx, DOPIdx, SrcIdx) = std::make_tuple(1, 2, 3);525 break;526 case AArch64::DestructiveUnaryPassthru:527 std::tie(PredIdx, DOPIdx, SrcIdx) = std::make_tuple(2, 3, 3);528 break;529 case AArch64::DestructiveTernaryCommWithRev:530 std::tie(PredIdx, DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 2, 3, 4);531 if (DstReg == MI.getOperand(3).getReg()) {532 // FMLA Zd, Pg, Za, Zd, Zm ==> FMAD Zdn, Pg, Zm, Za533 std::tie(PredIdx, DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 3, 4, 2);534 UseRev = true;535 } else if (DstReg == MI.getOperand(4).getReg()) {536 // FMLA Zd, Pg, Za, Zm, Zd ==> FMAD Zdn, Pg, Zm, Za537 std::tie(PredIdx, DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 4, 3, 2);538 UseRev = true;539 }540 break;541 case AArch64::Destructive2xRegImmUnpred:542 // EXT_ZZI_CONSTRUCTIVE Zd, Zs, Imm543 // ==> MOVPRFX Zd Zs; EXT_ZZI Zd, Zd, Zs, Imm544 std::tie(DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 1, 2);545 break;546 default:547 llvm_unreachable("Unsupported Destructive Operand type");548 }549 550 // MOVPRFX can only be used if the destination operand551 // is the destructive operand, not as any other operand,552 // so the Destructive Operand must be unique.553 bool DOPRegIsUnique = false;554 switch (DType) {555 case AArch64::DestructiveBinary:556 DOPRegIsUnique = DstReg != MI.getOperand(SrcIdx).getReg();557 break;558 case AArch64::DestructiveBinaryComm:559 case AArch64::DestructiveBinaryCommWithRev:560 DOPRegIsUnique =561 DstReg != MI.getOperand(DOPIdx).getReg() ||562 MI.getOperand(DOPIdx).getReg() != MI.getOperand(SrcIdx).getReg();563 break;564 case AArch64::DestructiveUnaryPassthru:565 case AArch64::DestructiveBinaryImm:566 case AArch64::Destructive2xRegImmUnpred:567 DOPRegIsUnique = true;568 break;569 case AArch64::DestructiveTernaryCommWithRev:570 DOPRegIsUnique =571 DstReg != MI.getOperand(DOPIdx).getReg() ||572 (MI.getOperand(DOPIdx).getReg() != MI.getOperand(SrcIdx).getReg() &&573 MI.getOperand(DOPIdx).getReg() != MI.getOperand(Src2Idx).getReg());574 break;575 }576 577 // Resolve the reverse opcode578 if (UseRev) {579 int NewOpcode;580 // e.g. DIV -> DIVR581 if ((NewOpcode = AArch64::getSVERevInstr(Opcode)) != -1)582 Opcode = NewOpcode;583 // e.g. DIVR -> DIV584 else if ((NewOpcode = AArch64::getSVENonRevInstr(Opcode)) != -1)585 Opcode = NewOpcode;586 }587 588 // Get the right MOVPRFX589 uint64_t ElementSize = TII->getElementSizeForOpcode(Opcode);590 unsigned MovPrfx, LSLZero, MovPrfxZero;591 switch (ElementSize) {592 case AArch64::ElementSizeNone:593 case AArch64::ElementSizeB:594 MovPrfx = AArch64::MOVPRFX_ZZ;595 LSLZero = AArch64::LSL_ZPmI_B;596 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_B;597 break;598 case AArch64::ElementSizeH:599 MovPrfx = AArch64::MOVPRFX_ZZ;600 LSLZero = AArch64::LSL_ZPmI_H;601 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_H;602 break;603 case AArch64::ElementSizeS:604 MovPrfx = AArch64::MOVPRFX_ZZ;605 LSLZero = AArch64::LSL_ZPmI_S;606 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_S;607 break;608 case AArch64::ElementSizeD:609 MovPrfx = AArch64::MOVPRFX_ZZ;610 LSLZero = AArch64::LSL_ZPmI_D;611 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_D;612 break;613 default:614 llvm_unreachable("Unsupported ElementSize");615 }616 617 // Preserve undef state until DOP's reg is defined.618 unsigned DOPRegState = MI.getOperand(DOPIdx).isUndef() ? RegState::Undef : 0;619 620 //621 // Create the destructive operation (if required)622 //623 MachineInstrBuilder PRFX, DOP;624 if (FalseZero) {625 // If we cannot prefix the requested instruction we'll instead emit a626 // prefixed_zeroing_mov for DestructiveBinary.627 assert((DOPRegIsUnique || DType == AArch64::DestructiveBinary ||628 DType == AArch64::DestructiveBinaryComm ||629 DType == AArch64::DestructiveBinaryCommWithRev) &&630 "The destructive operand should be unique");631 assert(ElementSize != AArch64::ElementSizeNone &&632 "This instruction is unpredicated");633 634 // Merge source operand into destination register635 PRFX = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(MovPrfxZero))636 .addReg(DstReg, RegState::Define)637 .addReg(MI.getOperand(PredIdx).getReg())638 .addReg(MI.getOperand(DOPIdx).getReg(), DOPRegState);639 640 // After the movprfx, the destructive operand is same as Dst641 DOPIdx = 0;642 DOPRegState = 0;643 644 // Create the additional LSL to zero the lanes when the DstReg is not645 // unique. Zeros the lanes in z0 that aren't active in p0 with sequence646 // movprfx z0.b, p0/z, z0.b; lsl z0.b, p0/m, z0.b, #0;647 if ((DType == AArch64::DestructiveBinary ||648 DType == AArch64::DestructiveBinaryComm ||649 DType == AArch64::DestructiveBinaryCommWithRev) &&650 !DOPRegIsUnique) {651 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LSLZero))652 .addReg(DstReg, RegState::Define)653 .add(MI.getOperand(PredIdx))654 .addReg(DstReg)655 .addImm(0);656 }657 } else if (DstReg != MI.getOperand(DOPIdx).getReg()) {658 assert(DOPRegIsUnique && "The destructive operand should be unique");659 PRFX = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(MovPrfx))660 .addReg(DstReg, RegState::Define)661 .addReg(MI.getOperand(DOPIdx).getReg(), DOPRegState);662 DOPIdx = 0;663 DOPRegState = 0;664 }665 666 //667 // Create the destructive operation668 //669 DOP = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opcode))670 .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead));671 DOPRegState = DOPRegState | RegState::Kill;672 673 switch (DType) {674 case AArch64::DestructiveUnaryPassthru:675 DOP.addReg(MI.getOperand(DOPIdx).getReg(), DOPRegState)676 .add(MI.getOperand(PredIdx))677 .add(MI.getOperand(SrcIdx));678 break;679 case AArch64::DestructiveBinary:680 case AArch64::DestructiveBinaryImm:681 case AArch64::DestructiveBinaryComm:682 case AArch64::DestructiveBinaryCommWithRev:683 DOP.add(MI.getOperand(PredIdx))684 .addReg(MI.getOperand(DOPIdx).getReg(), DOPRegState)685 .add(MI.getOperand(SrcIdx));686 break;687 case AArch64::DestructiveTernaryCommWithRev:688 DOP.add(MI.getOperand(PredIdx))689 .addReg(MI.getOperand(DOPIdx).getReg(), DOPRegState)690 .add(MI.getOperand(SrcIdx))691 .add(MI.getOperand(Src2Idx));692 break;693 case AArch64::Destructive2xRegImmUnpred:694 DOP.addReg(MI.getOperand(DOPIdx).getReg(), DOPRegState)695 .add(MI.getOperand(SrcIdx))696 .add(MI.getOperand(Src2Idx));697 break;698 }699 700 if (PRFX) {701 transferImpOps(MI, PRFX, DOP);702 finalizeBundle(MBB, PRFX->getIterator(), MBBI->getIterator());703 } else704 transferImpOps(MI, DOP, DOP);705 706 MI.eraseFromParent();707 return true;708}709 710bool AArch64ExpandPseudo::expandSetTagLoop(711 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,712 MachineBasicBlock::iterator &NextMBBI) {713 MachineInstr &MI = *MBBI;714 DebugLoc DL = MI.getDebugLoc();715 Register SizeReg = MI.getOperand(0).getReg();716 Register AddressReg = MI.getOperand(1).getReg();717 718 MachineFunction *MF = MBB.getParent();719 720 bool ZeroData = MI.getOpcode() == AArch64::STZGloop_wback;721 const unsigned OpCode1 =722 ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex;723 const unsigned OpCode2 =724 ZeroData ? AArch64::STZ2GPostIndex : AArch64::ST2GPostIndex;725 726 unsigned Size = MI.getOperand(2).getImm();727 assert(Size > 0 && Size % 16 == 0);728 if (Size % (16 * 2) != 0) {729 BuildMI(MBB, MBBI, DL, TII->get(OpCode1), AddressReg)730 .addReg(AddressReg)731 .addReg(AddressReg)732 .addImm(1);733 Size -= 16;734 }735 MachineBasicBlock::iterator I =736 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), SizeReg)737 .addImm(Size);738 expandMOVImm(MBB, I, 64);739 740 auto LoopBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());741 auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());742 743 MF->insert(++MBB.getIterator(), LoopBB);744 MF->insert(++LoopBB->getIterator(), DoneBB);745 746 BuildMI(LoopBB, DL, TII->get(OpCode2))747 .addDef(AddressReg)748 .addReg(AddressReg)749 .addReg(AddressReg)750 .addImm(2)751 .cloneMemRefs(MI)752 .setMIFlags(MI.getFlags());753 BuildMI(LoopBB, DL, TII->get(AArch64::SUBSXri))754 .addDef(SizeReg)755 .addReg(SizeReg)756 .addImm(16 * 2)757 .addImm(0);758 BuildMI(LoopBB, DL, TII->get(AArch64::Bcc))759 .addImm(AArch64CC::NE)760 .addMBB(LoopBB)761 .addReg(AArch64::NZCV, RegState::Implicit | RegState::Kill);762 763 LoopBB->addSuccessor(LoopBB);764 LoopBB->addSuccessor(DoneBB);765 766 DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());767 DoneBB->transferSuccessors(&MBB);768 769 MBB.addSuccessor(LoopBB);770 771 NextMBBI = MBB.end();772 MI.eraseFromParent();773 // Recompute liveness bottom up.774 LivePhysRegs LiveRegs;775 computeAndAddLiveIns(LiveRegs, *DoneBB);776 computeAndAddLiveIns(LiveRegs, *LoopBB);777 // Do an extra pass in the loop to get the loop carried dependencies right.778 // FIXME: is this necessary?779 LoopBB->clearLiveIns();780 computeAndAddLiveIns(LiveRegs, *LoopBB);781 DoneBB->clearLiveIns();782 computeAndAddLiveIns(LiveRegs, *DoneBB);783 784 return true;785}786 787bool AArch64ExpandPseudo::expandSVESpillFill(MachineBasicBlock &MBB,788 MachineBasicBlock::iterator MBBI,789 unsigned Opc, unsigned N) {790 assert((Opc == AArch64::LDR_ZXI || Opc == AArch64::STR_ZXI ||791 Opc == AArch64::LDR_PXI || Opc == AArch64::STR_PXI) &&792 "Unexpected opcode");793 unsigned RState = (Opc == AArch64::LDR_ZXI || Opc == AArch64::LDR_PXI)794 ? RegState::Define795 : 0;796 unsigned sub0 = (Opc == AArch64::LDR_ZXI || Opc == AArch64::STR_ZXI)797 ? AArch64::zsub0798 : AArch64::psub0;799 const TargetRegisterInfo *TRI =800 MBB.getParent()->getSubtarget().getRegisterInfo();801 MachineInstr &MI = *MBBI;802 for (unsigned Offset = 0; Offset < N; ++Offset) {803 int ImmOffset = MI.getOperand(2).getImm() + Offset;804 bool Kill = (Offset + 1 == N) ? MI.getOperand(1).isKill() : false;805 assert(ImmOffset >= -256 && ImmOffset < 256 &&806 "Immediate spill offset out of range");807 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc))808 .addReg(TRI->getSubReg(MI.getOperand(0).getReg(), sub0 + Offset),809 RState)810 .addReg(MI.getOperand(1).getReg(), getKillRegState(Kill))811 .addImm(ImmOffset);812 }813 MI.eraseFromParent();814 return true;815}816 817// Create a call with the passed opcode and explicit operands, copying over all818// the implicit operands from *MBBI, starting at the regmask.819static MachineInstr *createCallWithOps(MachineBasicBlock &MBB,820 MachineBasicBlock::iterator MBBI,821 const AArch64InstrInfo *TII,822 unsigned Opcode,823 ArrayRef<MachineOperand> ExplicitOps,824 unsigned RegMaskStartIdx) {825 // Build the MI, with explicit operands first (including the call target).826 MachineInstr *Call = BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(Opcode))827 .add(ExplicitOps)828 .getInstr();829 830 // Register arguments are added during ISel, but cannot be added as explicit831 // operands of the branch as it expects to be B <target> which is only one832 // operand. Instead they are implicit operands used by the branch.833 while (!MBBI->getOperand(RegMaskStartIdx).isRegMask()) {834 const MachineOperand &MOP = MBBI->getOperand(RegMaskStartIdx);835 assert(MOP.isReg() && "can only add register operands");836 Call->addOperand(MachineOperand::CreateReg(837 MOP.getReg(), /*Def=*/false, /*Implicit=*/true, /*isKill=*/false,838 /*isDead=*/false, /*isUndef=*/MOP.isUndef()));839 RegMaskStartIdx++;840 }841 for (const MachineOperand &MO :842 llvm::drop_begin(MBBI->operands(), RegMaskStartIdx))843 Call->addOperand(MO);844 845 return Call;846}847 848// Create a call to CallTarget, copying over all the operands from *MBBI,849// starting at the regmask.850static MachineInstr *createCall(MachineBasicBlock &MBB,851 MachineBasicBlock::iterator MBBI,852 const AArch64InstrInfo *TII,853 MachineOperand &CallTarget,854 unsigned RegMaskStartIdx) {855 unsigned Opc = CallTarget.isGlobal() ? AArch64::BL : AArch64::BLR;856 857 assert((CallTarget.isGlobal() || CallTarget.isReg()) &&858 "invalid operand for regular call");859 return createCallWithOps(MBB, MBBI, TII, Opc, CallTarget, RegMaskStartIdx);860}861 862bool AArch64ExpandPseudo::expandCALL_RVMARKER(863 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {864 // Expand CALL_RVMARKER pseudo to:865 // - a branch to the call target, followed by866 // - the special `mov x29, x29` marker, if necessary, and867 // - another branch, to the runtime function868 // Mark the sequence as bundle, to avoid passes moving other code in between.869 MachineInstr &MI = *MBBI;870 MachineOperand &RVTarget = MI.getOperand(0);871 bool DoEmitMarker = MI.getOperand(1).getImm();872 assert(RVTarget.isGlobal() && "invalid operand for attached call");873 874 MachineInstr *OriginalCall = nullptr;875 876 if (MI.getOpcode() == AArch64::BLRA_RVMARKER) {877 // ptrauth call.878 const MachineOperand &CallTarget = MI.getOperand(2);879 const MachineOperand &Key = MI.getOperand(3);880 const MachineOperand &IntDisc = MI.getOperand(4);881 const MachineOperand &AddrDisc = MI.getOperand(5);882 883 assert((Key.getImm() == AArch64PACKey::IA ||884 Key.getImm() == AArch64PACKey::IB) &&885 "Invalid auth call key");886 887 MachineOperand Ops[] = {CallTarget, Key, IntDisc, AddrDisc};888 889 OriginalCall = createCallWithOps(MBB, MBBI, TII, AArch64::BLRA, Ops,890 /*RegMaskStartIdx=*/6);891 } else {892 assert(MI.getOpcode() == AArch64::BLR_RVMARKER && "unknown rvmarker MI");893 OriginalCall = createCall(MBB, MBBI, TII, MI.getOperand(2),894 // Regmask starts after the RV and call targets.895 /*RegMaskStartIdx=*/3);896 }897 898 if (DoEmitMarker)899 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORRXrs))900 .addReg(AArch64::FP, RegState::Define)901 .addReg(AArch64::XZR)902 .addReg(AArch64::FP)903 .addImm(0);904 905 auto *RVCall = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::BL))906 .add(RVTarget)907 .getInstr();908 909 if (MI.shouldUpdateAdditionalCallInfo())910 MBB.getParent()->moveAdditionalCallInfo(&MI, OriginalCall);911 912 MI.eraseFromParent();913 finalizeBundle(MBB, OriginalCall->getIterator(),914 std::next(RVCall->getIterator()));915 return true;916}917 918bool AArch64ExpandPseudo::expandCALL_BTI(MachineBasicBlock &MBB,919 MachineBasicBlock::iterator MBBI) {920 // Expand CALL_BTI pseudo to:921 // - a branch to the call target922 // - a BTI instruction923 // Mark the sequence as a bundle, to avoid passes moving other code in924 // between.925 MachineInstr &MI = *MBBI;926 MachineInstr *Call = createCall(MBB, MBBI, TII, MI.getOperand(0),927 // Regmask starts after the call target.928 /*RegMaskStartIdx=*/1);929 930 Call->setCFIType(*MBB.getParent(), MI.getCFIType());931 932 MachineInstr *BTI =933 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::HINT))934 // BTI J so that setjmp can to BR to this.935 .addImm(36)936 .getInstr();937 938 if (MI.shouldUpdateAdditionalCallInfo())939 MBB.getParent()->moveAdditionalCallInfo(&MI, Call);940 941 MI.eraseFromParent();942 finalizeBundle(MBB, Call->getIterator(), std::next(BTI->getIterator()));943 return true;944}945 946bool AArch64ExpandPseudo::expandStoreSwiftAsyncContext(947 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {948 Register CtxReg = MBBI->getOperand(0).getReg();949 Register BaseReg = MBBI->getOperand(1).getReg();950 int Offset = MBBI->getOperand(2).getImm();951 DebugLoc DL(MBBI->getDebugLoc());952 auto &STI = MBB.getParent()->getSubtarget<AArch64Subtarget>();953 954 if (STI.getTargetTriple().getArchName() != "arm64e") {955 BuildMI(MBB, MBBI, DL, TII->get(AArch64::STRXui))956 .addUse(CtxReg)957 .addUse(BaseReg)958 .addImm(Offset / 8)959 .setMIFlag(MachineInstr::FrameSetup);960 MBBI->eraseFromParent();961 return true;962 }963 964 // We need to sign the context in an address-discriminated way. 0xc31a is a965 // fixed random value, chosen as part of the ABI.966 // add x16, xBase, #Offset967 // movk x16, #0xc31a, lsl #48968 // mov x17, x22/xzr969 // pacdb x17, x16970 // str x17, [xBase, #Offset]971 unsigned Opc = Offset >= 0 ? AArch64::ADDXri : AArch64::SUBXri;972 BuildMI(MBB, MBBI, DL, TII->get(Opc), AArch64::X16)973 .addUse(BaseReg)974 .addImm(abs(Offset))975 .addImm(0)976 .setMIFlag(MachineInstr::FrameSetup);977 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X16)978 .addUse(AArch64::X16)979 .addImm(0xc31a)980 .addImm(48)981 .setMIFlag(MachineInstr::FrameSetup);982 // We're not allowed to clobber X22 (and couldn't clobber XZR if we tried), so983 // move it somewhere before signing.984 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), AArch64::X17)985 .addUse(AArch64::XZR)986 .addUse(CtxReg)987 .addImm(0)988 .setMIFlag(MachineInstr::FrameSetup);989 BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACDB), AArch64::X17)990 .addUse(AArch64::X17)991 .addUse(AArch64::X16)992 .setMIFlag(MachineInstr::FrameSetup);993 BuildMI(MBB, MBBI, DL, TII->get(AArch64::STRXui))994 .addUse(AArch64::X17)995 .addUse(BaseReg)996 .addImm(Offset / 8)997 .setMIFlag(MachineInstr::FrameSetup);998 999 MBBI->eraseFromParent();1000 return true;1001}1002 1003AArch64ExpandPseudo::ConditionalBlocks1004AArch64ExpandPseudo::expandConditionalPseudo(MachineBasicBlock &MBB,1005 MachineBasicBlock::iterator MBBI,1006 DebugLoc DL,1007 MachineInstrBuilder &Branch) {1008 assert((std::next(MBBI) != MBB.end() ||1009 MBB.successors().begin() != MBB.successors().end()) &&1010 "Unexpected unreachable in block");1011 1012 // Split MBB and create two new blocks:1013 // - MBB now contains all instructions before the conditional pseudo.1014 // - CondBB contains the conditional pseudo instruction only.1015 // - EndBB contains all instructions after the conditional pseudo.1016 MachineInstr &PrevMI = *std::prev(MBBI);1017 MachineBasicBlock *CondBB = MBB.splitAt(PrevMI, /*UpdateLiveIns*/ true);1018 MachineBasicBlock *EndBB =1019 std::next(MBBI) == CondBB->end()1020 ? *CondBB->successors().begin()1021 : CondBB->splitAt(*MBBI, /*UpdateLiveIns*/ true);1022 1023 // Add the SMBB label to the branch instruction & create a branch to EndBB.1024 Branch.addMBB(CondBB);1025 BuildMI(&MBB, DL, TII->get(AArch64::B))1026 .addMBB(EndBB);1027 MBB.addSuccessor(EndBB);1028 1029 // Create branch from CondBB to EndBB. Users of this helper should insert new1030 // instructions at CondBB.back() -- i.e. before the branch.1031 BuildMI(CondBB, DL, TII->get(AArch64::B)).addMBB(EndBB);1032 return {*CondBB, *EndBB};1033}1034 1035MachineBasicBlock *1036AArch64ExpandPseudo::expandRestoreZASave(MachineBasicBlock &MBB,1037 MachineBasicBlock::iterator MBBI) {1038 MachineInstr &MI = *MBBI;1039 DebugLoc DL = MI.getDebugLoc();1040 1041 // Compare TPIDR2_EL0 against 0. Restore ZA if TPIDR2_EL0 is zero.1042 MachineInstrBuilder Branch =1043 BuildMI(MBB, MBBI, DL, TII->get(AArch64::CBZX)).add(MI.getOperand(0));1044 1045 auto [CondBB, EndBB] = expandConditionalPseudo(MBB, MBBI, DL, Branch);1046 // Replace the pseudo with a call (BL).1047 MachineInstrBuilder MIB =1048 BuildMI(CondBB, CondBB.back(), DL, TII->get(AArch64::BL));1049 // Copy operands (mainly the regmask) from the pseudo.1050 for (unsigned I = 2; I < MI.getNumOperands(); ++I)1051 MIB.add(MI.getOperand(I));1052 // Mark the TPIDR2 block pointer (X0) as an implicit use.1053 MIB.addReg(MI.getOperand(1).getReg(), RegState::Implicit);1054 1055 MI.eraseFromParent();1056 return &EndBB;1057}1058 1059static constexpr unsigned ZERO_ALL_ZA_MASK = 0b11111111;1060 1061MachineBasicBlock *1062AArch64ExpandPseudo::expandCommitZASave(MachineBasicBlock &MBB,1063 MachineBasicBlock::iterator MBBI) {1064 MachineInstr &MI = *MBBI;1065 DebugLoc DL = MI.getDebugLoc();1066 [[maybe_unused]] auto *RI = MBB.getParent()->getSubtarget().getRegisterInfo();1067 1068 // Compare TPIDR2_EL0 against 0. Commit ZA if TPIDR2_EL0 is non-zero.1069 MachineInstrBuilder Branch =1070 BuildMI(MBB, MBBI, DL, TII->get(AArch64::CBNZX)).add(MI.getOperand(0));1071 1072 auto [CondBB, EndBB] = expandConditionalPseudo(MBB, MBBI, DL, Branch);1073 // Replace the pseudo with a call (BL).1074 MachineInstrBuilder MIB =1075 BuildMI(CondBB, CondBB.back(), DL, TII->get(AArch64::BL));1076 // Copy operands (mainly the regmask) from the pseudo.1077 for (unsigned I = 3; I < MI.getNumOperands(); ++I)1078 MIB.add(MI.getOperand(I));1079 // Clear TPIDR2_EL0.1080 BuildMI(CondBB, CondBB.back(), DL, TII->get(AArch64::MSR))1081 .addImm(AArch64SysReg::TPIDR2_EL0)1082 .addReg(AArch64::XZR);1083 bool ZeroZA = MI.getOperand(1).getImm() != 0;1084 bool ZeroZT0 = MI.getOperand(2).getImm() != 0;1085 if (ZeroZA) {1086 assert(MI.definesRegister(AArch64::ZAB0, RI) && "should define ZA!");1087 BuildMI(CondBB, CondBB.back(), DL, TII->get(AArch64::ZERO_M))1088 .addImm(ZERO_ALL_ZA_MASK)1089 .addDef(AArch64::ZAB0, RegState::ImplicitDefine);1090 }1091 if (ZeroZT0) {1092 assert(MI.definesRegister(AArch64::ZT0, RI) && "should define ZT0!");1093 BuildMI(CondBB, CondBB.back(), DL, TII->get(AArch64::ZERO_T))1094 .addDef(AArch64::ZT0);1095 }1096 1097 MI.eraseFromParent();1098 return &EndBB;1099}1100 1101MachineBasicBlock *1102AArch64ExpandPseudo::expandCondSMToggle(MachineBasicBlock &MBB,1103 MachineBasicBlock::iterator MBBI) {1104 MachineInstr &MI = *MBBI;1105 // In the case of a smstart/smstop before a unreachable, just remove the pseudo.1106 // Exception handling code generated by Clang may introduce unreachables and it1107 // seems unnecessary to restore pstate.sm when that happens. Note that it is1108 // not just an optimisation, the code below expects a successor instruction/block1109 // in order to split the block at MBBI.1110 if (std::next(MBBI) == MBB.end() &&1111 MI.getParent()->successors().begin() ==1112 MI.getParent()->successors().end()) {1113 MI.eraseFromParent();1114 return &MBB;1115 }1116 1117 // Expand the pseudo into smstart or smstop instruction. The pseudo has the1118 // following operands:1119 //1120 // MSRpstatePseudo <za|sm|both>, <0|1>, condition[, pstate.sm], <regmask>1121 //1122 // The pseudo is expanded into a conditional smstart/smstop, with a1123 // check if pstate.sm (register) equals the expected value, and if not,1124 // invokes the smstart/smstop.1125 //1126 // As an example, the following block contains a normal call from a1127 // streaming-compatible function:1128 //1129 // OrigBB:1130 // MSRpstatePseudo 3, 0, IfCallerIsStreaming, %0, <regmask> <- Cond SMSTOP1131 // bl @normal_callee1132 // MSRpstatePseudo 3, 1, IfCallerIsStreaming, %0, <regmask> <- Cond SMSTART1133 //1134 // ...which will be transformed into:1135 //1136 // OrigBB:1137 // TBNZx %0:gpr64, 0, SMBB1138 // b EndBB1139 //1140 // SMBB:1141 // MSRpstatesvcrImm1 3, 0, <regmask> <- SMSTOP1142 //1143 // EndBB:1144 // bl @normal_callee1145 // MSRcond_pstatesvcrImm1 3, 1, <regmask> <- SMSTART1146 //1147 DebugLoc DL = MI.getDebugLoc();1148 1149 // Create the conditional branch based on the third operand of the1150 // instruction, which tells us if we are wrapping a normal or streaming1151 // function.1152 // We test the live value of pstate.sm and toggle pstate.sm if this is not the1153 // expected value for the callee (0 for a normal callee and 1 for a streaming1154 // callee).1155 unsigned Opc;1156 switch (MI.getOperand(2).getImm()) {1157 case AArch64SME::Always:1158 llvm_unreachable("Should have matched to instruction directly");1159 case AArch64SME::IfCallerIsStreaming:1160 Opc = AArch64::TBNZW;1161 break;1162 case AArch64SME::IfCallerIsNonStreaming:1163 Opc = AArch64::TBZW;1164 break;1165 }1166 auto PStateSM = MI.getOperand(3).getReg();1167 auto TRI = MBB.getParent()->getSubtarget().getRegisterInfo();1168 unsigned SMReg32 = TRI->getSubReg(PStateSM, AArch64::sub_32);1169 MachineInstrBuilder Tbx =1170 BuildMI(MBB, MBBI, DL, TII->get(Opc)).addReg(SMReg32).addImm(0);1171 1172 auto [CondBB, EndBB] = expandConditionalPseudo(MBB, MBBI, DL, Tbx);1173 // Create the SMSTART/SMSTOP (MSRpstatesvcrImm1) instruction in SMBB.1174 MachineInstrBuilder MIB = BuildMI(CondBB, CondBB.back(), MI.getDebugLoc(),1175 TII->get(AArch64::MSRpstatesvcrImm1));1176 // Copy all but the second and third operands of MSRcond_pstatesvcrImm1 (as1177 // these contain the CopyFromReg for the first argument and the flag to1178 // indicate whether the callee is streaming or normal).1179 MIB.add(MI.getOperand(0));1180 MIB.add(MI.getOperand(1));1181 for (unsigned i = 4; i < MI.getNumOperands(); ++i)1182 MIB.add(MI.getOperand(i));1183 1184 MI.eraseFromParent();1185 return &EndBB;1186}1187 1188bool AArch64ExpandPseudo::expandMultiVecPseudo(1189 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,1190 TargetRegisterClass ContiguousClass, TargetRegisterClass StridedClass,1191 unsigned ContiguousOp, unsigned StridedOpc) {1192 MachineInstr &MI = *MBBI;1193 Register Tuple = MI.getOperand(0).getReg();1194 1195 auto ContiguousRange = ContiguousClass.getRegisters();1196 auto StridedRange = StridedClass.getRegisters();1197 unsigned Opc;1198 if (llvm::is_contained(ContiguousRange, Tuple.asMCReg())) {1199 Opc = ContiguousOp;1200 } else if (llvm::is_contained(StridedRange, Tuple.asMCReg())) {1201 Opc = StridedOpc;1202 } else1203 llvm_unreachable("Cannot expand Multi-Vector pseudo");1204 1205 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc))1206 .add(MI.getOperand(0))1207 .add(MI.getOperand(1))1208 .add(MI.getOperand(2))1209 .add(MI.getOperand(3));1210 transferImpOps(MI, MIB, MIB);1211 MI.eraseFromParent();1212 return true;1213}1214 1215bool AArch64ExpandPseudo::expandFormTuplePseudo(1216 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,1217 MachineBasicBlock::iterator &NextMBBI, unsigned Size) {1218 assert((Size == 2 || Size == 4) && "Invalid Tuple Size");1219 MachineInstr &MI = *MBBI;1220 Register ReturnTuple = MI.getOperand(0).getReg();1221 1222 const TargetRegisterInfo *TRI =1223 MBB.getParent()->getSubtarget().getRegisterInfo();1224 for (unsigned I = 0; I < Size; ++I) {1225 Register FormTupleOpReg = MI.getOperand(I + 1).getReg();1226 Register ReturnTupleSubReg =1227 TRI->getSubReg(ReturnTuple, AArch64::zsub0 + I);1228 // Add copies to ensure the subregisters remain in the correct order1229 // for any contigious operation they are used by.1230 if (FormTupleOpReg != ReturnTupleSubReg)1231 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORR_ZZZ))1232 .addReg(ReturnTupleSubReg, RegState::Define)1233 .addReg(FormTupleOpReg)1234 .addReg(FormTupleOpReg);1235 }1236 1237 MI.eraseFromParent();1238 return true;1239}1240 1241/// If MBBI references a pseudo instruction that should be expanded here,1242/// do the expansion and return true. Otherwise return false.1243bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,1244 MachineBasicBlock::iterator MBBI,1245 MachineBasicBlock::iterator &NextMBBI) {1246 MachineInstr &MI = *MBBI;1247 unsigned Opcode = MI.getOpcode();1248 1249 // Check if we can expand the destructive op1250 int OrigInstr = AArch64::getSVEPseudoMap(MI.getOpcode());1251 if (OrigInstr != -1) {1252 auto &Orig = TII->get(OrigInstr);1253 if ((Orig.TSFlags & AArch64::DestructiveInstTypeMask) !=1254 AArch64::NotDestructive) {1255 return expand_DestructiveOp(MI, MBB, MBBI);1256 }1257 }1258 1259 switch (Opcode) {1260 default:1261 break;1262 1263 case AArch64::BSPv8i8:1264 case AArch64::BSPv16i8: {1265 Register DstReg = MI.getOperand(0).getReg();1266 if (DstReg == MI.getOperand(3).getReg()) {1267 // Expand to BIT1268 auto I = BuildMI(MBB, MBBI, MI.getDebugLoc(),1269 TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BITv8i81270 : AArch64::BITv16i8))1271 .add(MI.getOperand(0))1272 .add(MI.getOperand(3))1273 .add(MI.getOperand(2))1274 .add(MI.getOperand(1));1275 transferImpOps(MI, I, I);1276 } else if (DstReg == MI.getOperand(2).getReg()) {1277 // Expand to BIF1278 auto I = BuildMI(MBB, MBBI, MI.getDebugLoc(),1279 TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BIFv8i81280 : AArch64::BIFv16i8))1281 .add(MI.getOperand(0))1282 .add(MI.getOperand(2))1283 .add(MI.getOperand(3))1284 .add(MI.getOperand(1));1285 transferImpOps(MI, I, I);1286 } else {1287 // Expand to BSL, use additional move if required1288 if (DstReg == MI.getOperand(1).getReg()) {1289 auto I =1290 BuildMI(MBB, MBBI, MI.getDebugLoc(),1291 TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i81292 : AArch64::BSLv16i8))1293 .add(MI.getOperand(0))1294 .add(MI.getOperand(1))1295 .add(MI.getOperand(2))1296 .add(MI.getOperand(3));1297 transferImpOps(MI, I, I);1298 } else {1299 unsigned RegState =1300 getRenamableRegState(MI.getOperand(1).isRenamable()) |1301 getKillRegState(1302 MI.getOperand(1).isKill() &&1303 MI.getOperand(1).getReg() != MI.getOperand(2).getReg() &&1304 MI.getOperand(1).getReg() != MI.getOperand(3).getReg());1305 BuildMI(MBB, MBBI, MI.getDebugLoc(),1306 TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::ORRv8i81307 : AArch64::ORRv16i8))1308 .addReg(DstReg,1309 RegState::Define |1310 getRenamableRegState(MI.getOperand(0).isRenamable()))1311 .addReg(MI.getOperand(1).getReg(), RegState)1312 .addReg(MI.getOperand(1).getReg(), RegState);1313 auto I2 =1314 BuildMI(MBB, MBBI, MI.getDebugLoc(),1315 TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i81316 : AArch64::BSLv16i8))1317 .add(MI.getOperand(0))1318 .addReg(DstReg,1319 RegState::Kill | getRenamableRegState(1320 MI.getOperand(0).isRenamable()))1321 .add(MI.getOperand(2))1322 .add(MI.getOperand(3));1323 transferImpOps(MI, I2, I2);1324 }1325 }1326 MI.eraseFromParent();1327 return true;1328 }1329 1330 case AArch64::ADDWrr:1331 case AArch64::SUBWrr:1332 case AArch64::ADDXrr:1333 case AArch64::SUBXrr:1334 case AArch64::ADDSWrr:1335 case AArch64::SUBSWrr:1336 case AArch64::ADDSXrr:1337 case AArch64::SUBSXrr:1338 case AArch64::ANDWrr:1339 case AArch64::ANDXrr:1340 case AArch64::BICWrr:1341 case AArch64::BICXrr:1342 case AArch64::ANDSWrr:1343 case AArch64::ANDSXrr:1344 case AArch64::BICSWrr:1345 case AArch64::BICSXrr:1346 case AArch64::EONWrr:1347 case AArch64::EONXrr:1348 case AArch64::EORWrr:1349 case AArch64::EORXrr:1350 case AArch64::ORNWrr:1351 case AArch64::ORNXrr:1352 case AArch64::ORRWrr:1353 case AArch64::ORRXrr: {1354 unsigned Opcode;1355 switch (MI.getOpcode()) {1356 default:1357 return false;1358 case AArch64::ADDWrr: Opcode = AArch64::ADDWrs; break;1359 case AArch64::SUBWrr: Opcode = AArch64::SUBWrs; break;1360 case AArch64::ADDXrr: Opcode = AArch64::ADDXrs; break;1361 case AArch64::SUBXrr: Opcode = AArch64::SUBXrs; break;1362 case AArch64::ADDSWrr: Opcode = AArch64::ADDSWrs; break;1363 case AArch64::SUBSWrr: Opcode = AArch64::SUBSWrs; break;1364 case AArch64::ADDSXrr: Opcode = AArch64::ADDSXrs; break;1365 case AArch64::SUBSXrr: Opcode = AArch64::SUBSXrs; break;1366 case AArch64::ANDWrr: Opcode = AArch64::ANDWrs; break;1367 case AArch64::ANDXrr: Opcode = AArch64::ANDXrs; break;1368 case AArch64::BICWrr: Opcode = AArch64::BICWrs; break;1369 case AArch64::BICXrr: Opcode = AArch64::BICXrs; break;1370 case AArch64::ANDSWrr: Opcode = AArch64::ANDSWrs; break;1371 case AArch64::ANDSXrr: Opcode = AArch64::ANDSXrs; break;1372 case AArch64::BICSWrr: Opcode = AArch64::BICSWrs; break;1373 case AArch64::BICSXrr: Opcode = AArch64::BICSXrs; break;1374 case AArch64::EONWrr: Opcode = AArch64::EONWrs; break;1375 case AArch64::EONXrr: Opcode = AArch64::EONXrs; break;1376 case AArch64::EORWrr: Opcode = AArch64::EORWrs; break;1377 case AArch64::EORXrr: Opcode = AArch64::EORXrs; break;1378 case AArch64::ORNWrr: Opcode = AArch64::ORNWrs; break;1379 case AArch64::ORNXrr: Opcode = AArch64::ORNXrs; break;1380 case AArch64::ORRWrr: Opcode = AArch64::ORRWrs; break;1381 case AArch64::ORRXrr: Opcode = AArch64::ORRXrs; break;1382 }1383 MachineFunction &MF = *MBB.getParent();1384 // Try to create new inst without implicit operands added.1385 MachineInstr *NewMI = MF.CreateMachineInstr(1386 TII->get(Opcode), MI.getDebugLoc(), /*NoImplicit=*/true);1387 MBB.insert(MBBI, NewMI);1388 MachineInstrBuilder MIB1(MF, NewMI);1389 MIB1->setPCSections(MF, MI.getPCSections());1390 MIB1.addReg(MI.getOperand(0).getReg(), RegState::Define)1391 .add(MI.getOperand(1))1392 .add(MI.getOperand(2))1393 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));1394 transferImpOps(MI, MIB1, MIB1);1395 if (auto DebugNumber = MI.peekDebugInstrNum())1396 NewMI->setDebugInstrNum(DebugNumber);1397 MI.eraseFromParent();1398 return true;1399 }1400 1401 case AArch64::LOADgot: {1402 MachineFunction *MF = MBB.getParent();1403 Register DstReg = MI.getOperand(0).getReg();1404 const MachineOperand &MO1 = MI.getOperand(1);1405 unsigned Flags = MO1.getTargetFlags();1406 1407 if (MF->getTarget().getCodeModel() == CodeModel::Tiny) {1408 // Tiny codemodel expand to LDR1409 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),1410 TII->get(AArch64::LDRXl), DstReg);1411 1412 if (MO1.isGlobal()) {1413 MIB.addGlobalAddress(MO1.getGlobal(), 0, Flags);1414 } else if (MO1.isSymbol()) {1415 MIB.addExternalSymbol(MO1.getSymbolName(), Flags);1416 } else {1417 assert(MO1.isCPI() &&1418 "Only expect globals, externalsymbols, or constant pools");1419 MIB.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(), Flags);1420 }1421 } else {1422 // Small codemodel expand into ADRP + LDR.1423 MachineFunction &MF = *MI.getParent()->getParent();1424 DebugLoc DL = MI.getDebugLoc();1425 MachineInstrBuilder MIB1 =1426 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg);1427 1428 MachineInstrBuilder MIB2;1429 if (MF.getSubtarget<AArch64Subtarget>().isTargetILP32()) {1430 auto TRI = MBB.getParent()->getSubtarget().getRegisterInfo();1431 unsigned Reg32 = TRI->getSubReg(DstReg, AArch64::sub_32);1432 unsigned DstFlags = MI.getOperand(0).getTargetFlags();1433 MIB2 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::LDRWui))1434 .addDef(Reg32)1435 .addReg(DstReg, RegState::Kill)1436 .addReg(DstReg, DstFlags | RegState::Implicit);1437 } else {1438 Register DstReg = MI.getOperand(0).getReg();1439 MIB2 = BuildMI(MBB, MBBI, DL, TII->get(AArch64::LDRXui))1440 .add(MI.getOperand(0))1441 .addUse(DstReg, RegState::Kill);1442 }1443 1444 if (MO1.isGlobal()) {1445 MIB1.addGlobalAddress(MO1.getGlobal(), 0, Flags | AArch64II::MO_PAGE);1446 MIB2.addGlobalAddress(MO1.getGlobal(), 0,1447 Flags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);1448 } else if (MO1.isSymbol()) {1449 MIB1.addExternalSymbol(MO1.getSymbolName(), Flags | AArch64II::MO_PAGE);1450 MIB2.addExternalSymbol(MO1.getSymbolName(), Flags |1451 AArch64II::MO_PAGEOFF |1452 AArch64II::MO_NC);1453 } else {1454 assert(MO1.isCPI() &&1455 "Only expect globals, externalsymbols, or constant pools");1456 MIB1.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(),1457 Flags | AArch64II::MO_PAGE);1458 MIB2.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(),1459 Flags | AArch64II::MO_PAGEOFF |1460 AArch64II::MO_NC);1461 }1462 1463 // If the LOADgot instruction has a debug-instr-number, annotate the1464 // LDRWui instruction that it is expanded to with the same1465 // debug-instr-number to preserve debug information.1466 if (MI.peekDebugInstrNum() != 0)1467 MIB2->setDebugInstrNum(MI.peekDebugInstrNum());1468 transferImpOps(MI, MIB1, MIB2);1469 }1470 MI.eraseFromParent();1471 return true;1472 }1473 case AArch64::MOVaddrBA: {1474 MachineFunction &MF = *MI.getParent()->getParent();1475 if (MF.getSubtarget<AArch64Subtarget>().isTargetMachO()) {1476 // blockaddress expressions have to come from a constant pool because the1477 // largest addend (and hence offset within a function) allowed for ADRP is1478 // only 8MB.1479 const BlockAddress *BA = MI.getOperand(1).getBlockAddress();1480 assert(MI.getOperand(1).getOffset() == 0 && "unexpected offset");1481 1482 MachineConstantPool *MCP = MF.getConstantPool();1483 unsigned CPIdx = MCP->getConstantPoolIndex(BA, Align(8));1484 1485 Register DstReg = MI.getOperand(0).getReg();1486 auto MIB1 =1487 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg)1488 .addConstantPoolIndex(CPIdx, 0, AArch64II::MO_PAGE);1489 auto MIB2 = BuildMI(MBB, MBBI, MI.getDebugLoc(),1490 TII->get(AArch64::LDRXui), DstReg)1491 .addUse(DstReg)1492 .addConstantPoolIndex(1493 CPIdx, 0, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);1494 transferImpOps(MI, MIB1, MIB2);1495 MI.eraseFromParent();1496 return true;1497 }1498 }1499 [[fallthrough]];1500 case AArch64::MOVaddr:1501 case AArch64::MOVaddrJT:1502 case AArch64::MOVaddrCP:1503 case AArch64::MOVaddrTLS:1504 case AArch64::MOVaddrEXT: {1505 // Expand into ADRP + ADD.1506 Register DstReg = MI.getOperand(0).getReg();1507 assert(DstReg != AArch64::XZR);1508 MachineInstrBuilder MIB1 =1509 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg)1510 .add(MI.getOperand(1));1511 1512 if (MI.getOperand(1).getTargetFlags() & AArch64II::MO_TAGGED) {1513 // MO_TAGGED on the page indicates a tagged address. Set the tag now.1514 // We do so by creating a MOVK that sets bits 48-63 of the register to1515 // (global address + 0x100000000 - PC) >> 48. This assumes that we're in1516 // the small code model so we can assume a binary size of <= 4GB, which1517 // makes the untagged PC relative offset positive. The binary must also be1518 // loaded into address range [0, 2^48). Both of these properties need to1519 // be ensured at runtime when using tagged addresses.1520 auto Tag = MI.getOperand(1);1521 Tag.setTargetFlags(AArch64II::MO_PREL | AArch64II::MO_G3);1522 Tag.setOffset(0x100000000);1523 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MOVKXi), DstReg)1524 .addReg(DstReg)1525 .add(Tag)1526 .addImm(48);1527 }1528 1529 MachineInstrBuilder MIB2 =1530 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri))1531 .add(MI.getOperand(0))1532 .addReg(DstReg)1533 .add(MI.getOperand(2))1534 .addImm(0);1535 1536 transferImpOps(MI, MIB1, MIB2);1537 MI.eraseFromParent();1538 return true;1539 }1540 case AArch64::ADDlowTLS:1541 // Produce a plain ADD1542 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri))1543 .add(MI.getOperand(0))1544 .add(MI.getOperand(1))1545 .add(MI.getOperand(2))1546 .addImm(0);1547 MI.eraseFromParent();1548 return true;1549 1550 case AArch64::MOVbaseTLS: {1551 Register DstReg = MI.getOperand(0).getReg();1552 auto SysReg = AArch64SysReg::TPIDR_EL0;1553 MachineFunction *MF = MBB.getParent();1554 if (MF->getSubtarget<AArch64Subtarget>().useEL3ForTP())1555 SysReg = AArch64SysReg::TPIDR_EL3;1556 else if (MF->getSubtarget<AArch64Subtarget>().useEL2ForTP())1557 SysReg = AArch64SysReg::TPIDR_EL2;1558 else if (MF->getSubtarget<AArch64Subtarget>().useEL1ForTP())1559 SysReg = AArch64SysReg::TPIDR_EL1;1560 else if (MF->getSubtarget<AArch64Subtarget>().useROEL0ForTP())1561 SysReg = AArch64SysReg::TPIDRRO_EL0;1562 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MRS), DstReg)1563 .addImm(SysReg);1564 MI.eraseFromParent();1565 return true;1566 }1567 1568 case AArch64::MOVi32imm:1569 return expandMOVImm(MBB, MBBI, 32);1570 case AArch64::MOVi64imm:1571 return expandMOVImm(MBB, MBBI, 64);1572 case AArch64::RET_ReallyLR: {1573 // Hiding the LR use with RET_ReallyLR may lead to extra kills in the1574 // function and missing live-ins. We are fine in practice because callee1575 // saved register handling ensures the register value is restored before1576 // RET, but we need the undef flag here to appease the MachineVerifier1577 // liveness checks.1578 MachineInstrBuilder MIB =1579 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::RET))1580 .addReg(AArch64::LR, RegState::Undef);1581 transferImpOps(MI, MIB, MIB);1582 MI.eraseFromParent();1583 return true;1584 }1585 case AArch64::CMP_SWAP_8:1586 return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRB, AArch64::STLXRB,1587 AArch64::SUBSWrx,1588 AArch64_AM::getArithExtendImm(AArch64_AM::UXTB, 0),1589 AArch64::WZR, NextMBBI);1590 case AArch64::CMP_SWAP_16:1591 return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRH, AArch64::STLXRH,1592 AArch64::SUBSWrx,1593 AArch64_AM::getArithExtendImm(AArch64_AM::UXTH, 0),1594 AArch64::WZR, NextMBBI);1595 case AArch64::CMP_SWAP_32:1596 return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRW, AArch64::STLXRW,1597 AArch64::SUBSWrs,1598 AArch64_AM::getShifterImm(AArch64_AM::LSL, 0),1599 AArch64::WZR, NextMBBI);1600 case AArch64::CMP_SWAP_64:1601 return expandCMP_SWAP(MBB, MBBI,1602 AArch64::LDAXRX, AArch64::STLXRX, AArch64::SUBSXrs,1603 AArch64_AM::getShifterImm(AArch64_AM::LSL, 0),1604 AArch64::XZR, NextMBBI);1605 case AArch64::CMP_SWAP_128:1606 case AArch64::CMP_SWAP_128_RELEASE:1607 case AArch64::CMP_SWAP_128_ACQUIRE:1608 case AArch64::CMP_SWAP_128_MONOTONIC:1609 return expandCMP_SWAP_128(MBB, MBBI, NextMBBI);1610 1611 case AArch64::AESMCrrTied:1612 case AArch64::AESIMCrrTied: {1613 MachineInstrBuilder MIB =1614 BuildMI(MBB, MBBI, MI.getDebugLoc(),1615 TII->get(Opcode == AArch64::AESMCrrTied ? AArch64::AESMCrr :1616 AArch64::AESIMCrr))1617 .add(MI.getOperand(0))1618 .add(MI.getOperand(1));1619 transferImpOps(MI, MIB, MIB);1620 MI.eraseFromParent();1621 return true;1622 }1623 case AArch64::IRGstack: {1624 MachineFunction &MF = *MBB.getParent();1625 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();1626 const AArch64FrameLowering *TFI =1627 MF.getSubtarget<AArch64Subtarget>().getFrameLowering();1628 1629 // IRG does not allow immediate offset. getTaggedBasePointerOffset should1630 // almost always point to SP-after-prologue; if not, emit a longer1631 // instruction sequence.1632 int BaseOffset = -AFI->getTaggedBasePointerOffset();1633 Register FrameReg;1634 StackOffset FrameRegOffset = TFI->resolveFrameOffsetReference(1635 MF, BaseOffset, false /*isFixed*/, TargetStackID::Default /*StackID*/,1636 FrameReg,1637 /*PreferFP=*/false,1638 /*ForSimm=*/true);1639 Register SrcReg = FrameReg;1640 if (FrameRegOffset) {1641 // Use output register as temporary.1642 SrcReg = MI.getOperand(0).getReg();1643 emitFrameOffset(MBB, &MI, MI.getDebugLoc(), SrcReg, FrameReg,1644 FrameRegOffset, TII);1645 }1646 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::IRG))1647 .add(MI.getOperand(0))1648 .addUse(SrcReg)1649 .add(MI.getOperand(2));1650 MI.eraseFromParent();1651 return true;1652 }1653 case AArch64::TAGPstack: {1654 int64_t Offset = MI.getOperand(2).getImm();1655 BuildMI(MBB, MBBI, MI.getDebugLoc(),1656 TII->get(Offset >= 0 ? AArch64::ADDG : AArch64::SUBG))1657 .add(MI.getOperand(0))1658 .add(MI.getOperand(1))1659 .addImm(std::abs(Offset))1660 .add(MI.getOperand(4));1661 MI.eraseFromParent();1662 return true;1663 }1664 case AArch64::STGloop_wback:1665 case AArch64::STZGloop_wback:1666 return expandSetTagLoop(MBB, MBBI, NextMBBI);1667 case AArch64::STGloop:1668 case AArch64::STZGloop:1669 report_fatal_error(1670 "Non-writeback variants of STGloop / STZGloop should not "1671 "survive past PrologEpilogInserter.");1672 case AArch64::STR_ZZZZXI:1673 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:1674 return expandSVESpillFill(MBB, MBBI, AArch64::STR_ZXI, 4);1675 case AArch64::STR_ZZZXI:1676 return expandSVESpillFill(MBB, MBBI, AArch64::STR_ZXI, 3);1677 case AArch64::STR_ZZXI:1678 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:1679 return expandSVESpillFill(MBB, MBBI, AArch64::STR_ZXI, 2);1680 case AArch64::STR_PPXI:1681 return expandSVESpillFill(MBB, MBBI, AArch64::STR_PXI, 2);1682 case AArch64::LDR_ZZZZXI:1683 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:1684 return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 4);1685 case AArch64::LDR_ZZZXI:1686 return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 3);1687 case AArch64::LDR_ZZXI:1688 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:1689 return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 2);1690 case AArch64::LDR_PPXI:1691 return expandSVESpillFill(MBB, MBBI, AArch64::LDR_PXI, 2);1692 case AArch64::BLR_RVMARKER:1693 case AArch64::BLRA_RVMARKER:1694 return expandCALL_RVMARKER(MBB, MBBI);1695 case AArch64::BLR_BTI:1696 return expandCALL_BTI(MBB, MBBI);1697 case AArch64::StoreSwiftAsyncContext:1698 return expandStoreSwiftAsyncContext(MBB, MBBI);1699 case AArch64::RestoreZAPseudo:1700 case AArch64::CommitZASavePseudo:1701 case AArch64::MSRpstatePseudo: {1702 auto *NewMBB = [&] {1703 switch (Opcode) {1704 case AArch64::RestoreZAPseudo:1705 return expandRestoreZASave(MBB, MBBI);1706 case AArch64::CommitZASavePseudo:1707 return expandCommitZASave(MBB, MBBI);1708 case AArch64::MSRpstatePseudo:1709 return expandCondSMToggle(MBB, MBBI);1710 default:1711 llvm_unreachable("Unexpected conditional pseudo!");1712 }1713 }();1714 if (NewMBB != &MBB)1715 NextMBBI = MBB.end(); // The NextMBBI iterator is invalidated.1716 return true;1717 }1718 case AArch64::InOutZAUsePseudo:1719 case AArch64::RequiresZASavePseudo:1720 case AArch64::RequiresZT0SavePseudo:1721 case AArch64::SMEStateAllocPseudo:1722 case AArch64::COALESCER_BARRIER_FPR16:1723 case AArch64::COALESCER_BARRIER_FPR32:1724 case AArch64::COALESCER_BARRIER_FPR64:1725 case AArch64::COALESCER_BARRIER_FPR128:1726 MI.eraseFromParent();1727 return true;1728 case AArch64::LD1B_2Z_IMM_PSEUDO:1729 return expandMultiVecPseudo(1730 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,1731 AArch64::LD1B_2Z_IMM, AArch64::LD1B_2Z_STRIDED_IMM);1732 case AArch64::LD1H_2Z_IMM_PSEUDO:1733 return expandMultiVecPseudo(1734 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,1735 AArch64::LD1H_2Z_IMM, AArch64::LD1H_2Z_STRIDED_IMM);1736 case AArch64::LD1W_2Z_IMM_PSEUDO:1737 return expandMultiVecPseudo(1738 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,1739 AArch64::LD1W_2Z_IMM, AArch64::LD1W_2Z_STRIDED_IMM);1740 case AArch64::LD1D_2Z_IMM_PSEUDO:1741 return expandMultiVecPseudo(1742 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,1743 AArch64::LD1D_2Z_IMM, AArch64::LD1D_2Z_STRIDED_IMM);1744 case AArch64::LDNT1B_2Z_IMM_PSEUDO:1745 return expandMultiVecPseudo(1746 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,1747 AArch64::LDNT1B_2Z_IMM, AArch64::LDNT1B_2Z_STRIDED_IMM);1748 case AArch64::LDNT1H_2Z_IMM_PSEUDO:1749 return expandMultiVecPseudo(1750 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,1751 AArch64::LDNT1H_2Z_IMM, AArch64::LDNT1H_2Z_STRIDED_IMM);1752 case AArch64::LDNT1W_2Z_IMM_PSEUDO:1753 return expandMultiVecPseudo(1754 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,1755 AArch64::LDNT1W_2Z_IMM, AArch64::LDNT1W_2Z_STRIDED_IMM);1756 case AArch64::LDNT1D_2Z_IMM_PSEUDO:1757 return expandMultiVecPseudo(1758 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,1759 AArch64::LDNT1D_2Z_IMM, AArch64::LDNT1D_2Z_STRIDED_IMM);1760 case AArch64::LD1B_2Z_PSEUDO:1761 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR2RegClass,1762 AArch64::ZPR2StridedRegClass, AArch64::LD1B_2Z,1763 AArch64::LD1B_2Z_STRIDED);1764 case AArch64::LD1H_2Z_PSEUDO:1765 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR2RegClass,1766 AArch64::ZPR2StridedRegClass, AArch64::LD1H_2Z,1767 AArch64::LD1H_2Z_STRIDED);1768 case AArch64::LD1W_2Z_PSEUDO:1769 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR2RegClass,1770 AArch64::ZPR2StridedRegClass, AArch64::LD1W_2Z,1771 AArch64::LD1W_2Z_STRIDED);1772 case AArch64::LD1D_2Z_PSEUDO:1773 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR2RegClass,1774 AArch64::ZPR2StridedRegClass, AArch64::LD1D_2Z,1775 AArch64::LD1D_2Z_STRIDED);1776 case AArch64::LDNT1B_2Z_PSEUDO:1777 return expandMultiVecPseudo(1778 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,1779 AArch64::LDNT1B_2Z, AArch64::LDNT1B_2Z_STRIDED);1780 case AArch64::LDNT1H_2Z_PSEUDO:1781 return expandMultiVecPseudo(1782 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,1783 AArch64::LDNT1H_2Z, AArch64::LDNT1H_2Z_STRIDED);1784 case AArch64::LDNT1W_2Z_PSEUDO:1785 return expandMultiVecPseudo(1786 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,1787 AArch64::LDNT1W_2Z, AArch64::LDNT1W_2Z_STRIDED);1788 case AArch64::LDNT1D_2Z_PSEUDO:1789 return expandMultiVecPseudo(1790 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,1791 AArch64::LDNT1D_2Z, AArch64::LDNT1D_2Z_STRIDED);1792 case AArch64::LD1B_4Z_IMM_PSEUDO:1793 return expandMultiVecPseudo(1794 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,1795 AArch64::LD1B_4Z_IMM, AArch64::LD1B_4Z_STRIDED_IMM);1796 case AArch64::LD1H_4Z_IMM_PSEUDO:1797 return expandMultiVecPseudo(1798 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,1799 AArch64::LD1H_4Z_IMM, AArch64::LD1H_4Z_STRIDED_IMM);1800 case AArch64::LD1W_4Z_IMM_PSEUDO:1801 return expandMultiVecPseudo(1802 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,1803 AArch64::LD1W_4Z_IMM, AArch64::LD1W_4Z_STRIDED_IMM);1804 case AArch64::LD1D_4Z_IMM_PSEUDO:1805 return expandMultiVecPseudo(1806 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,1807 AArch64::LD1D_4Z_IMM, AArch64::LD1D_4Z_STRIDED_IMM);1808 case AArch64::LDNT1B_4Z_IMM_PSEUDO:1809 return expandMultiVecPseudo(1810 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,1811 AArch64::LDNT1B_4Z_IMM, AArch64::LDNT1B_4Z_STRIDED_IMM);1812 case AArch64::LDNT1H_4Z_IMM_PSEUDO:1813 return expandMultiVecPseudo(1814 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,1815 AArch64::LDNT1H_4Z_IMM, AArch64::LDNT1H_4Z_STRIDED_IMM);1816 case AArch64::LDNT1W_4Z_IMM_PSEUDO:1817 return expandMultiVecPseudo(1818 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,1819 AArch64::LDNT1W_4Z_IMM, AArch64::LDNT1W_4Z_STRIDED_IMM);1820 case AArch64::LDNT1D_4Z_IMM_PSEUDO:1821 return expandMultiVecPseudo(1822 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,1823 AArch64::LDNT1D_4Z_IMM, AArch64::LDNT1D_4Z_STRIDED_IMM);1824 case AArch64::LD1B_4Z_PSEUDO:1825 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR4RegClass,1826 AArch64::ZPR4StridedRegClass, AArch64::LD1B_4Z,1827 AArch64::LD1B_4Z_STRIDED);1828 case AArch64::LD1H_4Z_PSEUDO:1829 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR4RegClass,1830 AArch64::ZPR4StridedRegClass, AArch64::LD1H_4Z,1831 AArch64::LD1H_4Z_STRIDED);1832 case AArch64::LD1W_4Z_PSEUDO:1833 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR4RegClass,1834 AArch64::ZPR4StridedRegClass, AArch64::LD1W_4Z,1835 AArch64::LD1W_4Z_STRIDED);1836 case AArch64::LD1D_4Z_PSEUDO:1837 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR4RegClass,1838 AArch64::ZPR4StridedRegClass, AArch64::LD1D_4Z,1839 AArch64::LD1D_4Z_STRIDED);1840 case AArch64::LDNT1B_4Z_PSEUDO:1841 return expandMultiVecPseudo(1842 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,1843 AArch64::LDNT1B_4Z, AArch64::LDNT1B_4Z_STRIDED);1844 case AArch64::LDNT1H_4Z_PSEUDO:1845 return expandMultiVecPseudo(1846 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,1847 AArch64::LDNT1H_4Z, AArch64::LDNT1H_4Z_STRIDED);1848 case AArch64::LDNT1W_4Z_PSEUDO:1849 return expandMultiVecPseudo(1850 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,1851 AArch64::LDNT1W_4Z, AArch64::LDNT1W_4Z_STRIDED);1852 case AArch64::LDNT1D_4Z_PSEUDO:1853 return expandMultiVecPseudo(1854 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,1855 AArch64::LDNT1D_4Z, AArch64::LDNT1D_4Z_STRIDED);1856 case AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO:1857 return expandFormTuplePseudo(MBB, MBBI, NextMBBI, 2);1858 case AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO:1859 return expandFormTuplePseudo(MBB, MBBI, NextMBBI, 4);1860 }1861 return false;1862}1863 1864/// Iterate over the instructions in basic block MBB and expand any1865/// pseudo instructions. Return true if anything was modified.1866bool AArch64ExpandPseudo::expandMBB(MachineBasicBlock &MBB) {1867 bool Modified = false;1868 1869 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();1870 while (MBBI != E) {1871 MachineBasicBlock::iterator NMBBI = std::next(MBBI);1872 Modified |= expandMI(MBB, MBBI, NMBBI);1873 MBBI = NMBBI;1874 }1875 1876 return Modified;1877}1878 1879bool AArch64ExpandPseudo::runOnMachineFunction(MachineFunction &MF) {1880 TII = MF.getSubtarget<AArch64Subtarget>().getInstrInfo();1881 1882 bool Modified = false;1883 for (auto &MBB : MF)1884 Modified |= expandMBB(MBB);1885 return Modified;1886}1887 1888/// Returns an instance of the pseudo instruction expansion pass.1889FunctionPass *llvm::createAArch64ExpandPseudoPass() {1890 return new AArch64ExpandPseudo();1891}1892