947 lines · cpp
1//===------- X86ExpandPseudo.cpp - Expand pseudo instructions -------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains a pass that expands pseudo instructions into target10// instructions to allow proper scheduling, if-conversion, other late11// optimizations, or simply the encoding of the instructions.12//13//===----------------------------------------------------------------------===//14 15#include "X86.h"16#include "X86FrameLowering.h"17#include "X86InstrInfo.h"18#include "X86MachineFunctionInfo.h"19#include "X86Subtarget.h"20#include "llvm/CodeGen/LivePhysRegs.h"21#include "llvm/CodeGen/MachineFunctionPass.h"22#include "llvm/CodeGen/MachineInstrBuilder.h"23#include "llvm/CodeGen/Passes.h" // For IDs of passes that are preserved.24#include "llvm/IR/EHPersonalities.h"25#include "llvm/IR/GlobalValue.h"26#include "llvm/Target/TargetMachine.h"27using namespace llvm;28 29#define DEBUG_TYPE "x86-pseudo"30#define X86_EXPAND_PSEUDO_NAME "X86 pseudo instruction expansion pass"31 32namespace {33class X86ExpandPseudo : public MachineFunctionPass {34public:35 static char ID;36 X86ExpandPseudo() : MachineFunctionPass(ID) {}37 38 void getAnalysisUsage(AnalysisUsage &AU) const override {39 AU.setPreservesCFG();40 AU.addPreservedID(MachineLoopInfoID);41 AU.addPreservedID(MachineDominatorsID);42 MachineFunctionPass::getAnalysisUsage(AU);43 }44 45 const X86Subtarget *STI = nullptr;46 const X86InstrInfo *TII = nullptr;47 const X86RegisterInfo *TRI = nullptr;48 const X86MachineFunctionInfo *X86FI = nullptr;49 const X86FrameLowering *X86FL = nullptr;50 51 bool runOnMachineFunction(MachineFunction &MF) override;52 53 MachineFunctionProperties getRequiredProperties() const override {54 return MachineFunctionProperties().setNoVRegs();55 }56 57 StringRef getPassName() const override {58 return "X86 pseudo instruction expansion pass";59 }60 61private:62 void expandICallBranchFunnel(MachineBasicBlock *MBB,63 MachineBasicBlock::iterator MBBI);64 void expandCALL_RVMARKER(MachineBasicBlock &MBB,65 MachineBasicBlock::iterator MBBI);66 bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);67 bool expandMBB(MachineBasicBlock &MBB);68 69 /// This function expands pseudos which affects control flow.70 /// It is done in separate pass to simplify blocks navigation in main71 /// pass(calling expandMBB).72 bool expandPseudosWhichAffectControlFlow(MachineFunction &MF);73 74 /// Expand X86::VASTART_SAVE_XMM_REGS into set of xmm copying instructions,75 /// placed into separate block guarded by check for al register(for SystemV76 /// abi).77 void expandVastartSaveXmmRegs(78 MachineBasicBlock *EntryBlk,79 MachineBasicBlock::iterator VAStartPseudoInstr) const;80};81char X86ExpandPseudo::ID = 0;82 83} // End anonymous namespace.84 85INITIALIZE_PASS(X86ExpandPseudo, DEBUG_TYPE, X86_EXPAND_PSEUDO_NAME, false,86 false)87 88void X86ExpandPseudo::expandICallBranchFunnel(89 MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI) {90 MachineBasicBlock *JTMBB = MBB;91 MachineInstr *JTInst = &*MBBI;92 MachineFunction *MF = MBB->getParent();93 const BasicBlock *BB = MBB->getBasicBlock();94 auto InsPt = MachineFunction::iterator(MBB);95 ++InsPt;96 97 std::vector<std::pair<MachineBasicBlock *, unsigned>> TargetMBBs;98 const DebugLoc &DL = JTInst->getDebugLoc();99 MachineOperand Selector = JTInst->getOperand(0);100 const GlobalValue *CombinedGlobal = JTInst->getOperand(1).getGlobal();101 102 auto CmpTarget = [&](unsigned Target) {103 if (Selector.isReg())104 MBB->addLiveIn(Selector.getReg());105 BuildMI(*MBB, MBBI, DL, TII->get(X86::LEA64r), X86::R11)106 .addReg(X86::RIP)107 .addImm(1)108 .addReg(0)109 .addGlobalAddress(CombinedGlobal,110 JTInst->getOperand(2 + 2 * Target).getImm())111 .addReg(0);112 BuildMI(*MBB, MBBI, DL, TII->get(X86::CMP64rr))113 .add(Selector)114 .addReg(X86::R11);115 };116 117 auto CreateMBB = [&]() {118 auto *NewMBB = MF->CreateMachineBasicBlock(BB);119 MBB->addSuccessor(NewMBB);120 if (!MBB->isLiveIn(X86::EFLAGS))121 MBB->addLiveIn(X86::EFLAGS);122 return NewMBB;123 };124 125 auto EmitCondJump = [&](unsigned CC, MachineBasicBlock *ThenMBB) {126 BuildMI(*MBB, MBBI, DL, TII->get(X86::JCC_1)).addMBB(ThenMBB).addImm(CC);127 128 auto *ElseMBB = CreateMBB();129 MF->insert(InsPt, ElseMBB);130 MBB = ElseMBB;131 MBBI = MBB->end();132 };133 134 auto EmitCondJumpTarget = [&](unsigned CC, unsigned Target) {135 auto *ThenMBB = CreateMBB();136 TargetMBBs.push_back({ThenMBB, Target});137 EmitCondJump(CC, ThenMBB);138 };139 140 auto EmitTailCall = [&](unsigned Target) {141 BuildMI(*MBB, MBBI, DL, TII->get(X86::TAILJMPd64))142 .add(JTInst->getOperand(3 + 2 * Target));143 };144 145 std::function<void(unsigned, unsigned)> EmitBranchFunnel =146 [&](unsigned FirstTarget, unsigned NumTargets) {147 if (NumTargets == 1) {148 EmitTailCall(FirstTarget);149 return;150 }151 152 if (NumTargets == 2) {153 CmpTarget(FirstTarget + 1);154 EmitCondJumpTarget(X86::COND_B, FirstTarget);155 EmitTailCall(FirstTarget + 1);156 return;157 }158 159 if (NumTargets < 6) {160 CmpTarget(FirstTarget + 1);161 EmitCondJumpTarget(X86::COND_B, FirstTarget);162 EmitCondJumpTarget(X86::COND_E, FirstTarget + 1);163 EmitBranchFunnel(FirstTarget + 2, NumTargets - 2);164 return;165 }166 167 auto *ThenMBB = CreateMBB();168 CmpTarget(FirstTarget + (NumTargets / 2));169 EmitCondJump(X86::COND_B, ThenMBB);170 EmitCondJumpTarget(X86::COND_E, FirstTarget + (NumTargets / 2));171 EmitBranchFunnel(FirstTarget + (NumTargets / 2) + 1,172 NumTargets - (NumTargets / 2) - 1);173 174 MF->insert(InsPt, ThenMBB);175 MBB = ThenMBB;176 MBBI = MBB->end();177 EmitBranchFunnel(FirstTarget, NumTargets / 2);178 };179 180 EmitBranchFunnel(0, (JTInst->getNumOperands() - 2) / 2);181 for (auto P : TargetMBBs) {182 MF->insert(InsPt, P.first);183 BuildMI(P.first, DL, TII->get(X86::TAILJMPd64))184 .add(JTInst->getOperand(3 + 2 * P.second));185 }186 JTMBB->erase(JTInst);187}188 189void X86ExpandPseudo::expandCALL_RVMARKER(MachineBasicBlock &MBB,190 MachineBasicBlock::iterator MBBI) {191 // Expand CALL_RVMARKER pseudo to call instruction, followed by the special192 //"movq %rax, %rdi" marker.193 MachineInstr &MI = *MBBI;194 195 MachineInstr *OriginalCall;196 assert((MI.getOperand(1).isGlobal() || MI.getOperand(1).isReg()) &&197 "invalid operand for regular call");198 unsigned Opc = -1;199 if (MI.getOpcode() == X86::CALL64m_RVMARKER)200 Opc = X86::CALL64m;201 else if (MI.getOpcode() == X86::CALL64r_RVMARKER)202 Opc = X86::CALL64r;203 else if (MI.getOpcode() == X86::CALL64pcrel32_RVMARKER)204 Opc = X86::CALL64pcrel32;205 else206 llvm_unreachable("unexpected opcode");207 208 OriginalCall = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc)).getInstr();209 bool RAXImplicitDead = false;210 for (MachineOperand &Op : llvm::drop_begin(MI.operands())) {211 // RAX may be 'implicit dead', if there are no other users of the return212 // value. We introduce a new use, so change it to 'implicit def'.213 if (Op.isReg() && Op.isImplicit() && Op.isDead() &&214 TRI->regsOverlap(Op.getReg(), X86::RAX)) {215 Op.setIsDead(false);216 Op.setIsDef(true);217 RAXImplicitDead = true;218 }219 OriginalCall->addOperand(Op);220 }221 222 // Emit marker "movq %rax, %rdi". %rdi is not callee-saved, so it cannot be223 // live across the earlier call. The call to the ObjC runtime function returns224 // the first argument, so the value of %rax is unchanged after the ObjC225 // runtime call. On Windows targets, the runtime call follows the regular226 // x64 calling convention and expects the first argument in %rcx.227 auto TargetReg = STI->getTargetTriple().isOSWindows() ? X86::RCX : X86::RDI;228 auto *Marker = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(X86::MOV64rr))229 .addReg(TargetReg, RegState::Define)230 .addReg(X86::RAX)231 .getInstr();232 if (MI.shouldUpdateAdditionalCallInfo())233 MBB.getParent()->moveAdditionalCallInfo(&MI, Marker);234 235 // Emit call to ObjC runtime.236 const uint32_t *RegMask =237 TRI->getCallPreservedMask(*MBB.getParent(), CallingConv::C);238 MachineInstr *RtCall =239 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(X86::CALL64pcrel32))240 .addGlobalAddress(MI.getOperand(0).getGlobal(), 0, 0)241 .addRegMask(RegMask)242 .addReg(X86::RAX,243 RegState::Implicit |244 (RAXImplicitDead ? (RegState::Dead | RegState::Define)245 : RegState::Define))246 .getInstr();247 MI.eraseFromParent();248 249 auto &TM = MBB.getParent()->getTarget();250 // On Darwin platforms, wrap the expanded sequence in a bundle to prevent251 // later optimizations from breaking up the sequence.252 if (TM.getTargetTriple().isOSDarwin())253 finalizeBundle(MBB, OriginalCall->getIterator(),254 std::next(RtCall->getIterator()));255}256 257/// If \p MBBI is a pseudo instruction, this method expands258/// it to the corresponding (sequence of) actual instruction(s).259/// \returns true if \p MBBI has been expanded.260bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB,261 MachineBasicBlock::iterator MBBI) {262 MachineInstr &MI = *MBBI;263 unsigned Opcode = MI.getOpcode();264 const DebugLoc &DL = MBBI->getDebugLoc();265#define GET_EGPR_IF_ENABLED(OPC) (STI->hasEGPR() ? OPC##_EVEX : OPC)266 switch (Opcode) {267 default:268 return false;269 case X86::TCRETURNdi:270 case X86::TCRETURNdicc:271 case X86::TCRETURNri:272 case X86::TCRETURN_WIN64ri:273 case X86::TCRETURN_HIPE32ri:274 case X86::TCRETURNmi:275 case X86::TCRETURNdi64:276 case X86::TCRETURNdi64cc:277 case X86::TCRETURNri64:278 case X86::TCRETURNri64_ImpCall:279 case X86::TCRETURNmi64:280 case X86::TCRETURN_WINmi64: {281 bool isMem = Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64 ||282 Opcode == X86::TCRETURN_WINmi64;283 MachineOperand &JumpTarget = MBBI->getOperand(0);284 MachineOperand &StackAdjust = MBBI->getOperand(isMem ? X86::AddrNumOperands285 : 1);286 assert(StackAdjust.isImm() && "Expecting immediate value.");287 288 // Adjust stack pointer.289 int StackAdj = StackAdjust.getImm();290 int MaxTCDelta = X86FI->getTCReturnAddrDelta();291 int64_t Offset = 0;292 assert(MaxTCDelta <= 0 && "MaxTCDelta should never be positive");293 294 // Incoporate the retaddr area.295 Offset = StackAdj - MaxTCDelta;296 assert(Offset >= 0 && "Offset should never be negative");297 298 if (Opcode == X86::TCRETURNdicc || Opcode == X86::TCRETURNdi64cc) {299 assert(Offset == 0 && "Conditional tail call cannot adjust the stack.");300 }301 302 if (Offset) {303 // Check for possible merge with preceding ADD instruction.304 Offset = X86FL->mergeSPAdd(MBB, MBBI, Offset, true);305 X86FL->emitSPUpdate(MBB, MBBI, DL, Offset, /*InEpilogue=*/true);306 }307 308 // Use this predicate to set REX prefix for X86_64 targets.309 bool IsX64 = STI->isTargetWin64() || STI->isTargetUEFI64();310 // Jump to label or value in register.311 if (Opcode == X86::TCRETURNdi || Opcode == X86::TCRETURNdicc ||312 Opcode == X86::TCRETURNdi64 || Opcode == X86::TCRETURNdi64cc) {313 unsigned Op;314 switch (Opcode) {315 case X86::TCRETURNdi:316 Op = X86::TAILJMPd;317 break;318 case X86::TCRETURNdicc:319 Op = X86::TAILJMPd_CC;320 break;321 case X86::TCRETURNdi64cc:322 assert(!MBB.getParent()->hasWinCFI() &&323 "Conditional tail calls confuse "324 "the Win64 unwinder.");325 Op = X86::TAILJMPd64_CC;326 break;327 default:328 // Note: Win64 uses REX prefixes indirect jumps out of functions, but329 // not direct ones.330 Op = X86::TAILJMPd64;331 break;332 }333 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op));334 if (JumpTarget.isGlobal()) {335 MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),336 JumpTarget.getTargetFlags());337 } else {338 assert(JumpTarget.isSymbol());339 MIB.addExternalSymbol(JumpTarget.getSymbolName(),340 JumpTarget.getTargetFlags());341 }342 if (Op == X86::TAILJMPd_CC || Op == X86::TAILJMPd64_CC) {343 MIB.addImm(MBBI->getOperand(2).getImm());344 }345 346 } else if (Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64 ||347 Opcode == X86::TCRETURN_WINmi64) {348 unsigned Op = (Opcode == X86::TCRETURNmi)349 ? X86::TAILJMPm350 : (IsX64 ? X86::TAILJMPm64_REX : X86::TAILJMPm64);351 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op));352 for (unsigned i = 0; i != X86::AddrNumOperands; ++i)353 MIB.add(MBBI->getOperand(i));354 } else if (Opcode == X86::TCRETURNri64 ||355 Opcode == X86::TCRETURNri64_ImpCall ||356 Opcode == X86::TCRETURN_WIN64ri) {357 JumpTarget.setIsKill();358 BuildMI(MBB, MBBI, DL,359 TII->get(IsX64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64))360 .add(JumpTarget);361 } else {362 assert(!IsX64 && "Win64 and UEFI64 require REX for indirect jumps.");363 JumpTarget.setIsKill();364 BuildMI(MBB, MBBI, DL, TII->get(X86::TAILJMPr))365 .add(JumpTarget);366 }367 368 MachineInstr &NewMI = *std::prev(MBBI);369 NewMI.copyImplicitOps(*MBBI->getParent()->getParent(), *MBBI);370 NewMI.setCFIType(*MBB.getParent(), MI.getCFIType());371 372 // Update the call info.373 if (MBBI->isCandidateForAdditionalCallInfo())374 MBB.getParent()->moveAdditionalCallInfo(&*MBBI, &NewMI);375 376 // Delete the pseudo instruction TCRETURN.377 MBB.erase(MBBI);378 379 return true;380 }381 case X86::EH_RETURN:382 case X86::EH_RETURN64: {383 MachineOperand &DestAddr = MBBI->getOperand(0);384 assert(DestAddr.isReg() && "Offset should be in register!");385 const bool Uses64BitFramePtr = STI->isTarget64BitLP64();386 Register StackPtr = TRI->getStackRegister();387 BuildMI(MBB, MBBI, DL,388 TII->get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr), StackPtr)389 .addReg(DestAddr.getReg());390 // The EH_RETURN pseudo is really removed during the MC Lowering.391 return true;392 }393 case X86::IRET: {394 // Adjust stack to erase error code395 int64_t StackAdj = MBBI->getOperand(0).getImm();396 X86FL->emitSPUpdate(MBB, MBBI, DL, StackAdj, true);397 // Replace pseudo with machine iret398 unsigned RetOp = STI->is64Bit() ? X86::IRET64 : X86::IRET32;399 // Use UIRET if UINTR is present (except for building kernel)400 if (STI->is64Bit() && STI->hasUINTR() &&401 MBB.getParent()->getTarget().getCodeModel() != CodeModel::Kernel)402 RetOp = X86::UIRET;403 BuildMI(MBB, MBBI, DL, TII->get(RetOp));404 MBB.erase(MBBI);405 return true;406 }407 case X86::RET: {408 // Adjust stack to erase error code409 int64_t StackAdj = MBBI->getOperand(0).getImm();410 MachineInstrBuilder MIB;411 if (StackAdj == 0) {412 MIB = BuildMI(MBB, MBBI, DL,413 TII->get(STI->is64Bit() ? X86::RET64 : X86::RET32));414 } else if (isUInt<16>(StackAdj)) {415 MIB = BuildMI(MBB, MBBI, DL,416 TII->get(STI->is64Bit() ? X86::RETI64 : X86::RETI32))417 .addImm(StackAdj);418 } else {419 assert(!STI->is64Bit() &&420 "shouldn't need to do this for x86_64 targets!");421 // A ret can only handle immediates as big as 2**16-1. If we need to pop422 // off bytes before the return address, we must do it manually.423 BuildMI(MBB, MBBI, DL, TII->get(X86::POP32r)).addReg(X86::ECX, RegState::Define);424 X86FL->emitSPUpdate(MBB, MBBI, DL, StackAdj, /*InEpilogue=*/true);425 BuildMI(MBB, MBBI, DL, TII->get(X86::PUSH32r)).addReg(X86::ECX);426 MIB = BuildMI(MBB, MBBI, DL, TII->get(X86::RET32));427 }428 for (unsigned I = 1, E = MBBI->getNumOperands(); I != E; ++I)429 MIB.add(MBBI->getOperand(I));430 MBB.erase(MBBI);431 return true;432 }433 case X86::LCMPXCHG16B_SAVE_RBX: {434 // Perform the following transformation.435 // SaveRbx = pseudocmpxchg Addr, <4 opds for the address>, InArg, SaveRbx436 // =>437 // RBX = InArg438 // actualcmpxchg Addr439 // RBX = SaveRbx440 const MachineOperand &InArg = MBBI->getOperand(6);441 Register SaveRbx = MBBI->getOperand(7).getReg();442 443 // Copy the input argument of the pseudo into the argument of the444 // actual instruction.445 // NOTE: We don't copy the kill flag since the input might be the same reg446 // as one of the other operands of LCMPXCHG16B.447 TII->copyPhysReg(MBB, MBBI, DL, X86::RBX, InArg.getReg(), false);448 // Create the actual instruction.449 MachineInstr *NewInstr = BuildMI(MBB, MBBI, DL, TII->get(X86::LCMPXCHG16B));450 // Copy the operands related to the address. If we access a frame variable,451 // we need to replace the RBX base with SaveRbx, as RBX has another value.452 const MachineOperand &Base = MBBI->getOperand(1);453 if (Base.getReg() == X86::RBX || Base.getReg() == X86::EBX)454 NewInstr->addOperand(MachineOperand::CreateReg(455 Base.getReg() == X86::RBX456 ? SaveRbx457 : Register(TRI->getSubReg(SaveRbx, X86::sub_32bit)),458 /*IsDef=*/false));459 else460 NewInstr->addOperand(Base);461 for (unsigned Idx = 1 + 1; Idx < 1 + X86::AddrNumOperands; ++Idx)462 NewInstr->addOperand(MBBI->getOperand(Idx));463 // Finally, restore the value of RBX.464 TII->copyPhysReg(MBB, MBBI, DL, X86::RBX, SaveRbx,465 /*SrcIsKill*/ true);466 467 // Delete the pseudo.468 MBBI->eraseFromParent();469 return true;470 }471 // Loading/storing mask pairs requires two kmov operations. The second one of472 // these needs a 2 byte displacement relative to the specified address (with473 // 32 bit spill size). The pairs of 1bit masks up to 16 bit masks all use the474 // same spill size, they all are stored using MASKPAIR16STORE, loaded using475 // MASKPAIR16LOAD.476 //477 // The displacement value might wrap around in theory, thus the asserts in478 // both cases.479 case X86::MASKPAIR16LOAD: {480 int64_t Disp = MBBI->getOperand(1 + X86::AddrDisp).getImm();481 assert(Disp >= 0 && Disp <= INT32_MAX - 2 && "Unexpected displacement");482 Register Reg = MBBI->getOperand(0).getReg();483 bool DstIsDead = MBBI->getOperand(0).isDead();484 Register Reg0 = TRI->getSubReg(Reg, X86::sub_mask_0);485 Register Reg1 = TRI->getSubReg(Reg, X86::sub_mask_1);486 487 auto MIBLo =488 BuildMI(MBB, MBBI, DL, TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWkm)))489 .addReg(Reg0, RegState::Define | getDeadRegState(DstIsDead));490 auto MIBHi =491 BuildMI(MBB, MBBI, DL, TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWkm)))492 .addReg(Reg1, RegState::Define | getDeadRegState(DstIsDead));493 494 for (int i = 0; i < X86::AddrNumOperands; ++i) {495 MIBLo.add(MBBI->getOperand(1 + i));496 if (i == X86::AddrDisp)497 MIBHi.addImm(Disp + 2);498 else499 MIBHi.add(MBBI->getOperand(1 + i));500 }501 502 // Split the memory operand, adjusting the offset and size for the halves.503 MachineMemOperand *OldMMO = MBBI->memoperands().front();504 MachineFunction *MF = MBB.getParent();505 MachineMemOperand *MMOLo = MF->getMachineMemOperand(OldMMO, 0, 2);506 MachineMemOperand *MMOHi = MF->getMachineMemOperand(OldMMO, 2, 2);507 508 MIBLo.setMemRefs(MMOLo);509 MIBHi.setMemRefs(MMOHi);510 511 // Delete the pseudo.512 MBB.erase(MBBI);513 return true;514 }515 case X86::MASKPAIR16STORE: {516 int64_t Disp = MBBI->getOperand(X86::AddrDisp).getImm();517 assert(Disp >= 0 && Disp <= INT32_MAX - 2 && "Unexpected displacement");518 Register Reg = MBBI->getOperand(X86::AddrNumOperands).getReg();519 bool SrcIsKill = MBBI->getOperand(X86::AddrNumOperands).isKill();520 Register Reg0 = TRI->getSubReg(Reg, X86::sub_mask_0);521 Register Reg1 = TRI->getSubReg(Reg, X86::sub_mask_1);522 523 auto MIBLo =524 BuildMI(MBB, MBBI, DL, TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWmk)));525 auto MIBHi =526 BuildMI(MBB, MBBI, DL, TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWmk)));527 528 for (int i = 0; i < X86::AddrNumOperands; ++i) {529 MIBLo.add(MBBI->getOperand(i));530 if (i == X86::AddrDisp)531 MIBHi.addImm(Disp + 2);532 else533 MIBHi.add(MBBI->getOperand(i));534 }535 MIBLo.addReg(Reg0, getKillRegState(SrcIsKill));536 MIBHi.addReg(Reg1, getKillRegState(SrcIsKill));537 538 // Split the memory operand, adjusting the offset and size for the halves.539 MachineMemOperand *OldMMO = MBBI->memoperands().front();540 MachineFunction *MF = MBB.getParent();541 MachineMemOperand *MMOLo = MF->getMachineMemOperand(OldMMO, 0, 2);542 MachineMemOperand *MMOHi = MF->getMachineMemOperand(OldMMO, 2, 2);543 544 MIBLo.setMemRefs(MMOLo);545 MIBHi.setMemRefs(MMOHi);546 547 // Delete the pseudo.548 MBB.erase(MBBI);549 return true;550 }551 case X86::MWAITX_SAVE_RBX: {552 // Perform the following transformation.553 // SaveRbx = pseudomwaitx InArg, SaveRbx554 // =>555 // [E|R]BX = InArg556 // actualmwaitx557 // [E|R]BX = SaveRbx558 const MachineOperand &InArg = MBBI->getOperand(1);559 // Copy the input argument of the pseudo into the argument of the560 // actual instruction.561 TII->copyPhysReg(MBB, MBBI, DL, X86::EBX, InArg.getReg(), InArg.isKill());562 // Create the actual instruction.563 BuildMI(MBB, MBBI, DL, TII->get(X86::MWAITXrrr));564 // Finally, restore the value of RBX.565 Register SaveRbx = MBBI->getOperand(2).getReg();566 TII->copyPhysReg(MBB, MBBI, DL, X86::RBX, SaveRbx, /*SrcIsKill*/ true);567 // Delete the pseudo.568 MBBI->eraseFromParent();569 return true;570 }571 case TargetOpcode::ICALL_BRANCH_FUNNEL:572 expandICallBranchFunnel(&MBB, MBBI);573 return true;574 case X86::PLDTILECFGV: {575 MI.setDesc(TII->get(GET_EGPR_IF_ENABLED(X86::LDTILECFG)));576 return true;577 }578 case X86::PTILELOADDV:579 case X86::PTILELOADDT1V:580 case X86::PTILELOADDRSV:581 case X86::PTILELOADDRST1V:582 case X86::PTCVTROWD2PSrreV:583 case X86::PTCVTROWD2PSrriV:584 case X86::PTCVTROWPS2BF16HrreV:585 case X86::PTCVTROWPS2BF16HrriV:586 case X86::PTCVTROWPS2BF16LrreV:587 case X86::PTCVTROWPS2BF16LrriV:588 case X86::PTCVTROWPS2PHHrreV:589 case X86::PTCVTROWPS2PHHrriV:590 case X86::PTCVTROWPS2PHLrreV:591 case X86::PTCVTROWPS2PHLrriV:592 case X86::PTILEMOVROWrreV:593 case X86::PTILEMOVROWrriV: {594 for (unsigned i = 2; i > 0; --i)595 MI.removeOperand(i);596 unsigned Opc;597 switch (Opcode) {598 case X86::PTILELOADDRSV:599 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDRS);600 break;601 case X86::PTILELOADDRST1V:602 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDRST1);603 break;604 case X86::PTILELOADDV:605 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD);606 break;607 case X86::PTILELOADDT1V:608 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDT1);609 break;610 case X86::PTCVTROWD2PSrreV:611 Opc = X86::TCVTROWD2PSrte;612 break;613 case X86::PTCVTROWD2PSrriV:614 Opc = X86::TCVTROWD2PSrti;615 break;616 case X86::PTCVTROWPS2BF16HrreV:617 Opc = X86::TCVTROWPS2BF16Hrte;618 break;619 case X86::PTCVTROWPS2BF16HrriV:620 Opc = X86::TCVTROWPS2BF16Hrti;621 break;622 case X86::PTCVTROWPS2BF16LrreV:623 Opc = X86::TCVTROWPS2BF16Lrte;624 break;625 case X86::PTCVTROWPS2BF16LrriV:626 Opc = X86::TCVTROWPS2BF16Lrti;627 break;628 case X86::PTCVTROWPS2PHHrreV:629 Opc = X86::TCVTROWPS2PHHrte;630 break;631 case X86::PTCVTROWPS2PHHrriV:632 Opc = X86::TCVTROWPS2PHHrti;633 break;634 case X86::PTCVTROWPS2PHLrreV:635 Opc = X86::TCVTROWPS2PHLrte;636 break;637 case X86::PTCVTROWPS2PHLrriV:638 Opc = X86::TCVTROWPS2PHLrti;639 break;640 case X86::PTILEMOVROWrreV:641 Opc = X86::TILEMOVROWrte;642 break;643 case X86::PTILEMOVROWrriV:644 Opc = X86::TILEMOVROWrti;645 break;646 default:647 llvm_unreachable("Unexpected Opcode");648 }649 MI.setDesc(TII->get(Opc));650 return true;651 }652 case X86::PTCMMIMFP16PSV:653 case X86::PTCMMRLFP16PSV:654 case X86::PTDPBSSDV:655 case X86::PTDPBSUDV:656 case X86::PTDPBUSDV:657 case X86::PTDPBUUDV:658 case X86::PTDPBF16PSV:659 case X86::PTDPFP16PSV:660 case X86::PTMMULTF32PSV:661 case X86::PTDPBF8PSV:662 case X86::PTDPBHF8PSV:663 case X86::PTDPHBF8PSV:664 case X86::PTDPHF8PSV: {665 MI.untieRegOperand(4);666 for (unsigned i = 3; i > 0; --i)667 MI.removeOperand(i);668 unsigned Opc;669 switch (Opcode) {670 // clang-format off671 case X86::PTCMMIMFP16PSV: Opc = X86::TCMMIMFP16PS; break;672 case X86::PTCMMRLFP16PSV: Opc = X86::TCMMRLFP16PS; break;673 case X86::PTDPBSSDV: Opc = X86::TDPBSSD; break;674 case X86::PTDPBSUDV: Opc = X86::TDPBSUD; break;675 case X86::PTDPBUSDV: Opc = X86::TDPBUSD; break;676 case X86::PTDPBUUDV: Opc = X86::TDPBUUD; break;677 case X86::PTDPBF16PSV: Opc = X86::TDPBF16PS; break;678 case X86::PTDPFP16PSV: Opc = X86::TDPFP16PS; break;679 case X86::PTMMULTF32PSV: Opc = X86::TMMULTF32PS; break;680 case X86::PTDPBF8PSV: Opc = X86::TDPBF8PS; break;681 case X86::PTDPBHF8PSV: Opc = X86::TDPBHF8PS; break;682 case X86::PTDPHBF8PSV: Opc = X86::TDPHBF8PS; break;683 case X86::PTDPHF8PSV: Opc = X86::TDPHF8PS; break;684 // clang-format on685 default:686 llvm_unreachable("Unexpected Opcode");687 }688 MI.setDesc(TII->get(Opc));689 MI.tieOperands(0, 1);690 return true;691 }692 case X86::PTILESTOREDV: {693 for (int i = 1; i >= 0; --i)694 MI.removeOperand(i);695 MI.setDesc(TII->get(GET_EGPR_IF_ENABLED(X86::TILESTORED)));696 return true;697 }698#undef GET_EGPR_IF_ENABLED699 case X86::PTILEZEROV: {700 for (int i = 2; i > 0; --i) // Remove row, col701 MI.removeOperand(i);702 MI.setDesc(TII->get(X86::TILEZERO));703 return true;704 }705 case X86::CALL64pcrel32_RVMARKER:706 case X86::CALL64r_RVMARKER:707 case X86::CALL64m_RVMARKER:708 expandCALL_RVMARKER(MBB, MBBI);709 return true;710 case X86::CALL64r_ImpCall:711 MI.setDesc(TII->get(X86::CALL64r));712 return true;713 case X86::ADD32mi_ND:714 case X86::ADD64mi32_ND:715 case X86::SUB32mi_ND:716 case X86::SUB64mi32_ND:717 case X86::AND32mi_ND:718 case X86::AND64mi32_ND:719 case X86::OR32mi_ND:720 case X86::OR64mi32_ND:721 case X86::XOR32mi_ND:722 case X86::XOR64mi32_ND:723 case X86::ADC32mi_ND:724 case X86::ADC64mi32_ND:725 case X86::SBB32mi_ND:726 case X86::SBB64mi32_ND: {727 // It's possible for an EVEX-encoded legacy instruction to reach the 15-byte728 // instruction length limit: 4 bytes of EVEX prefix + 1 byte of opcode + 1729 // byte of ModRM + 1 byte of SIB + 4 bytes of displacement + 4 bytes of730 // immediate = 15 bytes in total, e.g.731 //732 // subq $184, %fs:257(%rbx, %rcx), %rax733 //734 // In such a case, no additional (ADSIZE or segment override) prefix can be735 // used. To resolve the issue, we split the “long” instruction into 2736 // instructions:737 //738 // movq %fs:257(%rbx, %rcx)，%rax739 // subq $184, %rax740 //741 // Therefore we consider the OPmi_ND to be a pseudo instruction to some742 // extent.743 const MachineOperand &ImmOp =744 MI.getOperand(MI.getNumExplicitOperands() - 1);745 // If the immediate is a expr, conservatively estimate 4 bytes.746 if (ImmOp.isImm() && isInt<8>(ImmOp.getImm()))747 return false;748 int MemOpNo = X86::getFirstAddrOperandIdx(MI);749 const MachineOperand &DispOp = MI.getOperand(MemOpNo + X86::AddrDisp);750 Register Base = MI.getOperand(MemOpNo + X86::AddrBaseReg).getReg();751 // If the displacement is a expr, conservatively estimate 4 bytes.752 if (Base && DispOp.isImm() && isInt<8>(DispOp.getImm()))753 return false;754 // There can only be one of three: SIB, segment override register, ADSIZE755 Register Index = MI.getOperand(MemOpNo + X86::AddrIndexReg).getReg();756 unsigned Count = !!MI.getOperand(MemOpNo + X86::AddrSegmentReg).getReg();757 if (X86II::needSIB(Base, Index, /*In64BitMode=*/true))758 ++Count;759 if (X86MCRegisterClasses[X86::GR32RegClassID].contains(Base) ||760 X86MCRegisterClasses[X86::GR32RegClassID].contains(Index))761 ++Count;762 if (Count < 2)763 return false;764 unsigned Opc, LoadOpc;765 switch (Opcode) {766#define MI_TO_RI(OP) \767 case X86::OP##32mi_ND: \768 Opc = X86::OP##32ri; \769 LoadOpc = X86::MOV32rm; \770 break; \771 case X86::OP##64mi32_ND: \772 Opc = X86::OP##64ri32; \773 LoadOpc = X86::MOV64rm; \774 break;775 776 default:777 llvm_unreachable("Unexpected Opcode");778 MI_TO_RI(ADD);779 MI_TO_RI(SUB);780 MI_TO_RI(AND);781 MI_TO_RI(OR);782 MI_TO_RI(XOR);783 MI_TO_RI(ADC);784 MI_TO_RI(SBB);785#undef MI_TO_RI786 }787 // Insert OPri.788 Register DestReg = MI.getOperand(0).getReg();789 BuildMI(MBB, std::next(MBBI), DL, TII->get(Opc), DestReg)790 .addReg(DestReg)791 .add(ImmOp);792 // Change OPmi_ND to MOVrm.793 for (unsigned I = MI.getNumImplicitOperands() + 1; I != 0; --I)794 MI.removeOperand(MI.getNumOperands() - 1);795 MI.setDesc(TII->get(LoadOpc));796 return true;797 }798 }799 llvm_unreachable("Previous switch has a fallthrough?");800}801 802// This function creates additional block for storing varargs guarded803// registers. It adds check for %al into entry block, to skip804// GuardedRegsBlk if xmm registers should not be stored.805//806// EntryBlk[VAStartPseudoInstr] EntryBlk807// | | .808// | | .809// | | GuardedRegsBlk810// | => | .811// | | .812// | TailBlk813// | |814// | |815//816void X86ExpandPseudo::expandVastartSaveXmmRegs(817 MachineBasicBlock *EntryBlk,818 MachineBasicBlock::iterator VAStartPseudoInstr) const {819 assert(VAStartPseudoInstr->getOpcode() == X86::VASTART_SAVE_XMM_REGS);820 821 MachineFunction *Func = EntryBlk->getParent();822 const TargetInstrInfo *TII = STI->getInstrInfo();823 const DebugLoc &DL = VAStartPseudoInstr->getDebugLoc();824 Register CountReg = VAStartPseudoInstr->getOperand(0).getReg();825 826 // Calculate liveins for newly created blocks.827 LivePhysRegs LiveRegs(*STI->getRegisterInfo());828 SmallVector<std::pair<MCPhysReg, const MachineOperand *>, 8> Clobbers;829 830 LiveRegs.addLiveIns(*EntryBlk);831 for (MachineInstr &MI : EntryBlk->instrs()) {832 if (MI.getOpcode() == VAStartPseudoInstr->getOpcode())833 break;834 835 LiveRegs.stepForward(MI, Clobbers);836 }837 838 // Create the new basic blocks. One block contains all the XMM stores,839 // and another block is the final destination regardless of whether any840 // stores were performed.841 const BasicBlock *LLVMBlk = EntryBlk->getBasicBlock();842 MachineFunction::iterator EntryBlkIter = ++EntryBlk->getIterator();843 MachineBasicBlock *GuardedRegsBlk = Func->CreateMachineBasicBlock(LLVMBlk);844 MachineBasicBlock *TailBlk = Func->CreateMachineBasicBlock(LLVMBlk);845 Func->insert(EntryBlkIter, GuardedRegsBlk);846 Func->insert(EntryBlkIter, TailBlk);847 848 // Transfer the remainder of EntryBlk and its successor edges to TailBlk.849 TailBlk->splice(TailBlk->begin(), EntryBlk,850 std::next(MachineBasicBlock::iterator(VAStartPseudoInstr)),851 EntryBlk->end());852 TailBlk->transferSuccessorsAndUpdatePHIs(EntryBlk);853 854 uint64_t FrameOffset = VAStartPseudoInstr->getOperand(4).getImm();855 uint64_t VarArgsRegsOffset = VAStartPseudoInstr->getOperand(6).getImm();856 857 // TODO: add support for YMM and ZMM here.858 unsigned MOVOpc = STI->hasAVX() ? X86::VMOVAPSmr : X86::MOVAPSmr;859 860 // In the XMM save block, save all the XMM argument registers.861 for (int64_t OpndIdx = 7, RegIdx = 0;862 OpndIdx < VAStartPseudoInstr->getNumOperands() - 1;863 OpndIdx++, RegIdx++) {864 auto NewMI = BuildMI(GuardedRegsBlk, DL, TII->get(MOVOpc));865 for (int i = 0; i < X86::AddrNumOperands; ++i) {866 if (i == X86::AddrDisp)867 NewMI.addImm(FrameOffset + VarArgsRegsOffset + RegIdx * 16);868 else869 NewMI.add(VAStartPseudoInstr->getOperand(i + 1));870 }871 NewMI.addReg(VAStartPseudoInstr->getOperand(OpndIdx).getReg());872 assert(VAStartPseudoInstr->getOperand(OpndIdx).getReg().isPhysical());873 }874 875 // The original block will now fall through to the GuardedRegsBlk.876 EntryBlk->addSuccessor(GuardedRegsBlk);877 // The GuardedRegsBlk will fall through to the TailBlk.878 GuardedRegsBlk->addSuccessor(TailBlk);879 880 if (!STI->isCallingConvWin64(Func->getFunction().getCallingConv())) {881 // If %al is 0, branch around the XMM save block.882 BuildMI(EntryBlk, DL, TII->get(X86::TEST8rr))883 .addReg(CountReg)884 .addReg(CountReg);885 BuildMI(EntryBlk, DL, TII->get(X86::JCC_1))886 .addMBB(TailBlk)887 .addImm(X86::COND_E);888 EntryBlk->addSuccessor(TailBlk);889 }890 891 // Add liveins to the created block.892 addLiveIns(*GuardedRegsBlk, LiveRegs);893 addLiveIns(*TailBlk, LiveRegs);894 895 // Delete the pseudo.896 VAStartPseudoInstr->eraseFromParent();897}898 899/// Expand all pseudo instructions contained in \p MBB.900/// \returns true if any expansion occurred for \p MBB.901bool X86ExpandPseudo::expandMBB(MachineBasicBlock &MBB) {902 bool Modified = false;903 904 // MBBI may be invalidated by the expansion.905 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();906 while (MBBI != E) {907 MachineBasicBlock::iterator NMBBI = std::next(MBBI);908 Modified |= expandMI(MBB, MBBI);909 MBBI = NMBBI;910 }911 912 return Modified;913}914 915bool X86ExpandPseudo::expandPseudosWhichAffectControlFlow(MachineFunction &MF) {916 // Currently pseudo which affects control flow is only917 // X86::VASTART_SAVE_XMM_REGS which is located in Entry block.918 // So we do not need to evaluate other blocks.919 for (MachineInstr &Instr : MF.front().instrs()) {920 if (Instr.getOpcode() == X86::VASTART_SAVE_XMM_REGS) {921 expandVastartSaveXmmRegs(&(MF.front()), Instr);922 return true;923 }924 }925 926 return false;927}928 929bool X86ExpandPseudo::runOnMachineFunction(MachineFunction &MF) {930 STI = &MF.getSubtarget<X86Subtarget>();931 TII = STI->getInstrInfo();932 TRI = STI->getRegisterInfo();933 X86FI = MF.getInfo<X86MachineFunctionInfo>();934 X86FL = STI->getFrameLowering();935 936 bool Modified = expandPseudosWhichAffectControlFlow(MF);937 938 for (MachineBasicBlock &MBB : MF)939 Modified |= expandMBB(MBB);940 return Modified;941}942 943/// Returns an instance of the pseudo instruction expansion pass.944FunctionPass *llvm::createX86ExpandPseudoPass() {945 return new X86ExpandPseudo();946}947