2031 lines · cpp
1//===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the X86MCCodeEmitter class.10//11//===----------------------------------------------------------------------===//12 13#include "MCTargetDesc/X86BaseInfo.h"14#include "MCTargetDesc/X86FixupKinds.h"15#include "MCTargetDesc/X86MCAsmInfo.h"16#include "MCTargetDesc/X86MCTargetDesc.h"17#include "llvm/ADT/SmallVector.h"18#include "llvm/BinaryFormat/ELF.h"19#include "llvm/MC/MCCodeEmitter.h"20#include "llvm/MC/MCContext.h"21#include "llvm/MC/MCExpr.h"22#include "llvm/MC/MCFixup.h"23#include "llvm/MC/MCInst.h"24#include "llvm/MC/MCInstrDesc.h"25#include "llvm/MC/MCInstrInfo.h"26#include "llvm/MC/MCRegisterInfo.h"27#include "llvm/MC/MCSubtargetInfo.h"28#include "llvm/MC/MCSymbol.h"29#include "llvm/Support/Casting.h"30#include "llvm/Support/ErrorHandling.h"31#include <cassert>32#include <cstdint>33#include <cstdlib>34 35using namespace llvm;36 37#define DEBUG_TYPE "mccodeemitter"38 39namespace {40 41enum PrefixKind { None, REX, REX2, XOP, VEX2, VEX3, EVEX };42 43static void emitByte(uint8_t C, SmallVectorImpl<char> &CB) { CB.push_back(C); }44 45class X86OpcodePrefixHelper {46 // REX (1 byte)47 // +-----+ +------+48 // | 40H | | WRXB |49 // +-----+ +------+50 51 // REX2 (2 bytes)52 // +-----+ +-------------------+53 // | D5H | | M | R'X'B' | WRXB |54 // +-----+ +-------------------+55 56 // XOP (3-byte)57 // +-----+ +--------------+ +-------------------+58 // | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp |59 // +-----+ +--------------+ +-------------------+60 61 // VEX2 (2 bytes)62 // +-----+ +-------------------+63 // | C5h | | R | vvvv | L | pp |64 // +-----+ +-------------------+65 66 // VEX3 (3 bytes)67 // +-----+ +--------------+ +-------------------+68 // | C4h | | RXB | m-mmmm | | W | vvvv | L | pp |69 // +-----+ +--------------+ +-------------------+70 71 // VEX_R: opcode externsion equivalent to REX.R in72 // 1's complement (inverted) form73 //74 // 1: Same as REX_R=0 (must be 1 in 32-bit mode)75 // 0: Same as REX_R=1 (64 bit mode only)76 77 // VEX_X: equivalent to REX.X, only used when a78 // register is used for index in SIB Byte.79 //80 // 1: Same as REX.X=0 (must be 1 in 32-bit mode)81 // 0: Same as REX.X=1 (64-bit mode only)82 83 // VEX_B:84 // 1: Same as REX_B=0 (ignored in 32-bit mode)85 // 0: Same as REX_B=1 (64 bit mode only)86 87 // VEX_W: opcode specific (use like REX.W, or used for88 // opcode extension, or ignored, depending on the opcode byte)89 90 // VEX_5M (VEX m-mmmmm field):91 //92 // 0b00000: Reserved for future use93 // 0b00001: implied 0F leading opcode94 // 0b00010: implied 0F 38 leading opcode bytes95 // 0b00011: implied 0F 3A leading opcode bytes96 // 0b00100: Reserved for future use97 // 0b00101: VEX MAP598 // 0b00110: VEX MAP699 // 0b00111: VEX MAP7100 // 0b00111-0b11111: Reserved for future use101 // 0b01000: XOP map select - 08h instructions with imm byte102 // 0b01001: XOP map select - 09h instructions with no imm byte103 // 0b01010: XOP map select - 0Ah instructions with imm dword104 105 // VEX_4V (VEX vvvv field): a register specifier106 // (in 1's complement form) or 1111 if unused.107 108 // VEX_PP: opcode extension providing equivalent109 // functionality of a SIMD prefix110 // 0b00: None111 // 0b01: 66112 // 0b10: F3113 // 0b11: F2114 115 // EVEX (4 bytes)116 // +-----+ +---------------+ +-------------------+ +------------------------+117 // | 62h | | RXBR' | B'mmm | | W | vvvv | U | pp | | z | L'L | b | v' | aaa |118 // +-----+ +---------------+ +-------------------+ +------------------------+119 120 // EVEX_L2/VEX_L (Vector Length):121 // L2 L122 // 0 0: scalar or 128-bit vector123 // 0 1: 256-bit vector124 // 1 0: 512-bit vector125 126 // 32-Register Support in 64-bit Mode Using EVEX with Embedded REX/REX2 Bits:127 //128 // +----------+---------+--------+-----------+---------+--------------+129 // | | 4 | 3 | [2:0] | Type | Common Usage |130 // +----------+---------+--------+-----------+---------+--------------+131 // | REG | EVEX_R' | EVEX_R | modrm.reg | GPR, VR | Dest or Src |132 // | VVVV | EVEX_v' | EVEX.vvvv | GPR, VR | Dest or Src |133 // | RM (VR) | EVEX_X | EVEX_B | modrm.r/m | VR | Dest or Src |134 // | RM (GPR) | EVEX_B' | EVEX_B | modrm.r/m | GPR | Dest or Src |135 // | BASE | EVEX_B' | EVEX_B | modrm.r/m | GPR | MA |136 // | INDEX | EVEX_U | EVEX_X | sib.index | GPR | MA |137 // | VIDX | EVEX_v' | EVEX_X | sib.index | VR | VSIB MA |138 // +----------+---------+--------+-----------+---------+--------------+139 //140 // * GPR - General-purpose register141 // * VR - Vector register142 // * VIDX - Vector index143 // * VSIB - Vector SIB144 // * MA - Memory addressing145 146private:147 unsigned W : 1;148 unsigned R : 1;149 unsigned X : 1;150 unsigned B : 1;151 unsigned M : 1;152 unsigned R2 : 1;153 unsigned X2 : 1;154 unsigned B2 : 1;155 unsigned VEX_4V : 4;156 unsigned VEX_L : 1;157 unsigned VEX_PP : 2;158 unsigned VEX_5M : 5;159 unsigned EVEX_z : 1;160 unsigned EVEX_L2 : 1;161 unsigned EVEX_b : 1;162 unsigned EVEX_V2 : 1;163 unsigned EVEX_aaa : 3;164 PrefixKind Kind = None;165 const MCRegisterInfo &MRI;166 167 unsigned getRegEncoding(const MCInst &MI, unsigned OpNum) const {168 return MRI.getEncodingValue(MI.getOperand(OpNum).getReg());169 }170 171 void setR(unsigned Encoding) { R = Encoding >> 3 & 1; }172 void setR2(unsigned Encoding) {173 R2 = Encoding >> 4 & 1;174 assert((!R2 || (Kind <= REX2 || Kind == EVEX)) && "invalid setting");175 }176 void setX(unsigned Encoding) { X = Encoding >> 3 & 1; }177 void setX2(unsigned Encoding) {178 assert((Kind <= REX2 || Kind == EVEX) && "invalid setting");179 X2 = Encoding >> 4 & 1;180 }181 void setB(unsigned Encoding) { B = Encoding >> 3 & 1; }182 void setB2(unsigned Encoding) {183 assert((Kind <= REX2 || Kind == EVEX) && "invalid setting");184 B2 = Encoding >> 4 & 1;185 }186 void set4V(unsigned Encoding) { VEX_4V = Encoding & 0xf; }187 void setV2(unsigned Encoding) { EVEX_V2 = Encoding >> 4 & 1; }188 189public:190 void setW(bool V) { W = V; }191 void setR(const MCInst &MI, unsigned OpNum) {192 setR(getRegEncoding(MI, OpNum));193 }194 void setX(const MCInst &MI, unsigned OpNum, unsigned Shift = 3) {195 MCRegister Reg = MI.getOperand(OpNum).getReg();196 // X is used to extend vector register only when shift is not 3.197 if (Shift != 3 && X86II::isApxExtendedReg(Reg))198 return;199 unsigned Encoding = MRI.getEncodingValue(Reg);200 X = Encoding >> Shift & 1;201 }202 void setB(const MCInst &MI, unsigned OpNum) {203 B = getRegEncoding(MI, OpNum) >> 3 & 1;204 }205 void set4V(const MCInst &MI, unsigned OpNum, bool IsImm = false) {206 // OF, SF, ZF and CF reuse VEX_4V bits but are not reversed207 if (IsImm)208 set4V(~(MI.getOperand(OpNum).getImm()));209 else210 set4V(getRegEncoding(MI, OpNum));211 }212 void setL(bool V) { VEX_L = V; }213 void setPP(unsigned V) { VEX_PP = V; }214 void set5M(unsigned V) { VEX_5M = V; }215 void setR2(const MCInst &MI, unsigned OpNum) {216 setR2(getRegEncoding(MI, OpNum));217 }218 void setRR2(const MCInst &MI, unsigned OpNum) {219 unsigned Encoding = getRegEncoding(MI, OpNum);220 setR(Encoding);221 setR2(Encoding);222 }223 void setM(bool V) { M = V; }224 void setXX2(const MCInst &MI, unsigned OpNum) {225 MCRegister Reg = MI.getOperand(OpNum).getReg();226 unsigned Encoding = MRI.getEncodingValue(Reg);227 setX(Encoding);228 // Index can be a vector register while X2 is used to extend GPR only.229 if (Kind <= REX2 || X86II::isApxExtendedReg(Reg))230 setX2(Encoding);231 }232 void setBB2(const MCInst &MI, unsigned OpNum) {233 MCRegister Reg = MI.getOperand(OpNum).getReg();234 unsigned Encoding = MRI.getEncodingValue(Reg);235 setB(Encoding);236 // Base can be a vector register while B2 is used to extend GPR only237 if (Kind <= REX2 || X86II::isApxExtendedReg(Reg))238 setB2(Encoding);239 }240 void setZ(bool V) { EVEX_z = V; }241 void setL2(bool V) { EVEX_L2 = V; }242 void setEVEX_b(bool V) { EVEX_b = V; }243 void setEVEX_U(bool V) { X2 = V; }244 void setV2(const MCInst &MI, unsigned OpNum, bool HasVEX_4V) {245 // Only needed with VSIB which don't use VVVV.246 if (HasVEX_4V)247 return;248 MCRegister Reg = MI.getOperand(OpNum).getReg();249 if (X86II::isApxExtendedReg(Reg))250 return;251 setV2(MRI.getEncodingValue(Reg));252 }253 void set4VV2(const MCInst &MI, unsigned OpNum) {254 unsigned Encoding = getRegEncoding(MI, OpNum);255 set4V(Encoding);256 setV2(Encoding);257 }258 void setAAA(const MCInst &MI, unsigned OpNum) {259 EVEX_aaa = getRegEncoding(MI, OpNum);260 }261 void setNF(bool V) { EVEX_aaa |= V << 2; }262 void setSC(const MCInst &MI, unsigned OpNum) {263 unsigned Encoding = MI.getOperand(OpNum).getImm();264 EVEX_V2 = ~(Encoding >> 3) & 0x1;265 EVEX_aaa = Encoding & 0x7;266 }267 268 X86OpcodePrefixHelper(const MCRegisterInfo &MRI)269 : W(0), R(0), X(0), B(0), M(0), R2(0), X2(0), B2(0), VEX_4V(0), VEX_L(0),270 VEX_PP(0), VEX_5M(0), EVEX_z(0), EVEX_L2(0), EVEX_b(0), EVEX_V2(0),271 EVEX_aaa(0), MRI(MRI) {}272 273 void setLowerBound(PrefixKind K) { Kind = K; }274 275 PrefixKind determineOptimalKind() {276 switch (Kind) {277 case None:278 // Not M bit here by intention b/c279 // 1. No guarantee that REX2 is supported by arch w/o explict EGPR280 // 2. REX2 is longer than 0FH281 Kind = (R2 | X2 | B2) ? REX2 : (W | R | X | B) ? REX : None;282 break;283 case REX:284 Kind = (R2 | X2 | B2) ? REX2 : REX;285 break;286 case REX2:287 case XOP:288 case VEX3:289 case EVEX:290 break;291 case VEX2:292 Kind = (W | X | B | (VEX_5M != 1)) ? VEX3 : VEX2;293 break;294 }295 return Kind;296 }297 298 void emit(SmallVectorImpl<char> &CB) const {299 uint8_t FirstPayload =300 ((~R) & 0x1) << 7 | ((~X) & 0x1) << 6 | ((~B) & 0x1) << 5;301 uint8_t LastPayload = ((~VEX_4V) & 0xf) << 3 | VEX_L << 2 | VEX_PP;302 switch (Kind) {303 case None:304 return;305 case REX:306 emitByte(0x40 | W << 3 | R << 2 | X << 1 | B, CB);307 return;308 case REX2:309 emitByte(0xD5, CB);310 emitByte(M << 7 | R2 << 6 | X2 << 5 | B2 << 4 | W << 3 | R << 2 | X << 1 |311 B,312 CB);313 return;314 case VEX2:315 emitByte(0xC5, CB);316 emitByte(((~R) & 1) << 7 | LastPayload, CB);317 return;318 case VEX3:319 case XOP:320 emitByte(Kind == VEX3 ? 0xC4 : 0x8F, CB);321 emitByte(FirstPayload | VEX_5M, CB);322 emitByte(W << 7 | LastPayload, CB);323 return;324 case EVEX:325 assert(VEX_5M && !(VEX_5M & 0x8) && "invalid mmm fields for EVEX!");326 emitByte(0x62, CB);327 emitByte(FirstPayload | ((~R2) & 0x1) << 4 | B2 << 3 | VEX_5M, CB);328 emitByte(W << 7 | ((~VEX_4V) & 0xf) << 3 | ((~X2) & 0x1) << 2 | VEX_PP,329 CB);330 emitByte(EVEX_z << 7 | EVEX_L2 << 6 | VEX_L << 5 | EVEX_b << 4 |331 ((~EVEX_V2) & 0x1) << 3 | EVEX_aaa,332 CB);333 return;334 }335 }336};337 338class X86MCCodeEmitter : public MCCodeEmitter {339 const MCInstrInfo &MCII;340 MCContext &Ctx;341 342public:343 X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx)344 : MCII(mcii), Ctx(ctx) {}345 X86MCCodeEmitter(const X86MCCodeEmitter &) = delete;346 X86MCCodeEmitter &operator=(const X86MCCodeEmitter &) = delete;347 ~X86MCCodeEmitter() override = default;348 349 void emitPrefix(const MCInst &MI, SmallVectorImpl<char> &CB,350 const MCSubtargetInfo &STI) const;351 352 void encodeInstruction(const MCInst &MI, SmallVectorImpl<char> &CB,353 SmallVectorImpl<MCFixup> &Fixups,354 const MCSubtargetInfo &STI) const override;355 356private:357 unsigned getX86RegNum(const MCOperand &MO) const;358 359 unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const;360 361 void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned FixupKind,362 bool IsPCRel, uint64_t StartByte,363 SmallVectorImpl<char> &CB,364 SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const;365 366 void emitRegModRMByte(const MCOperand &ModRMReg, unsigned RegOpcodeFld,367 SmallVectorImpl<char> &CB) const;368 369 void emitSIBByte(unsigned SS, unsigned Index, unsigned Base,370 SmallVectorImpl<char> &CB) const;371 372 void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField,373 uint64_t TSFlags, PrefixKind Kind, uint64_t StartByte,374 SmallVectorImpl<char> &CB,375 SmallVectorImpl<MCFixup> &Fixups,376 const MCSubtargetInfo &STI,377 bool ForceSIB = false) const;378 379 PrefixKind emitPrefixImpl(unsigned &CurOp, const MCInst &MI,380 const MCSubtargetInfo &STI,381 SmallVectorImpl<char> &CB) const;382 383 PrefixKind emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,384 const MCSubtargetInfo &STI,385 SmallVectorImpl<char> &CB) const;386 387 void emitSegmentOverridePrefix(unsigned SegOperand, const MCInst &MI,388 SmallVectorImpl<char> &CB) const;389 390 PrefixKind emitOpcodePrefix(int MemOperand, const MCInst &MI,391 const MCSubtargetInfo &STI,392 SmallVectorImpl<char> &CB) const;393 394 PrefixKind emitREXPrefix(int MemOperand, const MCInst &MI,395 const MCSubtargetInfo &STI,396 SmallVectorImpl<char> &CB) const;397};398 399} // end anonymous namespace400 401static uint8_t modRMByte(unsigned Mod, unsigned RegOpcode, unsigned RM) {402 assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!");403 return RM | (RegOpcode << 3) | (Mod << 6);404}405 406static void emitConstant(uint64_t Val, unsigned Size,407 SmallVectorImpl<char> &CB) {408 // Output the constant in little endian byte order.409 for (unsigned i = 0; i != Size; ++i) {410 emitByte(Val & 255, CB);411 Val >>= 8;412 }413}414 415/// Determine if this immediate can fit in a disp8 or a compressed disp8 for416/// EVEX instructions. \p will be set to the value to pass to the ImmOffset417/// parameter of emitImmediate.418static bool isDispOrCDisp8(uint64_t TSFlags, int Value, int &ImmOffset) {419 bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX;420 421 unsigned CD8_Scale =422 (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift;423 CD8_Scale = CD8_Scale ? 1U << (CD8_Scale - 1) : 0U;424 if (!HasEVEX || !CD8_Scale)425 return isInt<8>(Value);426 427 assert(isPowerOf2_32(CD8_Scale) && "Unexpected CD8 scale!");428 if (Value & (CD8_Scale - 1)) // Unaligned offset429 return false;430 431 int CDisp8 = Value / static_cast<int>(CD8_Scale);432 if (!isInt<8>(CDisp8))433 return false;434 435 // ImmOffset will be added to Value in emitImmediate leaving just CDisp8.436 ImmOffset = CDisp8 - Value;437 return true;438}439 440/// \returns the appropriate fixup kind to use for an immediate in an441/// instruction with the specified TSFlags.442static MCFixupKind getImmFixupKind(uint64_t TSFlags) {443 unsigned Size = X86II::getSizeOfImm(TSFlags);444 if (X86II::isImmSigned(TSFlags)) {445 switch (Size) {446 default:447 llvm_unreachable("Unsupported signed fixup size!");448 case 4:449 return X86::reloc_signed_4byte;450 }451 }452 switch (Size) {453 default:454 llvm_unreachable("Invalid generic fixup size!");455 case 1:456 return FK_Data_1;457 case 2:458 return FK_Data_2;459 case 4:460 return FK_Data_4;461 case 8:462 return FK_Data_8;463 }464}465 466enum GlobalOffsetTableExprKind { GOT_None, GOT_Normal, GOT_SymDiff };467 468/// Check if this expression starts with _GLOBAL_OFFSET_TABLE_ and if it is469/// of the form _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on470/// ELF i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that471/// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start of a472/// binary expression.473///474/// TODO: Move this to X86AsmBackend.cpp at relocation decision phase so that we475/// don't have to mess with MCExpr.476static GlobalOffsetTableExprKind477startsWithGlobalOffsetTable(const MCExpr *Expr) {478 const MCExpr *RHS = nullptr;479 if (Expr->getKind() == MCExpr::Binary) {480 const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Expr);481 Expr = BE->getLHS();482 RHS = BE->getRHS();483 }484 485 if (Expr->getKind() != MCExpr::SymbolRef)486 return GOT_None;487 488 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);489 const MCSymbol &S = Ref->getSymbol();490 if (S.getName() != "_GLOBAL_OFFSET_TABLE_")491 return GOT_None;492 if (RHS && RHS->getKind() == MCExpr::SymbolRef)493 return GOT_SymDiff;494 return GOT_Normal;495}496 497static bool hasSecRelSymbolRef(const MCExpr *Expr) {498 if (Expr->getKind() == MCExpr::SymbolRef) {499 auto *Ref = static_cast<const MCSymbolRefExpr *>(Expr);500 return Ref->getSpecifier() == X86::S_COFF_SECREL;501 }502 return false;503}504 505static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) {506 unsigned Opcode = MI.getOpcode();507 const MCInstrDesc &Desc = MCII.get(Opcode);508 if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 &&509 Opcode != X86::JCC_4) ||510 !(getImmFixupKind(Desc.TSFlags) == FK_Data_4 &&511 X86II::isImmPCRel(Desc.TSFlags)))512 return false;513 514 unsigned CurOp = X86II::getOperandBias(Desc);515 const MCOperand &Op = MI.getOperand(CurOp);516 if (!Op.isExpr())517 return false;518 519 auto *Ref = dyn_cast<MCSymbolRefExpr>(Op.getExpr());520 return Ref && Ref->getSpecifier() == X86::S_None;521}522 523unsigned X86MCCodeEmitter::getX86RegNum(const MCOperand &MO) const {524 return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg()) & 0x7;525}526 527unsigned X86MCCodeEmitter::getX86RegEncoding(const MCInst &MI,528 unsigned OpNum) const {529 return Ctx.getRegisterInfo()->getEncodingValue(MI.getOperand(OpNum).getReg());530}531 532void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc,533 unsigned FixupKind, bool PCRel,534 uint64_t StartByte,535 SmallVectorImpl<char> &CB,536 SmallVectorImpl<MCFixup> &Fixups,537 int ImmOffset) const {538 unsigned Size = 4;539 switch (FixupKind) {540 case FK_Data_1:541 Size = 1;542 break;543 case FK_Data_2:544 Size = 2;545 break;546 case FK_Data_8:547 Size = 8;548 break;549 }550 const MCExpr *Expr = nullptr;551 if (DispOp.isImm()) {552 // If this is a simple integer displacement that doesn't require a553 // relocation, emit it now.554 if (!(is_contained({FK_Data_1, FK_Data_2, FK_Data_4}, FixupKind) &&555 PCRel)) {556 emitConstant(DispOp.getImm() + ImmOffset, Size, CB);557 return;558 }559 Expr = MCConstantExpr::create(DispOp.getImm(), Ctx);560 } else {561 Expr = DispOp.getExpr();562 }563 564 // If we have an immoffset, add it to the expression.565 if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 ||566 FixupKind == X86::reloc_signed_4byte)) {567 GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr);568 if (Kind != GOT_None) {569 assert(ImmOffset == 0);570 571 if (Size == 8) {572 FixupKind = FirstLiteralRelocationKind + ELF::R_X86_64_GOTPC64;573 } else {574 assert(Size == 4);575 FixupKind = X86::reloc_global_offset_table;576 }577 578 if (Kind == GOT_Normal)579 ImmOffset = static_cast<int>(CB.size() - StartByte);580 } else if (Expr->getKind() == MCExpr::SymbolRef) {581 if (hasSecRelSymbolRef(Expr)) {582 FixupKind = FK_SecRel_4;583 }584 } else if (Expr->getKind() == MCExpr::Binary) {585 const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr);586 if (hasSecRelSymbolRef(Bin->getLHS()) ||587 hasSecRelSymbolRef(Bin->getRHS())) {588 FixupKind = FK_SecRel_4;589 }590 }591 }592 593 if (ImmOffset)594 Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx),595 Ctx, Expr->getLoc());596 597 // Emit a symbolic constant as a fixup and a few zero bytes.598 Fixups.push_back(MCFixup::create(static_cast<uint32_t>(CB.size() - StartByte),599 Expr, FixupKind, PCRel));600 emitConstant(0, Size, CB);601}602 603void X86MCCodeEmitter::emitRegModRMByte(const MCOperand &ModRMReg,604 unsigned RegOpcodeFld,605 SmallVectorImpl<char> &CB) const {606 emitByte(modRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)), CB);607}608 609void X86MCCodeEmitter::emitSIBByte(unsigned SS, unsigned Index, unsigned Base,610 SmallVectorImpl<char> &CB) const {611 // SIB byte is in the same format as the modRMByte.612 emitByte(modRMByte(SS, Index, Base), CB);613}614 615void X86MCCodeEmitter::emitMemModRMByte(616 const MCInst &MI, unsigned Op, unsigned RegOpcodeField, uint64_t TSFlags,617 PrefixKind Kind, uint64_t StartByte, SmallVectorImpl<char> &CB,618 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI,619 bool ForceSIB) const {620 const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);621 const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);622 const MCOperand &Scale = MI.getOperand(Op + X86::AddrScaleAmt);623 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);624 MCRegister BaseReg = Base.getReg();625 626 // Handle %rip relative addressing.627 if (BaseReg == X86::RIP ||628 BaseReg == X86::EIP) { // [disp32+rIP] in X86-64 mode629 assert(STI.hasFeature(X86::Is64Bit) &&630 "Rip-relative addressing requires 64-bit mode");631 assert(!IndexReg.getReg() && !ForceSIB && "Invalid rip-relative address");632 emitByte(modRMByte(0, RegOpcodeField, 5), CB);633 634 unsigned Opcode = MI.getOpcode();635 unsigned FixupKind = [&]() {636 // Enable relaxed relocation only for a MCSymbolRefExpr. We cannot use a637 // relaxed relocation if an offset is present (e.g. x@GOTPCREL+4).638 if (!(Disp.isExpr() && isa<MCSymbolRefExpr>(Disp.getExpr())))639 return X86::reloc_riprel_4byte;640 641 // Certain loads for GOT references can be relocated against the symbol642 // directly if the symbol ends up in the same linkage unit.643 switch (Opcode) {644 default:645 return X86::reloc_riprel_4byte;646 case X86::MOV64rm:647 // movq loads is a subset of reloc_riprel_4byte_relax_rex/rex2. It is a648 // special case because COFF and Mach-O don't support ELF's more649 // flexible R_X86_64_REX_GOTPCRELX/R_X86_64_CODE_4_GOTPCRELX relaxation.650 return Kind == REX2 ? X86::reloc_riprel_4byte_movq_load_rex2651 : X86::reloc_riprel_4byte_movq_load;652 case X86::ADC32rm:653 case X86::ADD32rm:654 case X86::AND32rm:655 case X86::CMP32rm:656 case X86::MOV32rm:657 case X86::OR32rm:658 case X86::SBB32rm:659 case X86::SUB32rm:660 case X86::TEST32mr:661 case X86::XOR32rm:662 case X86::CALL64m:663 case X86::JMP64m:664 case X86::TAILJMPm64:665 case X86::TEST64mr:666 case X86::ADC64rm:667 case X86::ADD64rm:668 case X86::AND64rm:669 case X86::CMP64rm:670 case X86::OR64rm:671 case X86::SBB64rm:672 case X86::SUB64rm:673 case X86::XOR64rm:674 case X86::LEA64r:675 return Kind == REX2 ? X86::reloc_riprel_4byte_relax_rex2676 : Kind == REX ? X86::reloc_riprel_4byte_relax_rex677 : X86::reloc_riprel_4byte_relax;678 case X86::ADD64rm_NF:679 case X86::ADD64rm_ND:680 case X86::ADD64mr_ND:681 case X86::ADD64mr_NF_ND:682 case X86::ADD64rm_NF_ND:683 return X86::reloc_riprel_4byte_relax_evex;684 }685 }();686 687 // rip-relative addressing is actually relative to the *next* instruction.688 // Since an immediate can follow the mod/rm byte for an instruction, this689 // means that we need to bias the displacement field of the instruction with690 // the size of the immediate field. If we have this case, add it into the691 // expression to emit.692 // Note: rip-relative addressing using immediate displacement values should693 // not be adjusted, assuming it was the user's intent.694 int ImmSize = !Disp.isImm() && X86II::hasImm(TSFlags)695 ? X86II::getSizeOfImm(TSFlags)696 : 0;697 698 emitImmediate(Disp, MI.getLoc(), FixupKind, true, StartByte, CB, Fixups,699 -ImmSize);700 return;701 }702 703 unsigned BaseRegNo = BaseReg ? getX86RegNum(Base) : -1U;704 705 bool IsAdSize16 = STI.hasFeature(X86::Is32Bit) &&706 (TSFlags & X86II::AdSizeMask) == X86II::AdSize16;707 708 // 16-bit addressing forms of the ModR/M byte have a different encoding for709 // the R/M field and are far more limited in which registers can be used.710 if (IsAdSize16 || X86_MC::is16BitMemOperand(MI, Op, STI)) {711 if (BaseReg) {712 // For 32-bit addressing, the row and column values in Table 2-2 are713 // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with714 // some special cases. And getX86RegNum reflects that numbering.715 // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A,716 // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only717 // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order,718 // while values 0-3 indicate the allowed combinations (base+index) of719 // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI.720 //721 // R16Table[] is a lookup from the normal RegNo, to the row values from722 // Table 2-1 for 16-bit addressing modes. Where zero means disallowed.723 static const unsigned R16Table[] = {0, 0, 0, 7, 0, 6, 4, 5};724 unsigned RMfield = R16Table[BaseRegNo];725 726 assert(RMfield && "invalid 16-bit base register");727 728 if (IndexReg.getReg()) {729 unsigned IndexReg16 = R16Table[getX86RegNum(IndexReg)];730 731 assert(IndexReg16 && "invalid 16-bit index register");732 // We must have one of SI/DI (4,5), and one of BP/BX (6,7).733 assert(((IndexReg16 ^ RMfield) & 2) &&734 "invalid 16-bit base/index register combination");735 assert(Scale.getImm() == 1 &&736 "invalid scale for 16-bit memory reference");737 738 // Allow base/index to appear in either order (although GAS doesn't).739 if (IndexReg16 & 2)740 RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1);741 else742 RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1);743 }744 745 if (Disp.isImm() && isInt<8>(Disp.getImm())) {746 if (Disp.getImm() == 0 && RMfield != 6) {747 // There is no displacement; just the register.748 emitByte(modRMByte(0, RegOpcodeField, RMfield), CB);749 return;750 }751 // Use the [REG]+disp8 form, including for [BP] which cannot be encoded.752 emitByte(modRMByte(1, RegOpcodeField, RMfield), CB);753 emitImmediate(Disp, MI.getLoc(), FK_Data_1, false, StartByte, CB,754 Fixups);755 return;756 }757 // This is the [REG]+disp16 case.758 emitByte(modRMByte(2, RegOpcodeField, RMfield), CB);759 } else {760 assert(!IndexReg.getReg() && "Unexpected index register!");761 // There is no BaseReg; this is the plain [disp16] case.762 emitByte(modRMByte(0, RegOpcodeField, 6), CB);763 }764 765 // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases.766 emitImmediate(Disp, MI.getLoc(), FK_Data_2, false, StartByte, CB, Fixups);767 return;768 }769 770 // Check for presence of {disp8} or {disp32} pseudo prefixes.771 bool UseDisp8 = MI.getFlags() & X86::IP_USE_DISP8;772 bool UseDisp32 = MI.getFlags() & X86::IP_USE_DISP32;773 774 // We only allow no displacement if no pseudo prefix is present.775 bool AllowNoDisp = !UseDisp8 && !UseDisp32;776 // Disp8 is allowed unless the {disp32} prefix is present.777 bool AllowDisp8 = !UseDisp32;778 779 // Determine whether a SIB byte is needed.780 if (!ForceSIB && !X86II::needSIB(BaseReg, IndexReg.getReg(),781 STI.hasFeature(X86::Is64Bit))) {782 if (!BaseReg) { // [disp32] in X86-32 mode783 emitByte(modRMByte(0, RegOpcodeField, 5), CB);784 emitImmediate(Disp, MI.getLoc(), FK_Data_4, false, StartByte, CB, Fixups);785 return;786 }787 788 // If the base is not EBP/ESP/R12/R13/R20/R21/R28/R29 and there is no789 // displacement, use simple indirect register encoding, this handles790 // addresses like [EAX]. The encoding for [EBP], [R13], [R20], [R21], [R28]791 // or [R29] with no displacement means [disp32] so we handle it by emitting792 // a displacement of 0 later.793 if (BaseRegNo != N86::EBP) {794 if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp) {795 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), CB);796 return;797 }798 799 // If the displacement is @tlscall, treat it as a zero.800 if (Disp.isExpr()) {801 auto *Sym = dyn_cast<MCSymbolRefExpr>(Disp.getExpr());802 if (Sym && Sym->getSpecifier() == X86::S_TLSCALL) {803 // This is exclusively used by call *a@tlscall(base). The relocation804 // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning.805 Fixups.push_back(MCFixup::create(0, Sym, FK_NONE));806 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), CB);807 return;808 }809 }810 }811 812 // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].813 // Including a compressed disp8 for EVEX instructions that support it.814 // This also handles the 0 displacement for [EBP], [R13], [R21] or [R29]. We815 // can't use disp8 if the {disp32} pseudo prefix is present.816 if (Disp.isImm() && AllowDisp8) {817 int ImmOffset = 0;818 if (isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {819 emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), CB);820 emitImmediate(Disp, MI.getLoc(), FK_Data_1, false, StartByte, CB,821 Fixups, ImmOffset);822 return;823 }824 }825 826 // Otherwise, emit the most general non-SIB encoding: [REG+disp32].827 // Displacement may be 0 for [EBP], [R13], [R21], [R29] case if {disp32}828 // pseudo prefix prevented using disp8 above.829 emitByte(modRMByte(2, RegOpcodeField, BaseRegNo), CB);830 unsigned Opcode = MI.getOpcode();831 unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax832 : X86::reloc_signed_4byte;833 emitImmediate(Disp, MI.getLoc(), MCFixupKind(FixupKind), false, StartByte,834 CB, Fixups);835 return;836 }837 838 // We need a SIB byte, so start by outputting the ModR/M byte first839 assert(IndexReg.getReg() != X86::ESP && IndexReg.getReg() != X86::RSP &&840 "Cannot use ESP as index reg!");841 842 bool ForceDisp32 = false;843 bool ForceDisp8 = false;844 int ImmOffset = 0;845 if (!BaseReg) {846 // If there is no base register, we emit the special case SIB byte with847 // MOD=0, BASE=5, to JUST get the index, scale, and displacement.848 BaseRegNo = 5;849 emitByte(modRMByte(0, RegOpcodeField, 4), CB);850 ForceDisp32 = true;851 } else if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp &&852 // Base reg can't be EBP/RBP/R13/R21/R29 as that would end up with853 // '5' as the base field, but that is the magic [*] nomenclature854 // that indicates no base when mod=0. For these cases we'll emit a855 // 0 displacement instead.856 BaseRegNo != N86::EBP) {857 // Emit no displacement ModR/M byte858 emitByte(modRMByte(0, RegOpcodeField, 4), CB);859 } else if (Disp.isImm() && AllowDisp8 &&860 isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {861 // Displacement fits in a byte or matches an EVEX compressed disp8, use862 // disp8 encoding. This also handles EBP/R13/R21/R29 base with 0863 // displacement unless {disp32} pseudo prefix was used.864 emitByte(modRMByte(1, RegOpcodeField, 4), CB);865 ForceDisp8 = true;866 } else {867 // Otherwise, emit the normal disp32 encoding.868 emitByte(modRMByte(2, RegOpcodeField, 4), CB);869 ForceDisp32 = true;870 }871 872 // Calculate what the SS field value should be...873 static const unsigned SSTable[] = {~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3};874 unsigned SS = SSTable[Scale.getImm()];875 876 unsigned IndexRegNo = IndexReg.getReg() ? getX86RegNum(IndexReg) : 4;877 878 emitSIBByte(SS, IndexRegNo, BaseRegNo, CB);879 880 // Do we need to output a displacement?881 if (ForceDisp8)882 emitImmediate(Disp, MI.getLoc(), FK_Data_1, false, StartByte, CB, Fixups,883 ImmOffset);884 else if (ForceDisp32)885 emitImmediate(Disp, MI.getLoc(), X86::reloc_signed_4byte, false, StartByte,886 CB, Fixups);887}888 889/// Emit all instruction prefixes.890///891/// \returns one of the REX, XOP, VEX2, VEX3, EVEX if any of them is used,892/// otherwise returns None.893PrefixKind X86MCCodeEmitter::emitPrefixImpl(unsigned &CurOp, const MCInst &MI,894 const MCSubtargetInfo &STI,895 SmallVectorImpl<char> &CB) const {896 uint64_t TSFlags = MCII.get(MI.getOpcode()).TSFlags;897 // Determine where the memory operand starts, if present.898 int MemoryOperand = X86II::getMemoryOperandNo(TSFlags);899 // Emit segment override opcode prefix as needed.900 if (MemoryOperand != -1) {901 MemoryOperand += CurOp;902 emitSegmentOverridePrefix(MemoryOperand + X86::AddrSegmentReg, MI, CB);903 }904 905 // Emit the repeat opcode prefix as needed.906 unsigned Flags = MI.getFlags();907 if (TSFlags & X86II::REP || Flags & X86::IP_HAS_REPEAT)908 emitByte(0xF3, CB);909 if (Flags & X86::IP_HAS_REPEAT_NE)910 emitByte(0xF2, CB);911 912 // Emit the address size opcode prefix as needed.913 if (X86_MC::needsAddressSizeOverride(MI, STI, MemoryOperand, TSFlags) ||914 Flags & X86::IP_HAS_AD_SIZE)915 emitByte(0x67, CB);916 917 uint64_t Form = TSFlags & X86II::FormMask;918 switch (Form) {919 default:920 break;921 case X86II::RawFrmDstSrc: {922 // Emit segment override opcode prefix as needed (not for %ds).923 if (MI.getOperand(2).getReg() != X86::DS)924 emitSegmentOverridePrefix(2, MI, CB);925 CurOp += 3; // Consume operands.926 break;927 }928 case X86II::RawFrmSrc: {929 // Emit segment override opcode prefix as needed (not for %ds).930 if (MI.getOperand(1).getReg() != X86::DS)931 emitSegmentOverridePrefix(1, MI, CB);932 CurOp += 2; // Consume operands.933 break;934 }935 case X86II::RawFrmDst: {936 ++CurOp; // Consume operand.937 break;938 }939 case X86II::RawFrmMemOffs: {940 // Emit segment override opcode prefix as needed.941 emitSegmentOverridePrefix(1, MI, CB);942 break;943 }944 }945 946 // REX prefix is optional, but if used must be immediately before the opcode947 // Encoding type for this instruction.948 return (TSFlags & X86II::EncodingMask)949 ? emitVEXOpcodePrefix(MemoryOperand, MI, STI, CB)950 : emitOpcodePrefix(MemoryOperand, MI, STI, CB);951}952 953// AVX instructions are encoded using an encoding scheme that combines954// prefix bytes, opcode extension field, operand encoding fields, and vector955// length encoding capability into a new prefix, referred to as VEX.956 957// The majority of the AVX-512 family of instructions (operating on958// 512/256/128-bit vector register operands) are encoded using a new prefix959// (called EVEX).960 961// XOP is a revised subset of what was originally intended as SSE5. It was962// changed to be similar but not overlapping with AVX.963 964/// Emit XOP, VEX2, VEX3 or EVEX prefix.965/// \returns the used prefix.966PrefixKind967X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,968 const MCSubtargetInfo &STI,969 SmallVectorImpl<char> &CB) const {970 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());971 uint64_t TSFlags = Desc.TSFlags;972 973 assert(!(TSFlags & X86II::LOCK) && "Can't have LOCK VEX.");974 975#ifndef NDEBUG976 unsigned NumOps = MI.getNumOperands();977 for (unsigned I = NumOps ? X86II::getOperandBias(Desc) : 0; I != NumOps;978 ++I) {979 const MCOperand &MO = MI.getOperand(I);980 if (!MO.isReg())981 continue;982 MCRegister Reg = MO.getReg();983 if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH)984 report_fatal_error(985 "Cannot encode high byte register in VEX/EVEX-prefixed instruction");986 }987#endif988 989 X86OpcodePrefixHelper Prefix(*Ctx.getRegisterInfo());990 switch (TSFlags & X86II::EncodingMask) {991 default:992 break;993 case X86II::XOP:994 Prefix.setLowerBound(XOP);995 break;996 case X86II::VEX:997 // VEX can be 2 byte or 3 byte, not determined yet if not explicit998 Prefix.setLowerBound((MI.getFlags() & X86::IP_USE_VEX3) ? VEX3 : VEX2);999 break;1000 case X86II::EVEX:1001 Prefix.setLowerBound(EVEX);1002 break;1003 }1004 1005 Prefix.setW(TSFlags & X86II::REX_W);1006 Prefix.setNF(TSFlags & X86II::EVEX_NF);1007 1008 bool HasEVEX_K = TSFlags & X86II::EVEX_K;1009 bool HasVEX_4V = TSFlags & X86II::VEX_4V;1010 bool IsND = X86II::hasNewDataDest(TSFlags); // IsND implies HasVEX_4V1011 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;1012 1013 switch (TSFlags & X86II::OpMapMask) {1014 default:1015 llvm_unreachable("Invalid prefix!");1016 case X86II::TB:1017 Prefix.set5M(0x1); // 0F1018 break;1019 case X86II::T8:1020 Prefix.set5M(0x2); // 0F 381021 break;1022 case X86II::TA:1023 Prefix.set5M(0x3); // 0F 3A1024 break;1025 case X86II::XOP8:1026 Prefix.set5M(0x8);1027 break;1028 case X86II::XOP9:1029 Prefix.set5M(0x9);1030 break;1031 case X86II::XOPA:1032 Prefix.set5M(0xA);1033 break;1034 case X86II::T_MAP4:1035 Prefix.set5M(0x4);1036 break;1037 case X86II::T_MAP5:1038 Prefix.set5M(0x5);1039 break;1040 case X86II::T_MAP6:1041 Prefix.set5M(0x6);1042 break;1043 case X86II::T_MAP7:1044 Prefix.set5M(0x7);1045 break;1046 }1047 1048 Prefix.setL(TSFlags & X86II::VEX_L);1049 Prefix.setL2(TSFlags & X86II::EVEX_L2);1050 switch (TSFlags & X86II::OpPrefixMask) {1051 case X86II::PD:1052 Prefix.setPP(0x1); // 661053 break;1054 case X86II::XS:1055 Prefix.setPP(0x2); // F31056 break;1057 case X86II::XD:1058 Prefix.setPP(0x3); // F21059 break;1060 }1061 1062 Prefix.setZ(HasEVEX_K && (TSFlags & X86II::EVEX_Z));1063 Prefix.setEVEX_b(TSFlags & X86II::EVEX_B);1064 Prefix.setEVEX_U(TSFlags & X86II::EVEX_U);1065 1066 bool EncodeRC = false;1067 uint8_t EVEX_rc = 0;1068 1069 unsigned CurOp = X86II::getOperandBias(Desc);1070 bool HasTwoConditionalOps = TSFlags & X86II::TwoConditionalOps;1071 1072 switch (TSFlags & X86II::FormMask) {1073 default:1074 llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!");1075 case X86II::MRMDestMem4VOp3CC: {1076 // src1(ModR/M), MemAddr, src2(VEX_4V)1077 Prefix.setRR2(MI, CurOp++);1078 Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);1079 Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);1080 CurOp += X86::AddrNumOperands;1081 Prefix.set4VV2(MI, CurOp++);1082 break;1083 }1084 case X86II::MRM_C0:1085 case X86II::RawFrm:1086 break;1087 case X86II::MRMDestMemCC:1088 case X86II::MRMDestMemFSIB:1089 case X86II::MRMDestMem: {1090 // MRMDestMem instructions forms:1091 // MemAddr, src1(ModR/M)1092 // MemAddr, src1(VEX_4V), src2(ModR/M)1093 // MemAddr, src1(ModR/M), imm81094 //1095 // NDD:1096 // dst(VEX_4V), MemAddr, src1(ModR/M)1097 Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);1098 Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);1099 Prefix.setV2(MI, MemOperand + X86::AddrIndexReg, HasVEX_4V);1100 1101 if (IsND)1102 Prefix.set4VV2(MI, CurOp++);1103 1104 CurOp += X86::AddrNumOperands;1105 1106 if (HasEVEX_K)1107 Prefix.setAAA(MI, CurOp++);1108 1109 if (!IsND && HasVEX_4V)1110 Prefix.set4VV2(MI, CurOp++);1111 1112 Prefix.setRR2(MI, CurOp++);1113 if (HasTwoConditionalOps) {1114 Prefix.set4V(MI, CurOp++, /*IsImm=*/true);1115 Prefix.setSC(MI, CurOp++);1116 }1117 break;1118 }1119 case X86II::MRMSrcMemCC:1120 case X86II::MRMSrcMemFSIB:1121 case X86II::MRMSrcMem: {1122 // MRMSrcMem instructions forms:1123 // src1(ModR/M), MemAddr1124 // src1(ModR/M), src2(VEX_4V), MemAddr1125 // src1(ModR/M), MemAddr, imm81126 // src1(ModR/M), MemAddr, src2(Imm[7:4])1127 //1128 // FMA4:1129 // dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])1130 //1131 // NDD:1132 // dst(VEX_4V), src1(ModR/M), MemAddr1133 if (IsND)1134 Prefix.set4VV2(MI, CurOp++);1135 1136 Prefix.setRR2(MI, CurOp++);1137 1138 if (HasEVEX_K)1139 Prefix.setAAA(MI, CurOp++);1140 1141 if (!IsND && HasVEX_4V)1142 Prefix.set4VV2(MI, CurOp++);1143 1144 Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);1145 Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);1146 Prefix.setV2(MI, MemOperand + X86::AddrIndexReg, HasVEX_4V);1147 CurOp += X86::AddrNumOperands;1148 if (HasTwoConditionalOps) {1149 Prefix.set4V(MI, CurOp++, /*IsImm=*/true);1150 Prefix.setSC(MI, CurOp++);1151 }1152 break;1153 }1154 case X86II::MRMSrcMem4VOp3: {1155 // Instruction format for 4VOp3:1156 // src1(ModR/M), MemAddr, src3(VEX_4V)1157 Prefix.setRR2(MI, CurOp++);1158 Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);1159 Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);1160 Prefix.set4VV2(MI, CurOp + X86::AddrNumOperands);1161 break;1162 }1163 case X86II::MRMSrcMemOp4: {1164 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),1165 Prefix.setR(MI, CurOp++);1166 Prefix.set4V(MI, CurOp++);1167 Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);1168 Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);1169 break;1170 }1171 case X86II::MRMXmCC:1172 case X86II::MRM0m:1173 case X86II::MRM1m:1174 case X86II::MRM2m:1175 case X86II::MRM3m:1176 case X86II::MRM4m:1177 case X86II::MRM5m:1178 case X86II::MRM6m:1179 case X86II::MRM7m: {1180 // MRM[0-9]m instructions forms:1181 // MemAddr1182 // src1(VEX_4V), MemAddr1183 if (HasVEX_4V)1184 Prefix.set4VV2(MI, CurOp++);1185 1186 if (HasEVEX_K)1187 Prefix.setAAA(MI, CurOp++);1188 1189 Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);1190 Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);1191 Prefix.setV2(MI, MemOperand + X86::AddrIndexReg, HasVEX_4V);1192 CurOp += X86::AddrNumOperands + 1; // Skip first imm.1193 if (HasTwoConditionalOps) {1194 Prefix.set4V(MI, CurOp++, /*IsImm=*/true);1195 Prefix.setSC(MI, CurOp++);1196 }1197 break;1198 }1199 case X86II::MRMSrcRegCC:1200 case X86II::MRMSrcReg: {1201 // MRMSrcReg instructions forms:1202 // dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])1203 // dst(ModR/M), src1(ModR/M)1204 // dst(ModR/M), src1(ModR/M), imm81205 //1206 // FMA4:1207 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),1208 //1209 // NDD:1210 // dst(VEX_4V), src1(ModR/M.reg), src2(ModR/M)1211 if (IsND)1212 Prefix.set4VV2(MI, CurOp++);1213 Prefix.setRR2(MI, CurOp++);1214 1215 if (HasEVEX_K)1216 Prefix.setAAA(MI, CurOp++);1217 1218 if (!IsND && HasVEX_4V)1219 Prefix.set4VV2(MI, CurOp++);1220 1221 Prefix.setBB2(MI, CurOp);1222 Prefix.setX(MI, CurOp, 4);1223 ++CurOp;1224 1225 if (HasTwoConditionalOps) {1226 Prefix.set4V(MI, CurOp++, /*IsImm=*/true);1227 Prefix.setSC(MI, CurOp++);1228 }1229 1230 if (TSFlags & X86II::EVEX_B) {1231 if (HasEVEX_RC) {1232 unsigned NumOps = Desc.getNumOperands();1233 unsigned RcOperand = NumOps - 1;1234 assert(RcOperand >= CurOp);1235 EVEX_rc = MI.getOperand(RcOperand).getImm();1236 assert(EVEX_rc <= 3 && "Invalid rounding control!");1237 }1238 EncodeRC = true;1239 }1240 break;1241 }1242 case X86II::MRMSrcReg4VOp3: {1243 // Instruction format for 4VOp3:1244 // src1(ModR/M), src2(ModR/M), src3(VEX_4V)1245 Prefix.setRR2(MI, CurOp++);1246 Prefix.setBB2(MI, CurOp++);1247 Prefix.set4VV2(MI, CurOp++);1248 break;1249 }1250 case X86II::MRMSrcRegOp4: {1251 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),1252 Prefix.setR(MI, CurOp++);1253 Prefix.set4V(MI, CurOp++);1254 // Skip second register source (encoded in Imm[7:4])1255 ++CurOp;1256 1257 Prefix.setB(MI, CurOp);1258 Prefix.setX(MI, CurOp, 4);1259 ++CurOp;1260 break;1261 }1262 case X86II::MRMDestRegCC:1263 case X86II::MRMDestReg: {1264 // MRMDestReg instructions forms:1265 // dst(ModR/M), src(ModR/M)1266 // dst(ModR/M), src(ModR/M), imm81267 // dst(ModR/M), src1(VEX_4V), src2(ModR/M)1268 //1269 // NDD:1270 // dst(VEX_4V), src1(ModR/M), src2(ModR/M)1271 if (IsND)1272 Prefix.set4VV2(MI, CurOp++);1273 Prefix.setBB2(MI, CurOp);1274 Prefix.setX(MI, CurOp, 4);1275 ++CurOp;1276 1277 if (HasEVEX_K)1278 Prefix.setAAA(MI, CurOp++);1279 1280 if (!IsND && HasVEX_4V)1281 Prefix.set4VV2(MI, CurOp++);1282 1283 Prefix.setRR2(MI, CurOp++);1284 if (HasTwoConditionalOps) {1285 Prefix.set4V(MI, CurOp++, /*IsImm=*/true);1286 Prefix.setSC(MI, CurOp++);1287 }1288 if (TSFlags & X86II::EVEX_B)1289 EncodeRC = true;1290 break;1291 }1292 case X86II::MRMr0: {1293 // MRMr0 instructions forms:1294 // 11:rrr:0001295 // dst(ModR/M)1296 Prefix.setRR2(MI, CurOp++);1297 break;1298 }1299 case X86II::MRMXrCC:1300 case X86II::MRM0r:1301 case X86II::MRM1r:1302 case X86II::MRM2r:1303 case X86II::MRM3r:1304 case X86II::MRM4r:1305 case X86II::MRM5r:1306 case X86II::MRM6r:1307 case X86II::MRM7r: {1308 // MRM0r-MRM7r instructions forms:1309 // dst(VEX_4V), src(ModR/M), imm81310 if (HasVEX_4V)1311 Prefix.set4VV2(MI, CurOp++);1312 1313 if (HasEVEX_K)1314 Prefix.setAAA(MI, CurOp++);1315 1316 Prefix.setBB2(MI, CurOp);1317 Prefix.setX(MI, CurOp, 4);1318 ++CurOp;1319 if (HasTwoConditionalOps) {1320 Prefix.set4V(MI, ++CurOp, /*IsImm=*/true);1321 Prefix.setSC(MI, ++CurOp);1322 }1323 break;1324 }1325 }1326 if (EncodeRC) {1327 Prefix.setL(EVEX_rc & 0x1);1328 Prefix.setL2(EVEX_rc & 0x2);1329 }1330 PrefixKind Kind = Prefix.determineOptimalKind();1331 Prefix.emit(CB);1332 return Kind;1333}1334 1335/// Emit REX prefix which specifies1336/// 1) 64-bit instructions,1337/// 2) non-default operand size, and1338/// 3) use of X86-64 extended registers.1339///1340/// \returns the used prefix (REX or None).1341PrefixKind X86MCCodeEmitter::emitREXPrefix(int MemOperand, const MCInst &MI,1342 const MCSubtargetInfo &STI,1343 SmallVectorImpl<char> &CB) const {1344 if (!STI.hasFeature(X86::Is64Bit))1345 return None;1346 X86OpcodePrefixHelper Prefix(*Ctx.getRegisterInfo());1347 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());1348 uint64_t TSFlags = Desc.TSFlags;1349 Prefix.setW(TSFlags & X86II::REX_W);1350 unsigned NumOps = MI.getNumOperands();1351 bool UsesHighByteReg = false;1352#ifndef NDEBUG1353 bool HasRegOp = false;1354#endif1355 unsigned CurOp = NumOps ? X86II::getOperandBias(Desc) : 0;1356 for (unsigned i = CurOp; i != NumOps; ++i) {1357 const MCOperand &MO = MI.getOperand(i);1358 if (MO.isReg()) {1359#ifndef NDEBUG1360 HasRegOp = true;1361#endif1362 MCRegister Reg = MO.getReg();1363 if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH)1364 UsesHighByteReg = true;1365 // If it accesses SPL, BPL, SIL, or DIL, then it requires a REX prefix.1366 if (X86II::isX86_64NonExtLowByteReg(Reg))1367 Prefix.setLowerBound(REX);1368 } else if (MO.isExpr() && STI.getTargetTriple().isX32()) {1369 // GOTTPOFF and TLSDESC relocations require a REX prefix to allow1370 // linker optimizations: even if the instructions we see may not require1371 // any prefix, they may be replaced by instructions that do. This is1372 // handled as a special case here so that it also works for hand-written1373 // assembly without the user needing to write REX, as with GNU as.1374 const auto *Ref = dyn_cast<MCSymbolRefExpr>(MO.getExpr());1375 if (Ref && (Ref->getSpecifier() == X86::S_GOTTPOFF ||1376 Ref->getSpecifier() == X86::S_TLSDESC)) {1377 Prefix.setLowerBound(REX);1378 }1379 }1380 }1381 if (MI.getFlags() & X86::IP_USE_REX)1382 Prefix.setLowerBound(REX);1383 if ((TSFlags & X86II::ExplicitOpPrefixMask) == X86II::ExplicitREX2Prefix ||1384 MI.getFlags() & X86::IP_USE_REX2)1385 Prefix.setLowerBound(REX2);1386 switch (TSFlags & X86II::FormMask) {1387 default:1388 assert(!HasRegOp && "Unexpected form in emitREXPrefix!");1389 break;1390 case X86II::RawFrm:1391 case X86II::RawFrmMemOffs:1392 case X86II::RawFrmSrc:1393 case X86II::RawFrmDst:1394 case X86II::RawFrmDstSrc:1395 break;1396 case X86II::AddRegFrm:1397 Prefix.setBB2(MI, CurOp++);1398 break;1399 case X86II::MRMSrcReg:1400 case X86II::MRMSrcRegCC:1401 Prefix.setRR2(MI, CurOp++);1402 Prefix.setBB2(MI, CurOp++);1403 break;1404 case X86II::MRMSrcMem:1405 case X86II::MRMSrcMemCC:1406 Prefix.setRR2(MI, CurOp++);1407 Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);1408 Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);1409 CurOp += X86::AddrNumOperands;1410 break;1411 case X86II::MRMDestReg:1412 Prefix.setBB2(MI, CurOp++);1413 Prefix.setRR2(MI, CurOp++);1414 break;1415 case X86II::MRMDestMem:1416 Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);1417 Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);1418 CurOp += X86::AddrNumOperands;1419 Prefix.setRR2(MI, CurOp++);1420 break;1421 case X86II::MRMXmCC:1422 case X86II::MRMXm:1423 case X86II::MRM0m:1424 case X86II::MRM1m:1425 case X86II::MRM2m:1426 case X86II::MRM3m:1427 case X86II::MRM4m:1428 case X86II::MRM5m:1429 case X86II::MRM6m:1430 case X86II::MRM7m:1431 Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg);1432 Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg);1433 break;1434 case X86II::MRMXrCC:1435 case X86II::MRMXr:1436 case X86II::MRM0r:1437 case X86II::MRM1r:1438 case X86II::MRM2r:1439 case X86II::MRM3r:1440 case X86II::MRM4r:1441 case X86II::MRM5r:1442 case X86II::MRM6r:1443 case X86II::MRM7r:1444 Prefix.setBB2(MI, CurOp++);1445 break;1446 }1447 Prefix.setM((TSFlags & X86II::OpMapMask) == X86II::TB);1448 PrefixKind Kind = Prefix.determineOptimalKind();1449 if (Kind && UsesHighByteReg)1450 report_fatal_error(1451 "Cannot encode high byte register in REX-prefixed instruction");1452 Prefix.emit(CB);1453 return Kind;1454}1455 1456/// Emit segment override opcode prefix as needed.1457void X86MCCodeEmitter::emitSegmentOverridePrefix(1458 unsigned SegOperand, const MCInst &MI, SmallVectorImpl<char> &CB) const {1459 // Check for explicit segment override on memory operand.1460 if (MCRegister Reg = MI.getOperand(SegOperand).getReg())1461 emitByte(X86::getSegmentOverridePrefixForReg(Reg), CB);1462}1463 1464/// Emit all instruction prefixes prior to the opcode.1465///1466/// \param MemOperand the operand # of the start of a memory operand if present.1467/// If not present, it is -1.1468///1469/// \returns the used prefix (REX or None).1470PrefixKind X86MCCodeEmitter::emitOpcodePrefix(int MemOperand, const MCInst &MI,1471 const MCSubtargetInfo &STI,1472 SmallVectorImpl<char> &CB) const {1473 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());1474 uint64_t TSFlags = Desc.TSFlags;1475 1476 // Emit the operand size opcode prefix as needed.1477 if ((TSFlags & X86II::OpSizeMask) ==1478 (STI.hasFeature(X86::Is16Bit) ? X86II::OpSize32 : X86II::OpSize16))1479 emitByte(0x66, CB);1480 1481 // Emit the LOCK opcode prefix.1482 if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK)1483 emitByte(0xF0, CB);1484 1485 // Emit the NOTRACK opcode prefix.1486 if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK)1487 emitByte(0x3E, CB);1488 1489 switch (TSFlags & X86II::OpPrefixMask) {1490 case X86II::PD: // 661491 emitByte(0x66, CB);1492 break;1493 case X86II::XS: // F31494 emitByte(0xF3, CB);1495 break;1496 case X86II::XD: // F21497 emitByte(0xF2, CB);1498 break;1499 }1500 1501 // Handle REX prefix.1502 assert((STI.hasFeature(X86::Is64Bit) || !(TSFlags & X86II::REX_W)) &&1503 "REX.W requires 64bit mode.");1504 PrefixKind Kind = emitREXPrefix(MemOperand, MI, STI, CB);1505 1506 // 0x0F escape code must be emitted just before the opcode.1507 switch (TSFlags & X86II::OpMapMask) {1508 case X86II::TB: // Two-byte opcode map1509 // Encoded by M bit in REX21510 if (Kind == REX2)1511 break;1512 [[fallthrough]];1513 case X86II::T8: // 0F 381514 case X86II::TA: // 0F 3A1515 case X86II::ThreeDNow: // 0F 0F, second 0F emitted by caller.1516 emitByte(0x0F, CB);1517 break;1518 }1519 1520 switch (TSFlags & X86II::OpMapMask) {1521 case X86II::T8: // 0F 381522 emitByte(0x38, CB);1523 break;1524 case X86II::TA: // 0F 3A1525 emitByte(0x3A, CB);1526 break;1527 }1528 1529 return Kind;1530}1531 1532void X86MCCodeEmitter::emitPrefix(const MCInst &MI, SmallVectorImpl<char> &CB,1533 const MCSubtargetInfo &STI) const {1534 unsigned Opcode = MI.getOpcode();1535 const MCInstrDesc &Desc = MCII.get(Opcode);1536 uint64_t TSFlags = Desc.TSFlags;1537 1538 // Pseudo instructions don't get encoded.1539 if (X86II::isPseudo(TSFlags))1540 return;1541 1542 unsigned CurOp = X86II::getOperandBias(Desc);1543 1544 emitPrefixImpl(CurOp, MI, STI, CB);1545}1546 1547void X86_MC::emitPrefix(MCCodeEmitter &MCE, const MCInst &MI,1548 SmallVectorImpl<char> &CB, const MCSubtargetInfo &STI) {1549 static_cast<X86MCCodeEmitter &>(MCE).emitPrefix(MI, CB, STI);1550}1551 1552void X86MCCodeEmitter::encodeInstruction(const MCInst &MI,1553 SmallVectorImpl<char> &CB,1554 SmallVectorImpl<MCFixup> &Fixups,1555 const MCSubtargetInfo &STI) const {1556 unsigned Opcode = MI.getOpcode();1557 const MCInstrDesc &Desc = MCII.get(Opcode);1558 uint64_t TSFlags = Desc.TSFlags;1559 1560 // Pseudo instructions don't get encoded.1561 if (X86II::isPseudo(TSFlags))1562 return;1563 1564 unsigned NumOps = Desc.getNumOperands();1565 unsigned CurOp = X86II::getOperandBias(Desc);1566 1567 uint64_t StartByte = CB.size();1568 1569 PrefixKind Kind = emitPrefixImpl(CurOp, MI, STI, CB);1570 1571 // It uses the VEX.VVVV field?1572 bool HasVEX_4V = TSFlags & X86II::VEX_4V;1573 bool HasVEX_I8Reg = (TSFlags & X86II::ImmMask) == X86II::Imm8Reg;1574 1575 // It uses the EVEX.aaa field?1576 bool HasEVEX_K = TSFlags & X86II::EVEX_K;1577 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;1578 1579 // Used if a register is encoded in 7:4 of immediate.1580 unsigned I8RegNum = 0;1581 1582 uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags);1583 1584 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)1585 BaseOpcode = 0x0F; // Weird 3DNow! encoding.1586 1587 unsigned OpcodeOffset = 0;1588 1589 bool IsND = X86II::hasNewDataDest(TSFlags);1590 bool HasTwoConditionalOps = TSFlags & X86II::TwoConditionalOps;1591 1592 uint64_t Form = TSFlags & X86II::FormMask;1593 switch (Form) {1594 default:1595 errs() << "FORM: " << Form << "\n";1596 llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!");1597 case X86II::Pseudo:1598 llvm_unreachable("Pseudo instruction shouldn't be emitted");1599 case X86II::RawFrmDstSrc:1600 case X86II::RawFrmSrc:1601 case X86II::RawFrmDst:1602 case X86II::PrefixByte:1603 emitByte(BaseOpcode, CB);1604 break;1605 case X86II::AddCCFrm: {1606 // This will be added to the opcode in the fallthrough.1607 OpcodeOffset = MI.getOperand(NumOps - 1).getImm();1608 assert(OpcodeOffset < 16 && "Unexpected opcode offset!");1609 --NumOps; // Drop the operand from the end.1610 [[fallthrough]];1611 case X86II::RawFrm:1612 emitByte(BaseOpcode + OpcodeOffset, CB);1613 1614 if (!STI.hasFeature(X86::Is64Bit) || !isPCRel32Branch(MI, MCII))1615 break;1616 1617 const MCOperand &Op = MI.getOperand(CurOp++);1618 emitImmediate(Op, MI.getLoc(), X86::reloc_branch_4byte_pcrel, true,1619 StartByte, CB, Fixups);1620 break;1621 }1622 case X86II::RawFrmMemOffs:1623 emitByte(BaseOpcode, CB);1624 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), getImmFixupKind(TSFlags),1625 X86II::isImmPCRel(TSFlags), StartByte, CB, Fixups);1626 ++CurOp; // skip segment operand1627 break;1628 case X86II::RawFrmImm8:1629 emitByte(BaseOpcode, CB);1630 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), getImmFixupKind(TSFlags),1631 X86II::isImmPCRel(TSFlags), StartByte, CB, Fixups);1632 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), FK_Data_1, false,1633 StartByte, CB, Fixups);1634 break;1635 case X86II::RawFrmImm16:1636 emitByte(BaseOpcode, CB);1637 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), getImmFixupKind(TSFlags),1638 X86II::isImmPCRel(TSFlags), StartByte, CB, Fixups);1639 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), FK_Data_2, false,1640 StartByte, CB, Fixups);1641 break;1642 1643 case X86II::AddRegFrm:1644 emitByte(BaseOpcode + getX86RegNum(MI.getOperand(CurOp++)), CB);1645 break;1646 1647 case X86II::MRMDestReg: {1648 emitByte(BaseOpcode, CB);1649 unsigned SrcRegNum = CurOp + 1;1650 1651 if (HasEVEX_K) // Skip writemask1652 ++SrcRegNum;1653 1654 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)1655 ++SrcRegNum;1656 if (IsND) // Skip the NDD operand encoded in EVEX_VVVV1657 ++CurOp;1658 1659 emitRegModRMByte(MI.getOperand(CurOp),1660 getX86RegNum(MI.getOperand(SrcRegNum)), CB);1661 CurOp = SrcRegNum + 1;1662 break;1663 }1664 case X86II::MRMDestRegCC: {1665 unsigned FirstOp = CurOp++;1666 unsigned SecondOp = CurOp++;1667 unsigned CC = MI.getOperand(CurOp++).getImm();1668 emitByte(BaseOpcode + CC, CB);1669 emitRegModRMByte(MI.getOperand(FirstOp),1670 getX86RegNum(MI.getOperand(SecondOp)), CB);1671 break;1672 }1673 case X86II::MRMDestMem4VOp3CC: {1674 unsigned CC = MI.getOperand(8).getImm();1675 emitByte(BaseOpcode + CC, CB);1676 unsigned SrcRegNum = CurOp + X86::AddrNumOperands;1677 emitMemModRMByte(MI, CurOp + 1, getX86RegNum(MI.getOperand(0)), TSFlags,1678 Kind, StartByte, CB, Fixups, STI, false);1679 CurOp = SrcRegNum + 3; // skip reg, VEX_V4 and CC1680 break;1681 }1682 case X86II::MRMDestMemFSIB:1683 case X86II::MRMDestMem: {1684 emitByte(BaseOpcode, CB);1685 unsigned SrcRegNum = CurOp + X86::AddrNumOperands;1686 1687 if (HasEVEX_K) // Skip writemask1688 ++SrcRegNum;1689 1690 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)1691 ++SrcRegNum;1692 1693 if (IsND) // Skip new data destination1694 ++CurOp;1695 1696 bool ForceSIB = (Form == X86II::MRMDestMemFSIB);1697 emitMemModRMByte(MI, CurOp, getX86RegNum(MI.getOperand(SrcRegNum)), TSFlags,1698 Kind, StartByte, CB, Fixups, STI, ForceSIB);1699 CurOp = SrcRegNum + 1;1700 break;1701 }1702 case X86II::MRMDestMemCC: {1703 unsigned MemOp = CurOp;1704 CurOp = MemOp + X86::AddrNumOperands;1705 unsigned RegOp = CurOp++;1706 unsigned CC = MI.getOperand(CurOp++).getImm();1707 emitByte(BaseOpcode + CC, CB);1708 emitMemModRMByte(MI, MemOp, getX86RegNum(MI.getOperand(RegOp)), TSFlags,1709 Kind, StartByte, CB, Fixups, STI);1710 break;1711 }1712 case X86II::MRMSrcReg: {1713 emitByte(BaseOpcode, CB);1714 unsigned SrcRegNum = CurOp + 1;1715 1716 if (HasEVEX_K) // Skip writemask1717 ++SrcRegNum;1718 1719 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)1720 ++SrcRegNum;1721 1722 if (IsND) // Skip new data destination1723 ++CurOp;1724 1725 emitRegModRMByte(MI.getOperand(SrcRegNum),1726 getX86RegNum(MI.getOperand(CurOp)), CB);1727 CurOp = SrcRegNum + 1;1728 if (HasVEX_I8Reg)1729 I8RegNum = getX86RegEncoding(MI, CurOp++);1730 // do not count the rounding control operand1731 if (HasEVEX_RC)1732 --NumOps;1733 break;1734 }1735 case X86II::MRMSrcReg4VOp3: {1736 emitByte(BaseOpcode, CB);1737 unsigned SrcRegNum = CurOp + 1;1738 1739 emitRegModRMByte(MI.getOperand(SrcRegNum),1740 getX86RegNum(MI.getOperand(CurOp)), CB);1741 CurOp = SrcRegNum + 1;1742 ++CurOp; // Encoded in VEX.VVVV1743 break;1744 }1745 case X86II::MRMSrcRegOp4: {1746 emitByte(BaseOpcode, CB);1747 unsigned SrcRegNum = CurOp + 1;1748 1749 // Skip 1st src (which is encoded in VEX_VVVV)1750 ++SrcRegNum;1751 1752 // Capture 2nd src (which is encoded in Imm[7:4])1753 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");1754 I8RegNum = getX86RegEncoding(MI, SrcRegNum++);1755 1756 emitRegModRMByte(MI.getOperand(SrcRegNum),1757 getX86RegNum(MI.getOperand(CurOp)), CB);1758 CurOp = SrcRegNum + 1;1759 break;1760 }1761 case X86II::MRMSrcRegCC: {1762 if (IsND) // Skip new data destination1763 ++CurOp;1764 unsigned FirstOp = CurOp++;1765 unsigned SecondOp = CurOp++;1766 1767 unsigned CC = MI.getOperand(CurOp++).getImm();1768 emitByte(BaseOpcode + CC, CB);1769 1770 emitRegModRMByte(MI.getOperand(SecondOp),1771 getX86RegNum(MI.getOperand(FirstOp)), CB);1772 break;1773 }1774 case X86II::MRMSrcMemFSIB:1775 case X86II::MRMSrcMem: {1776 unsigned FirstMemOp = CurOp + 1;1777 1778 if (IsND) // Skip new data destination1779 CurOp++;1780 1781 if (HasEVEX_K) // Skip writemask1782 ++FirstMemOp;1783 1784 if (HasVEX_4V)1785 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).1786 1787 emitByte(BaseOpcode, CB);1788 1789 bool ForceSIB = (Form == X86II::MRMSrcMemFSIB);1790 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),1791 TSFlags, Kind, StartByte, CB, Fixups, STI, ForceSIB);1792 CurOp = FirstMemOp + X86::AddrNumOperands;1793 if (HasVEX_I8Reg)1794 I8RegNum = getX86RegEncoding(MI, CurOp++);1795 break;1796 }1797 case X86II::MRMSrcMem4VOp3: {1798 unsigned FirstMemOp = CurOp + 1;1799 1800 emitByte(BaseOpcode, CB);1801 1802 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),1803 TSFlags, Kind, StartByte, CB, Fixups, STI);1804 CurOp = FirstMemOp + X86::AddrNumOperands;1805 ++CurOp; // Encoded in VEX.VVVV.1806 break;1807 }1808 case X86II::MRMSrcMemOp4: {1809 unsigned FirstMemOp = CurOp + 1;1810 1811 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).1812 1813 // Capture second register source (encoded in Imm[7:4])1814 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");1815 I8RegNum = getX86RegEncoding(MI, FirstMemOp++);1816 1817 emitByte(BaseOpcode, CB);1818 1819 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),1820 TSFlags, Kind, StartByte, CB, Fixups, STI);1821 CurOp = FirstMemOp + X86::AddrNumOperands;1822 break;1823 }1824 case X86II::MRMSrcMemCC: {1825 if (IsND) // Skip new data destination1826 ++CurOp;1827 unsigned RegOp = CurOp++;1828 unsigned FirstMemOp = CurOp;1829 CurOp = FirstMemOp + X86::AddrNumOperands;1830 1831 unsigned CC = MI.getOperand(CurOp++).getImm();1832 emitByte(BaseOpcode + CC, CB);1833 1834 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(RegOp)),1835 TSFlags, Kind, StartByte, CB, Fixups, STI);1836 break;1837 }1838 1839 case X86II::MRMXrCC: {1840 unsigned RegOp = CurOp++;1841 1842 unsigned CC = MI.getOperand(CurOp++).getImm();1843 emitByte(BaseOpcode + CC, CB);1844 emitRegModRMByte(MI.getOperand(RegOp), 0, CB);1845 break;1846 }1847 1848 case X86II::MRMXr:1849 case X86II::MRM0r:1850 case X86II::MRM1r:1851 case X86II::MRM2r:1852 case X86II::MRM3r:1853 case X86II::MRM4r:1854 case X86II::MRM5r:1855 case X86II::MRM6r:1856 case X86II::MRM7r:1857 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).1858 ++CurOp;1859 if (HasEVEX_K) // Skip writemask1860 ++CurOp;1861 emitByte(BaseOpcode, CB);1862 emitRegModRMByte(MI.getOperand(CurOp++),1863 (Form == X86II::MRMXr) ? 0 : Form - X86II::MRM0r, CB);1864 break;1865 case X86II::MRMr0:1866 emitByte(BaseOpcode, CB);1867 emitByte(modRMByte(3, getX86RegNum(MI.getOperand(CurOp++)), 0), CB);1868 break;1869 1870 case X86II::MRMXmCC: {1871 unsigned FirstMemOp = CurOp;1872 CurOp = FirstMemOp + X86::AddrNumOperands;1873 1874 unsigned CC = MI.getOperand(CurOp++).getImm();1875 emitByte(BaseOpcode + CC, CB);1876 1877 emitMemModRMByte(MI, FirstMemOp, 0, TSFlags, Kind, StartByte, CB, Fixups,1878 STI);1879 break;1880 }1881 1882 case X86II::MRMXm:1883 case X86II::MRM0m:1884 case X86II::MRM1m:1885 case X86II::MRM2m:1886 case X86II::MRM3m:1887 case X86II::MRM4m:1888 case X86II::MRM5m:1889 case X86II::MRM6m:1890 case X86II::MRM7m:1891 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).1892 ++CurOp;1893 if (HasEVEX_K) // Skip writemask1894 ++CurOp;1895 emitByte(BaseOpcode, CB);1896 emitMemModRMByte(MI, CurOp,1897 (Form == X86II::MRMXm) ? 0 : Form - X86II::MRM0m, TSFlags,1898 Kind, StartByte, CB, Fixups, STI);1899 CurOp += X86::AddrNumOperands;1900 break;1901 1902 case X86II::MRM0X:1903 case X86II::MRM1X:1904 case X86II::MRM2X:1905 case X86II::MRM3X:1906 case X86II::MRM4X:1907 case X86II::MRM5X:1908 case X86II::MRM6X:1909 case X86II::MRM7X:1910 emitByte(BaseOpcode, CB);1911 emitByte(0xC0 + ((Form - X86II::MRM0X) << 3), CB);1912 break;1913 1914 case X86II::MRM_C0:1915 case X86II::MRM_C1:1916 case X86II::MRM_C2:1917 case X86II::MRM_C3:1918 case X86II::MRM_C4:1919 case X86II::MRM_C5:1920 case X86II::MRM_C6:1921 case X86II::MRM_C7:1922 case X86II::MRM_C8:1923 case X86II::MRM_C9:1924 case X86II::MRM_CA:1925 case X86II::MRM_CB:1926 case X86II::MRM_CC:1927 case X86II::MRM_CD:1928 case X86II::MRM_CE:1929 case X86II::MRM_CF:1930 case X86II::MRM_D0:1931 case X86II::MRM_D1:1932 case X86II::MRM_D2:1933 case X86II::MRM_D3:1934 case X86II::MRM_D4:1935 case X86II::MRM_D5:1936 case X86II::MRM_D6:1937 case X86II::MRM_D7:1938 case X86II::MRM_D8:1939 case X86II::MRM_D9:1940 case X86II::MRM_DA:1941 case X86II::MRM_DB:1942 case X86II::MRM_DC:1943 case X86II::MRM_DD:1944 case X86II::MRM_DE:1945 case X86II::MRM_DF:1946 case X86II::MRM_E0:1947 case X86II::MRM_E1:1948 case X86II::MRM_E2:1949 case X86II::MRM_E3:1950 case X86II::MRM_E4:1951 case X86II::MRM_E5:1952 case X86II::MRM_E6:1953 case X86II::MRM_E7:1954 case X86II::MRM_E8:1955 case X86II::MRM_E9:1956 case X86II::MRM_EA:1957 case X86II::MRM_EB:1958 case X86II::MRM_EC:1959 case X86II::MRM_ED:1960 case X86II::MRM_EE:1961 case X86II::MRM_EF:1962 case X86II::MRM_F0:1963 case X86II::MRM_F1:1964 case X86II::MRM_F2:1965 case X86II::MRM_F3:1966 case X86II::MRM_F4:1967 case X86II::MRM_F5:1968 case X86II::MRM_F6:1969 case X86II::MRM_F7:1970 case X86II::MRM_F8:1971 case X86II::MRM_F9:1972 case X86II::MRM_FA:1973 case X86II::MRM_FB:1974 case X86II::MRM_FC:1975 case X86II::MRM_FD:1976 case X86II::MRM_FE:1977 case X86II::MRM_FF:1978 emitByte(BaseOpcode, CB);1979 emitByte(0xC0 + Form - X86II::MRM_C0, CB);1980 break;1981 }1982 1983 if (HasVEX_I8Reg) {1984 // The last source register of a 4 operand instruction in AVX is encoded1985 // in bits[7:4] of a immediate byte.1986 assert(I8RegNum < 16 && "Register encoding out of range");1987 I8RegNum <<= 4;1988 if (CurOp != NumOps) {1989 unsigned Val = MI.getOperand(CurOp++).getImm();1990 assert(Val < 16 && "Immediate operand value out of range");1991 I8RegNum |= Val;1992 }1993 emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), FK_Data_1, false,1994 StartByte, CB, Fixups);1995 } else {1996 // If there is a remaining operand, it must be a trailing immediate. Emit it1997 // according to the right size for the instruction. Some instructions1998 // (SSE4a extrq and insertq) have two trailing immediates.1999 2000 // Skip two trainling conditional operands encoded in EVEX prefix2001 unsigned RemainingOps = NumOps - CurOp - 2 * HasTwoConditionalOps;2002 while (RemainingOps) {2003 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),2004 getImmFixupKind(Desc.TSFlags),2005 X86II::isImmPCRel(Desc.TSFlags), StartByte, CB, Fixups);2006 --RemainingOps;2007 }2008 CurOp += 2 * HasTwoConditionalOps;2009 }2010 2011 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)2012 emitByte(X86II::getBaseOpcodeFor(TSFlags), CB);2013 2014 if (CB.size() - StartByte > 15)2015 Ctx.reportError(MI.getLoc(), "instruction length exceeds the limit of 15");2016#ifndef NDEBUG2017 // FIXME: Verify.2018 if (/*!Desc.isVariadic() &&*/ CurOp != NumOps) {2019 errs() << "Cannot encode all operands of: ";2020 MI.dump();2021 errs() << '\n';2022 abort();2023 }2024#endif2025}2026 2027MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII,2028 MCContext &Ctx) {2029 return new X86MCCodeEmitter(MCII, Ctx);2030}2031