737 lines · cpp
1//===-- AMDGPUMCCodeEmitter.cpp - AMDGPU Code Emitter ---------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// The AMDGPU code emitter produces machine code that can be executed11/// directly on the GPU device.12//13//===----------------------------------------------------------------------===//14 15#include "MCTargetDesc/AMDGPUFixupKinds.h"16#include "MCTargetDesc/AMDGPUMCExpr.h"17#include "MCTargetDesc/AMDGPUMCTargetDesc.h"18#include "SIDefines.h"19#include "Utils/AMDGPUBaseInfo.h"20#include "llvm/ADT/APInt.h"21#include "llvm/MC/MCCodeEmitter.h"22#include "llvm/MC/MCContext.h"23#include "llvm/MC/MCExpr.h"24#include "llvm/MC/MCInstrInfo.h"25#include "llvm/MC/MCRegisterInfo.h"26#include "llvm/MC/MCSubtargetInfo.h"27#include "llvm/Support/Casting.h"28#include "llvm/Support/EndianStream.h"29#include <optional>30 31using namespace llvm;32 33namespace {34 35class AMDGPUMCCodeEmitter : public MCCodeEmitter {36 const MCRegisterInfo &MRI;37 const MCInstrInfo &MCII;38 39public:40 AMDGPUMCCodeEmitter(const MCInstrInfo &MCII, const MCRegisterInfo &MRI)41 : MRI(MRI), MCII(MCII) {}42 43 /// Encode the instruction and write it to the OS.44 void encodeInstruction(const MCInst &MI, SmallVectorImpl<char> &CB,45 SmallVectorImpl<MCFixup> &Fixups,46 const MCSubtargetInfo &STI) const override;47 48 void getMachineOpValue(const MCInst &MI, const MCOperand &MO, APInt &Op,49 SmallVectorImpl<MCFixup> &Fixups,50 const MCSubtargetInfo &STI) const;51 52 void getMachineOpValueT16(const MCInst &MI, unsigned OpNo, APInt &Op,53 SmallVectorImpl<MCFixup> &Fixups,54 const MCSubtargetInfo &STI) const;55 56 void getMachineOpValueT16Lo128(const MCInst &MI, unsigned OpNo, APInt &Op,57 SmallVectorImpl<MCFixup> &Fixups,58 const MCSubtargetInfo &STI) const;59 60 /// Use a fixup to encode the simm16 field for SOPP branch61 /// instructions.62 void getSOPPBrEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,63 SmallVectorImpl<MCFixup> &Fixups,64 const MCSubtargetInfo &STI) const;65 66 void getSMEMOffsetEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,67 SmallVectorImpl<MCFixup> &Fixups,68 const MCSubtargetInfo &STI) const;69 70 void getSDWASrcEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,71 SmallVectorImpl<MCFixup> &Fixups,72 const MCSubtargetInfo &STI) const;73 74 void getSDWAVopcDstEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,75 SmallVectorImpl<MCFixup> &Fixups,76 const MCSubtargetInfo &STI) const;77 78 void getAVOperandEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,79 SmallVectorImpl<MCFixup> &Fixups,80 const MCSubtargetInfo &STI) const;81 82private:83 uint64_t getImplicitOpSelHiEncoding(int Opcode) const;84 void getMachineOpValueCommon(const MCInst &MI, const MCOperand &MO,85 unsigned OpNo, APInt &Op,86 SmallVectorImpl<MCFixup> &Fixups,87 const MCSubtargetInfo &STI) const;88 89 /// Encode an fp or int literal.90 std::optional<uint64_t>91 getLitEncoding(const MCInstrDesc &Desc, const MCOperand &MO, unsigned OpNo,92 const MCSubtargetInfo &STI,93 bool HasMandatoryLiteral = false) const;94 95 void getBinaryCodeForInstr(const MCInst &MI, SmallVectorImpl<MCFixup> &Fixups,96 APInt &Inst, APInt &Scratch,97 const MCSubtargetInfo &STI) const;98};99 100} // end anonymous namespace101 102MCCodeEmitter *llvm::createAMDGPUMCCodeEmitter(const MCInstrInfo &MCII,103 MCContext &Ctx) {104 return new AMDGPUMCCodeEmitter(MCII, *Ctx.getRegisterInfo());105}106 107static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,108 const MCExpr *Value, uint16_t Kind, bool PCRel = false) {109 Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));110}111 112// Returns the encoding value to use if the given integer is an integer inline113// immediate value, or 0 if it is not.114template <typename IntTy>115static uint32_t getIntInlineImmEncoding(IntTy Imm) {116 if (Imm >= 0 && Imm <= 64)117 return 128 + Imm;118 119 if (Imm >= -16 && Imm <= -1)120 return 192 + std::abs(Imm);121 122 return 0;123}124 125static uint32_t getLit16Encoding(uint16_t Val, const MCSubtargetInfo &STI) {126 uint16_t IntImm = getIntInlineImmEncoding(static_cast<int16_t>(Val));127 if (IntImm != 0)128 return IntImm;129 130 if (Val == 0x3800) // 0.5131 return 240;132 133 if (Val == 0xB800) // -0.5134 return 241;135 136 if (Val == 0x3C00) // 1.0137 return 242;138 139 if (Val == 0xBC00) // -1.0140 return 243;141 142 if (Val == 0x4000) // 2.0143 return 244;144 145 if (Val == 0xC000) // -2.0146 return 245;147 148 if (Val == 0x4400) // 4.0149 return 246;150 151 if (Val == 0xC400) // -4.0152 return 247;153 154 if (Val == 0x3118 && // 1.0 / (2.0 * pi)155 STI.hasFeature(AMDGPU::FeatureInv2PiInlineImm))156 return 248;157 158 return 255;159}160 161static uint32_t getLitBF16Encoding(uint16_t Val) {162 uint16_t IntImm = getIntInlineImmEncoding(static_cast<int16_t>(Val));163 if (IntImm != 0)164 return IntImm;165 166 // clang-format off167 switch (Val) {168 case 0x3F00: return 240; // 0.5169 case 0xBF00: return 241; // -0.5170 case 0x3F80: return 242; // 1.0171 case 0xBF80: return 243; // -1.0172 case 0x4000: return 244; // 2.0173 case 0xC000: return 245; // -2.0174 case 0x4080: return 246; // 4.0175 case 0xC080: return 247; // -4.0176 case 0x3E22: return 248; // 1.0 / (2.0 * pi)177 default: return 255;178 }179 // clang-format on180}181 182static uint32_t getLit32Encoding(uint32_t Val, const MCSubtargetInfo &STI) {183 uint32_t IntImm = getIntInlineImmEncoding(static_cast<int32_t>(Val));184 if (IntImm != 0)185 return IntImm;186 187 if (Val == llvm::bit_cast<uint32_t>(0.5f))188 return 240;189 190 if (Val == llvm::bit_cast<uint32_t>(-0.5f))191 return 241;192 193 if (Val == llvm::bit_cast<uint32_t>(1.0f))194 return 242;195 196 if (Val == llvm::bit_cast<uint32_t>(-1.0f))197 return 243;198 199 if (Val == llvm::bit_cast<uint32_t>(2.0f))200 return 244;201 202 if (Val == llvm::bit_cast<uint32_t>(-2.0f))203 return 245;204 205 if (Val == llvm::bit_cast<uint32_t>(4.0f))206 return 246;207 208 if (Val == llvm::bit_cast<uint32_t>(-4.0f))209 return 247;210 211 if (Val == 0x3e22f983 && // 1.0 / (2.0 * pi)212 STI.hasFeature(AMDGPU::FeatureInv2PiInlineImm))213 return 248;214 215 return 255;216}217 218static uint32_t getLit16IntEncoding(uint32_t Val, const MCSubtargetInfo &STI) {219 return getLit32Encoding(Val, STI);220}221 222static uint32_t getLit64Encoding(const MCInstrDesc &Desc, uint64_t Val,223 const MCSubtargetInfo &STI, bool IsFP) {224 uint32_t IntImm = getIntInlineImmEncoding(static_cast<int64_t>(Val));225 if (IntImm != 0)226 return IntImm;227 228 if (Val == llvm::bit_cast<uint64_t>(0.5))229 return 240;230 231 if (Val == llvm::bit_cast<uint64_t>(-0.5))232 return 241;233 234 if (Val == llvm::bit_cast<uint64_t>(1.0))235 return 242;236 237 if (Val == llvm::bit_cast<uint64_t>(-1.0))238 return 243;239 240 if (Val == llvm::bit_cast<uint64_t>(2.0))241 return 244;242 243 if (Val == llvm::bit_cast<uint64_t>(-2.0))244 return 245;245 246 if (Val == llvm::bit_cast<uint64_t>(4.0))247 return 246;248 249 if (Val == llvm::bit_cast<uint64_t>(-4.0))250 return 247;251 252 if (Val == 0x3fc45f306dc9c882 && // 1.0 / (2.0 * pi)253 STI.hasFeature(AMDGPU::FeatureInv2PiInlineImm))254 return 248;255 256 // The rest part needs to align with AMDGPUInstPrinter::printLiteral64.257 258 bool CanUse64BitLiterals =259 STI.hasFeature(AMDGPU::Feature64BitLiterals) &&260 !(Desc.TSFlags & (SIInstrFlags::VOP3 | SIInstrFlags::VOP3P));261 if (IsFP) {262 return CanUse64BitLiterals && Lo_32(Val) ? 254 : 255;263 }264 265 return CanUse64BitLiterals && (!isInt<32>(Val) || !isUInt<32>(Val)) ? 254266 : 255;267}268 269std::optional<uint64_t> AMDGPUMCCodeEmitter::getLitEncoding(270 const MCInstrDesc &Desc, const MCOperand &MO, unsigned OpNo,271 const MCSubtargetInfo &STI, bool HasMandatoryLiteral) const {272 const MCOperandInfo &OpInfo = Desc.operands()[OpNo];273 int64_t Imm = 0;274 if (MO.isExpr()) {275 if (!MO.getExpr()->evaluateAsAbsolute(Imm) ||276 AMDGPU::isLitExpr(MO.getExpr())) {277 if (OpInfo.OperandType == AMDGPU::OPERAND_KIMM16 ||278 OpInfo.OperandType == AMDGPU::OPERAND_KIMM32 ||279 OpInfo.OperandType == AMDGPU::OPERAND_KIMM64)280 return Imm;281 if (STI.hasFeature(AMDGPU::Feature64BitLiterals) &&282 AMDGPU::getOperandSize(OpInfo) == 8)283 return 254;284 return 255;285 }286 } else {287 assert(!MO.isDFPImm());288 289 if (!MO.isImm())290 return {};291 292 Imm = MO.getImm();293 }294 295 switch (OpInfo.OperandType) {296 case AMDGPU::OPERAND_REG_IMM_INT32:297 case AMDGPU::OPERAND_REG_IMM_FP32:298 case AMDGPU::OPERAND_REG_INLINE_C_INT32:299 case AMDGPU::OPERAND_REG_INLINE_C_FP32:300 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:301 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:302 case AMDGPU::OPERAND_REG_IMM_V2INT32:303 case AMDGPU::OPERAND_REG_IMM_V2FP32:304 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:305 return getLit32Encoding(static_cast<uint32_t>(Imm), STI);306 307 case AMDGPU::OPERAND_REG_IMM_INT64:308 case AMDGPU::OPERAND_REG_INLINE_C_INT64:309 return getLit64Encoding(Desc, static_cast<uint64_t>(Imm), STI, false);310 311 case AMDGPU::OPERAND_REG_INLINE_C_FP64:312 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:313 return getLit64Encoding(Desc, static_cast<uint64_t>(Imm), STI, true);314 315 case AMDGPU::OPERAND_REG_IMM_FP64: {316 auto Enc = getLit64Encoding(Desc, static_cast<uint64_t>(Imm), STI, true);317 return (HasMandatoryLiteral && Enc == 255) ? 254 : Enc;318 }319 320 case AMDGPU::OPERAND_REG_IMM_INT16:321 case AMDGPU::OPERAND_REG_INLINE_C_INT16:322 return getLit16IntEncoding(static_cast<uint32_t>(Imm), STI);323 324 case AMDGPU::OPERAND_REG_IMM_FP16:325 case AMDGPU::OPERAND_REG_INLINE_C_FP16:326 // FIXME Is this correct? What do inline immediates do on SI for f16 src327 // which does not have f16 support?328 return getLit16Encoding(static_cast<uint16_t>(Imm), STI);329 330 case AMDGPU::OPERAND_REG_IMM_BF16:331 case AMDGPU::OPERAND_REG_INLINE_C_BF16:332 // We don't actually need to check Inv2Pi here because BF16 instructions can333 // only be emitted for targets that already support the feature.334 return getLitBF16Encoding(static_cast<uint16_t>(Imm));335 336 case AMDGPU::OPERAND_REG_IMM_V2INT16:337 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:338 return AMDGPU::getInlineEncodingV2I16(static_cast<uint32_t>(Imm))339 .value_or(255);340 341 case AMDGPU::OPERAND_REG_IMM_V2FP16:342 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:343 return AMDGPU::getInlineEncodingV2F16(static_cast<uint32_t>(Imm))344 .value_or(255);345 346 case AMDGPU::OPERAND_REG_IMM_V2BF16:347 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:348 return AMDGPU::getInlineEncodingV2BF16(static_cast<uint32_t>(Imm))349 .value_or(255);350 351 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:352 return 255;353 354 case AMDGPU::OPERAND_KIMM32:355 case AMDGPU::OPERAND_KIMM16:356 case AMDGPU::OPERAND_KIMM64:357 return Imm;358 default:359 llvm_unreachable("invalid operand size");360 }361}362 363uint64_t AMDGPUMCCodeEmitter::getImplicitOpSelHiEncoding(int Opcode) const {364 using namespace AMDGPU::VOP3PEncoding;365 366 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::op_sel_hi)) {367 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src2))368 return 0;369 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src1))370 return OP_SEL_HI_2;371 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src0))372 return OP_SEL_HI_1 | OP_SEL_HI_2;373 }374 return OP_SEL_HI_0 | OP_SEL_HI_1 | OP_SEL_HI_2;375}376 377static bool isVCMPX64(const MCInstrDesc &Desc) {378 return (Desc.TSFlags & SIInstrFlags::VOP3) &&379 Desc.hasImplicitDefOfPhysReg(AMDGPU::EXEC);380}381 382void AMDGPUMCCodeEmitter::encodeInstruction(const MCInst &MI,383 SmallVectorImpl<char> &CB,384 SmallVectorImpl<MCFixup> &Fixups,385 const MCSubtargetInfo &STI) const {386 int Opcode = MI.getOpcode();387 APInt Encoding, Scratch;388 getBinaryCodeForInstr(MI, Fixups, Encoding, Scratch, STI);389 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());390 unsigned bytes = Desc.getSize();391 392 // Set unused op_sel_hi bits to 1 for VOP3P and MAI instructions.393 // Note that accvgpr_read/write are MAI, have src0, but do not use op_sel.394 if (((Desc.TSFlags & SIInstrFlags::VOP3P) ||395 Opcode == AMDGPU::V_ACCVGPR_READ_B32_vi ||396 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_vi) &&397 // Matrix B format operand reuses op_sel_hi.398 !AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::matrix_b_fmt) &&399 // Matrix B scale operand reuses op_sel_hi.400 !AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::matrix_b_scale) &&401 // Matrix B reuse operand reuses op_sel_hi.402 !AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::matrix_b_reuse)) {403 Encoding |= getImplicitOpSelHiEncoding(Opcode);404 }405 406 // GFX10+ v_cmpx opcodes promoted to VOP3 have implied dst=EXEC.407 // Documentation requires dst to be encoded as EXEC (0x7E),408 // but it looks like the actual value encoded for dst operand409 // is ignored by HW. It was decided to define dst as "do not care"410 // in td files to allow disassembler accept any dst value.411 // However, dst is encoded as EXEC for compatibility with SP3.412 if (AMDGPU::isGFX10Plus(STI) && isVCMPX64(Desc)) {413 assert((Encoding & 0xFF) == 0);414 Encoding |= MRI.getEncodingValue(AMDGPU::EXEC_LO) &415 AMDGPU::HWEncoding::LO256_REG_IDX_MASK;416 }417 418 for (unsigned i = 0; i < bytes; i++) {419 CB.push_back((uint8_t)Encoding.extractBitsAsZExtValue(8, 8 * i));420 }421 422 // NSA encoding.423 if (AMDGPU::isGFX10Plus(STI) && Desc.TSFlags & SIInstrFlags::MIMG) {424 int vaddr0 = AMDGPU::getNamedOperandIdx(MI.getOpcode(),425 AMDGPU::OpName::vaddr0);426 int srsrc = AMDGPU::getNamedOperandIdx(MI.getOpcode(),427 AMDGPU::OpName::srsrc);428 assert(vaddr0 >= 0 && srsrc > vaddr0);429 unsigned NumExtraAddrs = srsrc - vaddr0 - 1;430 unsigned NumPadding = (-NumExtraAddrs) & 3;431 432 for (unsigned i = 0; i < NumExtraAddrs; ++i) {433 getMachineOpValue(MI, MI.getOperand(vaddr0 + 1 + i), Encoding, Fixups,434 STI);435 CB.push_back((uint8_t)Encoding.getLimitedValue());436 }437 CB.append(NumPadding, 0);438 }439 440 if ((bytes > 8 && STI.hasFeature(AMDGPU::FeatureVOP3Literal)) ||441 (bytes > 4 && !STI.hasFeature(AMDGPU::FeatureVOP3Literal)))442 return;443 444 // Do not print literals from SISrc Operands for insts with mandatory literals445 if (AMDGPU::hasNamedOperand(MI.getOpcode(), AMDGPU::OpName::imm))446 return;447 448 // Check for additional literals449 for (unsigned i = 0, e = Desc.getNumOperands(); i < e; ++i) {450 451 // Check if this operand should be encoded as [SV]Src452 if (!AMDGPU::isSISrcOperand(Desc, i))453 continue;454 455 // Is this operand a literal immediate?456 const MCOperand &Op = MI.getOperand(i);457 auto Enc = getLitEncoding(Desc, Op, i, STI);458 if (!Enc || (*Enc != 255 && *Enc != 254))459 continue;460 461 // Yes! Encode it462 int64_t Imm = 0;463 464 bool IsLit = false;465 if (Op.isImm())466 Imm = Op.getImm();467 else if (Op.isExpr()) {468 if (const auto *C = dyn_cast<MCConstantExpr>(Op.getExpr())) {469 Imm = C->getValue();470 } else if (AMDGPU::isLitExpr(Op.getExpr())) {471 IsLit = true;472 Imm = AMDGPU::getLitValue(Op.getExpr());473 }474 } else // Exprs will be replaced with a fixup value.475 llvm_unreachable("Must be immediate or expr");476 477 if (*Enc == 254) {478 assert(STI.hasFeature(AMDGPU::Feature64BitLiterals));479 support::endian::write<uint64_t>(CB, Imm, llvm::endianness::little);480 } else {481 auto OpType =482 static_cast<AMDGPU::OperandType>(Desc.operands()[i].OperandType);483 Imm = AMDGPU::encode32BitLiteral(Imm, OpType, IsLit);484 support::endian::write<uint32_t>(CB, Imm, llvm::endianness::little);485 }486 487 // Only one literal value allowed488 break;489 }490}491 492void AMDGPUMCCodeEmitter::getSOPPBrEncoding(const MCInst &MI, unsigned OpNo,493 APInt &Op,494 SmallVectorImpl<MCFixup> &Fixups,495 const MCSubtargetInfo &STI) const {496 const MCOperand &MO = MI.getOperand(OpNo);497 498 if (MO.isExpr()) {499 const MCExpr *Expr = MO.getExpr();500 addFixup(Fixups, 0, Expr, AMDGPU::fixup_si_sopp_br, true);501 Op = APInt::getZero(96);502 } else {503 getMachineOpValue(MI, MO, Op, Fixups, STI);504 }505}506 507void AMDGPUMCCodeEmitter::getSMEMOffsetEncoding(508 const MCInst &MI, unsigned OpNo, APInt &Op,509 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {510 auto Offset = MI.getOperand(OpNo).getImm();511 // VI only supports 20-bit unsigned offsets.512 assert(!AMDGPU::isVI(STI) || isUInt<20>(Offset));513 Op = Offset;514}515 516void AMDGPUMCCodeEmitter::getSDWASrcEncoding(const MCInst &MI, unsigned OpNo,517 APInt &Op,518 SmallVectorImpl<MCFixup> &Fixups,519 const MCSubtargetInfo &STI) const {520 using namespace AMDGPU::SDWA;521 522 uint64_t RegEnc = 0;523 524 const MCOperand &MO = MI.getOperand(OpNo);525 526 if (MO.isReg()) {527 MCRegister Reg = MO.getReg();528 RegEnc |= MRI.getEncodingValue(Reg);529 RegEnc &= SDWA9EncValues::SRC_VGPR_MASK;530 if (AMDGPU::isSGPR(AMDGPU::mc2PseudoReg(Reg), &MRI)) {531 RegEnc |= SDWA9EncValues::SRC_SGPR_MASK;532 }533 Op = RegEnc;534 return;535 } else {536 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());537 auto Enc = getLitEncoding(Desc, MO, OpNo, STI);538 if (Enc && *Enc != 255) {539 Op = *Enc | SDWA9EncValues::SRC_SGPR_MASK;540 return;541 }542 }543 544 llvm_unreachable("Unsupported operand kind");545}546 547void AMDGPUMCCodeEmitter::getSDWAVopcDstEncoding(548 const MCInst &MI, unsigned OpNo, APInt &Op,549 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {550 using namespace AMDGPU::SDWA;551 552 uint64_t RegEnc = 0;553 554 const MCOperand &MO = MI.getOperand(OpNo);555 556 MCRegister Reg = MO.getReg();557 if (Reg != AMDGPU::VCC && Reg != AMDGPU::VCC_LO) {558 RegEnc |= MRI.getEncodingValue(Reg);559 RegEnc &= SDWA9EncValues::VOPC_DST_SGPR_MASK;560 RegEnc |= SDWA9EncValues::VOPC_DST_VCC_MASK;561 }562 Op = RegEnc;563}564 565void AMDGPUMCCodeEmitter::getAVOperandEncoding(566 const MCInst &MI, unsigned OpNo, APInt &Op,567 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {568 MCRegister Reg = MI.getOperand(OpNo).getReg();569 unsigned Enc = MRI.getEncodingValue(Reg);570 unsigned Idx = Enc & AMDGPU::HWEncoding::LO256_REG_IDX_MASK;571 bool IsVGPROrAGPR =572 Enc & (AMDGPU::HWEncoding::IS_VGPR | AMDGPU::HWEncoding::IS_AGPR);573 574 // VGPR and AGPR have the same encoding, but SrcA and SrcB operands of mfma575 // instructions use acc[0:1] modifier bits to distinguish. These bits are576 // encoded as a virtual 9th bit of the register for these operands.577 bool IsAGPR = Enc & AMDGPU::HWEncoding::IS_AGPR;578 579 Op = Idx | (IsVGPROrAGPR << 8) | (IsAGPR << 9);580}581 582static bool needsPCRel(const MCExpr *Expr) {583 switch (Expr->getKind()) {584 case MCExpr::SymbolRef: {585 auto *SE = cast<MCSymbolRefExpr>(Expr);586 auto Spec = AMDGPU::getSpecifier(SE);587 return Spec != AMDGPUMCExpr::S_ABS32_LO &&588 Spec != AMDGPUMCExpr::S_ABS32_HI && Spec != AMDGPUMCExpr::S_ABS64;589 }590 case MCExpr::Binary: {591 auto *BE = cast<MCBinaryExpr>(Expr);592 if (BE->getOpcode() == MCBinaryExpr::Sub)593 return false;594 return needsPCRel(BE->getLHS()) || needsPCRel(BE->getRHS());595 }596 case MCExpr::Unary:597 return needsPCRel(cast<MCUnaryExpr>(Expr)->getSubExpr());598 case MCExpr::Specifier:599 case MCExpr::Target:600 case MCExpr::Constant:601 return false;602 }603 llvm_unreachable("invalid kind");604}605 606void AMDGPUMCCodeEmitter::getMachineOpValue(const MCInst &MI,607 const MCOperand &MO, APInt &Op,608 SmallVectorImpl<MCFixup> &Fixups,609 const MCSubtargetInfo &STI) const {610 if (MO.isReg()){611 unsigned Enc = MRI.getEncodingValue(MO.getReg());612 unsigned Idx = Enc & AMDGPU::HWEncoding::LO256_REG_IDX_MASK;613 bool IsVGPROrAGPR =614 Enc & (AMDGPU::HWEncoding::IS_VGPR | AMDGPU::HWEncoding::IS_AGPR);615 Op = Idx | (IsVGPROrAGPR << 8);616 return;617 }618 unsigned OpNo = &MO - MI.begin();619 getMachineOpValueCommon(MI, MO, OpNo, Op, Fixups, STI);620}621 622void AMDGPUMCCodeEmitter::getMachineOpValueT16(623 const MCInst &MI, unsigned OpNo, APInt &Op,624 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {625 const MCOperand &MO = MI.getOperand(OpNo);626 if (MO.isReg()) {627 unsigned Enc = MRI.getEncodingValue(MO.getReg());628 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;629 bool IsVGPR = Enc & AMDGPU::HWEncoding::IS_VGPR;630 Op = Idx | (IsVGPR << 8);631 return;632 }633 getMachineOpValueCommon(MI, MO, OpNo, Op, Fixups, STI);634 // VGPRs include the suffix/op_sel bit in the register encoding, but635 // immediates and SGPRs include it in src_modifiers. Therefore, copy the636 // op_sel bit from the src operands into src_modifier operands if Op is637 // src_modifiers and the corresponding src is a VGPR638 int SrcMOIdx = -1;639 assert(OpNo < INT_MAX);640 if ((int)OpNo == AMDGPU::getNamedOperandIdx(MI.getOpcode(),641 AMDGPU::OpName::src0_modifiers)) {642 SrcMOIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);643 int VDstMOIdx =644 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdst);645 if (VDstMOIdx != -1) {646 auto DstReg = MI.getOperand(VDstMOIdx).getReg();647 if (AMDGPU::isHi16Reg(DstReg, MRI))648 Op |= SISrcMods::DST_OP_SEL;649 }650 } else if ((int)OpNo == AMDGPU::getNamedOperandIdx(651 MI.getOpcode(), AMDGPU::OpName::src1_modifiers))652 SrcMOIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src1);653 else if ((int)OpNo == AMDGPU::getNamedOperandIdx(654 MI.getOpcode(), AMDGPU::OpName::src2_modifiers))655 SrcMOIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src2);656 if (SrcMOIdx == -1)657 return;658 659 const MCOperand &SrcMO = MI.getOperand(SrcMOIdx);660 if (!SrcMO.isReg())661 return;662 auto SrcReg = SrcMO.getReg();663 if (AMDGPU::isSGPR(SrcReg, &MRI))664 return;665 if (AMDGPU::isHi16Reg(SrcReg, MRI))666 Op |= SISrcMods::OP_SEL_0;667}668 669void AMDGPUMCCodeEmitter::getMachineOpValueT16Lo128(670 const MCInst &MI, unsigned OpNo, APInt &Op,671 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {672 const MCOperand &MO = MI.getOperand(OpNo);673 if (MO.isReg()) {674 uint16_t Encoding = MRI.getEncodingValue(MO.getReg());675 unsigned RegIdx = Encoding & AMDGPU::HWEncoding::LO256_REG_IDX_MASK;676 bool IsHi = Encoding & AMDGPU::HWEncoding::IS_HI16;677 bool IsVGPR = Encoding & AMDGPU::HWEncoding::IS_VGPR;678 assert((!IsVGPR || isUInt<7>(RegIdx)) && "VGPR0-VGPR127 expected!");679 Op = (IsVGPR ? 0x100 : 0) | (IsHi ? 0x80 : 0) | RegIdx;680 return;681 }682 getMachineOpValueCommon(MI, MO, OpNo, Op, Fixups, STI);683}684 685void AMDGPUMCCodeEmitter::getMachineOpValueCommon(686 const MCInst &MI, const MCOperand &MO, unsigned OpNo, APInt &Op,687 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {688 bool isLikeImm = false;689 int64_t Val;690 691 if (MO.isImm()) {692 Val = MO.getImm();693 isLikeImm = true;694 } else if (MO.isExpr() && MO.getExpr()->evaluateAsAbsolute(Val)) {695 isLikeImm = true;696 } else if (MO.isExpr()) {697 // FIXME: If this is expression is PCRel or not should not depend on what698 // the expression looks like. Given that this is just a general expression,699 // it should probably be FK_Data_4 and whatever is producing700 //701 // s_add_u32 s2, s2, (extern_const_addrspace+16702 //703 // And expecting a PCRel should instead produce704 //705 // .Ltmp1:706 // s_add_u32 s2, s2, (extern_const_addrspace+16)-.Ltmp1707 bool PCRel = needsPCRel(MO.getExpr());708 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());709 uint32_t Offset = Desc.getSize();710 assert(Offset == 4 || Offset == 8);711 unsigned Size = AMDGPU::getOperandSize(Desc, OpNo);712 MCFixupKind Kind = MCFixup::getDataKindForSize(Size);713 addFixup(Fixups, Offset, MO.getExpr(), Kind, PCRel);714 }715 716 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());717 if (AMDGPU::isSISrcOperand(Desc, OpNo)) {718 bool HasMandatoryLiteral =719 AMDGPU::hasNamedOperand(MI.getOpcode(), AMDGPU::OpName::imm);720 if (auto Enc = getLitEncoding(Desc, MO, OpNo, STI, HasMandatoryLiteral)) {721 Op = *Enc;722 return;723 }724 725 llvm_unreachable("Operand not supported for SISrc");726 }727 728 if (isLikeImm) {729 Op = Val;730 return;731 }732 733 llvm_unreachable("Encoding of this operand type is not supported yet.");734}735 736#include "AMDGPUGenMCCodeEmitter.inc"737