756 lines · cpp
1//===- utils/TableGen/X86FoldTablesEmitter.cpp - X86 backend-*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This tablegen backend is responsible for emitting the memory fold tables of10// the X86 backend instructions.11//12//===----------------------------------------------------------------------===//13 14#include "Common/CodeGenInstruction.h"15#include "Common/CodeGenTarget.h"16#include "X86RecognizableInstr.h"17#include "llvm/ADT/StringSwitch.h"18#include "llvm/Support/X86FoldTablesUtils.h"19#include "llvm/TableGen/Record.h"20#include "llvm/TableGen/TableGenBackend.h"21#include <set>22 23using namespace llvm;24using namespace X86Disassembler;25 26namespace {27// Represents an entry in the manual mapped instructions set.28struct ManualMapEntry {29 const char *RegInstStr;30 const char *MemInstStr;31 uint16_t Strategy;32};33} // namespace34 35// List of instructions requiring explicitly aligned memory.36static constexpr const char *ExplicitAlign[] = {37 "MOVDQA", "MOVAPS", "MOVAPD", "MOVNTPS", "MOVNTPD", "MOVNTDQ", "MOVNTDQA"};38 39// List of instructions NOT requiring explicit memory alignment.40static constexpr const char *ExplicitUnalign[] = {41 "MOVDQU", "MOVUPS", "MOVUPD", "PCMPESTRM",42 "PCMPESTRI", "PCMPISTRM", "PCMPISTRI"};43 44static const ManualMapEntry ManualMapSet[] = {45#define ENTRY(REG, MEM, FLAGS) {#REG, #MEM, FLAGS},46#include "X86ManualFoldTables.def"47};48 49static const std::set<StringRef> NoFoldSet = {50#define NOFOLD(INSN) #INSN,51#include "X86ManualFoldTables.def"52};53 54static bool isExplicitAlign(const CodeGenInstruction *Inst) {55 return any_of(ExplicitAlign, [Inst](const char *InstStr) {56 return Inst->getName().contains(InstStr);57 });58}59 60static bool isExplicitUnalign(const CodeGenInstruction *Inst) {61 return any_of(ExplicitUnalign, [Inst](const char *InstStr) {62 return Inst->getName().contains(InstStr);63 });64}65 66namespace {67class X86FoldTablesEmitter {68 const RecordKeeper &Records;69 const CodeGenTarget Target;70 71 // Represents an entry in the folding table72 class X86FoldTableEntry {73 const CodeGenInstruction *RegInst;74 const CodeGenInstruction *MemInst;75 76 public:77 bool NoReverse = false;78 bool NoForward = false;79 bool FoldLoad = false;80 bool FoldStore = false;81 enum BcastType {82 BCAST_NONE,83 BCAST_W,84 BCAST_D,85 BCAST_Q,86 BCAST_SS,87 BCAST_SD,88 BCAST_SH,89 };90 BcastType BroadcastKind = BCAST_NONE;91 92 Align Alignment;93 94 X86FoldTableEntry() = default;95 X86FoldTableEntry(const CodeGenInstruction *RegInst,96 const CodeGenInstruction *MemInst)97 : RegInst(RegInst), MemInst(MemInst) {}98 99 void print(raw_ostream &OS) const {100 OS.indent(2);101 OS << "{X86::" << RegInst->getName() << ", ";102 OS << "X86::" << MemInst->getName() << ", ";103 104 std::string Attrs;105 if (FoldLoad)106 Attrs += "TB_FOLDED_LOAD|";107 if (FoldStore)108 Attrs += "TB_FOLDED_STORE|";109 if (NoReverse)110 Attrs += "TB_NO_REVERSE|";111 if (NoForward)112 Attrs += "TB_NO_FORWARD|";113 if (Alignment != Align(1))114 Attrs += "TB_ALIGN_" + std::to_string(Alignment.value()) + "|";115 switch (BroadcastKind) {116 case BCAST_NONE:117 break;118 case BCAST_W:119 Attrs += "TB_BCAST_W|";120 break;121 case BCAST_D:122 Attrs += "TB_BCAST_D|";123 break;124 case BCAST_Q:125 Attrs += "TB_BCAST_Q|";126 break;127 case BCAST_SS:128 Attrs += "TB_BCAST_SS|";129 break;130 case BCAST_SD:131 Attrs += "TB_BCAST_SD|";132 break;133 case BCAST_SH:134 Attrs += "TB_BCAST_SH|";135 break;136 }137 138 StringRef SimplifiedAttrs = StringRef(Attrs).rtrim("|");139 if (SimplifiedAttrs.empty())140 SimplifiedAttrs = "0";141 142 OS << SimplifiedAttrs << "},\n";143 }144 145#ifndef NDEBUG146 // Check that Uses and Defs are same after memory fold.147 void checkCorrectness() const {148 auto &RegInstRec = *RegInst->TheDef;149 auto &MemInstRec = *MemInst->TheDef;150 auto ListOfUsesReg = RegInstRec.getValueAsListOfDefs("Uses");151 auto ListOfUsesMem = MemInstRec.getValueAsListOfDefs("Uses");152 auto ListOfDefsReg = RegInstRec.getValueAsListOfDefs("Defs");153 auto ListOfDefsMem = MemInstRec.getValueAsListOfDefs("Defs");154 if (ListOfUsesReg != ListOfUsesMem || ListOfDefsReg != ListOfDefsMem)155 report_fatal_error("Uses/Defs couldn't be changed after folding " +156 RegInstRec.getName() + " to " +157 MemInstRec.getName());158 }159#endif160 };161 162 // NOTE: We check the fold tables are sorted in X86InstrFoldTables.cpp by the163 // enum of the instruction, which is computed in164 // CodeGenTarget::ComputeInstrsByEnum. So we should use the same comparator165 // here.166 // FIXME: Could we share the code with CodeGenTarget::ComputeInstrsByEnum?167 struct CompareInstrsByEnum {168 bool operator()(const CodeGenInstruction *LHS,169 const CodeGenInstruction *RHS) const {170 assert(LHS && RHS && "LHS and RHS shouldn't be nullptr");171 const auto &D1 = *LHS->TheDef;172 const auto &D2 = *RHS->TheDef;173 return std::tuple(!D1.getValueAsBit("isPseudo"), D1.getName()) <174 std::tuple(!D2.getValueAsBit("isPseudo"), D2.getName());175 }176 };177 178 using FoldTable = std::map<const CodeGenInstruction *, X86FoldTableEntry,179 CompareInstrsByEnum>;180 // Table2Addr - Holds instructions which their memory form performs181 // load+store.182 //183 // Table#i - Holds instructions which the their memory form184 // performs a load OR a store, and their #i'th operand is folded.185 //186 // BroadcastTable#i - Holds instructions which the their memory form performs187 // a broadcast load and their #i'th operand is folded.188 FoldTable Table2Addr;189 FoldTable Table0;190 FoldTable Table1;191 FoldTable Table2;192 FoldTable Table3;193 FoldTable Table4;194 FoldTable BroadcastTable1;195 FoldTable BroadcastTable2;196 FoldTable BroadcastTable3;197 FoldTable BroadcastTable4;198 199public:200 X86FoldTablesEmitter(const RecordKeeper &R) : Records(R), Target(R) {}201 202 // run - Generate the 6 X86 memory fold tables.203 void run(raw_ostream &OS);204 205private:206 // Decides to which table to add the entry with the given instructions.207 // S sets the strategy of adding the TB_NO_REVERSE flag.208 void updateTables(const CodeGenInstruction *RegInst,209 const CodeGenInstruction *MemInst, uint16_t S = 0,210 bool IsManual = false, bool IsBroadcast = false);211 212 // Generates X86FoldTableEntry with the given instructions and fill it with213 // the appropriate flags, then adds it to a memory fold table.214 void addEntryWithFlags(FoldTable &Table, const CodeGenInstruction *RegInst,215 const CodeGenInstruction *MemInst, uint16_t S,216 unsigned FoldedIdx, bool IsManual);217 // Generates X86FoldTableEntry with the given instructions and adds it to a218 // broadcast table.219 void addBroadcastEntry(FoldTable &Table, const CodeGenInstruction *RegInst,220 const CodeGenInstruction *MemInst);221 222 // Print the given table as a static const C++ array of type223 // X86FoldTableEntry.224 void printTable(const FoldTable &Table, StringRef TableName,225 raw_ostream &OS) {226 OS << "static const X86FoldTableEntry " << TableName << "[] = {\n";227 228 for (auto &E : Table)229 E.second.print(OS);230 231 OS << "};\n\n";232 }233};234} // namespace235 236// Return true if one of the instruction's operands is a RST register class237static bool hasRSTRegClass(const CodeGenInstruction *Inst) {238 return any_of(Inst->Operands, [](const CGIOperandList::OperandInfo &OpIn) {239 return OpIn.Rec->getName() == "RST" || OpIn.Rec->getName() == "RSTi";240 });241}242 243// Return true if one of the instruction's operands is a ptr_rc_tailcall244static bool hasPtrTailcallRegClass(const CodeGenInstruction *Inst) {245 return any_of(Inst->Operands, [](const CGIOperandList::OperandInfo &OpIn) {246 return OpIn.Rec->getName() == "ptr_rc_tailcall";247 });248}249 250static bool mayFoldFromForm(uint8_t Form) {251 switch (Form) {252 default:253 return Form >= X86Local::MRM0r && Form <= X86Local::MRM7r;254 case X86Local::MRMXr:255 case X86Local::MRMXrCC:256 case X86Local::MRMDestReg:257 case X86Local::MRMSrcReg:258 case X86Local::MRMSrcReg4VOp3:259 case X86Local::MRMSrcRegOp4:260 case X86Local::MRMSrcRegCC:261 return true;262 }263}264 265static bool mayFoldToForm(uint8_t Form) {266 switch (Form) {267 default:268 return Form >= X86Local::MRM0m && Form <= X86Local::MRM7m;269 case X86Local::MRMXm:270 case X86Local::MRMXmCC:271 case X86Local::MRMDestMem:272 case X86Local::MRMSrcMem:273 case X86Local::MRMSrcMem4VOp3:274 case X86Local::MRMSrcMemOp4:275 case X86Local::MRMSrcMemCC:276 return true;277 }278}279 280static bool mayFoldFromLeftToRight(uint8_t LHS, uint8_t RHS) {281 switch (LHS) {282 default:283 llvm_unreachable("Unexpected Form!");284 case X86Local::MRM0r:285 return RHS == X86Local::MRM0m;286 case X86Local::MRM1r:287 return RHS == X86Local::MRM1m;288 case X86Local::MRM2r:289 return RHS == X86Local::MRM2m;290 case X86Local::MRM3r:291 return RHS == X86Local::MRM3m;292 case X86Local::MRM4r:293 return RHS == X86Local::MRM4m;294 case X86Local::MRM5r:295 return RHS == X86Local::MRM5m;296 case X86Local::MRM6r:297 return RHS == X86Local::MRM6m;298 case X86Local::MRM7r:299 return RHS == X86Local::MRM7m;300 case X86Local::MRMXr:301 return RHS == X86Local::MRMXm;302 case X86Local::MRMXrCC:303 return RHS == X86Local::MRMXmCC;304 case X86Local::MRMDestReg:305 return RHS == X86Local::MRMDestMem;306 case X86Local::MRMSrcReg:307 return RHS == X86Local::MRMSrcMem;308 case X86Local::MRMSrcReg4VOp3:309 return RHS == X86Local::MRMSrcMem4VOp3;310 case X86Local::MRMSrcRegOp4:311 return RHS == X86Local::MRMSrcMemOp4;312 case X86Local::MRMSrcRegCC:313 return RHS == X86Local::MRMSrcMemCC;314 }315}316 317static bool isNOREXRegClass(const Record *Op) {318 return Op->getName().contains("_NOREX");319}320 321// Function object - Operator() returns true if the given Reg instruction322// matches the Mem instruction of this object.323namespace {324class IsMatch {325 const CodeGenInstruction *MemInst;326 const X86Disassembler::RecognizableInstrBase MemRI;327 bool IsBroadcast;328 const unsigned Variant;329 330public:331 IsMatch(const CodeGenInstruction *Inst, bool IsBroadcast, unsigned V)332 : MemInst(Inst), MemRI(*MemInst), IsBroadcast(IsBroadcast), Variant(V) {}333 334 bool operator()(const CodeGenInstruction *RegInst) {335 X86Disassembler::RecognizableInstrBase RegRI(*RegInst);336 const Record *RegRec = RegInst->TheDef;337 const Record *MemRec = MemInst->TheDef;338 339 // EVEX_B means different things for memory and register forms.340 // register form: rounding control or SAE341 // memory form: broadcast342 if (IsBroadcast && (RegRI.HasEVEX_B || !MemRI.HasEVEX_B))343 return false;344 // EVEX_B indicates NDD for MAP4 instructions345 if (!IsBroadcast && (RegRI.HasEVEX_B || MemRI.HasEVEX_B) &&346 RegRI.OpMap != X86Local::T_MAP4)347 return false;348 349 if (!mayFoldFromLeftToRight(RegRI.Form, MemRI.Form))350 return false;351 352 // X86 encoding is crazy, e.g353 //354 // f3 0f c7 30 vmxon (%rax)355 // f3 0f c7 f0 senduipi %rax356 //357 // This two instruction have similiar encoding fields but are unrelated358 if (X86Disassembler::getMnemonic(MemInst, Variant) !=359 X86Disassembler::getMnemonic(RegInst, Variant))360 return false;361 362 // Return false if any of the following fields of does not match.363 if (std::tuple(RegRI.Encoding, RegRI.Opcode, RegRI.OpPrefix, RegRI.OpMap,364 RegRI.OpSize, RegRI.AdSize, RegRI.HasREX_W, RegRI.HasVEX_4V,365 RegRI.HasVEX_L, RegRI.IgnoresVEX_L, RegRI.IgnoresW,366 RegRI.HasEVEX_K, RegRI.HasEVEX_KZ, RegRI.HasEVEX_L2,367 RegRI.HasEVEX_NF, RegRec->getValueAsBit("hasEVEX_RC"),368 RegRec->getValueAsBit("hasLockPrefix"),369 RegRec->getValueAsBit("hasNoTrackPrefix")) !=370 std::tuple(MemRI.Encoding, MemRI.Opcode, MemRI.OpPrefix, MemRI.OpMap,371 MemRI.OpSize, MemRI.AdSize, MemRI.HasREX_W, MemRI.HasVEX_4V,372 MemRI.HasVEX_L, MemRI.IgnoresVEX_L, MemRI.IgnoresW,373 MemRI.HasEVEX_K, MemRI.HasEVEX_KZ, MemRI.HasEVEX_L2,374 MemRI.HasEVEX_NF, MemRec->getValueAsBit("hasEVEX_RC"),375 MemRec->getValueAsBit("hasLockPrefix"),376 MemRec->getValueAsBit("hasNoTrackPrefix")))377 return false;378 379 // Make sure the sizes of the operands of both instructions suit each other.380 // This is needed for instructions with intrinsic version (_Int).381 // Where the only difference is the size of the operands.382 // For example: VUCOMISDZrm and VUCOMISDrm_Int383 // Also for instructions that their EVEX version was upgraded to work with384 // k-registers. For example VPCMPEQBrm (xmm output register) and385 // VPCMPEQBZ128rm (k register output register).386 unsigned MemOutSize = MemRec->getValueAsDag("OutOperandList")->getNumArgs();387 unsigned RegOutSize = RegRec->getValueAsDag("OutOperandList")->getNumArgs();388 unsigned MemInSize = MemRec->getValueAsDag("InOperandList")->getNumArgs();389 unsigned RegInSize = RegRec->getValueAsDag("InOperandList")->getNumArgs();390 391 // Instructions with one output in their memory form use the memory folded392 // operand as source and destination (Read-Modify-Write).393 unsigned RegStartIdx =394 (MemOutSize + 1 == RegOutSize) && (MemInSize == RegInSize) ? 1 : 0;395 396 bool FoundFoldedOp = false;397 for (unsigned I = 0, E = MemInst->Operands.size(); I != E; I++) {398 const Record *MemOpRec = MemInst->Operands[I].Rec;399 const Record *RegOpRec = RegInst->Operands[I + RegStartIdx].Rec;400 401 if (MemOpRec == RegOpRec)402 continue;403 404 if (isRegisterOperand(MemOpRec) && isRegisterOperand(RegOpRec) &&405 ((getRegOperandSize(MemOpRec) != getRegOperandSize(RegOpRec)) ||406 (isNOREXRegClass(MemOpRec) != isNOREXRegClass(RegOpRec))))407 return false;408 409 if (isMemoryOperand(MemOpRec) && isMemoryOperand(RegOpRec) &&410 (getMemOperandSize(MemOpRec) != getMemOperandSize(RegOpRec)))411 return false;412 413 if (isImmediateOperand(MemOpRec) && isImmediateOperand(RegOpRec) &&414 (MemOpRec->getValueAsDef("Type") != RegOpRec->getValueAsDef("Type")))415 return false;416 417 // Only one operand can be folded.418 if (FoundFoldedOp)419 return false;420 421 assert(isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec));422 FoundFoldedOp = true;423 }424 425 return FoundFoldedOp;426 }427};428 429} // end anonymous namespace430 431void X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table,432 const CodeGenInstruction *RegInst,433 const CodeGenInstruction *MemInst,434 uint16_t S, unsigned FoldedIdx,435 bool IsManual) {436 437 assert((IsManual || Table.find(RegInst) == Table.end()) &&438 "Override entry unexpectedly");439 X86FoldTableEntry Result = X86FoldTableEntry(RegInst, MemInst);440 const Record *RegRec = RegInst->TheDef;441 Result.NoReverse = S & TB_NO_REVERSE;442 Result.NoForward = S & TB_NO_FORWARD;443 Result.FoldLoad = S & TB_FOLDED_LOAD;444 Result.FoldStore = S & TB_FOLDED_STORE;445 Result.Alignment = Align(1ULL << ((S & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT));446 if (IsManual) {447 Table[RegInst] = Result;448 return;449 }450 451 const Record *RegOpRec = RegInst->Operands[FoldedIdx].Rec;452 const Record *MemOpRec = MemInst->Operands[FoldedIdx].Rec;453 454 // Unfolding code generates a load/store instruction according to the size of455 // the register in the register form instruction.456 // If the register's size is greater than the memory's operand size, do not457 // allow unfolding.458 459 // the unfolded load size will be based on the register size. If that’s bigger460 // than the memory operand size, the unfolded load will load more memory and461 // potentially cause a memory fault.462 if (getRegOperandSize(RegOpRec) > getMemOperandSize(MemOpRec))463 Result.NoReverse = true;464 465 // Check no-kz version's isMoveReg466 StringRef RegInstName = RegRec->getName();467 unsigned DropLen =468 RegInstName.ends_with("rkz") ? 2 : (RegInstName.ends_with("rk") ? 1 : 0);469 const Record *BaseDef =470 DropLen ? Records.getDef(RegInstName.drop_back(DropLen)) : nullptr;471 bool IsMoveReg =472 BaseDef ? Target.getInstruction(BaseDef).isMoveReg : RegInst->isMoveReg;473 // A masked load can not be unfolded to a full load, otherwise it would access474 // unexpected memory. A simple store can not be unfolded.475 if (IsMoveReg && (BaseDef || Result.FoldStore))476 Result.NoReverse = true;477 478 uint8_t Enc = byteFromBitsInit(RegRec->getValueAsBitsInit("OpEncBits"));479 if (isExplicitAlign(RegInst)) {480 // The instruction require explicitly aligned memory.481 const BitsInit *VectSize = RegRec->getValueAsBitsInit("VectSize");482 Result.Alignment = Align(byteFromBitsInit(VectSize));483 } else if (!Enc && !isExplicitUnalign(RegInst) &&484 getMemOperandSize(MemOpRec) > 64) {485 // Instructions with XOP/VEX/EVEX encoding do not require alignment while486 // SSE packed vector instructions require a 16 byte alignment.487 Result.Alignment = Align(16);488 }489 // Expand is only ever created as a masked instruction. It is not safe to490 // unfold a masked expand because we don't know if it came from an expand load491 // intrinsic or folding a plain load. If it is from a expand load intrinsic,492 // Unfolding to plain load would read more elements and could trigger a fault.493 if (RegRec->getName().contains("EXPAND"))494 Result.NoReverse = true;495 496 Table[RegInst] = Result;497}498 499void X86FoldTablesEmitter::addBroadcastEntry(500 FoldTable &Table, const CodeGenInstruction *RegInst,501 const CodeGenInstruction *MemInst) {502 503 assert(Table.find(RegInst) == Table.end() && "Override entry unexpectedly");504 X86FoldTableEntry Result = X86FoldTableEntry(RegInst, MemInst);505 506 const DagInit *In = MemInst->TheDef->getValueAsDag("InOperandList");507 for (unsigned I = 0, E = In->getNumArgs(); I != E; ++I) {508 Result.BroadcastKind =509 StringSwitch<X86FoldTableEntry::BcastType>(In->getArg(I)->getAsString())510 .Case("i16mem", X86FoldTableEntry::BCAST_W)511 .Case("i32mem", X86FoldTableEntry::BCAST_D)512 .Case("i64mem", X86FoldTableEntry::BCAST_Q)513 .Case("f16mem", X86FoldTableEntry::BCAST_SH)514 .Case("f32mem", X86FoldTableEntry::BCAST_SS)515 .Case("f64mem", X86FoldTableEntry::BCAST_SD)516 .Default(X86FoldTableEntry::BCAST_NONE);517 if (Result.BroadcastKind != X86FoldTableEntry::BCAST_NONE)518 break;519 }520 assert(Result.BroadcastKind != X86FoldTableEntry::BCAST_NONE &&521 "Unknown memory operand for broadcast");522 523 Table[RegInst] = Result;524}525 526void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInst,527 const CodeGenInstruction *MemInst,528 uint16_t S, bool IsManual,529 bool IsBroadcast) {530 531 const Record *RegRec = RegInst->TheDef;532 const Record *MemRec = MemInst->TheDef;533 unsigned MemOutSize = MemRec->getValueAsDag("OutOperandList")->getNumArgs();534 unsigned RegOutSize = RegRec->getValueAsDag("OutOperandList")->getNumArgs();535 unsigned MemInSize = MemRec->getValueAsDag("InOperandList")->getNumArgs();536 unsigned RegInSize = RegRec->getValueAsDag("InOperandList")->getNumArgs();537 538 // Instructions which Read-Modify-Write should be added to Table2Addr.539 if (!MemOutSize && RegOutSize == 1 && MemInSize == RegInSize) {540 assert(!IsBroadcast && "Read-Modify-Write can not be broadcast");541 // X86 would not unfold Read-Modify-Write instructions so add TB_NO_REVERSE.542 addEntryWithFlags(Table2Addr, RegInst, MemInst, S | TB_NO_REVERSE, 0,543 IsManual);544 return;545 }546 547 // Only table0 entries should explicitly specify a load or store flag.548 // If the instruction writes to the folded operand, it will appear as549 // an output in the register form instruction and as an input in the550 // memory form instruction. If the instruction reads from the folded551 // operand, it will appear as in input in both forms.552 if (MemInSize == RegInSize && MemOutSize == RegOutSize) {553 // Load-Folding cases.554 // If the i'th register form operand is a register and the i'th memory form555 // operand is a memory operand, add instructions to Table#i.556 for (unsigned I = RegOutSize, E = RegInst->Operands.size(); I < E; I++) {557 const Record *RegOpRec = RegInst->Operands[I].Rec;558 const Record *MemOpRec = MemInst->Operands[I].Rec;559 // RegClassByHwMode: For instructions like TAILJMPr, TAILJMPr64,560 // TAILJMPr64_REX561 if ((isRegisterOperand(RegOpRec) ||562 (RegOpRec->isSubClassOf("RegClassByHwMode"))) &&563 isMemoryOperand(MemOpRec)) {564 switch (I) {565 case 0:566 assert(!IsBroadcast && "BroadcastTable0 needs to be added");567 addEntryWithFlags(Table0, RegInst, MemInst, S | TB_FOLDED_LOAD, 0,568 IsManual);569 return;570 case 1:571 IsBroadcast572 ? addBroadcastEntry(BroadcastTable1, RegInst, MemInst)573 : addEntryWithFlags(Table1, RegInst, MemInst, S, 1, IsManual);574 return;575 case 2:576 IsBroadcast577 ? addBroadcastEntry(BroadcastTable2, RegInst, MemInst)578 : addEntryWithFlags(Table2, RegInst, MemInst, S, 2, IsManual);579 return;580 case 3:581 IsBroadcast582 ? addBroadcastEntry(BroadcastTable3, RegInst, MemInst)583 : addEntryWithFlags(Table3, RegInst, MemInst, S, 3, IsManual);584 return;585 case 4:586 IsBroadcast587 ? addBroadcastEntry(BroadcastTable4, RegInst, MemInst)588 : addEntryWithFlags(Table4, RegInst, MemInst, S, 4, IsManual);589 return;590 }591 }592 }593 } else if (MemInSize == RegInSize + 1 && MemOutSize + 1 == RegOutSize) {594 // Store-Folding cases.595 // If the memory form instruction performs a store, the *output*596 // register of the register form instructions disappear and instead a597 // memory *input* operand appears in the memory form instruction.598 // For example:599 // MOVAPSrr => (outs VR128:$dst), (ins VR128:$src)600 // MOVAPSmr => (outs), (ins f128mem:$dst, VR128:$src)601 const Record *RegOpRec = RegInst->Operands[RegOutSize - 1].Rec;602 const Record *MemOpRec = MemInst->Operands[RegOutSize - 1].Rec;603 if (isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec) &&604 getRegOperandSize(RegOpRec) == getMemOperandSize(MemOpRec)) {605 assert(!IsBroadcast && "Store can not be broadcast");606 addEntryWithFlags(Table0, RegInst, MemInst, S | TB_FOLDED_STORE, 0,607 IsManual);608 }609 }610}611 612void X86FoldTablesEmitter::run(raw_ostream &OS) {613 // Holds all memory instructions614 std::vector<const CodeGenInstruction *> MemInsts;615 // Holds all register instructions - divided according to opcode.616 std::map<uint8_t, std::vector<const CodeGenInstruction *>> RegInsts;617 618 ArrayRef<const CodeGenInstruction *> NumberedInstructions =619 Target.getInstructions();620 621 for (const CodeGenInstruction *Inst : NumberedInstructions) {622 const Record *Rec = Inst->TheDef;623 if (!Rec->isSubClassOf("X86Inst") || Rec->getValueAsBit("isAsmParserOnly"))624 continue;625 626 if (NoFoldSet.find(Rec->getName()) != NoFoldSet.end())627 continue;628 629 // Promoted legacy instruction is in EVEX space, and has REX2-encoding630 // alternative. It's added due to HW design and never emitted by compiler.631 if (byteFromBitsInit(Rec->getValueAsBitsInit("OpMapBits")) ==632 X86Local::T_MAP4 &&633 byteFromBitsInit(Rec->getValueAsBitsInit("explicitOpPrefixBits")) ==634 X86Local::ExplicitEVEX)635 continue;636 637 // - Instructions including RST register class operands are not relevant638 // for memory folding (for further details check the explanation in639 // lib/Target/X86/X86InstrFPStack.td file).640 // - Some instructions (listed in the manual map above) use the register641 // class ptr_rc_tailcall, which can be of a size 32 or 64, to ensure642 // safe mapping of these instruction we manually map them and exclude643 // them from the automation.644 if (hasRSTRegClass(Inst) || hasPtrTailcallRegClass(Inst))645 continue;646 647 // Add all the memory form instructions to MemInsts, and all the register648 // form instructions to RegInsts[Opc], where Opc is the opcode of each649 // instructions. this helps reducing the runtime of the backend.650 const BitsInit *FormBits = Rec->getValueAsBitsInit("FormBits");651 uint8_t Form = byteFromBitsInit(FormBits);652 if (mayFoldToForm(Form))653 MemInsts.push_back(Inst);654 else if (mayFoldFromForm(Form)) {655 uint8_t Opc = byteFromBitsInit(Rec->getValueAsBitsInit("Opcode"));656 RegInsts[Opc].push_back(Inst);657 }658 }659 660 // Create a copy b/c the register instruction will removed when a new entry is661 // added into memory fold tables.662 auto RegInstsForBroadcast = RegInsts;663 664 const Record *AsmWriter = Target.getAsmWriter();665 unsigned Variant = AsmWriter->getValueAsInt("Variant");666 auto FixUp = [&](const CodeGenInstruction *RegInst) {667 StringRef RegInstName = RegInst->getName();668 if (RegInstName.ends_with("_REV") || RegInstName.ends_with("_alt"))669 if (auto *RegAltRec = Records.getDef(RegInstName.drop_back(4)))670 RegInst = &Target.getInstruction(RegAltRec);671 return RegInst;672 };673 // For each memory form instruction, try to find its register form674 // instruction.675 for (const CodeGenInstruction *MemInst : MemInsts) {676 uint8_t Opc =677 byteFromBitsInit(MemInst->TheDef->getValueAsBitsInit("Opcode"));678 679 auto RegInstsIt = RegInsts.find(Opc);680 if (RegInstsIt == RegInsts.end())681 continue;682 683 // Two forms (memory & register) of the same instruction must have the same684 // opcode.685 std::vector<const CodeGenInstruction *> &OpcRegInsts = RegInstsIt->second;686 687 // Memory fold tables688 auto Match =689 find_if(OpcRegInsts, IsMatch(MemInst, /*IsBroadcast=*/false, Variant));690 if (Match != OpcRegInsts.end()) {691 updateTables(FixUp(*Match), MemInst);692 OpcRegInsts.erase(Match);693 }694 695 // Broadcast tables696 StringRef MemInstName = MemInst->getName();697 if (!MemInstName.contains("mb") && !MemInstName.contains("mib"))698 continue;699 RegInstsIt = RegInstsForBroadcast.find(Opc);700 assert(RegInstsIt != RegInstsForBroadcast.end() &&701 "Unexpected control flow");702 std::vector<const CodeGenInstruction *> &OpcRegInstsForBroadcast =703 RegInstsIt->second;704 Match = find_if(OpcRegInstsForBroadcast,705 IsMatch(MemInst, /*IsBroadcast=*/true, Variant));706 if (Match != OpcRegInstsForBroadcast.end()) {707 updateTables(FixUp(*Match), MemInst, 0, /*IsManual=*/false,708 /*IsBroadcast=*/true);709 OpcRegInstsForBroadcast.erase(Match);710 }711 }712 713 // Add the manually mapped instructions listed above.714 for (const ManualMapEntry &Entry : ManualMapSet) {715 const Record *RegInstIter = Records.getDef(Entry.RegInstStr);716 const Record *MemInstIter = Records.getDef(Entry.MemInstStr);717 718 updateTables(&(Target.getInstruction(RegInstIter)),719 &(Target.getInstruction(MemInstIter)), Entry.Strategy, true);720 }721 722#ifndef NDEBUG723 auto CheckMemFoldTable = [](const FoldTable &Table) -> void {724 for (const auto &Record : Table) {725 auto &FoldEntry = Record.second;726 FoldEntry.checkCorrectness();727 }728 };729 CheckMemFoldTable(Table2Addr);730 CheckMemFoldTable(Table0);731 CheckMemFoldTable(Table1);732 CheckMemFoldTable(Table2);733 CheckMemFoldTable(Table3);734 CheckMemFoldTable(Table4);735 CheckMemFoldTable(BroadcastTable1);736 CheckMemFoldTable(BroadcastTable2);737 CheckMemFoldTable(BroadcastTable3);738 CheckMemFoldTable(BroadcastTable4);739#endif740#define PRINT_TABLE(TABLE) printTable(TABLE, #TABLE, OS);741 // Print all tables.742 PRINT_TABLE(Table2Addr)743 PRINT_TABLE(Table0)744 PRINT_TABLE(Table1)745 PRINT_TABLE(Table2)746 PRINT_TABLE(Table3)747 PRINT_TABLE(Table4)748 PRINT_TABLE(BroadcastTable1)749 PRINT_TABLE(BroadcastTable2)750 PRINT_TABLE(BroadcastTable3)751 PRINT_TABLE(BroadcastTable4)752}753 754static TableGen::Emitter::OptClass<X86FoldTablesEmitter>755 X("gen-x86-fold-tables", "Generate X86 fold tables");756