714 lines · cpp
1//===-- X86FastPreTileConfig.cpp - Fast Tile Register Configure------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file Pass to preconfig the shape of physical tile registers10/// It inserts ldtilecfg ahead of each group of tile registers. The algorithm11/// walk each instruction of basic block in reverse order. All the tile12/// registers that live out the basic block would be spilled and reloaded13/// before its user. It also check the depenedency of the shape to ensure14/// the shape is defined before ldtilecfg.15//16//===----------------------------------------------------------------------===//17 18#include "X86.h"19#include "X86InstrBuilder.h"20#include "X86MachineFunctionInfo.h"21#include "X86RegisterInfo.h"22#include "X86Subtarget.h"23#include "llvm/ADT/PostOrderIterator.h"24#include "llvm/ADT/Statistic.h"25#include "llvm/CodeGen/MachineFrameInfo.h"26#include "llvm/CodeGen/MachineFunctionPass.h"27#include "llvm/CodeGen/MachineInstr.h"28#include "llvm/CodeGen/MachineRegisterInfo.h"29#include "llvm/CodeGen/Passes.h"30#include "llvm/CodeGen/TargetInstrInfo.h"31#include "llvm/CodeGen/TargetRegisterInfo.h"32#include "llvm/Support/Debug.h"33 34using namespace llvm;35 36#define DEBUG_TYPE "fastpretileconfig"37 38STATISTIC(NumStores, "Number of stores added");39STATISTIC(NumLoads, "Number of loads added");40 41namespace {42 43class X86FastPreTileConfig : public MachineFunctionPass {44 MachineFunction *MF = nullptr;45 const X86Subtarget *ST = nullptr;46 const TargetInstrInfo *TII = nullptr;47 MachineRegisterInfo *MRI = nullptr;48 X86MachineFunctionInfo *X86FI = nullptr;49 MachineFrameInfo *MFI = nullptr;50 const TargetRegisterInfo *TRI = nullptr;51 MachineBasicBlock *MBB = nullptr;52 int CfgSS = -1;53 struct PHIInfo {54 Register Row;55 Register Col;56 Register StackAddr;57 };58 DenseMap<MachineInstr *, struct PHIInfo> VisitedPHIs;59 60 /// Maps virtual regs to the frame index where these values are spilled.61 IndexedMap<int, VirtReg2IndexFunctor> StackSlotForVirtReg;62 63 /// Has a bit set for tile virtual register for which it was determined64 /// that it is alive across blocks.65 BitVector MayLiveAcrossBlocks;66 67 int getStackSpaceFor(Register VirtReg);68 void InitializeTileConfigStackSpace();69 bool mayLiveOut(Register VirtReg, MachineInstr *CfgMI);70 void spill(MachineBasicBlock::iterator Before, Register VirtReg, bool Kill);71 void reload(MachineBasicBlock::iterator UseMI, Register VirtReg,72 MachineOperand *RowMO, MachineOperand *ColMO);73 void canonicalizePHIs(MachineBasicBlock &MBB);74 void convertPHI(MachineBasicBlock *MBB, MachineInstr &PHI);75 void convertPHIs(MachineBasicBlock &MBB);76 bool configBasicBlock(MachineBasicBlock &MBB);77 78public:79 X86FastPreTileConfig() : MachineFunctionPass(ID), StackSlotForVirtReg(-1) {}80 81 /// Return the pass name.82 StringRef getPassName() const override {83 return "Fast Tile Register Preconfigure";84 }85 86 /// Perform tile register configure.87 bool runOnMachineFunction(MachineFunction &MFunc) override;88 89 static char ID;90};91 92} // end anonymous namespace93 94char X86FastPreTileConfig::ID = 0;95 96INITIALIZE_PASS_BEGIN(X86FastPreTileConfig, DEBUG_TYPE,97 "Fast Tile Register Preconfigure", false, false)98INITIALIZE_PASS_END(X86FastPreTileConfig, DEBUG_TYPE,99 "Fast Tile Register Preconfigure", false, false)100 101static bool dominates(MachineBasicBlock &MBB,102 MachineBasicBlock::const_iterator A,103 MachineBasicBlock::const_iterator B) {104 auto MBBEnd = MBB.end();105 if (B == MBBEnd)106 return true;107 108 MachineBasicBlock::const_iterator I = MBB.begin();109 for (; &*I != A && &*I != B; ++I)110 ;111 112 return &*I == A;113}114 115/// This allocates space for the specified virtual register to be held on the116/// stack.117int X86FastPreTileConfig::getStackSpaceFor(Register VirtReg) {118 // Find the location Reg would belong...119 int SS = StackSlotForVirtReg[VirtReg];120 // Already has space allocated?121 if (SS != -1)122 return SS;123 124 // Allocate a new stack object for this spill location...125 const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);126 unsigned Size = TRI->getSpillSize(RC);127 Align Alignment = TRI->getSpillAlign(RC);128 int FrameIdx = MFI->CreateSpillStackObject(Size, Alignment);129 130 // Assign the slot.131 StackSlotForVirtReg[VirtReg] = FrameIdx;132 return FrameIdx;133}134 135/// Returns false if \p VirtReg is known to not live out of the current config.136/// If \p VirtReg live out of the current MBB, it must live out of the current137/// config138bool X86FastPreTileConfig::mayLiveOut(Register VirtReg, MachineInstr *CfgMI) {139 if (MayLiveAcrossBlocks.test(VirtReg.virtRegIndex()))140 return true;141 142 for (const MachineInstr &UseInst : MRI->use_nodbg_instructions(VirtReg)) {143 if (UseInst.getParent() != MBB) {144 MayLiveAcrossBlocks.set(VirtReg.virtRegIndex());145 return true;146 }147 148 // The use and def are in the same MBB. If the tile register is149 // reconfigured, it is crobbered and we need to spill and reload150 // tile register.151 if (CfgMI) {152 if (dominates(*MBB, *CfgMI, UseInst)) {153 MayLiveAcrossBlocks.set(VirtReg.virtRegIndex());154 return true;155 }156 }157 }158 159 return false;160}161 162void X86FastPreTileConfig::InitializeTileConfigStackSpace() {163 MachineBasicBlock &MBB = MF->front();164 MachineInstr *MI = &*MBB.getFirstNonPHI();165 DebugLoc DL;166 if (ST->hasAVX512()) {167 Register Zmm = MRI->createVirtualRegister(&X86::VR512RegClass);168 BuildMI(MBB, MI, DL, TII->get(X86::AVX512_512_SET0), Zmm);169 addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr)), CfgSS)170 .addReg(Zmm);171 } else if (ST->hasAVX2()) {172 Register Ymm = MRI->createVirtualRegister(&X86::VR256RegClass);173 BuildMI(MBB, MI, DL, TII->get(X86::AVX_SET0), Ymm);174 addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSYmr)), CfgSS)175 .addReg(Ymm);176 addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSYmr)), CfgSS,177 32)178 .addReg(Ymm);179 } else {180 assert(ST->hasSSE2() && "AMX should assume SSE2 enabled");181 unsigned StoreOpc = ST->hasAVX() ? X86::VMOVUPSmr : X86::MOVUPSmr;182 Register Xmm = MRI->createVirtualRegister(&X86::VR128RegClass);183 BuildMI(MBB, MI, DL, TII->get(X86::V_SET0), Xmm);184 addFrameReference(BuildMI(MBB, MI, DL, TII->get(StoreOpc)), CfgSS)185 .addReg(Xmm);186 addFrameReference(BuildMI(MBB, MI, DL, TII->get(StoreOpc)), CfgSS, 16)187 .addReg(Xmm);188 addFrameReference(BuildMI(MBB, MI, DL, TII->get(StoreOpc)), CfgSS, 32)189 .addReg(Xmm);190 addFrameReference(BuildMI(MBB, MI, DL, TII->get(StoreOpc)), CfgSS, 48)191 .addReg(Xmm);192 }193 // Fill in the palette first.194 addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV8mi)), CfgSS)195 .addImm(1);196}197 198/// Insert spill instruction for \p AssignedReg before \p Before.199/// TODO: Update DBG_VALUEs with \p VirtReg operands with the stack slot.200void X86FastPreTileConfig::spill(MachineBasicBlock::iterator Before,201 Register VirtReg, bool Kill) {202 LLVM_DEBUG(dbgs() << "Spilling " << printReg(VirtReg, TRI) << " \n");203 int FI = getStackSpaceFor(VirtReg);204 LLVM_DEBUG(dbgs() << " to stack slot #" << FI << '\n');205 206 const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);207 // Don't need shape information for tile store, becasue it is adjacent to208 // the tile def instruction.209 TII->storeRegToStackSlot(*MBB, Before, VirtReg, Kill, FI, &RC, Register());210 ++NumStores;211 212 // TODO: update DBG_VALUEs213}214 215/// Insert reload instruction for \p PhysReg before \p Before.216void X86FastPreTileConfig::reload(MachineBasicBlock::iterator UseMI,217 Register OrigReg, MachineOperand *RowMO,218 MachineOperand *ColMO) {219 int FI = getStackSpaceFor(OrigReg);220 const TargetRegisterClass &RC = *MRI->getRegClass(OrigReg);221 Register TileReg;222 // Fold copy to tileload223 // BB1:224 // spill src to s225 //226 // BB2:227 // t = copy src228 // -->229 // t = tileload (s)230 if (UseMI->isCopy())231 TileReg = UseMI->getOperand(0).getReg();232 else233 TileReg = MRI->createVirtualRegister(&RC);234 // Can't use TII->loadRegFromStackSlot(), because we need the shape235 // information for reload.236 // tileloadd (%sp, %idx), %tmm237 unsigned Opc = X86::PTILELOADDV;238 Register StrideReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);239 // FIXME: MBB is not the parent of UseMI.240 MachineInstr *NewMI = BuildMI(*UseMI->getParent(), UseMI, DebugLoc(),241 TII->get(X86::MOV64ri), StrideReg)242 .addImm(64);243 NewMI = addFrameReference(244 BuildMI(*UseMI->getParent(), UseMI, DebugLoc(), TII->get(Opc), TileReg)245 .addReg(RowMO->getReg())246 .addReg(ColMO->getReg()),247 FI);248 MachineOperand &MO = NewMI->getOperand(5);249 MO.setReg(StrideReg);250 MO.setIsKill(true);251 RowMO->setIsKill(false);252 ColMO->setIsKill(false);253 // Erase copy instruction after it is folded.254 if (UseMI->isCopy()) {255 UseMI->eraseFromParent();256 } else {257 // Replace the register in the user MI.258 for (auto &MO : UseMI->operands()) {259 if (MO.isReg() && MO.getReg() == OrigReg)260 MO.setReg(TileReg);261 }262 }263 264 ++NumLoads;265 LLVM_DEBUG(dbgs() << "Reloading " << printReg(OrigReg, TRI) << " into "266 << printReg(TileReg, TRI) << '\n');267}268 269static bool isTileRegister(MachineRegisterInfo *MRI, Register Reg) {270 if (Reg.isVirtual() &&271 (MRI->getRegClass(Reg)->getID() == X86::TILERegClassID)) {272 return true;273 }274 275 if (Reg >= X86::TMM0 && Reg <= X86::TMM7)276 return true;277 278 return false;279}280 281static bool isTileDef(MachineRegisterInfo *MRI, MachineInstr &MI) {282 // The instruction must have 3 operands: tile def, row, col.283 if (MI.isDebugInstr() || MI.getNumOperands() < 3 || !MI.isPseudo())284 return false;285 MachineOperand &MO = MI.getOperand(0);286 287 if (!MO.isReg())288 return false;289 290 return isTileRegister(MRI, MO.getReg());291}292 293static ShapeT getShape(MachineRegisterInfo *MRI, Register TileReg) {294 MachineInstr *MI = MRI->getVRegDef(TileReg);295 if (isTileDef(MRI, *MI)) {296 MachineOperand *RowMO = &MI->getOperand(1);297 MachineOperand *ColMO = &MI->getOperand(2);298 return ShapeT(RowMO, ColMO, MRI);299 } else if (MI->isCopy()) {300 TileReg = MI->getOperand(1).getReg();301 return getShape(MRI, TileReg);302 }303 304 // The def should not be PHI node, because we walk the MBB in reverse post305 // order.306 assert(MI->isPHI() && "Unexpected PHI when get shape.");307 llvm_unreachable("Unexpected MI when get shape.");308}309 310// BB0:311// spill t0 to s0312// BB1:313// spill t1 to s1314//315// BB2:316// t = phi [t0, bb0] [t1, bb1]317// -->318// row = phi [r0, bb0] [r1, bb1]319// col = phi [c0, bb0] [c1, bb1]320// s = phi [s0, bb0] [s1, bb1]321// t = tileload row, col, s322// The new instruction is inserted at the end of the phi node. The order323// of the original phi node is not ensured.324void X86FastPreTileConfig::convertPHI(MachineBasicBlock *MBB,325 MachineInstr &PHI) {326 // 1. Create instruction to get stack slot address of each incoming block.327 // 2. Create PHI node for the stack address.328 // 3. Create PHI node for shape. If one of the incoming shape is immediate329 // use the immediate and delete the PHI node.330 // 4. Create tileload instruction from the stack address.331 Register StackAddrReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);332 MachineInstrBuilder AddrPHI = BuildMI(*MBB, ++PHI.getIterator(), DebugLoc(),333 TII->get(X86::PHI), StackAddrReg);334 Register RowReg = MRI->createVirtualRegister(&X86::GR16RegClass);335 MachineInstrBuilder RowPHI = BuildMI(*MBB, ++PHI.getIterator(), DebugLoc(),336 TII->get(X86::PHI), RowReg);337 Register ColReg = MRI->createVirtualRegister(&X86::GR16RegClass);338 MachineInstrBuilder ColPHI = BuildMI(*MBB, ++PHI.getIterator(), DebugLoc(),339 TII->get(X86::PHI), ColReg);340 // Record the mapping of phi node and its row/column information.341 VisitedPHIs[&PHI] = {RowReg, ColReg, StackAddrReg};342 343 for (unsigned I = 1, E = PHI.getNumOperands(); I != E; I += 2) {344 // Get the 2 incoming value of tile register and MBB.345 Register InTileReg = PHI.getOperand(I).getReg();346 // Mark it as liveout, so that it will be spilled when visit347 // the incoming MBB. Otherwise since phi will be deleted, it348 // would miss spill when visit incoming MBB.349 MayLiveAcrossBlocks.set(InTileReg.virtRegIndex());350 MachineBasicBlock *InMBB = PHI.getOperand(I + 1).getMBB();351 352 MachineInstr *TileDefMI = MRI->getVRegDef(InTileReg);353 MachineBasicBlock::iterator InsertPos;354 if (TileDefMI->isPHI()) {355 InsertPos = TileDefMI->getParent()->getFirstNonPHI();356 if (auto It = VisitedPHIs.find(TileDefMI);357 It != VisitedPHIs.end()) { // circular phi reference358 // def t1359 // / \360 // def t2 t3 = phi(t1, t4) <--361 // \ / |362 // t4 = phi(t2, t3)-------------363 //364 // For each (row, column and stack address) append phi incoming value.365 // Create r3 = phi(r1, r4)366 // Create r4 = phi(r2, r3)367 Register InRowReg = It->second.Row;368 Register InColReg = It->second.Col;369 Register InStackAddrReg = It->second.StackAddr;370 RowPHI.addReg(InRowReg).addMBB(InMBB);371 ColPHI.addReg(InColReg).addMBB(InMBB);372 AddrPHI.addReg(InStackAddrReg).addMBB(InMBB);373 continue;374 } else {375 // Recursively convert PHI to tileload376 convertPHI(TileDefMI->getParent(), *TileDefMI);377 // The PHI node is coverted to tileload instruction. Get the stack378 // address from tileload operands.379 MachineInstr *TileLoad = MRI->getVRegDef(InTileReg);380 assert(TileLoad && TileLoad->getOpcode() == X86::PTILELOADDV);381 Register InRowReg = TileLoad->getOperand(1).getReg();382 Register InColReg = TileLoad->getOperand(2).getReg();383 Register InStackAddrReg = TileLoad->getOperand(3).getReg();384 RowPHI.addReg(InRowReg).addMBB(InMBB);385 ColPHI.addReg(InColReg).addMBB(InMBB);386 AddrPHI.addReg(InStackAddrReg).addMBB(InMBB);387 }388 } else {389 InsertPos = TileDefMI->getIterator();390 391 // Fill the incoming operand of row/column phi instruction.392 ShapeT Shape = getShape(MRI, InTileReg);393 Shape.getRow()->setIsKill(false);394 Shape.getCol()->setIsKill(false);395 RowPHI.addReg(Shape.getRow()->getReg()).addMBB(InMBB);396 ColPHI.addReg(Shape.getCol()->getReg()).addMBB(InMBB);397 398 // The incoming tile register live out of its def BB, it would be spilled.399 // Create MI to get the spill stack slot address for the tile register400 int FI = getStackSpaceFor(InTileReg);401 Register InStackAddrReg =402 MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);403 addOffset(BuildMI(*TileDefMI->getParent(), InsertPos, DebugLoc(),404 TII->get(X86::LEA64r), InStackAddrReg)405 .addFrameIndex(FI),406 0);407 AddrPHI.addReg(InStackAddrReg).addMBB(InMBB);408 }409 }410 411 MachineBasicBlock::iterator InsertPos = MBB->getFirstNonPHI();412 Register StrideReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);413 BuildMI(*MBB, InsertPos, DebugLoc(), TII->get(X86::MOV64ri), StrideReg)414 .addImm(64);415 Register TileReg = PHI.getOperand(0).getReg();416 MachineInstr *NewMI = addDirectMem(417 BuildMI(*MBB, InsertPos, DebugLoc(), TII->get(X86::PTILELOADDV), TileReg)418 .addReg(RowReg)419 .addReg(ColReg),420 StackAddrReg);421 MachineOperand &MO = NewMI->getOperand(5);422 MO.setReg(StrideReg);423 MO.setIsKill(true);424 PHI.eraseFromParent();425 VisitedPHIs.erase(&PHI);426}427 428static bool isTileRegDef(MachineRegisterInfo *MRI, MachineInstr &MI) {429 MachineOperand &MO = MI.getOperand(0);430 if (MO.isReg() && MO.getReg().isVirtual() && isTileRegister(MRI, MO.getReg()))431 return true;432 return false;433}434 435void X86FastPreTileConfig::canonicalizePHIs(MachineBasicBlock &MBB) {436 SmallVector<MachineInstr *, 8> PHIs;437 438 for (MachineInstr &MI : MBB) {439 if (!MI.isPHI())440 break;441 if (!isTileRegDef(MRI, MI))442 continue;443 PHIs.push_back(&MI);444 }445 // Canonicalize the phi node first. One tile phi may depeneds previous446 // phi node. For below case, we need convert %t4.447 //448 // BB0:449 // %t3 = phi (t1 BB1, t2 BB0)450 // %t4 = phi (t5 BB1, t3 BB0)451 // -->452 // %t3 = phi (t1 BB1, t2 BB0)453 // %t4 = phi (t5 BB1, t2 BB0)454 //455 while (!PHIs.empty()) {456 MachineInstr *PHI = PHIs.pop_back_val();457 458 // Find the operand that is incoming from the same MBB and the def459 // is also phi node.460 MachineOperand *InMO = nullptr;461 MachineInstr *DefMI = nullptr;462 for (unsigned I = 1, E = PHI->getNumOperands(); I != E; I += 2) {463 Register InTileReg = PHI->getOperand(I).getReg();464 MachineBasicBlock *InMBB = PHI->getOperand(I + 1).getMBB();465 DefMI = MRI->getVRegDef(InTileReg);466 if (InMBB != &MBB || !DefMI->isPHI())467 continue;468 469 InMO = &PHI->getOperand(I);470 break;471 }472 // If can't find such operand, do nothing.473 if (!InMO)474 continue;475 476 // Current phi node depends on previous phi node. Break the477 // dependency.478 Register DefTileReg;479 for (unsigned I = 1, E = DefMI->getNumOperands(); I != E; I += 2) {480 MachineBasicBlock *InMBB = PHI->getOperand(I + 1).getMBB();481 if (InMBB != &MBB)482 continue;483 DefTileReg = DefMI->getOperand(I).getReg();484 InMO->setReg(DefTileReg);485 break;486 }487 }488}489 490void X86FastPreTileConfig::convertPHIs(MachineBasicBlock &MBB) {491 SmallVector<MachineInstr *, 8> PHIs;492 for (MachineInstr &MI : MBB) {493 if (!MI.isPHI())494 break;495 if (!isTileRegDef(MRI, MI))496 continue;497 PHIs.push_back(&MI);498 }499 while (!PHIs.empty()) {500 MachineInstr *MI = PHIs.pop_back_val();501 VisitedPHIs.clear();502 convertPHI(&MBB, *MI);503 }504}505 506// PreTileConfig should configure the tile registers based on basic507// block.508bool X86FastPreTileConfig::configBasicBlock(MachineBasicBlock &MBB) {509 this->MBB = &MBB;510 bool Change = false;511 MachineInstr *LastShapeMI = nullptr;512 MachineInstr *LastTileCfg = nullptr;513 bool HasUnconfigTile = false;514 515 auto Config = [&](MachineInstr &Before) {516 if (CfgSS == -1)517 CfgSS = MFI->CreateStackObject(ST->getTileConfigSize(),518 ST->getTileConfigAlignment(), false);519 LastTileCfg = addFrameReference(520 BuildMI(MBB, Before, DebugLoc(), TII->get(X86::PLDTILECFGV)), CfgSS);521 LastShapeMI = nullptr;522 Change = true;523 };524 auto HasTileOperand = [](MachineRegisterInfo *MRI, MachineInstr &MI) {525 for (const MachineOperand &MO : MI.operands()) {526 if (!MO.isReg())527 continue;528 Register Reg = MO.getReg();529 if (Reg.isVirtual() && isTileRegister(MRI, Reg))530 return true;531 }532 return false;533 };534 for (MachineInstr &MI : reverse(MBB)) {535 // We have transformed phi node before configuring BB.536 if (MI.isPHI())537 break;538 // Don't collect the shape of used tile, the tile should be defined539 // before the tile use. Spill and reload would happen if there is only540 // tile use after ldtilecfg, so the shape can be collected from reload.541 // Take below code for example. %t would be reloaded before tilestore542 // call543 // ....544 // tilestore %r, %c, %t545 // -->546 // call547 // ldtilecfg548 // %t = tileload %r, %c549 // tilestore %r, %c, %t550 if (HasTileOperand(MRI, MI))551 HasUnconfigTile = true;552 // According to AMX ABI, all the tile registers including config register553 // are volatile. Caller need to save/restore config register.554 if (MI.isCall() && HasUnconfigTile) {555 MachineBasicBlock::iterator I;556 if (LastShapeMI && dominates(MBB, MI, LastShapeMI))557 I = ++LastShapeMI->getIterator();558 else {559 // Call can overwrite registers like rax, ensure the tile config560 // instruction is sinked closer to first instruction that uses tile.561 auto UseIt = MI.getIterator();562 while (UseIt != MBB.end()) {563 if (HasTileOperand(MRI, *UseIt))564 break;565 ++UseIt;566 }567 I = UseIt;568 }569 Config(*I);570 HasUnconfigTile = false;571 continue;572 }573 if (!isTileDef(MRI, MI))574 continue;575 //576 //---------------------------------------------------------------------577 // Don't handle COPY instruction. If the src and dst of the COPY can be578 // in the same config in below case, we just check the shape of t0.579 // def row0580 // def col0581 // ldtilecfg582 // t0 = tielzero(row0, col0)583 // t1 = copy t0584 // ...585 // If the src and dst of the COPY can NOT be in the same config in below586 // case. Reload would be generated befor the copy instruction.587 // def row0588 // def col0589 // t0 = tielzero(row0, col0)590 // spill t0591 // ...592 // def row1593 // def col1594 // ldtilecfg595 // t1 = tilezero(row1, col1)596 // reload t0597 // t1 = copy t0598 //---------------------------------------------------------------------599 //600 // If MI dominate the last shape def instruction, we need insert601 // ldtilecfg after LastShapeMI now. The config doesn't include602 // current MI.603 // def row0604 // def col0605 // tilezero(row0, col0) <- MI606 // def row1607 // def col1608 // ldtilecfg <- insert609 // tilezero(row1, col1)610 if (LastShapeMI && dominates(MBB, MI, LastShapeMI))611 Config(*(++LastShapeMI->getIterator()));612 MachineOperand *RowMO = &MI.getOperand(1);613 MachineOperand *ColMO = &MI.getOperand(2);614 MachineInstr *RowMI = MRI->getVRegDef(RowMO->getReg());615 MachineInstr *ColMI = MRI->getVRegDef(ColMO->getReg());616 // If the shape is defined in current MBB, check the domination.617 // FIXME how about loop?618 if (RowMI->getParent() == &MBB) {619 if (!LastShapeMI)620 LastShapeMI = RowMI;621 else if (dominates(MBB, LastShapeMI, RowMI))622 LastShapeMI = RowMI;623 }624 if (ColMI->getParent() == &MBB) {625 if (!LastShapeMI)626 LastShapeMI = ColMI;627 else if (dominates(MBB, LastShapeMI, ColMI))628 LastShapeMI = ColMI;629 }630 631 // If there is user live out of the tilecfg, spill it and reload in632 // before the user.633 Register TileReg = MI.getOperand(0).getReg();634 if (mayLiveOut(TileReg, LastTileCfg))635 spill(++MI.getIterator(), TileReg, false);636 for (MachineInstr &UseMI : MRI->use_instructions(TileReg)) {637 if (UseMI.getParent() == &MBB) {638 // check user should not across ldtilecfg639 if (!LastTileCfg || !dominates(MBB, LastTileCfg, UseMI))640 continue;641 // reload befor UseMI642 reload(UseMI.getIterator(), TileReg, RowMO, ColMO);643 } else {644 // Don't reload for phi instruction, we handle phi reload separately.645 // TODO: merge the reload for the same user MBB.646 if (!UseMI.isPHI())647 reload(UseMI.getIterator(), TileReg, RowMO, ColMO);648 }649 }650 }651 652 // Configure tile registers at the head of the MBB653 if (HasUnconfigTile) {654 MachineInstr *Before;655 if (LastShapeMI == nullptr || LastShapeMI->isPHI())656 Before = &*MBB.getFirstNonPHI();657 else658 Before = &*(++LastShapeMI->getIterator());659 660 Config(*Before);661 }662 663 return Change;664}665 666bool X86FastPreTileConfig::runOnMachineFunction(MachineFunction &MFunc) {667 X86FI = MFunc.getInfo<X86MachineFunctionInfo>();668 // Early exit in the common case of non-AMX code.669 if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA)670 return false;671 672 MF = &MFunc;673 MRI = &MFunc.getRegInfo();674 ST = &MFunc.getSubtarget<X86Subtarget>();675 TII = ST->getInstrInfo();676 MFI = &MFunc.getFrameInfo();677 TRI = ST->getRegisterInfo();678 CfgSS = -1;679 680 unsigned NumVirtRegs = MRI->getNumVirtRegs();681 682 StackSlotForVirtReg.resize(NumVirtRegs);683 MayLiveAcrossBlocks.clear();684 // We will create register during config. *3 is to make sure685 // the virtual register number doesn't exceed the size of686 // the bit vector.687 MayLiveAcrossBlocks.resize(NumVirtRegs * 3);688 bool Change = false;689 assert(MRI->isSSA());690 691 // Canonicalize the phi node first.692 for (MachineBasicBlock &MBB : MFunc)693 canonicalizePHIs(MBB);694 695 // Loop over all of the basic blocks in reverse post order and insert696 // ldtilecfg for tile registers. The reserse post order is to facilitate697 // PHI node convert.698 ReversePostOrderTraversal<MachineFunction *> RPOT(MF);699 for (MachineBasicBlock *MBB : RPOT) {700 convertPHIs(*MBB);701 Change |= configBasicBlock(*MBB);702 }703 704 if (Change)705 InitializeTileConfigStackSpace();706 707 StackSlotForVirtReg.clear();708 return Change;709}710 711FunctionPass *llvm::createX86FastPreTileConfigPass() {712 return new X86FastPreTileConfig();713}714