843 lines · cpp
1//===--- X86DomainReassignment.cpp - Selectively switch register classes---===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This pass attempts to find instruction chains (closures) in one domain,10// and convert them to equivalent instructions in a different domain,11// if profitable.12//13//===----------------------------------------------------------------------===//14 15#include "X86.h"16#include "X86InstrInfo.h"17#include "X86Subtarget.h"18#include "llvm/ADT/DenseMap.h"19#include "llvm/ADT/STLExtras.h"20#include "llvm/ADT/SmallVector.h"21#include "llvm/ADT/Statistic.h"22#include "llvm/CodeGen/MachineFunctionPass.h"23#include "llvm/CodeGen/MachineInstrBuilder.h"24#include "llvm/CodeGen/MachineRegisterInfo.h"25#include "llvm/CodeGen/TargetRegisterInfo.h"26#include "llvm/Support/Debug.h"27#include "llvm/Support/Printable.h"28#include <bitset>29 30using namespace llvm;31 32#define DEBUG_TYPE "x86-domain-reassignment"33 34STATISTIC(NumClosuresConverted, "Number of closures converted by the pass");35 36static cl::opt<bool> DisableX86DomainReassignment(37 "disable-x86-domain-reassignment", cl::Hidden,38 cl::desc("X86: Disable Virtual Register Reassignment."), cl::init(false));39 40namespace {41enum RegDomain { NoDomain = -1, GPRDomain, MaskDomain, OtherDomain, NumDomains };42 43static bool isMask(const TargetRegisterClass *RC,44 const TargetRegisterInfo *TRI) {45 return X86::VK16RegClass.hasSubClassEq(RC);46}47 48static RegDomain getDomain(const TargetRegisterClass *RC,49 const TargetRegisterInfo *TRI) {50 if (TRI->isGeneralPurposeRegisterClass(RC))51 return GPRDomain;52 if (isMask(RC, TRI))53 return MaskDomain;54 return OtherDomain;55}56 57/// Return a register class equivalent to \p SrcRC, in \p Domain.58static const TargetRegisterClass *getDstRC(const TargetRegisterClass *SrcRC,59 RegDomain Domain) {60 assert(Domain == MaskDomain && "add domain");61 if (X86::GR8RegClass.hasSubClassEq(SrcRC))62 return &X86::VK8RegClass;63 if (X86::GR16RegClass.hasSubClassEq(SrcRC))64 return &X86::VK16RegClass;65 if (X86::GR32RegClass.hasSubClassEq(SrcRC))66 return &X86::VK32RegClass;67 if (X86::GR64RegClass.hasSubClassEq(SrcRC))68 return &X86::VK64RegClass;69 llvm_unreachable("add register class");70 return nullptr;71}72 73/// Abstract Instruction Converter class.74class InstrConverterBase {75protected:76 unsigned SrcOpcode;77 78public:79 InstrConverterBase(unsigned SrcOpcode) : SrcOpcode(SrcOpcode) {}80 81 virtual ~InstrConverterBase() = default;82 83 /// \returns true if \p MI is legal to convert.84 virtual bool isLegal(const MachineInstr *MI,85 const TargetInstrInfo *TII) const {86 assert(MI->getOpcode() == SrcOpcode &&87 "Wrong instruction passed to converter");88 return true;89 }90 91 /// Applies conversion to \p MI.92 ///93 /// \returns true if \p MI is no longer need, and can be deleted.94 virtual bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,95 MachineRegisterInfo *MRI) const = 0;96 97 /// \returns the cost increment incurred by converting \p MI.98 virtual double getExtraCost(const MachineInstr *MI,99 MachineRegisterInfo *MRI) const = 0;100};101 102/// An Instruction Converter which ignores the given instruction.103/// For example, PHI instructions can be safely ignored since only the registers104/// need to change.105class InstrIgnore : public InstrConverterBase {106public:107 InstrIgnore(unsigned SrcOpcode) : InstrConverterBase(SrcOpcode) {}108 109 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,110 MachineRegisterInfo *MRI) const override {111 assert(isLegal(MI, TII) && "Cannot convert instruction");112 return false;113 }114 115 double getExtraCost(const MachineInstr *MI,116 MachineRegisterInfo *MRI) const override {117 return 0;118 }119};120 121/// An Instruction Converter which replaces an instruction with another.122class InstrReplacer : public InstrConverterBase {123public:124 /// Opcode of the destination instruction.125 unsigned DstOpcode;126 127 InstrReplacer(unsigned SrcOpcode, unsigned DstOpcode)128 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}129 130 bool isLegal(const MachineInstr *MI,131 const TargetInstrInfo *TII) const override {132 if (!InstrConverterBase::isLegal(MI, TII))133 return false;134 // It's illegal to replace an instruction that implicitly defines a register135 // with an instruction that doesn't, unless that register dead.136 for (const auto &MO : MI->implicit_operands())137 if (MO.isReg() && MO.isDef() && !MO.isDead() &&138 !TII->get(DstOpcode).hasImplicitDefOfPhysReg(MO.getReg()))139 return false;140 return true;141 }142 143 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,144 MachineRegisterInfo *MRI) const override {145 assert(isLegal(MI, TII) && "Cannot convert instruction");146 MachineInstrBuilder Bld =147 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(DstOpcode));148 // Transfer explicit operands from original instruction. Implicit operands149 // are handled by BuildMI.150 for (auto &Op : MI->explicit_operands())151 Bld.add(Op);152 return true;153 }154 155 double getExtraCost(const MachineInstr *MI,156 MachineRegisterInfo *MRI) const override {157 // Assuming instructions have the same cost.158 return 0;159 }160};161 162/// An Instruction Converter which replaces an instruction with another, and163/// adds a COPY from the new instruction's destination to the old one's.164class InstrReplacerDstCOPY : public InstrConverterBase {165public:166 unsigned DstOpcode;167 168 InstrReplacerDstCOPY(unsigned SrcOpcode, unsigned DstOpcode)169 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}170 171 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,172 MachineRegisterInfo *MRI) const override {173 assert(isLegal(MI, TII) && "Cannot convert instruction");174 MachineBasicBlock *MBB = MI->getParent();175 const DebugLoc &DL = MI->getDebugLoc();176 177 Register Reg =178 MRI->createVirtualRegister(TII->getRegClass(TII->get(DstOpcode), 0));179 MachineInstrBuilder Bld = BuildMI(*MBB, MI, DL, TII->get(DstOpcode), Reg);180 for (const MachineOperand &MO : llvm::drop_begin(MI->operands()))181 Bld.add(MO);182 183 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::COPY))184 .add(MI->getOperand(0))185 .addReg(Reg);186 187 return true;188 }189 190 double getExtraCost(const MachineInstr *MI,191 MachineRegisterInfo *MRI) const override {192 // Assuming instructions have the same cost, and that COPY is in the same193 // domain so it will be eliminated.194 return 0;195 }196};197 198/// An Instruction Converter for replacing COPY instructions.199class InstrCOPYReplacer : public InstrReplacer {200public:201 RegDomain DstDomain;202 203 InstrCOPYReplacer(unsigned SrcOpcode, RegDomain DstDomain, unsigned DstOpcode)204 : InstrReplacer(SrcOpcode, DstOpcode), DstDomain(DstDomain) {}205 206 bool isLegal(const MachineInstr *MI,207 const TargetInstrInfo *TII) const override {208 if (!InstrConverterBase::isLegal(MI, TII))209 return false;210 211 // Don't allow copies to/flow GR8/GR16 physical registers.212 // FIXME: Is there some better way to support this?213 Register DstReg = MI->getOperand(0).getReg();214 if (DstReg.isPhysical() && (X86::GR8RegClass.contains(DstReg) ||215 X86::GR16RegClass.contains(DstReg)))216 return false;217 Register SrcReg = MI->getOperand(1).getReg();218 if (SrcReg.isPhysical() && (X86::GR8RegClass.contains(SrcReg) ||219 X86::GR16RegClass.contains(SrcReg)))220 return false;221 222 return true;223 }224 225 double getExtraCost(const MachineInstr *MI,226 MachineRegisterInfo *MRI) const override {227 assert(MI->getOpcode() == TargetOpcode::COPY && "Expected a COPY");228 229 for (const auto &MO : MI->operands()) {230 // Physical registers will not be converted. Assume that converting the231 // COPY to the destination domain will eventually result in a actual232 // instruction.233 if (MO.getReg().isPhysical())234 return 1;235 236 RegDomain OpDomain = getDomain(MRI->getRegClass(MO.getReg()),237 MRI->getTargetRegisterInfo());238 // Converting a cross domain COPY to a same domain COPY should eliminate239 // an insturction240 if (OpDomain == DstDomain)241 return -1;242 }243 return 0;244 }245};246 247/// An Instruction Converter which replaces an instruction with a COPY.248class InstrReplaceWithCopy : public InstrConverterBase {249public:250 // Source instruction operand Index, to be used as the COPY source.251 unsigned SrcOpIdx;252 253 InstrReplaceWithCopy(unsigned SrcOpcode, unsigned SrcOpIdx)254 : InstrConverterBase(SrcOpcode), SrcOpIdx(SrcOpIdx) {}255 256 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,257 MachineRegisterInfo *MRI) const override {258 assert(isLegal(MI, TII) && "Cannot convert instruction");259 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),260 TII->get(TargetOpcode::COPY))261 .add({MI->getOperand(0), MI->getOperand(SrcOpIdx)});262 return true;263 }264 265 double getExtraCost(const MachineInstr *MI,266 MachineRegisterInfo *MRI) const override {267 return 0;268 }269};270 271// Key type to be used by the Instruction Converters map.272// A converter is identified by <destination domain, source opcode>273typedef std::pair<int, unsigned> InstrConverterBaseKeyTy;274 275typedef DenseMap<InstrConverterBaseKeyTy, std::unique_ptr<InstrConverterBase>>276 InstrConverterBaseMap;277 278/// A closure is a set of virtual register representing all of the edges in279/// the closure, as well as all of the instructions connected by those edges.280///281/// A closure may encompass virtual registers in the same register bank that282/// have different widths. For example, it may contain 32-bit GPRs as well as283/// 64-bit GPRs.284///285/// A closure that computes an address (i.e. defines a virtual register that is286/// used in a memory operand) excludes the instructions that contain memory287/// operands using the address. Such an instruction will be included in a288/// different closure that manipulates the loaded or stored value.289class Closure {290private:291 /// Virtual registers in the closure.292 DenseSet<Register> Edges;293 294 /// Instructions in the closure.295 SmallVector<MachineInstr *, 8> Instrs;296 297 /// Domains which this closure can legally be reassigned to.298 std::bitset<NumDomains> LegalDstDomains;299 300 /// An ID to uniquely identify this closure, even when it gets301 /// moved around302 unsigned ID;303 304public:305 Closure(unsigned ID, std::initializer_list<RegDomain> LegalDstDomainList) : ID(ID) {306 for (RegDomain D : LegalDstDomainList)307 LegalDstDomains.set(D);308 }309 310 /// Mark this closure as illegal for reassignment to all domains.311 void setAllIllegal() { LegalDstDomains.reset(); }312 313 /// \returns true if this closure has domains which are legal to reassign to.314 bool hasLegalDstDomain() const { return LegalDstDomains.any(); }315 316 /// \returns true if is legal to reassign this closure to domain \p RD.317 bool isLegal(RegDomain RD) const { return LegalDstDomains[RD]; }318 319 /// Mark this closure as illegal for reassignment to domain \p RD.320 void setIllegal(RegDomain RD) { LegalDstDomains[RD] = false; }321 322 bool empty() const { return Edges.empty(); }323 324 bool insertEdge(Register Reg) { return Edges.insert(Reg).second; }325 326 using const_edge_iterator = DenseSet<Register>::const_iterator;327 iterator_range<const_edge_iterator> edges() const { return Edges; }328 329 void addInstruction(MachineInstr *I) {330 Instrs.push_back(I);331 }332 333 ArrayRef<MachineInstr *> instructions() const {334 return Instrs;335 }336 337 LLVM_DUMP_METHOD void dump(const MachineRegisterInfo *MRI) const {338 dbgs() << "Registers: ";339 bool First = true;340 for (Register Reg : Edges) {341 if (!First)342 dbgs() << ", ";343 First = false;344 dbgs() << printReg(Reg, MRI->getTargetRegisterInfo(), 0, MRI);345 }346 dbgs() << "\n" << "Instructions:";347 for (MachineInstr *MI : Instrs) {348 dbgs() << "\n ";349 MI->print(dbgs());350 }351 dbgs() << "\n";352 }353 354 unsigned getID() const {355 return ID;356 }357 358};359 360class X86DomainReassignment : public MachineFunctionPass {361 const X86Subtarget *STI = nullptr;362 MachineRegisterInfo *MRI = nullptr;363 const X86InstrInfo *TII = nullptr;364 365 /// All edges that are included in some closure366 DenseMap<Register, unsigned> EnclosedEdges;367 368 /// All instructions that are included in some closure.369 DenseMap<MachineInstr *, unsigned> EnclosedInstrs;370 371public:372 static char ID;373 374 X86DomainReassignment() : MachineFunctionPass(ID) { }375 376 bool runOnMachineFunction(MachineFunction &MF) override;377 378 void getAnalysisUsage(AnalysisUsage &AU) const override {379 AU.setPreservesCFG();380 MachineFunctionPass::getAnalysisUsage(AU);381 }382 383 StringRef getPassName() const override {384 return "X86 Domain Reassignment Pass";385 }386 387private:388 /// A map of available Instruction Converters.389 InstrConverterBaseMap Converters;390 391 /// Initialize Converters map.392 void initConverters();393 394 /// Starting from \Reg, expand the closure as much as possible.395 void buildClosure(Closure &, Register Reg);396 397 /// Enqueue \p Reg to be considered for addition to the closure.398 /// Return false if the closure becomes invalid.399 bool visitRegister(Closure &, Register Reg, RegDomain &Domain,400 SmallVectorImpl<Register> &Worklist);401 402 /// Reassign the closure to \p Domain.403 void reassign(const Closure &C, RegDomain Domain) const;404 405 /// Add \p MI to the closure.406 /// Return false if the closure becomes invalid.407 bool encloseInstr(Closure &C, MachineInstr *MI);408 409 /// /returns true if it is profitable to reassign the closure to \p Domain.410 bool isReassignmentProfitable(const Closure &C, RegDomain Domain) const;411 412 /// Calculate the total cost of reassigning the closure to \p Domain.413 double calculateCost(const Closure &C, RegDomain Domain) const;414};415 416char X86DomainReassignment::ID = 0;417 418} // End anonymous namespace.419 420bool X86DomainReassignment::visitRegister(Closure &C, Register Reg,421 RegDomain &Domain,422 SmallVectorImpl<Register> &Worklist) {423 if (!Reg.isVirtual())424 return true;425 426 auto I = EnclosedEdges.find(Reg);427 if (I != EnclosedEdges.end()) {428 if (I->second != C.getID()) {429 C.setAllIllegal();430 return false;431 }432 return true;433 }434 435 if (!MRI->hasOneDef(Reg))436 return true;437 438 RegDomain RD = getDomain(MRI->getRegClass(Reg), MRI->getTargetRegisterInfo());439 // First edge in closure sets the domain.440 if (Domain == NoDomain)441 Domain = RD;442 443 if (Domain != RD)444 return true;445 446 Worklist.push_back(Reg);447 return true;448}449 450bool X86DomainReassignment::encloseInstr(Closure &C, MachineInstr *MI) {451 auto [I, Inserted] = EnclosedInstrs.try_emplace(MI, C.getID());452 if (!Inserted) {453 if (I->second != C.getID()) {454 // Instruction already belongs to another closure, avoid conflicts between455 // closure and mark this closure as illegal.456 C.setAllIllegal();457 return false;458 }459 return true;460 }461 462 C.addInstruction(MI);463 464 // Mark closure as illegal for reassignment to domains, if there is no465 // converter for the instruction or if the converter cannot convert the466 // instruction.467 for (int i = 0; i != NumDomains; ++i) {468 if (C.isLegal((RegDomain)i)) {469 auto I = Converters.find({i, MI->getOpcode()});470 if (I == Converters.end() || !I->second->isLegal(MI, TII))471 C.setIllegal((RegDomain)i);472 }473 }474 return C.hasLegalDstDomain();475}476 477double X86DomainReassignment::calculateCost(const Closure &C,478 RegDomain DstDomain) const {479 assert(C.isLegal(DstDomain) && "Cannot calculate cost for illegal closure");480 481 double Cost = 0.0;482 for (auto *MI : C.instructions())483 Cost += Converters.find({DstDomain, MI->getOpcode()})484 ->second->getExtraCost(MI, MRI);485 return Cost;486}487 488bool X86DomainReassignment::isReassignmentProfitable(const Closure &C,489 RegDomain Domain) const {490 return calculateCost(C, Domain) < 0.0;491}492 493void X86DomainReassignment::reassign(const Closure &C, RegDomain Domain) const {494 assert(C.isLegal(Domain) && "Cannot convert illegal closure");495 496 // Iterate all instructions in the closure, convert each one using the497 // appropriate converter.498 SmallVector<MachineInstr *, 8> ToErase;499 for (auto *MI : C.instructions())500 if (Converters.find({Domain, MI->getOpcode()})501 ->second->convertInstr(MI, TII, MRI))502 ToErase.push_back(MI);503 504 // Iterate all registers in the closure, replace them with registers in the505 // destination domain.506 for (Register Reg : C.edges()) {507 MRI->setRegClass(Reg, getDstRC(MRI->getRegClass(Reg), Domain));508 for (auto &MO : MRI->use_operands(Reg)) {509 if (MO.isReg())510 // Remove all subregister references as they are not valid in the511 // destination domain.512 MO.setSubReg(0);513 }514 }515 516 for (auto *MI : ToErase)517 MI->eraseFromParent();518}519 520/// \returns true when \p Reg is used as part of an address calculation in \p521/// MI.522static bool usedAsAddr(const MachineInstr &MI, Register Reg,523 const TargetInstrInfo *TII) {524 if (!MI.mayLoadOrStore())525 return false;526 527 const MCInstrDesc &Desc = TII->get(MI.getOpcode());528 int MemOpStart = X86II::getMemoryOperandNo(Desc.TSFlags);529 if (MemOpStart == -1)530 return false;531 532 MemOpStart += X86II::getOperandBias(Desc);533 for (unsigned MemOpIdx = MemOpStart,534 MemOpEnd = MemOpStart + X86::AddrNumOperands;535 MemOpIdx < MemOpEnd; ++MemOpIdx) {536 const MachineOperand &Op = MI.getOperand(MemOpIdx);537 if (Op.isReg() && Op.getReg() == Reg)538 return true;539 }540 return false;541}542 543void X86DomainReassignment::buildClosure(Closure &C, Register Reg) {544 SmallVector<Register, 4> Worklist;545 RegDomain Domain = NoDomain;546 visitRegister(C, Reg, Domain, Worklist);547 while (!Worklist.empty()) {548 Register CurReg = Worklist.pop_back_val();549 550 // Register already in this closure.551 if (!C.insertEdge(CurReg))552 continue;553 EnclosedEdges[Reg] = C.getID();554 555 MachineInstr *DefMI = MRI->getVRegDef(CurReg);556 if (!encloseInstr(C, DefMI))557 return;558 559 // Add register used by the defining MI to the worklist.560 // Do not add registers which are used in address calculation, they will be561 // added to a different closure.562 int OpEnd = DefMI->getNumOperands();563 const MCInstrDesc &Desc = DefMI->getDesc();564 int MemOp = X86II::getMemoryOperandNo(Desc.TSFlags);565 if (MemOp != -1)566 MemOp += X86II::getOperandBias(Desc);567 for (int OpIdx = 0; OpIdx < OpEnd; ++OpIdx) {568 if (OpIdx == MemOp) {569 // skip address calculation.570 OpIdx += (X86::AddrNumOperands - 1);571 continue;572 }573 auto &Op = DefMI->getOperand(OpIdx);574 if (!Op.isReg() || !Op.isUse())575 continue;576 if (!visitRegister(C, Op.getReg(), Domain, Worklist))577 return;578 }579 580 // Expand closure through register uses.581 for (auto &UseMI : MRI->use_nodbg_instructions(CurReg)) {582 // We would like to avoid converting closures which calculare addresses,583 // as this should remain in GPRs.584 if (usedAsAddr(UseMI, CurReg, TII)) {585 C.setAllIllegal();586 return;587 }588 if (!encloseInstr(C, &UseMI))589 return;590 591 for (auto &DefOp : UseMI.defs()) {592 if (!DefOp.isReg())593 continue;594 595 Register DefReg = DefOp.getReg();596 if (!DefReg.isVirtual()) {597 C.setAllIllegal();598 return;599 }600 if (!visitRegister(C, DefReg, Domain, Worklist))601 return;602 }603 }604 }605}606 607void X86DomainReassignment::initConverters() {608 Converters[{MaskDomain, TargetOpcode::PHI}] =609 std::make_unique<InstrIgnore>(TargetOpcode::PHI);610 611 Converters[{MaskDomain, TargetOpcode::IMPLICIT_DEF}] =612 std::make_unique<InstrIgnore>(TargetOpcode::IMPLICIT_DEF);613 614 Converters[{MaskDomain, TargetOpcode::INSERT_SUBREG}] =615 std::make_unique<InstrReplaceWithCopy>(TargetOpcode::INSERT_SUBREG, 2);616 617 Converters[{MaskDomain, TargetOpcode::COPY}] =618 std::make_unique<InstrCOPYReplacer>(TargetOpcode::COPY, MaskDomain,619 TargetOpcode::COPY);620 621 auto createReplacerDstCOPY = [&](unsigned From, unsigned To) {622 Converters[{MaskDomain, From}] =623 std::make_unique<InstrReplacerDstCOPY>(From, To);624 };625 626#define GET_EGPR_IF_ENABLED(OPC) STI->hasEGPR() ? OPC##_EVEX : OPC627 createReplacerDstCOPY(X86::MOVZX32rm16, GET_EGPR_IF_ENABLED(X86::KMOVWkm));628 createReplacerDstCOPY(X86::MOVZX64rm16, GET_EGPR_IF_ENABLED(X86::KMOVWkm));629 630 createReplacerDstCOPY(X86::MOVZX32rr16, GET_EGPR_IF_ENABLED(X86::KMOVWkk));631 createReplacerDstCOPY(X86::MOVZX64rr16, GET_EGPR_IF_ENABLED(X86::KMOVWkk));632 633 if (STI->hasDQI()) {634 createReplacerDstCOPY(X86::MOVZX16rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm));635 createReplacerDstCOPY(X86::MOVZX32rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm));636 createReplacerDstCOPY(X86::MOVZX64rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm));637 638 createReplacerDstCOPY(X86::MOVZX16rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk));639 createReplacerDstCOPY(X86::MOVZX32rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk));640 createReplacerDstCOPY(X86::MOVZX64rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk));641 }642 643 auto createReplacer = [&](unsigned From, unsigned To) {644 Converters[{MaskDomain, From}] = std::make_unique<InstrReplacer>(From, To);645 };646 647 createReplacer(X86::MOV16rm, GET_EGPR_IF_ENABLED(X86::KMOVWkm));648 createReplacer(X86::MOV16mr, GET_EGPR_IF_ENABLED(X86::KMOVWmk));649 createReplacer(X86::MOV16rr, GET_EGPR_IF_ENABLED(X86::KMOVWkk));650 createReplacer(X86::SHR16ri, X86::KSHIFTRWki);651 createReplacer(X86::SHL16ri, X86::KSHIFTLWki);652 createReplacer(X86::NOT16r, X86::KNOTWkk);653 createReplacer(X86::OR16rr, X86::KORWkk);654 createReplacer(X86::AND16rr, X86::KANDWkk);655 createReplacer(X86::XOR16rr, X86::KXORWkk);656 657 bool HasNDD = STI->hasNDD();658 if (HasNDD) {659 createReplacer(X86::SHR16ri_ND, X86::KSHIFTRWki);660 createReplacer(X86::SHL16ri_ND, X86::KSHIFTLWki);661 createReplacer(X86::NOT16r_ND, X86::KNOTWkk);662 createReplacer(X86::OR16rr_ND, X86::KORWkk);663 createReplacer(X86::AND16rr_ND, X86::KANDWkk);664 createReplacer(X86::XOR16rr_ND, X86::KXORWkk);665 }666 667 if (STI->hasBWI()) {668 createReplacer(X86::MOV32rm, GET_EGPR_IF_ENABLED(X86::KMOVDkm));669 createReplacer(X86::MOV64rm, GET_EGPR_IF_ENABLED(X86::KMOVQkm));670 671 createReplacer(X86::MOV32mr, GET_EGPR_IF_ENABLED(X86::KMOVDmk));672 createReplacer(X86::MOV64mr, GET_EGPR_IF_ENABLED(X86::KMOVQmk));673 674 createReplacer(X86::MOV32rr, GET_EGPR_IF_ENABLED(X86::KMOVDkk));675 createReplacer(X86::MOV64rr, GET_EGPR_IF_ENABLED(X86::KMOVQkk));676 677 createReplacer(X86::SHR32ri, X86::KSHIFTRDki);678 createReplacer(X86::SHR64ri, X86::KSHIFTRQki);679 680 createReplacer(X86::SHL32ri, X86::KSHIFTLDki);681 createReplacer(X86::SHL64ri, X86::KSHIFTLQki);682 683 createReplacer(X86::ADD32rr, X86::KADDDkk);684 createReplacer(X86::ADD64rr, X86::KADDQkk);685 686 createReplacer(X86::NOT32r, X86::KNOTDkk);687 createReplacer(X86::NOT64r, X86::KNOTQkk);688 689 createReplacer(X86::OR32rr, X86::KORDkk);690 createReplacer(X86::OR64rr, X86::KORQkk);691 692 createReplacer(X86::AND32rr, X86::KANDDkk);693 createReplacer(X86::AND64rr, X86::KANDQkk);694 695 createReplacer(X86::ANDN32rr, X86::KANDNDkk);696 createReplacer(X86::ANDN64rr, X86::KANDNQkk);697 698 createReplacer(X86::XOR32rr, X86::KXORDkk);699 createReplacer(X86::XOR64rr, X86::KXORQkk);700 701 if (HasNDD) {702 createReplacer(X86::SHR32ri_ND, X86::KSHIFTRDki);703 createReplacer(X86::SHL32ri_ND, X86::KSHIFTLDki);704 createReplacer(X86::ADD32rr_ND, X86::KADDDkk);705 createReplacer(X86::NOT32r_ND, X86::KNOTDkk);706 createReplacer(X86::OR32rr_ND, X86::KORDkk);707 createReplacer(X86::AND32rr_ND, X86::KANDDkk);708 createReplacer(X86::XOR32rr_ND, X86::KXORDkk);709 createReplacer(X86::SHR64ri_ND, X86::KSHIFTRQki);710 createReplacer(X86::SHL64ri_ND, X86::KSHIFTLQki);711 createReplacer(X86::ADD64rr_ND, X86::KADDQkk);712 createReplacer(X86::NOT64r_ND, X86::KNOTQkk);713 createReplacer(X86::OR64rr_ND, X86::KORQkk);714 createReplacer(X86::AND64rr_ND, X86::KANDQkk);715 createReplacer(X86::XOR64rr_ND, X86::KXORQkk);716 }717 718 // TODO: KTEST is not a replacement for TEST due to flag differences. Need719 // to prove only Z flag is used.720 // createReplacer(X86::TEST32rr, X86::KTESTDkk);721 // createReplacer(X86::TEST64rr, X86::KTESTQkk);722 }723 724 if (STI->hasDQI()) {725 createReplacer(X86::ADD8rr, X86::KADDBkk);726 createReplacer(X86::ADD16rr, X86::KADDWkk);727 728 createReplacer(X86::AND8rr, X86::KANDBkk);729 730 createReplacer(X86::MOV8rm, GET_EGPR_IF_ENABLED(X86::KMOVBkm));731 createReplacer(X86::MOV8mr, GET_EGPR_IF_ENABLED(X86::KMOVBmk));732 createReplacer(X86::MOV8rr, GET_EGPR_IF_ENABLED(X86::KMOVBkk));733 734 createReplacer(X86::NOT8r, X86::KNOTBkk);735 736 createReplacer(X86::OR8rr, X86::KORBkk);737 738 createReplacer(X86::SHR8ri, X86::KSHIFTRBki);739 createReplacer(X86::SHL8ri, X86::KSHIFTLBki);740 741 // TODO: KTEST is not a replacement for TEST due to flag differences. Need742 // to prove only Z flag is used.743 // createReplacer(X86::TEST8rr, X86::KTESTBkk);744 // createReplacer(X86::TEST16rr, X86::KTESTWkk);745 746 createReplacer(X86::XOR8rr, X86::KXORBkk);747 748 if (HasNDD) {749 createReplacer(X86::ADD8rr_ND, X86::KADDBkk);750 createReplacer(X86::ADD16rr_ND, X86::KADDWkk);751 createReplacer(X86::AND8rr_ND, X86::KANDBkk);752 createReplacer(X86::NOT8r_ND, X86::KNOTBkk);753 createReplacer(X86::OR8rr_ND, X86::KORBkk);754 createReplacer(X86::SHR8ri_ND, X86::KSHIFTRBki);755 createReplacer(X86::SHL8ri_ND, X86::KSHIFTLBki);756 createReplacer(X86::XOR8rr_ND, X86::KXORBkk);757 }758 }759#undef GET_EGPR_IF_ENABLED760}761 762bool X86DomainReassignment::runOnMachineFunction(MachineFunction &MF) {763 if (skipFunction(MF.getFunction()))764 return false;765 if (DisableX86DomainReassignment)766 return false;767 768 LLVM_DEBUG(769 dbgs() << "***** Machine Function before Domain Reassignment *****\n");770 LLVM_DEBUG(MF.print(dbgs()));771 772 STI = &MF.getSubtarget<X86Subtarget>();773 // GPR->K is the only transformation currently supported, bail out early if no774 // AVX512.775 // TODO: We're also bailing of AVX512BW isn't supported since we use VK32 and776 // VK64 for GR32/GR64, but those aren't legal classes on KNL. If the register777 // coalescer doesn't clean it up and we generate a spill we will crash.778 if (!STI->hasAVX512() || !STI->hasBWI())779 return false;780 781 MRI = &MF.getRegInfo();782 assert(MRI->isSSA() && "Expected MIR to be in SSA form");783 784 TII = STI->getInstrInfo();785 initConverters();786 bool Changed = false;787 788 EnclosedEdges.clear();789 EnclosedInstrs.clear();790 791 std::vector<Closure> Closures;792 793 // Go over all virtual registers and calculate a closure.794 unsigned ClosureID = 0;795 for (unsigned Idx = 0; Idx < MRI->getNumVirtRegs(); ++Idx) {796 Register Reg = Register::index2VirtReg(Idx);797 798 // Skip unused VRegs.799 if (MRI->reg_nodbg_empty(Reg))800 continue;801 802 // GPR only current source domain supported.803 if (!MRI->getTargetRegisterInfo()->isGeneralPurposeRegisterClass(804 MRI->getRegClass(Reg)))805 continue;806 807 // Register already in closure.808 if (EnclosedEdges.contains(Reg))809 continue;810 811 // Calculate closure starting with Reg.812 Closure C(ClosureID++, {MaskDomain});813 buildClosure(C, Reg);814 815 // Collect all closures that can potentially be converted.816 if (!C.empty() && C.isLegal(MaskDomain))817 Closures.push_back(std::move(C));818 }819 820 for (Closure &C : Closures) {821 LLVM_DEBUG(C.dump(MRI));822 if (isReassignmentProfitable(C, MaskDomain)) {823 reassign(C, MaskDomain);824 ++NumClosuresConverted;825 Changed = true;826 }827 }828 829 LLVM_DEBUG(830 dbgs() << "***** Machine Function after Domain Reassignment *****\n");831 LLVM_DEBUG(MF.print(dbgs()));832 833 return Changed;834}835 836INITIALIZE_PASS(X86DomainReassignment, "x86-domain-reassignment",837 "X86 Domain Reassignment Pass", false, false)838 839/// Returns an instance of the Domain Reassignment pass.840FunctionPass *llvm::createX86DomainReassignmentPass() {841 return new X86DomainReassignment();842}843