361 lines · cpp
1//===- X86CompressEVEX.cpp ------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This pass compresses instructions from EVEX space to legacy/VEX/EVEX space10// when possible in order to reduce code size or facilitate HW decoding.11//12// Possible compression:13// a. AVX512 instruction (EVEX) -> AVX instruction (VEX)14// b. Promoted instruction (EVEX) -> pre-promotion instruction (legacy/VEX)15// c. NDD (EVEX) -> non-NDD (legacy)16// d. NF_ND (EVEX) -> NF (EVEX)17// e. NonNF (EVEX) -> NF (EVEX)18//19// Compression a, b and c can always reduce code size, with some exceptions20// such as promoted 16-bit CRC32 which is as long as the legacy version.21//22// legacy:23// crc32w %si, %eax ## encoding: [0x66,0xf2,0x0f,0x38,0xf1,0xc6]24// promoted:25// crc32w %si, %eax ## encoding: [0x62,0xf4,0x7d,0x08,0xf1,0xc6]26//27// From performance perspective, these should be same (same uops and same EXE28// ports). From a FMV perspective, an older legacy encoding is preferred b/c it29// can execute in more places (broader HW install base). So we will still do30// the compression.31//32// Compression d can help hardware decode (HW may skip reading the NDD33// register) although the instruction length remains unchanged.34//35// Compression e can help hardware skip updating EFLAGS although the instruction36// length remains unchanged.37//===----------------------------------------------------------------------===//38 39#include "MCTargetDesc/X86BaseInfo.h"40#include "X86.h"41#include "X86InstrInfo.h"42#include "X86Subtarget.h"43#include "llvm/ADT/StringRef.h"44#include "llvm/CodeGen/MachineFunction.h"45#include "llvm/CodeGen/MachineFunctionPass.h"46#include "llvm/CodeGen/MachineInstr.h"47#include "llvm/CodeGen/MachineOperand.h"48#include "llvm/MC/MCInstrDesc.h"49#include "llvm/Pass.h"50#include <atomic>51#include <cassert>52#include <cstdint>53 54using namespace llvm;55 56#define COMP_EVEX_DESC "Compressing EVEX instrs when possible"57#define COMP_EVEX_NAME "x86-compress-evex"58 59#define DEBUG_TYPE COMP_EVEX_NAME60 61extern cl::opt<bool> X86EnableAPXForRelocation;62 63namespace {64// Including the generated EVEX compression tables.65#define GET_X86_COMPRESS_EVEX_TABLE66#include "X86GenInstrMapping.inc"67 68class CompressEVEXPass : public MachineFunctionPass {69public:70 static char ID;71 CompressEVEXPass() : MachineFunctionPass(ID) {}72 StringRef getPassName() const override { return COMP_EVEX_DESC; }73 74 bool runOnMachineFunction(MachineFunction &MF) override;75 76 // This pass runs after regalloc and doesn't support VReg operands.77 MachineFunctionProperties getRequiredProperties() const override {78 return MachineFunctionProperties().setNoVRegs();79 }80};81 82} // end anonymous namespace83 84char CompressEVEXPass::ID = 0;85 86static bool usesExtendedRegister(const MachineInstr &MI) {87 auto isHiRegIdx = [](MCRegister Reg) {88 // Check for XMM register with indexes between 16 - 31.89 if (Reg >= X86::XMM16 && Reg <= X86::XMM31)90 return true;91 // Check for YMM register with indexes between 16 - 31.92 if (Reg >= X86::YMM16 && Reg <= X86::YMM31)93 return true;94 // Check for GPR with indexes between 16 - 31.95 if (X86II::isApxExtendedReg(Reg))96 return true;97 return false;98 };99 100 // Check that operands are not ZMM regs or101 // XMM/YMM regs with hi indexes between 16 - 31.102 for (const MachineOperand &MO : MI.explicit_operands()) {103 if (!MO.isReg())104 continue;105 106 MCRegister Reg = MO.getReg().asMCReg();107 assert(!X86II::isZMMReg(Reg) &&108 "ZMM instructions should not be in the EVEX->VEX tables");109 if (isHiRegIdx(Reg))110 return true;111 }112 113 return false;114}115 116// Do any custom cleanup needed to finalize the conversion.117static bool performCustomAdjustments(MachineInstr &MI, unsigned NewOpc) {118 (void)NewOpc;119 unsigned Opc = MI.getOpcode();120 switch (Opc) {121 case X86::VALIGNDZ128rri:122 case X86::VALIGNDZ128rmi:123 case X86::VALIGNQZ128rri:124 case X86::VALIGNQZ128rmi: {125 assert((NewOpc == X86::VPALIGNRrri || NewOpc == X86::VPALIGNRrmi) &&126 "Unexpected new opcode!");127 unsigned Scale =128 (Opc == X86::VALIGNQZ128rri || Opc == X86::VALIGNQZ128rmi) ? 8 : 4;129 MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands() - 1);130 Imm.setImm(Imm.getImm() * Scale);131 break;132 }133 case X86::VSHUFF32X4Z256rmi:134 case X86::VSHUFF32X4Z256rri:135 case X86::VSHUFF64X2Z256rmi:136 case X86::VSHUFF64X2Z256rri:137 case X86::VSHUFI32X4Z256rmi:138 case X86::VSHUFI32X4Z256rri:139 case X86::VSHUFI64X2Z256rmi:140 case X86::VSHUFI64X2Z256rri: {141 assert((NewOpc == X86::VPERM2F128rri || NewOpc == X86::VPERM2I128rri ||142 NewOpc == X86::VPERM2F128rmi || NewOpc == X86::VPERM2I128rmi) &&143 "Unexpected new opcode!");144 MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands() - 1);145 int64_t ImmVal = Imm.getImm();146 // Set bit 5, move bit 1 to bit 4, copy bit 0.147 Imm.setImm(0x20 | ((ImmVal & 2) << 3) | (ImmVal & 1));148 break;149 }150 case X86::VRNDSCALEPDZ128rri:151 case X86::VRNDSCALEPDZ128rmi:152 case X86::VRNDSCALEPSZ128rri:153 case X86::VRNDSCALEPSZ128rmi:154 case X86::VRNDSCALEPDZ256rri:155 case X86::VRNDSCALEPDZ256rmi:156 case X86::VRNDSCALEPSZ256rri:157 case X86::VRNDSCALEPSZ256rmi:158 case X86::VRNDSCALESDZrri:159 case X86::VRNDSCALESDZrmi:160 case X86::VRNDSCALESSZrri:161 case X86::VRNDSCALESSZrmi:162 case X86::VRNDSCALESDZrri_Int:163 case X86::VRNDSCALESDZrmi_Int:164 case X86::VRNDSCALESSZrri_Int:165 case X86::VRNDSCALESSZrmi_Int:166 const MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands() - 1);167 int64_t ImmVal = Imm.getImm();168 // Ensure that only bits 3:0 of the immediate are used.169 if ((ImmVal & 0xf) != ImmVal)170 return false;171 break;172 }173 174 return true;175}176 177static bool CompressEVEXImpl(MachineInstr &MI, MachineBasicBlock &MBB,178 const X86Subtarget &ST) {179 uint64_t TSFlags = MI.getDesc().TSFlags;180 181 // Check for EVEX instructions only.182 if ((TSFlags & X86II::EncodingMask) != X86II::EVEX)183 return false;184 185 // Instructions with mask or 512-bit vector can't be converted to VEX.186 if (TSFlags & (X86II::EVEX_K | X86II::EVEX_L2))187 return false;188 189 auto IsRedundantNewDataDest = [&](unsigned &Opc) {190 // $rbx = ADD64rr_ND $rbx, $rax / $rbx = ADD64rr_ND $rax, $rbx191 // ->192 // $rbx = ADD64rr $rbx, $rax193 const MCInstrDesc &Desc = MI.getDesc();194 Register Reg0 = MI.getOperand(0).getReg();195 const MachineOperand &Op1 = MI.getOperand(1);196 if (!Op1.isReg() || X86::getFirstAddrOperandIdx(MI) == 1 ||197 X86::isCFCMOVCC(MI.getOpcode()))198 return false;199 Register Reg1 = Op1.getReg();200 if (Reg1 == Reg0)201 return true;202 203 // Op1 and Op2 may be commutable for ND instructions.204 if (!Desc.isCommutable() || Desc.getNumOperands() < 3 ||205 !MI.getOperand(2).isReg() || MI.getOperand(2).getReg() != Reg0)206 return false;207 // Opcode may change after commute, e.g. SHRD -> SHLD208 ST.getInstrInfo()->commuteInstruction(MI, false, 1, 2);209 Opc = MI.getOpcode();210 return true;211 };212 213 // EVEX_B has several meanings.214 // AVX512:215 // register form: rounding control or SAE216 // memory form: broadcast217 //218 // APX:219 // MAP4: NDD220 //221 // For AVX512 cases, EVEX prefix is needed in order to carry this information222 // thus preventing the transformation to VEX encoding.223 bool IsND = X86II::hasNewDataDest(TSFlags);224 if (TSFlags & X86II::EVEX_B && !IsND)225 return false;226 unsigned Opc = MI.getOpcode();227 // MOVBE*rr is special because it has semantic of NDD but not set EVEX_B.228 bool IsNDLike = IsND || Opc == X86::MOVBE32rr || Opc == X86::MOVBE64rr;229 bool IsRedundantNDD = IsNDLike ? IsRedundantNewDataDest(Opc) : false;230 231 auto GetCompressedOpc = [&](unsigned Opc) -> unsigned {232 ArrayRef<X86TableEntry> Table = ArrayRef(X86CompressEVEXTable);233 const auto I = llvm::lower_bound(Table, Opc);234 if (I == Table.end() || I->OldOpc != Opc)235 return 0;236 237 if (usesExtendedRegister(MI) || !checkPredicate(I->NewOpc, &ST) ||238 !performCustomAdjustments(MI, I->NewOpc))239 return 0;240 return I->NewOpc;241 };242 243 Register Dst = MI.getOperand(0).getReg();244 if (IsRedundantNDD) {245 // Redundant NDD ops cannot be safely compressed if either:246 // - the legacy op would introduce a partial write that BreakFalseDeps247 // identified as a potential stall, or248 // - the op is writing to a subregister of a live register, i.e. the249 // full (zeroed) result is used.250 // Both cases are indicated by an implicit def of the superregister.251 if (Dst &&252 (X86::GR16RegClass.contains(Dst) || X86::GR8RegClass.contains(Dst))) {253 Register Super = getX86SubSuperRegister(Dst, 64);254 if (MI.definesRegister(Super, /*TRI=*/nullptr))255 IsRedundantNDD = false;256 }257 258 // ADDrm/mr instructions with NDD + relocation had been transformed to the259 // instructions without NDD in X86SuppressAPXForRelocation pass. That is to260 // keep backward compatibility with linkers without APX support.261 if (!X86EnableAPXForRelocation)262 assert(!isAddMemInstrWithRelocation(MI) &&263 "Unexpected NDD instruction with relocation!");264 } else if (Opc == X86::ADD32ri_ND || Opc == X86::ADD64ri32_ND ||265 Opc == X86::ADD32rr_ND || Opc == X86::ADD64rr_ND) {266 // Non-redundant NDD ADD can be compressed to LEA when:267 // - No EGPR register used and268 // - EFLAGS is dead.269 if (!usesExtendedRegister(MI) &&270 MI.registerDefIsDead(X86::EFLAGS, /*TRI=*/nullptr)) {271 Register Src1 = MI.getOperand(1).getReg();272 const MachineOperand &Src2 = MI.getOperand(2);273 bool Is32BitReg = Opc == X86::ADD32ri_ND || Opc == X86::ADD32rr_ND;274 const MCInstrDesc &NewDesc =275 ST.getInstrInfo()->get(Is32BitReg ? X86::LEA64_32r : X86::LEA64r);276 if (Is32BitReg)277 Src1 = getX86SubSuperRegister(Src1, 64);278 MachineInstrBuilder MIB = BuildMI(MBB, MI, MI.getDebugLoc(), NewDesc, Dst)279 .addReg(Src1)280 .addImm(1);281 if (Opc == X86::ADD32ri_ND || Opc == X86::ADD64ri32_ND)282 MIB.addReg(0).add(Src2);283 else if (Is32BitReg)284 MIB.addReg(getX86SubSuperRegister(Src2.getReg(), 64)).addImm(0);285 else286 MIB.add(Src2).addImm(0);287 MIB.addReg(0);288 MI.removeFromParent();289 return true;290 }291 }292 293 // NonNF -> NF only if it's not a compressible NDD instruction and eflags is294 // dead.295 unsigned NewOpc = IsRedundantNDD296 ? X86::getNonNDVariant(Opc)297 : ((IsNDLike && ST.hasNF() &&298 MI.registerDefIsDead(X86::EFLAGS, /*TRI=*/nullptr))299 ? X86::getNFVariant(Opc)300 : GetCompressedOpc(Opc));301 302 if (!NewOpc)303 return false;304 305 const MCInstrDesc &NewDesc = ST.getInstrInfo()->get(NewOpc);306 MI.setDesc(NewDesc);307 unsigned AsmComment;308 switch (NewDesc.TSFlags & X86II::EncodingMask) {309 case X86II::LEGACY:310 AsmComment = X86::AC_EVEX_2_LEGACY;311 break;312 case X86II::VEX:313 AsmComment = X86::AC_EVEX_2_VEX;314 break;315 case X86II::EVEX:316 AsmComment = X86::AC_EVEX_2_EVEX;317 assert(IsND && (NewDesc.TSFlags & X86II::EVEX_NF) &&318 "Unknown EVEX2EVEX compression");319 break;320 default:321 llvm_unreachable("Unknown EVEX compression");322 }323 MI.setAsmPrinterFlag(AsmComment);324 if (IsRedundantNDD)325 MI.tieOperands(0, 1);326 327 return true;328}329 330bool CompressEVEXPass::runOnMachineFunction(MachineFunction &MF) {331 LLVM_DEBUG(dbgs() << "Start X86CompressEVEXPass\n";);332#ifndef NDEBUG333 // Make sure the tables are sorted.334 static std::atomic<bool> TableChecked(false);335 if (!TableChecked.load(std::memory_order_relaxed)) {336 assert(llvm::is_sorted(X86CompressEVEXTable) &&337 "X86CompressEVEXTable is not sorted!");338 TableChecked.store(true, std::memory_order_relaxed);339 }340#endif341 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();342 if (!ST.hasAVX512() && !ST.hasEGPR() && !ST.hasNDD())343 return false;344 345 bool Changed = false;346 347 for (MachineBasicBlock &MBB : MF) {348 // Traverse the basic block.349 for (MachineInstr &MI : llvm::make_early_inc_range(MBB))350 Changed |= CompressEVEXImpl(MI, MBB, ST);351 }352 LLVM_DEBUG(dbgs() << "End X86CompressEVEXPass\n";);353 return Changed;354}355 356INITIALIZE_PASS(CompressEVEXPass, COMP_EVEX_NAME, COMP_EVEX_DESC, false, false)357 358FunctionPass *llvm::createX86CompressEVEXPass() {359 return new CompressEVEXPass();360}361