1000 lines · cpp
1//===- X86LegalizerInfo.cpp --------------------------------------*- C++ -*-==//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8/// \file9/// This file implements the targeting of the Machinelegalizer class for X86.10/// \todo This should be generated by TableGen.11//===----------------------------------------------------------------------===//12 13#include "X86LegalizerInfo.h"14#include "X86Subtarget.h"15#include "X86TargetMachine.h"16#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"17#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"18#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"19#include "llvm/CodeGen/MachineConstantPool.h"20#include "llvm/CodeGen/MachineFrameInfo.h"21#include "llvm/CodeGen/TargetOpcodes.h"22#include "llvm/CodeGen/ValueTypes.h"23#include "llvm/IR/DerivedTypes.h"24#include "llvm/IR/IntrinsicsX86.h"25#include "llvm/IR/Type.h"26 27using namespace llvm;28using namespace TargetOpcode;29using namespace LegalizeActions;30using namespace LegalityPredicates;31 32X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,33 const X86TargetMachine &TM)34 : Subtarget(STI) {35 36 bool Is64Bit = Subtarget.is64Bit();37 bool HasCMOV = Subtarget.canUseCMOV();38 bool HasSSE1 = Subtarget.hasSSE1();39 bool HasSSE2 = Subtarget.hasSSE2();40 bool HasSSE41 = Subtarget.hasSSE41();41 bool HasAVX = Subtarget.hasAVX();42 bool HasAVX2 = Subtarget.hasAVX2();43 bool HasAVX512 = Subtarget.hasAVX512();44 bool HasVLX = Subtarget.hasVLX();45 bool HasDQI = Subtarget.hasAVX512() && Subtarget.hasDQI();46 bool HasBWI = Subtarget.hasAVX512() && Subtarget.hasBWI();47 bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87();48 bool HasPOPCNT = Subtarget.hasPOPCNT();49 bool HasLZCNT = Subtarget.hasLZCNT();50 bool HasBMI = Subtarget.hasBMI();51 52 const LLT p0 = LLT::pointer(0, TM.getPointerSizeInBits(0));53 const LLT s1 = LLT::scalar(1);54 const LLT s8 = LLT::scalar(8);55 const LLT s16 = LLT::scalar(16);56 const LLT s32 = LLT::scalar(32);57 const LLT s64 = LLT::scalar(64);58 const LLT s80 = LLT::scalar(80);59 const LLT s128 = LLT::scalar(128);60 const LLT sMaxScalar = Subtarget.is64Bit() ? s64 : s32;61 const LLT v2s32 = LLT::fixed_vector(2, 32);62 const LLT v4s8 = LLT::fixed_vector(4, 8);63 64 const LLT v16s8 = LLT::fixed_vector(16, 8);65 const LLT v8s16 = LLT::fixed_vector(8, 16);66 const LLT v4s32 = LLT::fixed_vector(4, 32);67 const LLT v2s64 = LLT::fixed_vector(2, 64);68 const LLT v2p0 = LLT::fixed_vector(2, p0);69 70 const LLT v32s8 = LLT::fixed_vector(32, 8);71 const LLT v16s16 = LLT::fixed_vector(16, 16);72 const LLT v8s32 = LLT::fixed_vector(8, 32);73 const LLT v4s64 = LLT::fixed_vector(4, 64);74 const LLT v4p0 = LLT::fixed_vector(4, p0);75 76 const LLT v64s8 = LLT::fixed_vector(64, 8);77 const LLT v32s16 = LLT::fixed_vector(32, 16);78 const LLT v16s32 = LLT::fixed_vector(16, 32);79 const LLT v8s64 = LLT::fixed_vector(8, 64);80 81 const LLT s8MaxVector = HasAVX512 ? v64s8 : HasAVX ? v32s8 : v16s8;82 const LLT s16MaxVector = HasAVX512 ? v32s16 : HasAVX ? v16s16 : v8s16;83 const LLT s32MaxVector = HasAVX512 ? v16s32 : HasAVX ? v8s32 : v4s32;84 const LLT s64MaxVector = HasAVX512 ? v8s64 : HasAVX ? v4s64 : v2s64;85 86 // todo: AVX512 bool vector predicate types87 88 // implicit/constants89 // 32/64-bits needs support for s64/s128 to handle cases:90 // s64 = EXTEND (G_IMPLICIT_DEF s32) -> s64 = G_IMPLICIT_DEF91 // s128 = EXTEND (G_IMPLICIT_DEF s32/s64) -> s128 = G_IMPLICIT_DEF92 getActionDefinitionsBuilder(93 {G_IMPLICIT_DEF, G_PHI, G_FREEZE, G_CONSTANT_FOLD_BARRIER})94 .legalFor({p0, s1, s8, s16, s32, s64})95 .legalFor(UseX87, {s80})96 .legalFor(Is64Bit, {s128})97 .legalFor(HasSSE2, {v16s8, v8s16, v4s32, v2s64})98 .legalFor(HasAVX, {v32s8, v16s16, v8s32, v4s64})99 .legalFor(HasAVX512, {v64s8, v32s16, v16s32, v8s64})100 .widenScalarOrEltToNextPow2(0, /*Min=*/8)101 .clampScalarOrElt(0, s8, sMaxScalar)102 .moreElementsToNextPow2(0)103 .clampMinNumElements(0, s8, 16)104 .clampMinNumElements(0, s16, 8)105 .clampMinNumElements(0, s32, 4)106 .clampMinNumElements(0, s64, 2)107 .clampMaxNumElements(0, s8, HasAVX512 ? 64 : (HasAVX ? 32 : 16))108 .clampMaxNumElements(0, s16, HasAVX512 ? 32 : (HasAVX ? 16 : 8))109 .clampMaxNumElements(0, s32, HasAVX512 ? 16 : (HasAVX ? 8 : 4))110 .clampMaxNumElements(0, s64, HasAVX512 ? 8 : (HasAVX ? 4 : 2))111 .clampMaxNumElements(0, p0,112 Is64Bit ? s64MaxVector.getNumElements()113 : s32MaxVector.getNumElements())114 .scalarizeIf(scalarOrEltWiderThan(0, 64), 0);115 116 getActionDefinitionsBuilder(G_CONSTANT)117 .legalFor({p0, s8, s16, s32})118 .legalFor(Is64Bit, {s64})119 .widenScalarToNextPow2(0, /*Min=*/8)120 .clampScalar(0, s8, sMaxScalar);121 122 getActionDefinitionsBuilder({G_LROUND, G_LLROUND, G_FCOS, G_FCOSH, G_FACOS,123 G_FSIN, G_FSINH, G_FASIN, G_FTAN, G_FTANH,124 G_FATAN, G_FATAN2, G_FPOW, G_FEXP, G_FEXP2,125 G_FEXP10, G_FLOG, G_FLOG2, G_FLOG10, G_FPOWI,126 G_FSINCOS, G_FCEIL, G_FFLOOR})127 .libcall();128 129 getActionDefinitionsBuilder(G_FSQRT)130 .legalFor(HasSSE1 || UseX87, {s32})131 .legalFor(HasSSE2 || UseX87, {s64})132 .legalFor(UseX87, {s80});133 134 getActionDefinitionsBuilder({G_GET_ROUNDING, G_SET_ROUNDING})135 .customFor({s32});136 137 // merge/unmerge138 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {139 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;140 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;141 getActionDefinitionsBuilder(Op)142 .widenScalarToNextPow2(LitTyIdx, /*Min=*/8)143 .widenScalarToNextPow2(BigTyIdx, /*Min=*/16)144 .minScalar(LitTyIdx, s8)145 .minScalar(BigTyIdx, s32)146 .legalIf([=](const LegalityQuery &Q) {147 switch (Q.Types[BigTyIdx].getSizeInBits()) {148 case 16:149 case 32:150 case 64:151 case 128:152 case 256:153 case 512:154 break;155 default:156 return false;157 }158 switch (Q.Types[LitTyIdx].getSizeInBits()) {159 case 8:160 case 16:161 case 32:162 case 64:163 case 128:164 case 256:165 return true;166 default:167 return false;168 }169 });170 }171 172 getActionDefinitionsBuilder({G_UMIN, G_UMAX, G_SMIN, G_SMAX})173 .widenScalarToNextPow2(0, /*Min=*/32)174 .lower();175 176 // integer addition/subtraction177 getActionDefinitionsBuilder({G_ADD, G_SUB})178 .legalFor({s8, s16, s32})179 .legalFor(Is64Bit, {s64})180 .legalFor(HasSSE2, {v16s8, v8s16, v4s32, v2s64})181 .legalFor(HasAVX2, {v32s8, v16s16, v8s32, v4s64})182 .legalFor(HasAVX512, {v16s32, v8s64})183 .legalFor(HasBWI, {v64s8, v32s16})184 .clampMinNumElements(0, s8, 16)185 .clampMinNumElements(0, s16, 8)186 .clampMinNumElements(0, s32, 4)187 .clampMinNumElements(0, s64, 2)188 .clampMaxNumElements(0, s8, HasBWI ? 64 : (HasAVX2 ? 32 : 16))189 .clampMaxNumElements(0, s16, HasBWI ? 32 : (HasAVX2 ? 16 : 8))190 .clampMaxNumElements(0, s32, HasAVX512 ? 16 : (HasAVX2 ? 8 : 4))191 .clampMaxNumElements(0, s64, HasAVX512 ? 8 : (HasAVX2 ? 4 : 2))192 .widenScalarToNextPow2(0, /*Min=*/32)193 .clampScalar(0, s8, sMaxScalar)194 .scalarize(0);195 196 getActionDefinitionsBuilder({G_UADDE, G_UADDO, G_USUBE, G_USUBO})197 .legalFor({{s8, s8}, {s16, s8}, {s32, s8}})198 .legalFor(Is64Bit, {{s64, s8}})199 .widenScalarToNextPow2(0, /*Min=*/32)200 .clampScalar(0, s8, sMaxScalar)201 .clampScalar(1, s8, s8)202 .scalarize(0);203 204 // integer multiply205 getActionDefinitionsBuilder(G_MUL)206 .legalFor({s8, s16, s32})207 .legalFor(Is64Bit, {s64})208 .legalFor(HasSSE2, {v8s16})209 .legalFor(HasSSE41, {v4s32})210 .legalFor(HasAVX2, {v16s16, v8s32})211 .legalFor(HasAVX512, {v16s32})212 .legalFor(HasDQI, {v8s64})213 .legalFor(HasDQI && HasVLX, {v2s64, v4s64})214 .legalFor(HasBWI, {v32s16})215 .clampMinNumElements(0, s16, 8)216 .clampMinNumElements(0, s32, 4)217 .clampMinNumElements(0, s64, HasVLX ? 2 : 8)218 .clampMaxNumElements(0, s16, HasBWI ? 32 : (HasAVX2 ? 16 : 8))219 .clampMaxNumElements(0, s32, HasAVX512 ? 16 : (HasAVX2 ? 8 : 4))220 .clampMaxNumElements(0, s64, 8)221 .widenScalarToNextPow2(0, /*Min=*/32)222 .clampScalar(0, s8, sMaxScalar)223 .scalarize(0);224 225 getActionDefinitionsBuilder({G_SMULH, G_UMULH})226 .legalFor({s8, s16, s32})227 .legalFor(Is64Bit, {s64})228 .widenScalarToNextPow2(0, /*Min=*/32)229 .clampScalar(0, s8, sMaxScalar)230 .scalarize(0);231 232 // integer divisions233 getActionDefinitionsBuilder({G_SDIV, G_SREM, G_UDIV, G_UREM})234 .legalFor({s8, s16, s32})235 .legalFor(Is64Bit, {s64})236 .libcallFor({s64})237 .clampScalar(0, s8, sMaxScalar);238 239 // integer shifts240 getActionDefinitionsBuilder({G_SHL, G_LSHR, G_ASHR})241 .legalFor({{s8, s8}, {s16, s8}, {s32, s8}})242 .legalFor(Is64Bit, {{s64, s8}})243 .clampScalar(0, s8, sMaxScalar)244 .clampScalar(1, s8, s8);245 246 // integer logic247 getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})248 .legalFor({s8, s16, s32})249 .legalFor(Is64Bit, {s64})250 .legalFor(HasSSE2, {v16s8, v8s16, v4s32, v2s64})251 .legalFor(HasAVX, {v32s8, v16s16, v8s32, v4s64})252 .legalFor(HasAVX512, {v64s8, v32s16, v16s32, v8s64})253 .clampMinNumElements(0, s8, 16)254 .clampMinNumElements(0, s16, 8)255 .clampMinNumElements(0, s32, 4)256 .clampMinNumElements(0, s64, 2)257 .clampMaxNumElements(0, s8, HasAVX512 ? 64 : (HasAVX ? 32 : 16))258 .clampMaxNumElements(0, s16, HasAVX512 ? 32 : (HasAVX ? 16 : 8))259 .clampMaxNumElements(0, s32, HasAVX512 ? 16 : (HasAVX ? 8 : 4))260 .clampMaxNumElements(0, s64, HasAVX512 ? 8 : (HasAVX ? 4 : 2))261 .widenScalarToNextPow2(0, /*Min=*/32)262 .clampScalar(0, s8, sMaxScalar)263 .scalarize(0);264 265 // integer comparison266 const std::initializer_list<LLT> IntTypes32 = {s8, s16, s32, p0};267 const std::initializer_list<LLT> IntTypes64 = {s8, s16, s32, s64, p0};268 269 getActionDefinitionsBuilder(G_ICMP)270 .legalForCartesianProduct({s8}, Is64Bit ? IntTypes64 : IntTypes32)271 .clampScalar(0, s8, s8)272 .clampScalar(1, s8, sMaxScalar);273 274 // bswap275 getActionDefinitionsBuilder(G_BSWAP)276 .legalFor({s32})277 .legalFor(Is64Bit, {s64})278 .widenScalarToNextPow2(0, /*Min=*/32)279 .clampScalar(0, s32, sMaxScalar);280 281 // popcount282 getActionDefinitionsBuilder(G_CTPOP)283 .legalFor(HasPOPCNT, {{s16, s16}, {s32, s32}})284 .legalFor(HasPOPCNT && Is64Bit, {{s64, s64}})285 .widenScalarToNextPow2(1, /*Min=*/16)286 .clampScalar(1, s16, sMaxScalar)287 .scalarSameSizeAs(0, 1);288 289 // count leading zeros (LZCNT)290 getActionDefinitionsBuilder(G_CTLZ)291 .legalFor(HasLZCNT, {{s16, s16}, {s32, s32}})292 .legalFor(HasLZCNT && Is64Bit, {{s64, s64}})293 .widenScalarToNextPow2(1, /*Min=*/16)294 .clampScalar(1, s16, sMaxScalar)295 .scalarSameSizeAs(0, 1);296 297 // count trailing zeros298 getActionDefinitionsBuilder(G_CTTZ_ZERO_UNDEF)299 .legalFor({{s16, s16}, {s32, s32}})300 .legalFor(Is64Bit, {{s64, s64}})301 .widenScalarToNextPow2(1, /*Min=*/16)302 .clampScalar(1, s16, sMaxScalar)303 .scalarSameSizeAs(0, 1);304 305 getActionDefinitionsBuilder(G_CTTZ)306 .legalFor(HasBMI, {{s16, s16}, {s32, s32}})307 .legalFor(HasBMI && Is64Bit, {{s64, s64}})308 .widenScalarToNextPow2(1, /*Min=*/16)309 .clampScalar(1, s16, sMaxScalar)310 .scalarSameSizeAs(0, 1);311 312 getActionDefinitionsBuilder(G_BRCOND).legalFor({s1});313 314 // pointer handling315 const std::initializer_list<LLT> PtrTypes32 = {s1, s8, s16, s32};316 const std::initializer_list<LLT> PtrTypes64 = {s1, s8, s16, s32, s64};317 318 getActionDefinitionsBuilder(G_PTRTOINT)319 .legalForCartesianProduct(Is64Bit ? PtrTypes64 : PtrTypes32, {p0})320 .maxScalar(0, sMaxScalar)321 .widenScalarToNextPow2(0, /*Min*/ 8);322 323 getActionDefinitionsBuilder(G_INTTOPTR).legalFor({{p0, sMaxScalar}});324 325 getActionDefinitionsBuilder(G_CONSTANT_POOL).legalFor({p0});326 327 getActionDefinitionsBuilder(G_PTR_ADD)328 .legalFor({{p0, s32}})329 .legalFor(Is64Bit, {{p0, s64}})330 .widenScalarToNextPow2(1, /*Min*/ 32)331 .clampScalar(1, s32, sMaxScalar);332 333 getActionDefinitionsBuilder({G_FRAME_INDEX, G_GLOBAL_VALUE}).legalFor({p0});334 335 // load/store: add more corner cases336 for (unsigned Op : {G_LOAD, G_STORE}) {337 auto &Action = getActionDefinitionsBuilder(Op);338 Action.legalForTypesWithMemDesc({{s8, p0, s8, 1},339 {s16, p0, s16, 1},340 {s32, p0, s32, 1},341 {s80, p0, s80, 1},342 {p0, p0, p0, 1},343 {v4s8, p0, v4s8, 1}});344 if (Is64Bit)345 Action.legalForTypesWithMemDesc(346 {{s64, p0, s64, 1}, {v2s32, p0, v2s32, 1}});347 348 if (HasSSE1)349 Action.legalForTypesWithMemDesc({{v4s32, p0, v4s32, 1}});350 if (HasSSE2)351 Action.legalForTypesWithMemDesc({{v16s8, p0, v16s8, 1},352 {v8s16, p0, v8s16, 1},353 {v2s64, p0, v2s64, 1},354 {v2p0, p0, v2p0, 1}});355 if (HasAVX)356 Action.legalForTypesWithMemDesc({{v32s8, p0, v32s8, 1},357 {v16s16, p0, v16s16, 1},358 {v8s32, p0, v8s32, 1},359 {v4s64, p0, v4s64, 1},360 {v4p0, p0, v4p0, 1}});361 if (HasAVX512)362 Action.legalForTypesWithMemDesc({{v64s8, p0, v64s8, 1},363 {v32s16, p0, v32s16, 1},364 {v16s32, p0, v16s32, 1},365 {v8s64, p0, v8s64, 1}});366 367 // X86 supports extending loads but not stores for GPRs368 if (Op == G_LOAD) {369 Action.legalForTypesWithMemDesc({{s8, p0, s1, 1},370 {s16, p0, s8, 1},371 {s32, p0, s8, 1},372 {s32, p0, s16, 1}});373 if (Is64Bit)374 Action.legalForTypesWithMemDesc(375 {{s64, p0, s8, 1}, {s64, p0, s16, 1}, {s64, p0, s32, 1}});376 } else {377 Action.customIf([=](const LegalityQuery &Query) {378 return Query.Types[0] != Query.MMODescrs[0].MemoryTy;379 });380 }381 Action.widenScalarToNextPow2(0, /*Min=*/8)382 .clampScalar(0, s8, sMaxScalar)383 .scalarize(0);384 }385 386 for (unsigned Op : {G_SEXTLOAD, G_ZEXTLOAD}) {387 auto &Action = getActionDefinitionsBuilder(Op);388 Action.legalForTypesWithMemDesc(389 {{s16, p0, s8, 1}, {s32, p0, s8, 1}, {s32, p0, s16, 1}});390 if (Is64Bit)391 Action.legalForTypesWithMemDesc(392 {{s64, p0, s8, 1}, {s64, p0, s16, 1}, {s64, p0, s32, 1}});393 // TODO - SSE41/AVX2/AVX512F/AVX512BW vector extensions394 }395 396 // sext, zext, and anyext397 getActionDefinitionsBuilder(G_ANYEXT)398 .legalFor({s8, s16, s32, s128})399 .legalFor(Is64Bit, {s64})400 .widenScalarToNextPow2(0, /*Min=*/8)401 .clampScalar(0, s8, sMaxScalar)402 .widenScalarToNextPow2(1, /*Min=*/8)403 .clampScalar(1, s8, sMaxScalar)404 .scalarize(0);405 406 getActionDefinitionsBuilder({G_SEXT, G_ZEXT})407 .legalFor({s8, s16, s32})408 .legalFor(Is64Bit, {s64})409 .widenScalarToNextPow2(0, /*Min=*/8)410 .clampScalar(0, s8, sMaxScalar)411 .widenScalarToNextPow2(1, /*Min=*/8)412 .clampScalar(1, s8, sMaxScalar)413 .scalarize(0);414 415 getActionDefinitionsBuilder(G_SEXT_INREG).lower();416 417 // fp constants418 getActionDefinitionsBuilder(G_FCONSTANT)419 .legalFor({s32, s64})420 .legalFor(UseX87, {s80});421 422 // fp arithmetic423 getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV})424 .legalFor({s32, s64})425 .legalFor(HasSSE1, {v4s32})426 .legalFor(HasSSE2, {v2s64})427 .legalFor(HasAVX, {v8s32, v4s64})428 .legalFor(HasAVX512, {v16s32, v8s64})429 .legalFor(UseX87, {s80});430 431 getActionDefinitionsBuilder(G_FABS)432 .legalFor(UseX87, {s80})433 .legalFor(UseX87 && !Is64Bit, {s64})434 .lower();435 436 // fp comparison437 getActionDefinitionsBuilder(G_FCMP)438 .legalFor(HasSSE1 || UseX87, {s8, s32})439 .legalFor(HasSSE2 || UseX87, {s8, s64})440 .legalFor(UseX87, {s8, s80})441 .clampScalar(0, s8, s8)442 .clampScalar(1, s32, HasSSE2 ? s64 : s32)443 .widenScalarToNextPow2(1);444 445 // fp conversions446 getActionDefinitionsBuilder(G_FPEXT)447 .legalFor(HasSSE2, {{s64, s32}})448 .legalFor(HasAVX, {{v4s64, v4s32}})449 .legalFor(HasAVX512, {{v8s64, v8s32}});450 451 getActionDefinitionsBuilder(G_FPTRUNC)452 .legalFor(HasSSE2, {{s32, s64}})453 .legalFor(HasAVX, {{v4s32, v4s64}})454 .legalFor(HasAVX512, {{v8s32, v8s64}});455 456 getActionDefinitionsBuilder(G_SITOFP)457 .legalFor(HasSSE1, {{s32, s32}})458 .legalFor(HasSSE1 && Is64Bit, {{s32, s64}})459 .legalFor(HasSSE2, {{s64, s32}})460 .legalFor(HasSSE2 && Is64Bit, {{s64, s64}})461 .clampScalar(1, (UseX87 && !HasSSE1) ? s16 : s32, sMaxScalar)462 .widenScalarToNextPow2(1)463 .customForCartesianProduct(UseX87, {s32, s64, s80}, {s16, s32, s64})464 .clampScalar(0, s32, HasSSE2 ? s64 : s32)465 .widenScalarToNextPow2(0);466 467 getActionDefinitionsBuilder(G_FPTOSI)468 .legalFor(HasSSE1, {{s32, s32}})469 .legalFor(HasSSE1 && Is64Bit, {{s64, s32}})470 .legalFor(HasSSE2, {{s32, s64}})471 .legalFor(HasSSE2 && Is64Bit, {{s64, s64}})472 .clampScalar(0, (UseX87 && !HasSSE1) ? s16 : s32, sMaxScalar)473 .widenScalarToNextPow2(0)474 .customForCartesianProduct(UseX87, {s16, s32, s64}, {s32, s64, s80})475 .clampScalar(1, s32, HasSSE2 ? s64 : s32)476 .widenScalarToNextPow2(1);477 478 // For G_UITOFP and G_FPTOUI without AVX512, we have to custom legalize types479 // <= s32 manually. Otherwise, in custom handler there is no way to480 // understand whether s32 is an original type and we need to promote it to481 // s64 or s32 is obtained after widening and we shouldn't widen it to s64.482 //483 // For AVX512 we simply widen types as there is direct mapping from opcodes484 // to asm instructions.485 getActionDefinitionsBuilder(G_UITOFP)486 .legalFor(HasAVX512, {{s32, s32}, {s32, s64}, {s64, s32}, {s64, s64}})487 .customIf([=](const LegalityQuery &Query) {488 return !HasAVX512 &&489 ((HasSSE1 && typeIs(0, s32)(Query)) ||490 (HasSSE2 && typeIs(0, s64)(Query))) &&491 scalarNarrowerThan(1, Is64Bit ? 64 : 32)(Query);492 })493 .lowerIf([=](const LegalityQuery &Query) {494 // Lower conversions from s64495 return !HasAVX512 &&496 ((HasSSE1 && typeIs(0, s32)(Query)) ||497 (HasSSE2 && typeIs(0, s64)(Query))) &&498 (Is64Bit && typeIs(1, s64)(Query));499 })500 .clampScalar(0, s32, HasSSE2 ? s64 : s32)501 .widenScalarToNextPow2(0)502 .clampScalar(1, s32, sMaxScalar)503 .widenScalarToNextPow2(1);504 505 getActionDefinitionsBuilder(G_FPTOUI)506 .legalFor(HasAVX512, {{s32, s32}, {s32, s64}, {s64, s32}, {s64, s64}})507 .customIf([=](const LegalityQuery &Query) {508 return !HasAVX512 &&509 ((HasSSE1 && typeIs(1, s32)(Query)) ||510 (HasSSE2 && typeIs(1, s64)(Query))) &&511 scalarNarrowerThan(0, Is64Bit ? 64 : 32)(Query);512 })513 // TODO: replace with customized legalization using514 // specifics of cvttsd2si. The selection of this node requires515 // a vector type. Either G_SCALAR_TO_VECTOR is needed or more advanced516 // support of G_BUILD_VECTOR/G_INSERT_VECTOR_ELT is required beforehand.517 .lowerIf([=](const LegalityQuery &Query) {518 return !HasAVX512 &&519 ((HasSSE1 && typeIs(1, s32)(Query)) ||520 (HasSSE2 && typeIs(1, s64)(Query))) &&521 (Is64Bit && typeIs(0, s64)(Query));522 })523 .clampScalar(0, s32, sMaxScalar)524 .widenScalarToNextPow2(0)525 .clampScalar(1, s32, HasSSE2 ? s64 : s32)526 .widenScalarToNextPow2(1);527 528 // vector ops529 getActionDefinitionsBuilder(G_BUILD_VECTOR)530 .customIf([=](const LegalityQuery &Query) {531 return (HasSSE1 && typeInSet(0, {v4s32})(Query)) ||532 (HasSSE2 && typeInSet(0, {v2s64, v8s16, v16s8})(Query)) ||533 (HasAVX && typeInSet(0, {v4s64, v8s32, v16s16, v32s8})(Query)) ||534 (HasAVX512 && typeInSet(0, {v8s64, v16s32, v32s16, v64s8}));535 })536 .clampNumElements(0, v16s8, s8MaxVector)537 .clampNumElements(0, v8s16, s16MaxVector)538 .clampNumElements(0, v4s32, s32MaxVector)539 .clampNumElements(0, v2s64, s64MaxVector)540 .moreElementsToNextPow2(0);541 542 getActionDefinitionsBuilder({G_EXTRACT, G_INSERT})543 .legalIf([=](const LegalityQuery &Query) {544 unsigned SubIdx = Query.Opcode == G_EXTRACT ? 0 : 1;545 unsigned FullIdx = Query.Opcode == G_EXTRACT ? 1 : 0;546 return (HasAVX && typePairInSet(SubIdx, FullIdx,547 {{v16s8, v32s8},548 {v8s16, v16s16},549 {v4s32, v8s32},550 {v2s64, v4s64}})(Query)) ||551 (HasAVX512 && typePairInSet(SubIdx, FullIdx,552 {{v16s8, v64s8},553 {v32s8, v64s8},554 {v8s16, v32s16},555 {v16s16, v32s16},556 {v4s32, v16s32},557 {v8s32, v16s32},558 {v2s64, v8s64},559 {v4s64, v8s64}})(Query));560 });561 562 // todo: only permit dst types up to max legal vector register size?563 getActionDefinitionsBuilder(G_CONCAT_VECTORS)564 .legalFor(565 HasSSE1,566 {{v32s8, v16s8}, {v16s16, v8s16}, {v8s32, v4s32}, {v4s64, v2s64}})567 .legalFor(HasAVX, {{v64s8, v16s8},568 {v64s8, v32s8},569 {v32s16, v8s16},570 {v32s16, v16s16},571 {v16s32, v4s32},572 {v16s32, v8s32},573 {v8s64, v2s64},574 {v8s64, v4s64}});575 576 // todo: vectors and address spaces577 getActionDefinitionsBuilder(G_SELECT)578 .legalFor({{s16, s32}, {s32, s32}, {p0, s32}})579 .legalFor(!HasCMOV, {{s8, s32}})580 .legalFor(Is64Bit, {{s64, s32}})581 .legalFor(UseX87, {{s80, s32}})582 .clampScalar(1, s32, s32)583 .widenScalarToNextPow2(0, /*Min=*/8)584 .clampScalar(0, HasCMOV ? s16 : s8, sMaxScalar);585 586 // memory intrinsics587 getActionDefinitionsBuilder({G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();588 589 getActionDefinitionsBuilder({G_DYN_STACKALLOC, G_STACKSAVE, G_STACKRESTORE})590 .lower();591 592 // fp intrinsics593 getActionDefinitionsBuilder({G_INTRINSIC_ROUNDEVEN, G_INTRINSIC_TRUNC})594 .scalarize(0)595 .minScalar(0, LLT::scalar(32))596 .libcall();597 598 getLegacyLegalizerInfo().computeTables();599 verify(*STI.getInstrInfo());600}601 602bool X86LegalizerInfo::legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,603 LostDebugLocObserver &LocObserver) const {604 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;605 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();606 switch (MI.getOpcode()) {607 default:608 // No idea what to do.609 return false;610 case TargetOpcode::G_BUILD_VECTOR:611 return legalizeBuildVector(MI, MRI, Helper);612 case TargetOpcode::G_FPTOUI:613 return legalizeFPTOUI(MI, MRI, Helper);614 case TargetOpcode::G_UITOFP:615 return legalizeUITOFP(MI, MRI, Helper);616 case TargetOpcode::G_STORE:617 return legalizeNarrowingStore(MI, MRI, Helper);618 case TargetOpcode::G_SITOFP:619 return legalizeSITOFP(MI, MRI, Helper);620 case TargetOpcode::G_FPTOSI:621 return legalizeFPTOSI(MI, MRI, Helper);622 case TargetOpcode::G_GET_ROUNDING:623 return legalizeGETROUNDING(MI, MRI, Helper);624 case TargetOpcode::G_SET_ROUNDING:625 return legalizeSETROUNDING(MI, MRI, Helper);626 }627 llvm_unreachable("expected switch to return");628}629 630bool X86LegalizerInfo::legalizeSITOFP(MachineInstr &MI,631 MachineRegisterInfo &MRI,632 LegalizerHelper &Helper) const {633 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;634 MachineFunction &MF = *MI.getMF();635 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();636 637 assert((SrcTy.getSizeInBits() == 16 || SrcTy.getSizeInBits() == 32 ||638 SrcTy.getSizeInBits() == 64) &&639 "Unexpected source type for SITOFP in X87 mode.");640 641 TypeSize MemSize = SrcTy.getSizeInBytes();642 MachinePointerInfo PtrInfo;643 Align Alignmt = Helper.getStackTemporaryAlignment(SrcTy);644 auto SlotPointer = Helper.createStackTemporary(MemSize, Alignmt, PtrInfo);645 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(646 PtrInfo, MachineMemOperand::MOStore, MemSize, Align(MemSize));647 648 // Store the integer value on the FPU stack.649 MIRBuilder.buildStore(Src, SlotPointer, *StoreMMO);650 651 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(652 PtrInfo, MachineMemOperand::MOLoad, MemSize, Align(MemSize));653 MIRBuilder.buildInstr(X86::G_FILD)654 .addDef(Dst)655 .addUse(SlotPointer.getReg(0))656 .addMemOperand(LoadMMO);657 658 MI.eraseFromParent();659 return true;660}661 662bool X86LegalizerInfo::legalizeFPTOSI(MachineInstr &MI,663 MachineRegisterInfo &MRI,664 LegalizerHelper &Helper) const {665 MachineFunction &MF = *MI.getMF();666 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;667 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();668 669 TypeSize MemSize = DstTy.getSizeInBytes();670 MachinePointerInfo PtrInfo;671 Align Alignmt = Helper.getStackTemporaryAlignment(DstTy);672 auto SlotPointer = Helper.createStackTemporary(MemSize, Alignmt, PtrInfo);673 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(674 PtrInfo, MachineMemOperand::MOStore, MemSize, Align(MemSize));675 676 MIRBuilder.buildInstr(X86::G_FIST)677 .addUse(Src)678 .addUse(SlotPointer.getReg(0))679 .addMemOperand(StoreMMO);680 681 MIRBuilder.buildLoad(Dst, SlotPointer, PtrInfo, Align(MemSize));682 MI.eraseFromParent();683 return true;684}685 686bool X86LegalizerInfo::legalizeBuildVector(MachineInstr &MI,687 MachineRegisterInfo &MRI,688 LegalizerHelper &Helper) const {689 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;690 const auto &BuildVector = cast<GBuildVector>(MI);691 Register Dst = BuildVector.getReg(0);692 LLT DstTy = MRI.getType(Dst);693 MachineFunction &MF = MIRBuilder.getMF();694 LLVMContext &Ctx = MF.getFunction().getContext();695 uint64_t DstTySize = DstTy.getScalarSizeInBits();696 697 SmallVector<Constant *, 4> CstIdxs;698 for (unsigned i = 0; i < BuildVector.getNumSources(); ++i) {699 Register Source = BuildVector.getSourceReg(i);700 701 auto ValueAndReg = getIConstantVRegValWithLookThrough(Source, MRI);702 if (ValueAndReg) {703 CstIdxs.emplace_back(ConstantInt::get(Ctx, ValueAndReg->Value));704 continue;705 }706 707 auto FPValueAndReg = getFConstantVRegValWithLookThrough(Source, MRI);708 if (FPValueAndReg) {709 CstIdxs.emplace_back(ConstantFP::get(Ctx, FPValueAndReg->Value));710 continue;711 }712 713 if (getOpcodeDef<GImplicitDef>(Source, MRI)) {714 CstIdxs.emplace_back(UndefValue::get(Type::getIntNTy(Ctx, DstTySize)));715 continue;716 }717 return false;718 }719 720 Constant *ConstVal = ConstantVector::get(CstIdxs);721 722 const DataLayout &DL = MIRBuilder.getDataLayout();723 unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();724 Align Alignment(DL.getABITypeAlign(ConstVal->getType()));725 auto Addr = MIRBuilder.buildConstantPool(726 LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace)),727 MF.getConstantPool()->getConstantPoolIndex(ConstVal, Alignment));728 MachineMemOperand *MMO =729 MF.getMachineMemOperand(MachinePointerInfo::getConstantPool(MF),730 MachineMemOperand::MOLoad, DstTy, Alignment);731 732 MIRBuilder.buildLoad(Dst, Addr, *MMO);733 MI.eraseFromParent();734 return true;735}736 737bool X86LegalizerInfo::legalizeFPTOUI(MachineInstr &MI,738 MachineRegisterInfo &MRI,739 LegalizerHelper &Helper) const {740 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;741 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();742 unsigned DstSizeInBits = DstTy.getScalarSizeInBits();743 const LLT s32 = LLT::scalar(32);744 const LLT s64 = LLT::scalar(64);745 746 // Simply reuse FPTOSI when it is possible to widen the type747 if (DstSizeInBits <= 32) {748 auto Casted = MIRBuilder.buildFPTOSI(DstTy == s32 ? s64 : s32, Src);749 MIRBuilder.buildTrunc(Dst, Casted);750 MI.eraseFromParent();751 return true;752 }753 754 return false;755}756 757bool X86LegalizerInfo::legalizeUITOFP(MachineInstr &MI,758 MachineRegisterInfo &MRI,759 LegalizerHelper &Helper) const {760 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;761 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();762 const LLT s32 = LLT::scalar(32);763 const LLT s64 = LLT::scalar(64);764 765 // Simply reuse SITOFP when it is possible to widen the type766 if (SrcTy.getSizeInBits() <= 32) {767 auto Ext = MIRBuilder.buildZExt(SrcTy == s32 ? s64 : s32, Src);768 MIRBuilder.buildSITOFP(Dst, Ext);769 MI.eraseFromParent();770 return true;771 }772 773 return false;774}775 776bool X86LegalizerInfo::legalizeNarrowingStore(MachineInstr &MI,777 MachineRegisterInfo &MRI,778 LegalizerHelper &Helper) const {779 auto &Store = cast<GStore>(MI);780 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;781 MachineMemOperand &MMO = **Store.memoperands_begin();782 MachineFunction &MF = MIRBuilder.getMF();783 LLT ValTy = MRI.getType(Store.getValueReg());784 auto *NewMMO = MF.getMachineMemOperand(&MMO, MMO.getPointerInfo(), ValTy);785 786 Helper.Observer.changingInstr(Store);787 Store.setMemRefs(MF, {NewMMO});788 Helper.Observer.changedInstr(Store);789 return true;790}791 792bool X86LegalizerInfo::legalizeGETROUNDING(MachineInstr &MI,793 MachineRegisterInfo &MRI,794 LegalizerHelper &Helper) const {795 /*796 The rounding mode is in bits 11:10 of FPSR, and has the following797 settings:798 00 Round to nearest799 01 Round to -inf800 10 Round to +inf801 11 Round to 0802 803 GET_ROUNDING, on the other hand, expects the following:804 -1 Undefined805 0 Round to 0806 1 Round to nearest807 2 Round to +inf808 3 Round to -inf809 810 To perform the conversion, we use a packed lookup table of the four 2-bit811 values that we can index by FPSP[11:10]812 0x2d --> (0b00,10,11,01) --> (0,2,3,1) >> FPSR[11:10]813 814 (0x2d >> ((FPSR >> 9) & 6)) & 3815 */816 817 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;818 MachineFunction &MF = MIRBuilder.getMF();819 Register Dst = MI.getOperand(0).getReg();820 LLT DstTy = MRI.getType(Dst);821 const LLT s8 = LLT::scalar(8);822 const LLT s16 = LLT::scalar(16);823 const LLT s32 = LLT::scalar(32);824 825 // Save FP Control Word to stack slot826 int MemSize = 2;827 Align Alignment = Align(2);828 MachinePointerInfo PtrInfo;829 auto StackTemp = Helper.createStackTemporary(TypeSize::getFixed(MemSize),830 Alignment, PtrInfo);831 Register StackPtr = StackTemp.getReg(0);832 833 auto StoreMMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,834 MemSize, Alignment);835 836 // Store FP Control Word to stack slot using G_FNSTCW16837 MIRBuilder.buildInstr(X86::G_FNSTCW16)838 .addUse(StackPtr)839 .addMemOperand(StoreMMO);840 841 // Load FP Control Word from stack slot842 auto LoadMMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,843 MemSize, Alignment);844 845 auto CWD32 =846 MIRBuilder.buildZExt(s32, MIRBuilder.buildLoad(s16, StackPtr, *LoadMMO));847 auto Shifted8 = MIRBuilder.buildTrunc(848 s8, MIRBuilder.buildLShr(s32, CWD32, MIRBuilder.buildConstant(s8, 9)));849 auto Masked32 = MIRBuilder.buildZExt(850 s32, MIRBuilder.buildAnd(s8, Shifted8, MIRBuilder.buildConstant(s8, 6)));851 852 // LUT is a packed lookup table (0x2d) used to map the 2-bit x87 FPU rounding853 // mode (from bits 11:10 of the control word) to the values expected by854 // GET_ROUNDING. The mapping is performed by shifting LUT right by the855 // extracted rounding mode and masking the result with 3 to obtain the final856 auto LUT = MIRBuilder.buildConstant(s32, 0x2d);857 auto LUTShifted = MIRBuilder.buildLShr(s32, LUT, Masked32);858 auto RetVal =859 MIRBuilder.buildAnd(s32, LUTShifted, MIRBuilder.buildConstant(s32, 3));860 auto RetValTrunc = MIRBuilder.buildZExtOrTrunc(DstTy, RetVal);861 862 MIRBuilder.buildCopy(Dst, RetValTrunc);863 864 MI.eraseFromParent();865 return true;866}867 868bool X86LegalizerInfo::legalizeSETROUNDING(MachineInstr &MI,869 MachineRegisterInfo &MRI,870 LegalizerHelper &Helper) const {871 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;872 MachineFunction &MF = MIRBuilder.getMF();873 Register Src = MI.getOperand(0).getReg();874 const LLT s8 = LLT::scalar(8);875 const LLT s16 = LLT::scalar(16);876 const LLT s32 = LLT::scalar(32);877 878 // Allocate stack slot for control word and MXCSR (4 bytes).879 int MemSize = 4;880 Align Alignment = Align(4);881 MachinePointerInfo PtrInfo;882 auto StackTemp = Helper.createStackTemporary(TypeSize::getFixed(MemSize),883 Alignment, PtrInfo);884 Register StackPtr = StackTemp.getReg(0);885 886 auto StoreMMO =887 MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, 2, Align(2));888 MIRBuilder.buildInstr(X86::G_FNSTCW16)889 .addUse(StackPtr)890 .addMemOperand(StoreMMO);891 892 auto LoadMMO =893 MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, 2, Align(2));894 auto CWD16 = MIRBuilder.buildLoad(s16, StackPtr, *LoadMMO);895 896 // Clear RM field (bits 11:10)897 auto ClearedCWD =898 MIRBuilder.buildAnd(s16, CWD16, MIRBuilder.buildConstant(s16, 0xf3ff));899 900 // Check if Src is a constant901 auto *SrcDef = MRI.getVRegDef(Src);902 Register RMBits;903 Register MXCSRRMBits;904 905 if (SrcDef && SrcDef->getOpcode() == TargetOpcode::G_CONSTANT) {906 uint64_t RM = getIConstantFromReg(Src, MRI).getZExtValue();907 int FieldVal = X86::getRoundingModeX86(RM);908 909 if (FieldVal == X86::rmInvalid) {910 FieldVal = X86::rmToNearest;911 LLVMContext &C = MF.getFunction().getContext();912 C.diagnose(DiagnosticInfoUnsupported(913 MF.getFunction(), "rounding mode is not supported by X86 hardware",914 DiagnosticLocation(MI.getDebugLoc()), DS_Error));915 return false;916 }917 918 FieldVal = FieldVal << 3;919 RMBits = MIRBuilder.buildConstant(s16, FieldVal).getReg(0);920 MXCSRRMBits = MIRBuilder.buildConstant(s32, FieldVal).getReg(0);921 } else {922 // Convert Src (rounding mode) to bits for control word923 // (0xc9 << (2 * Src + 4)) & 0xc00924 auto Src32 = MIRBuilder.buildZExtOrTrunc(s32, Src);925 auto ShiftAmt = MIRBuilder.buildAdd(926 s32, MIRBuilder.buildShl(s32, Src32, MIRBuilder.buildConstant(s32, 1)),927 MIRBuilder.buildConstant(s32, 4));928 auto ShiftAmt8 = MIRBuilder.buildTrunc(s8, ShiftAmt);929 auto Shifted = MIRBuilder.buildShl(s16, MIRBuilder.buildConstant(s16, 0xc9),930 ShiftAmt8);931 RMBits =932 MIRBuilder.buildAnd(s16, Shifted, MIRBuilder.buildConstant(s16, 0xc00))933 .getReg(0);934 935 // For non-constant case, we still need to compute MXCSR bits dynamically936 auto RMBits32 = MIRBuilder.buildZExt(s32, RMBits);937 MXCSRRMBits =938 MIRBuilder.buildShl(s32, RMBits32, MIRBuilder.buildConstant(s32, 3))939 .getReg(0);940 }941 // Update rounding mode bits942 auto NewCWD =943 MIRBuilder.buildOr(s16, ClearedCWD, RMBits, MachineInstr::Disjoint);944 945 // Store new FP Control Word to stack946 auto StoreNewMMO =947 MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, 2, Align(2));948 MIRBuilder.buildStore(NewCWD, StackPtr, *StoreNewMMO);949 950 // Load FP control word from the slot using G_FLDCW16951 auto LoadNewMMO =952 MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, 2, Align(2));953 MIRBuilder.buildInstr(X86::G_FLDCW16)954 .addUse(StackPtr)955 .addMemOperand(LoadNewMMO);956 957 if (Subtarget.hasSSE1()) {958 // Store MXCSR to stack (use STMXCSR)959 auto StoreMXCSRMMO = MF.getMachineMemOperand(960 PtrInfo, MachineMemOperand::MOStore, 4, Align(4));961 MIRBuilder.buildInstr(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)962 .addIntrinsicID(Intrinsic::x86_sse_stmxcsr)963 .addUse(StackPtr)964 .addMemOperand(StoreMXCSRMMO);965 966 // Load MXCSR from stack967 auto LoadMXCSRMMO = MF.getMachineMemOperand(968 PtrInfo, MachineMemOperand::MOLoad, 4, Align(4));969 auto MXCSR = MIRBuilder.buildLoad(s32, StackPtr, *LoadMXCSRMMO);970 971 // Clear RM field (bits 14:13)972 auto ClearedMXCSR = MIRBuilder.buildAnd(973 s32, MXCSR, MIRBuilder.buildConstant(s32, 0xffff9fff));974 975 // Update rounding mode bits976 auto NewMXCSR = MIRBuilder.buildOr(s32, ClearedMXCSR, MXCSRRMBits);977 978 // Store new MXCSR to stack979 auto StoreNewMXCSRMMO = MF.getMachineMemOperand(980 PtrInfo, MachineMemOperand::MOStore, 4, Align(4));981 MIRBuilder.buildStore(NewMXCSR, StackPtr, *StoreNewMXCSRMMO);982 983 // Load MXCSR from stack (use LDMXCSR)984 auto LoadNewMXCSRMMO = MF.getMachineMemOperand(985 PtrInfo, MachineMemOperand::MOLoad, 4, Align(4));986 MIRBuilder.buildInstr(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)987 .addIntrinsicID(Intrinsic::x86_sse_ldmxcsr)988 .addUse(StackPtr)989 .addMemOperand(LoadNewMXCSRMMO);990 }991 992 MI.eraseFromParent();993 return true;994}995 996bool X86LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,997 MachineInstr &MI) const {998 return true;999}1000