806 lines · cpp
1//===- SPIRVLegalizerInfo.cpp --- SPIR-V Legalization Rules ------*- C++ -*-==//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the targeting of the Machinelegalizer class for SPIR-V.10//11//===----------------------------------------------------------------------===//12 13#include "SPIRVLegalizerInfo.h"14#include "SPIRV.h"15#include "SPIRVGlobalRegistry.h"16#include "SPIRVSubtarget.h"17#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"18#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"19#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"20#include "llvm/CodeGen/MachineInstr.h"21#include "llvm/CodeGen/MachineRegisterInfo.h"22#include "llvm/CodeGen/TargetOpcodes.h"23#include "llvm/IR/IntrinsicsSPIRV.h"24#include "llvm/Support/Debug.h"25#include "llvm/Support/MathExtras.h"26 27using namespace llvm;28using namespace llvm::LegalizeActions;29using namespace llvm::LegalityPredicates;30 31#define DEBUG_TYPE "spirv-legalizer"32 33LegalityPredicate typeOfExtendedScalars(unsigned TypeIdx, bool IsExtendedInts) {34 return [IsExtendedInts, TypeIdx](const LegalityQuery &Query) {35 const LLT Ty = Query.Types[TypeIdx];36 return IsExtendedInts && Ty.isValid() && Ty.isScalar();37 };38}39 40SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {41 using namespace TargetOpcode;42 43 this->ST = &ST;44 GR = ST.getSPIRVGlobalRegistry();45 46 const LLT s1 = LLT::scalar(1);47 const LLT s8 = LLT::scalar(8);48 const LLT s16 = LLT::scalar(16);49 const LLT s32 = LLT::scalar(32);50 const LLT s64 = LLT::scalar(64);51 52 const LLT v16s64 = LLT::fixed_vector(16, 64);53 const LLT v16s32 = LLT::fixed_vector(16, 32);54 const LLT v16s16 = LLT::fixed_vector(16, 16);55 const LLT v16s8 = LLT::fixed_vector(16, 8);56 const LLT v16s1 = LLT::fixed_vector(16, 1);57 58 const LLT v8s64 = LLT::fixed_vector(8, 64);59 const LLT v8s32 = LLT::fixed_vector(8, 32);60 const LLT v8s16 = LLT::fixed_vector(8, 16);61 const LLT v8s8 = LLT::fixed_vector(8, 8);62 const LLT v8s1 = LLT::fixed_vector(8, 1);63 64 const LLT v4s64 = LLT::fixed_vector(4, 64);65 const LLT v4s32 = LLT::fixed_vector(4, 32);66 const LLT v4s16 = LLT::fixed_vector(4, 16);67 const LLT v4s8 = LLT::fixed_vector(4, 8);68 const LLT v4s1 = LLT::fixed_vector(4, 1);69 70 const LLT v3s64 = LLT::fixed_vector(3, 64);71 const LLT v3s32 = LLT::fixed_vector(3, 32);72 const LLT v3s16 = LLT::fixed_vector(3, 16);73 const LLT v3s8 = LLT::fixed_vector(3, 8);74 const LLT v3s1 = LLT::fixed_vector(3, 1);75 76 const LLT v2s64 = LLT::fixed_vector(2, 64);77 const LLT v2s32 = LLT::fixed_vector(2, 32);78 const LLT v2s16 = LLT::fixed_vector(2, 16);79 const LLT v2s8 = LLT::fixed_vector(2, 8);80 const LLT v2s1 = LLT::fixed_vector(2, 1);81 82 const unsigned PSize = ST.getPointerSize();83 const LLT p0 = LLT::pointer(0, PSize); // Function84 const LLT p1 = LLT::pointer(1, PSize); // CrossWorkgroup85 const LLT p2 = LLT::pointer(2, PSize); // UniformConstant86 const LLT p3 = LLT::pointer(3, PSize); // Workgroup87 const LLT p4 = LLT::pointer(4, PSize); // Generic88 const LLT p5 =89 LLT::pointer(5, PSize); // Input, SPV_INTEL_usm_storage_classes (Device)90 const LLT p6 = LLT::pointer(6, PSize); // SPV_INTEL_usm_storage_classes (Host)91 const LLT p7 = LLT::pointer(7, PSize); // Input92 const LLT p8 = LLT::pointer(8, PSize); // Output93 const LLT p10 = LLT::pointer(10, PSize); // Private94 const LLT p11 = LLT::pointer(11, PSize); // StorageBuffer95 const LLT p12 = LLT::pointer(12, PSize); // Uniform96 97 // TODO: remove copy-pasting here by using concatenation in some way.98 auto allPtrsScalarsAndVectors = {99 p0, p1, p2, p3, p4, p5, p6, p7, p8,100 p10, p11, p12, s1, s8, s16, s32, s64, v2s1,101 v2s8, v2s16, v2s32, v2s64, v3s1, v3s8, v3s16, v3s32, v3s64,102 v4s1, v4s8, v4s16, v4s32, v4s64, v8s1, v8s8, v8s16, v8s32,103 v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};104 105 auto allVectors = {v2s1, v2s8, v2s16, v2s32, v2s64, v3s1, v3s8,106 v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32,107 v4s64, v8s1, v8s8, v8s16, v8s32, v8s64, v16s1,108 v16s8, v16s16, v16s32, v16s64};109 110 auto allShaderVectors = {v2s1, v2s8, v2s16, v2s32, v2s64,111 v3s1, v3s8, v3s16, v3s32, v3s64,112 v4s1, v4s8, v4s16, v4s32, v4s64};113 114 auto allScalarsAndVectors = {115 s1, s8, s16, s32, s64, v2s1, v2s8, v2s16, v2s32, v2s64,116 v3s1, v3s8, v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32, v4s64,117 v8s1, v8s8, v8s16, v8s32, v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};118 119 auto allIntScalarsAndVectors = {s8, s16, s32, s64, v2s8, v2s16,120 v2s32, v2s64, v3s8, v3s16, v3s32, v3s64,121 v4s8, v4s16, v4s32, v4s64, v8s8, v8s16,122 v8s32, v8s64, v16s8, v16s16, v16s32, v16s64};123 124 auto allBoolScalarsAndVectors = {s1, v2s1, v3s1, v4s1, v8s1, v16s1};125 126 auto allIntScalars = {s8, s16, s32, s64};127 128 auto allFloatScalars = {s16, s32, s64};129 130 auto allFloatScalarsAndVectors = {131 s16, s32, s64, v2s16, v2s32, v2s64, v3s16, v3s32, v3s64,132 v4s16, v4s32, v4s64, v8s16, v8s32, v8s64, v16s16, v16s32, v16s64};133 134 auto allFloatAndIntScalarsAndPtrs = {s8, s16, s32, s64, p0, p1, p2, p3,135 p4, p5, p6, p7, p8, p10, p11, p12};136 137 auto allPtrs = {p0, p1, p2, p3, p4, p5, p6, p7, p8, p10, p11, p12};138 139 auto &allowedVectorTypes = ST.isShader() ? allShaderVectors : allVectors;140 141 bool IsExtendedInts =142 ST.canUseExtension(143 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers) ||144 ST.canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions) ||145 ST.canUseExtension(SPIRV::Extension::SPV_INTEL_int4);146 auto extendedScalarsAndVectors =147 [IsExtendedInts](const LegalityQuery &Query) {148 const LLT Ty = Query.Types[0];149 return IsExtendedInts && Ty.isValid() && !Ty.isPointerOrPointerVector();150 };151 auto extendedScalarsAndVectorsProduct = [IsExtendedInts](152 const LegalityQuery &Query) {153 const LLT Ty1 = Query.Types[0], Ty2 = Query.Types[1];154 return IsExtendedInts && Ty1.isValid() && Ty2.isValid() &&155 !Ty1.isPointerOrPointerVector() && !Ty2.isPointerOrPointerVector();156 };157 auto extendedPtrsScalarsAndVectors =158 [IsExtendedInts](const LegalityQuery &Query) {159 const LLT Ty = Query.Types[0];160 return IsExtendedInts && Ty.isValid();161 };162 163 // The universal validation rules in the SPIR-V specification state that164 // vector sizes are typically limited to 2, 3, or 4. However, larger vector165 // sizes (8 and 16) are enabled when the Kernel capability is present. For166 // shader execution models, vector sizes are strictly limited to 4. In167 // non-shader contexts, vector sizes of 8 and 16 are also permitted, but168 // arbitrary sizes (e.g., 6 or 11) are not.169 uint32_t MaxVectorSize = ST.isShader() ? 4 : 16;170 171 for (auto Opc : getTypeFoldingSupportedOpcodes()) {172 if (Opc != G_EXTRACT_VECTOR_ELT)173 getActionDefinitionsBuilder(Opc).custom();174 }175 176 getActionDefinitionsBuilder(G_INTRINSIC_W_SIDE_EFFECTS).custom();177 178 getActionDefinitionsBuilder(G_SHUFFLE_VECTOR)179 .legalForCartesianProduct(allowedVectorTypes, allowedVectorTypes)180 .moreElementsToNextPow2(0)181 .lowerIf(vectorElementCountIsGreaterThan(0, MaxVectorSize))182 .moreElementsToNextPow2(1)183 .lowerIf(vectorElementCountIsGreaterThan(1, MaxVectorSize))184 .alwaysLegal();185 186 getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)187 .moreElementsToNextPow2(1)188 .fewerElementsIf(vectorElementCountIsGreaterThan(1, MaxVectorSize),189 LegalizeMutations::changeElementCountTo(190 1, ElementCount::getFixed(MaxVectorSize)))191 .custom();192 193 // Illegal G_UNMERGE_VALUES instructions should be handled194 // during the combine phase.195 getActionDefinitionsBuilder(G_BUILD_VECTOR)196 .legalIf(vectorElementCountIsLessThanOrEqualTo(0, MaxVectorSize))197 .fewerElementsIf(vectorElementCountIsGreaterThan(0, MaxVectorSize),198 LegalizeMutations::changeElementCountTo(199 0, ElementCount::getFixed(MaxVectorSize)));200 201 // When entering the legalizer, there should be no G_BITCAST instructions.202 // They should all be calls to the `spv_bitcast` intrinsic. The call to203 // the intrinsic will be converted to a G_BITCAST during legalization if204 // the vectors are not legal. After using the rules to legalize a G_BITCAST,205 // we turn it back into a call to the intrinsic with a custom rule to avoid206 // potential machine verifier failures.207 getActionDefinitionsBuilder(G_BITCAST)208 .moreElementsToNextPow2(0)209 .moreElementsToNextPow2(1)210 .fewerElementsIf(vectorElementCountIsGreaterThan(0, MaxVectorSize),211 LegalizeMutations::changeElementCountTo(212 0, ElementCount::getFixed(MaxVectorSize)))213 .lowerIf(vectorElementCountIsGreaterThan(1, MaxVectorSize))214 .custom();215 216 getActionDefinitionsBuilder(G_CONCAT_VECTORS)217 .legalIf(vectorElementCountIsLessThanOrEqualTo(0, MaxVectorSize))218 .moreElementsToNextPow2(0)219 .lowerIf(vectorElementCountIsGreaterThan(0, MaxVectorSize))220 .alwaysLegal();221 222 getActionDefinitionsBuilder(G_SPLAT_VECTOR)223 .legalIf(vectorElementCountIsLessThanOrEqualTo(0, MaxVectorSize))224 .moreElementsToNextPow2(0)225 .fewerElementsIf(vectorElementCountIsGreaterThan(0, MaxVectorSize),226 LegalizeMutations::changeElementSizeTo(0, MaxVectorSize))227 .alwaysLegal();228 229 // Vector Reduction Operations230 getActionDefinitionsBuilder(231 {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX,232 G_VECREDUCE_ADD, G_VECREDUCE_MUL, G_VECREDUCE_FMUL, G_VECREDUCE_FMIN,233 G_VECREDUCE_FMAX, G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM,234 G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})235 .legalFor(allowedVectorTypes)236 .scalarize(1)237 .lower();238 239 getActionDefinitionsBuilder({G_VECREDUCE_SEQ_FADD, G_VECREDUCE_SEQ_FMUL})240 .scalarize(2)241 .lower();242 243 // Illegal G_UNMERGE_VALUES instructions should be handled244 // during the combine phase.245 getActionDefinitionsBuilder(G_UNMERGE_VALUES)246 .legalIf(vectorElementCountIsLessThanOrEqualTo(1, MaxVectorSize));247 248 getActionDefinitionsBuilder({G_MEMCPY, G_MEMMOVE})249 .legalIf(all(typeInSet(0, allPtrs), typeInSet(1, allPtrs)));250 251 getActionDefinitionsBuilder(G_MEMSET).legalIf(252 all(typeInSet(0, allPtrs), typeInSet(1, allIntScalars)));253 254 getActionDefinitionsBuilder(G_ADDRSPACE_CAST)255 .legalForCartesianProduct(allPtrs, allPtrs);256 257 getActionDefinitionsBuilder({G_LOAD, G_STORE}).legalIf(typeInSet(1, allPtrs));258 259 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS,260 G_BITREVERSE, G_SADDSAT, G_UADDSAT, G_SSUBSAT,261 G_USUBSAT, G_SCMP, G_UCMP})262 .legalFor(allIntScalarsAndVectors)263 .legalIf(extendedScalarsAndVectors);264 265 getActionDefinitionsBuilder({G_FMA, G_STRICT_FMA})266 .legalFor(allFloatScalarsAndVectors);267 268 getActionDefinitionsBuilder(G_STRICT_FLDEXP)269 .legalForCartesianProduct(allFloatScalarsAndVectors, allIntScalars);270 271 getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})272 .legalForCartesianProduct(allIntScalarsAndVectors,273 allFloatScalarsAndVectors);274 275 getActionDefinitionsBuilder({G_FPTOSI_SAT, G_FPTOUI_SAT})276 .legalForCartesianProduct(allIntScalarsAndVectors,277 allFloatScalarsAndVectors);278 279 getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})280 .legalForCartesianProduct(allFloatScalarsAndVectors,281 allScalarsAndVectors);282 283 getActionDefinitionsBuilder(G_CTPOP)284 .legalForCartesianProduct(allIntScalarsAndVectors)285 .legalIf(extendedScalarsAndVectorsProduct);286 287 // Extensions.288 getActionDefinitionsBuilder({G_TRUNC, G_ZEXT, G_SEXT, G_ANYEXT})289 .legalForCartesianProduct(allScalarsAndVectors)290 .legalIf(extendedScalarsAndVectorsProduct);291 292 getActionDefinitionsBuilder(G_PHI)293 .legalFor(allPtrsScalarsAndVectors)294 .legalIf(extendedPtrsScalarsAndVectors);295 296 getActionDefinitionsBuilder(G_BITCAST).legalIf(297 all(typeInSet(0, allPtrsScalarsAndVectors),298 typeInSet(1, allPtrsScalarsAndVectors)));299 300 getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE})301 .legalFor({s1})302 .legalFor(allFloatAndIntScalarsAndPtrs)303 .legalFor(allowedVectorTypes)304 .moreElementsToNextPow2(0)305 .fewerElementsIf(vectorElementCountIsGreaterThan(0, MaxVectorSize),306 LegalizeMutations::changeElementCountTo(307 0, ElementCount::getFixed(MaxVectorSize)));308 309 getActionDefinitionsBuilder({G_STACKSAVE, G_STACKRESTORE}).alwaysLegal();310 311 getActionDefinitionsBuilder(G_INTTOPTR)312 .legalForCartesianProduct(allPtrs, allIntScalars)313 .legalIf(314 all(typeInSet(0, allPtrs), typeOfExtendedScalars(1, IsExtendedInts)));315 getActionDefinitionsBuilder(G_PTRTOINT)316 .legalForCartesianProduct(allIntScalars, allPtrs)317 .legalIf(318 all(typeOfExtendedScalars(0, IsExtendedInts), typeInSet(1, allPtrs)));319 getActionDefinitionsBuilder(G_PTR_ADD)320 .legalForCartesianProduct(allPtrs, allIntScalars)321 .legalIf(322 all(typeInSet(0, allPtrs), typeOfExtendedScalars(1, IsExtendedInts)));323 324 // ST.canDirectlyComparePointers() for pointer args is supported in325 // legalizeCustom().326 getActionDefinitionsBuilder(G_ICMP).customIf(327 all(typeInSet(0, allBoolScalarsAndVectors),328 typeInSet(1, allPtrsScalarsAndVectors)));329 330 getActionDefinitionsBuilder(G_FCMP).legalIf(331 all(typeInSet(0, allBoolScalarsAndVectors),332 typeInSet(1, allFloatScalarsAndVectors)));333 334 getActionDefinitionsBuilder({G_ATOMICRMW_OR, G_ATOMICRMW_ADD, G_ATOMICRMW_AND,335 G_ATOMICRMW_MAX, G_ATOMICRMW_MIN,336 G_ATOMICRMW_SUB, G_ATOMICRMW_XOR,337 G_ATOMICRMW_UMAX, G_ATOMICRMW_UMIN})338 .legalForCartesianProduct(allIntScalars, allPtrs);339 340 getActionDefinitionsBuilder(341 {G_ATOMICRMW_FADD, G_ATOMICRMW_FSUB, G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})342 .legalForCartesianProduct(allFloatScalars, allPtrs);343 344 getActionDefinitionsBuilder(G_ATOMICRMW_XCHG)345 .legalForCartesianProduct(allFloatAndIntScalarsAndPtrs, allPtrs);346 347 getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG_WITH_SUCCESS).lower();348 // TODO: add proper legalization rules.349 getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG).alwaysLegal();350 351 getActionDefinitionsBuilder(352 {G_UADDO, G_SADDO, G_USUBO, G_SSUBO, G_UMULO, G_SMULO})353 .alwaysLegal();354 355 getActionDefinitionsBuilder({G_LROUND, G_LLROUND})356 .legalForCartesianProduct(allFloatScalarsAndVectors,357 allIntScalarsAndVectors);358 359 // FP conversions.360 getActionDefinitionsBuilder({G_FPTRUNC, G_FPEXT})361 .legalForCartesianProduct(allFloatScalarsAndVectors);362 363 // Pointer-handling.364 getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0});365 366 getActionDefinitionsBuilder(G_GLOBAL_VALUE).legalFor(allPtrs);367 368 // Control-flow. In some cases (e.g. constants) s1 may be promoted to s32.369 getActionDefinitionsBuilder(G_BRCOND).legalFor({s1, s32});370 371 getActionDefinitionsBuilder(G_FFREXP).legalForCartesianProduct(372 allFloatScalarsAndVectors, {s32, v2s32, v3s32, v4s32, v8s32, v16s32});373 374 // TODO: Review the target OpenCL and GLSL Extended Instruction Set specs to375 // tighten these requirements. Many of these math functions are only legal on376 // specific bitwidths, so they are not selectable for377 // allFloatScalarsAndVectors.378 getActionDefinitionsBuilder({G_STRICT_FSQRT,379 G_FPOW,380 G_FEXP,381 G_FMODF,382 G_FEXP2,383 G_FLOG,384 G_FLOG2,385 G_FLOG10,386 G_FABS,387 G_FMINNUM,388 G_FMAXNUM,389 G_FCEIL,390 G_FCOS,391 G_FSIN,392 G_FTAN,393 G_FACOS,394 G_FASIN,395 G_FATAN,396 G_FATAN2,397 G_FCOSH,398 G_FSINH,399 G_FTANH,400 G_FSQRT,401 G_FFLOOR,402 G_FRINT,403 G_FNEARBYINT,404 G_INTRINSIC_ROUND,405 G_INTRINSIC_TRUNC,406 G_FMINIMUM,407 G_FMAXIMUM,408 G_INTRINSIC_ROUNDEVEN})409 .legalFor(allFloatScalarsAndVectors);410 411 getActionDefinitionsBuilder(G_FCOPYSIGN)412 .legalForCartesianProduct(allFloatScalarsAndVectors,413 allFloatScalarsAndVectors);414 415 getActionDefinitionsBuilder(G_FPOWI).legalForCartesianProduct(416 allFloatScalarsAndVectors, allIntScalarsAndVectors);417 418 if (ST.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {419 getActionDefinitionsBuilder(420 {G_CTTZ, G_CTTZ_ZERO_UNDEF, G_CTLZ, G_CTLZ_ZERO_UNDEF})421 .legalForCartesianProduct(allIntScalarsAndVectors,422 allIntScalarsAndVectors);423 424 // Struct return types become a single scalar, so cannot easily legalize.425 getActionDefinitionsBuilder({G_SMULH, G_UMULH}).alwaysLegal();426 }427 428 getActionDefinitionsBuilder(G_IS_FPCLASS).custom();429 430 getLegacyLegalizerInfo().computeTables();431 verify(*ST.getInstrInfo());432}433 434static bool legalizeExtractVectorElt(LegalizerHelper &Helper, MachineInstr &MI,435 SPIRVGlobalRegistry *GR) {436 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;437 Register DstReg = MI.getOperand(0).getReg();438 Register SrcReg = MI.getOperand(1).getReg();439 Register IdxReg = MI.getOperand(2).getReg();440 441 MIRBuilder442 .buildIntrinsic(Intrinsic::spv_extractelt, ArrayRef<Register>{DstReg})443 .addUse(SrcReg)444 .addUse(IdxReg);445 MI.eraseFromParent();446 return true;447}448 449static Register convertPtrToInt(Register Reg, LLT ConvTy, SPIRVType *SpvType,450 LegalizerHelper &Helper,451 MachineRegisterInfo &MRI,452 SPIRVGlobalRegistry *GR) {453 Register ConvReg = MRI.createGenericVirtualRegister(ConvTy);454 MRI.setRegClass(ConvReg, GR->getRegClass(SpvType));455 GR->assignSPIRVTypeToVReg(SpvType, ConvReg, Helper.MIRBuilder.getMF());456 Helper.MIRBuilder.buildInstr(TargetOpcode::G_PTRTOINT)457 .addDef(ConvReg)458 .addUse(Reg);459 return ConvReg;460}461 462bool SPIRVLegalizerInfo::legalizeCustom(463 LegalizerHelper &Helper, MachineInstr &MI,464 LostDebugLocObserver &LocObserver) const {465 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();466 switch (MI.getOpcode()) {467 default:468 // TODO: implement legalization for other opcodes.469 return true;470 case TargetOpcode::G_BITCAST:471 return legalizeBitcast(Helper, MI);472 case TargetOpcode::G_EXTRACT_VECTOR_ELT:473 return legalizeExtractVectorElt(Helper, MI, GR);474 case TargetOpcode::G_INTRINSIC:475 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:476 return legalizeIntrinsic(Helper, MI);477 case TargetOpcode::G_IS_FPCLASS:478 return legalizeIsFPClass(Helper, MI, LocObserver);479 case TargetOpcode::G_ICMP: {480 assert(GR->getSPIRVTypeForVReg(MI.getOperand(0).getReg()));481 auto &Op0 = MI.getOperand(2);482 auto &Op1 = MI.getOperand(3);483 Register Reg0 = Op0.getReg();484 Register Reg1 = Op1.getReg();485 CmpInst::Predicate Cond =486 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());487 if ((!ST->canDirectlyComparePointers() ||488 (Cond != CmpInst::ICMP_EQ && Cond != CmpInst::ICMP_NE)) &&489 MRI.getType(Reg0).isPointer() && MRI.getType(Reg1).isPointer()) {490 LLT ConvT = LLT::scalar(ST->getPointerSize());491 Type *LLVMTy = IntegerType::get(MI.getMF()->getFunction().getContext(),492 ST->getPointerSize());493 SPIRVType *SpirvTy = GR->getOrCreateSPIRVType(494 LLVMTy, Helper.MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true);495 Op0.setReg(convertPtrToInt(Reg0, ConvT, SpirvTy, Helper, MRI, GR));496 Op1.setReg(convertPtrToInt(Reg1, ConvT, SpirvTy, Helper, MRI, GR));497 }498 return true;499 }500 }501}502 503bool SPIRVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,504 MachineInstr &MI) const {505 LLVM_DEBUG(dbgs() << "legalizeIntrinsic: " << MI);506 507 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;508 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();509 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();510 511 auto IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();512 if (IntrinsicID == Intrinsic::spv_bitcast) {513 LLVM_DEBUG(dbgs() << "Found a bitcast instruction\n");514 Register DstReg = MI.getOperand(0).getReg();515 Register SrcReg = MI.getOperand(2).getReg();516 LLT DstTy = MRI.getType(DstReg);517 LLT SrcTy = MRI.getType(SrcReg);518 519 int32_t MaxVectorSize = ST.isShader() ? 4 : 16;520 521 bool DstNeedsLegalization = false;522 bool SrcNeedsLegalization = false;523 524 if (DstTy.isVector()) {525 if (DstTy.getNumElements() > 4 &&526 !isPowerOf2_32(DstTy.getNumElements())) {527 DstNeedsLegalization = true;528 }529 530 if (DstTy.getNumElements() > MaxVectorSize) {531 DstNeedsLegalization = true;532 }533 }534 535 if (SrcTy.isVector()) {536 if (SrcTy.getNumElements() > 4 &&537 !isPowerOf2_32(SrcTy.getNumElements())) {538 SrcNeedsLegalization = true;539 }540 541 if (SrcTy.getNumElements() > MaxVectorSize) {542 SrcNeedsLegalization = true;543 }544 }545 546 // If an spv_bitcast needs to be legalized, we convert it to G_BITCAST to547 // allow using the generic legalization rules.548 if (DstNeedsLegalization || SrcNeedsLegalization) {549 LLVM_DEBUG(dbgs() << "Replacing with a G_BITCAST\n");550 MIRBuilder.buildBitcast(DstReg, SrcReg);551 MI.eraseFromParent();552 }553 return true;554 }555 return true;556}557 558bool SPIRVLegalizerInfo::legalizeBitcast(LegalizerHelper &Helper,559 MachineInstr &MI) const {560 // Once the G_BITCAST is using vectors that are allowed, we turn it back into561 // an spv_bitcast to avoid verifier problems when the register types are the562 // same for the source and the result. Note that the SPIR-V types associated563 // with the bitcast can be different even if the register types are the same.564 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;565 Register DstReg = MI.getOperand(0).getReg();566 Register SrcReg = MI.getOperand(1).getReg();567 SmallVector<Register, 1> DstRegs = {DstReg};568 MIRBuilder.buildIntrinsic(Intrinsic::spv_bitcast, DstRegs).addUse(SrcReg);569 MI.eraseFromParent();570 return true;571}572 573// Note this code was copied from LegalizerHelper::lowerISFPCLASS and adjusted574// to ensure that all instructions created during the lowering have SPIR-V types575// assigned to them.576bool SPIRVLegalizerInfo::legalizeIsFPClass(577 LegalizerHelper &Helper, MachineInstr &MI,578 LostDebugLocObserver &LocObserver) const {579 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();580 FPClassTest Mask = static_cast<FPClassTest>(MI.getOperand(2).getImm());581 582 auto &MIRBuilder = Helper.MIRBuilder;583 auto &MF = MIRBuilder.getMF();584 MachineRegisterInfo &MRI = MF.getRegInfo();585 586 Type *LLVMDstTy =587 IntegerType::get(MIRBuilder.getContext(), DstTy.getScalarSizeInBits());588 if (DstTy.isVector())589 LLVMDstTy = VectorType::get(LLVMDstTy, DstTy.getElementCount());590 SPIRVType *SPIRVDstTy = GR->getOrCreateSPIRVType(591 LLVMDstTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite,592 /*EmitIR*/ true);593 594 unsigned BitSize = SrcTy.getScalarSizeInBits();595 const fltSemantics &Semantics = getFltSemanticForLLT(SrcTy.getScalarType());596 597 LLT IntTy = LLT::scalar(BitSize);598 Type *LLVMIntTy = IntegerType::get(MIRBuilder.getContext(), BitSize);599 if (SrcTy.isVector()) {600 IntTy = LLT::vector(SrcTy.getElementCount(), IntTy);601 LLVMIntTy = VectorType::get(LLVMIntTy, SrcTy.getElementCount());602 }603 SPIRVType *SPIRVIntTy = GR->getOrCreateSPIRVType(604 LLVMIntTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite,605 /*EmitIR*/ true);606 607 // Clang doesn't support capture of structured bindings:608 LLT DstTyCopy = DstTy;609 const auto assignSPIRVTy = [&](MachineInstrBuilder &&MI) {610 // Assign this MI's (assumed only) destination to one of the two types we611 // expect: either the G_IS_FPCLASS's destination type, or the integer type612 // bitcast from the source type.613 LLT MITy = MRI.getType(MI.getReg(0));614 assert((MITy == IntTy || MITy == DstTyCopy) &&615 "Unexpected LLT type while lowering G_IS_FPCLASS");616 auto *SPVTy = MITy == IntTy ? SPIRVIntTy : SPIRVDstTy;617 GR->assignSPIRVTypeToVReg(SPVTy, MI.getReg(0), MF);618 return MI;619 };620 621 // Helper to build and assign a constant in one go622 const auto buildSPIRVConstant = [&](LLT Ty, auto &&C) -> MachineInstrBuilder {623 if (!Ty.isFixedVector())624 return assignSPIRVTy(MIRBuilder.buildConstant(Ty, C));625 auto ScalarC = MIRBuilder.buildConstant(Ty.getScalarType(), C);626 assert((Ty == IntTy || Ty == DstTyCopy) &&627 "Unexpected LLT type while lowering constant for G_IS_FPCLASS");628 SPIRVType *VecEltTy = GR->getOrCreateSPIRVType(629 (Ty == IntTy ? LLVMIntTy : LLVMDstTy)->getScalarType(), MIRBuilder,630 SPIRV::AccessQualifier::ReadWrite,631 /*EmitIR*/ true);632 GR->assignSPIRVTypeToVReg(VecEltTy, ScalarC.getReg(0), MF);633 return assignSPIRVTy(MIRBuilder.buildSplatBuildVector(Ty, ScalarC));634 };635 636 if (Mask == fcNone) {637 MIRBuilder.buildCopy(DstReg, buildSPIRVConstant(DstTy, 0));638 MI.eraseFromParent();639 return true;640 }641 if (Mask == fcAllFlags) {642 MIRBuilder.buildCopy(DstReg, buildSPIRVConstant(DstTy, 1));643 MI.eraseFromParent();644 return true;645 }646 647 // Note that rather than creating a COPY here (between a floating-point and648 // integer type of the same size) we create a SPIR-V bitcast immediately. We649 // can't create a G_BITCAST because the LLTs are the same, and we can't seem650 // to correctly lower COPYs to SPIR-V bitcasts at this moment.651 Register ResVReg = MRI.createGenericVirtualRegister(IntTy);652 MRI.setRegClass(ResVReg, GR->getRegClass(SPIRVIntTy));653 GR->assignSPIRVTypeToVReg(SPIRVIntTy, ResVReg, Helper.MIRBuilder.getMF());654 auto AsInt = MIRBuilder.buildInstr(SPIRV::OpBitcast)655 .addDef(ResVReg)656 .addUse(GR->getSPIRVTypeID(SPIRVIntTy))657 .addUse(SrcReg);658 AsInt = assignSPIRVTy(std::move(AsInt));659 660 // Various masks.661 APInt SignBit = APInt::getSignMask(BitSize);662 APInt ValueMask = APInt::getSignedMaxValue(BitSize); // All bits but sign.663 APInt Inf = APFloat::getInf(Semantics).bitcastToAPInt(); // Exp and int bit.664 APInt ExpMask = Inf;665 APInt AllOneMantissa = APFloat::getLargest(Semantics).bitcastToAPInt() & ~Inf;666 APInt QNaNBitMask =667 APInt::getOneBitSet(BitSize, AllOneMantissa.getActiveBits() - 1);668 APInt InversionMask = APInt::getAllOnes(DstTy.getScalarSizeInBits());669 670 auto SignBitC = buildSPIRVConstant(IntTy, SignBit);671 auto ValueMaskC = buildSPIRVConstant(IntTy, ValueMask);672 auto InfC = buildSPIRVConstant(IntTy, Inf);673 auto ExpMaskC = buildSPIRVConstant(IntTy, ExpMask);674 auto ZeroC = buildSPIRVConstant(IntTy, 0);675 676 auto Abs = assignSPIRVTy(MIRBuilder.buildAnd(IntTy, AsInt, ValueMaskC));677 auto Sign = assignSPIRVTy(678 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_NE, DstTy, AsInt, Abs));679 680 auto Res = buildSPIRVConstant(DstTy, 0);681 682 const auto appendToRes = [&](MachineInstrBuilder &&ToAppend) {683 Res = assignSPIRVTy(684 MIRBuilder.buildOr(DstTyCopy, Res, assignSPIRVTy(std::move(ToAppend))));685 };686 687 // Tests that involve more than one class should be processed first.688 if ((Mask & fcFinite) == fcFinite) {689 // finite(V) ==> abs(V) u< exp_mask690 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, Abs,691 ExpMaskC));692 Mask &= ~fcFinite;693 } else if ((Mask & fcFinite) == fcPosFinite) {694 // finite(V) && V > 0 ==> V u< exp_mask695 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, AsInt,696 ExpMaskC));697 Mask &= ~fcPosFinite;698 } else if ((Mask & fcFinite) == fcNegFinite) {699 // finite(V) && V < 0 ==> abs(V) u< exp_mask && signbit == 1700 auto Cmp = assignSPIRVTy(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT,701 DstTy, Abs, ExpMaskC));702 appendToRes(MIRBuilder.buildAnd(DstTy, Cmp, Sign));703 Mask &= ~fcNegFinite;704 }705 706 if (FPClassTest PartialCheck = Mask & (fcZero | fcSubnormal)) {707 // fcZero | fcSubnormal => test all exponent bits are 0708 // TODO: Handle sign bit specific cases709 // TODO: Handle inverted case710 if (PartialCheck == (fcZero | fcSubnormal)) {711 auto ExpBits = assignSPIRVTy(MIRBuilder.buildAnd(IntTy, AsInt, ExpMaskC));712 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,713 ExpBits, ZeroC));714 Mask &= ~PartialCheck;715 }716 }717 718 // Check for individual classes.719 if (FPClassTest PartialCheck = Mask & fcZero) {720 if (PartialCheck == fcPosZero)721 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,722 AsInt, ZeroC));723 else if (PartialCheck == fcZero)724 appendToRes(725 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, Abs, ZeroC));726 else // fcNegZero727 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,728 AsInt, SignBitC));729 }730 731 if (FPClassTest PartialCheck = Mask & fcSubnormal) {732 // issubnormal(V) ==> unsigned(abs(V) - 1) u< (all mantissa bits set)733 // issubnormal(V) && V>0 ==> unsigned(V - 1) u< (all mantissa bits set)734 auto V = (PartialCheck == fcPosSubnormal) ? AsInt : Abs;735 auto OneC = buildSPIRVConstant(IntTy, 1);736 auto VMinusOne = MIRBuilder.buildSub(IntTy, V, OneC);737 auto SubnormalRes = assignSPIRVTy(738 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, VMinusOne,739 buildSPIRVConstant(IntTy, AllOneMantissa)));740 if (PartialCheck == fcNegSubnormal)741 SubnormalRes = MIRBuilder.buildAnd(DstTy, SubnormalRes, Sign);742 appendToRes(std::move(SubnormalRes));743 }744 745 if (FPClassTest PartialCheck = Mask & fcInf) {746 if (PartialCheck == fcPosInf)747 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,748 AsInt, InfC));749 else if (PartialCheck == fcInf)750 appendToRes(751 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, Abs, InfC));752 else { // fcNegInf753 APInt NegInf = APFloat::getInf(Semantics, true).bitcastToAPInt();754 auto NegInfC = buildSPIRVConstant(IntTy, NegInf);755 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,756 AsInt, NegInfC));757 }758 }759 760 if (FPClassTest PartialCheck = Mask & fcNan) {761 auto InfWithQnanBitC =762 buildSPIRVConstant(IntTy, std::move(Inf) | QNaNBitMask);763 if (PartialCheck == fcNan) {764 // isnan(V) ==> abs(V) u> int(inf)765 appendToRes(766 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGT, DstTy, Abs, InfC));767 } else if (PartialCheck == fcQNan) {768 // isquiet(V) ==> abs(V) u>= (unsigned(Inf) | quiet_bit)769 appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGE, DstTy, Abs,770 InfWithQnanBitC));771 } else { // fcSNan772 // issignaling(V) ==> abs(V) u> unsigned(Inf) &&773 // abs(V) u< (unsigned(Inf) | quiet_bit)774 auto IsNan = assignSPIRVTy(775 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGT, DstTy, Abs, InfC));776 auto IsNotQnan = assignSPIRVTy(MIRBuilder.buildICmp(777 CmpInst::Predicate::ICMP_ULT, DstTy, Abs, InfWithQnanBitC));778 appendToRes(MIRBuilder.buildAnd(DstTy, IsNan, IsNotQnan));779 }780 }781 782 if (FPClassTest PartialCheck = Mask & fcNormal) {783 // isnormal(V) ==> (0 u< exp u< max_exp) ==> (unsigned(exp-1) u<784 // (max_exp-1))785 APInt ExpLSB = ExpMask & ~(ExpMask.shl(1));786 auto ExpMinusOne = assignSPIRVTy(787 MIRBuilder.buildSub(IntTy, Abs, buildSPIRVConstant(IntTy, ExpLSB)));788 APInt MaxExpMinusOne = std::move(ExpMask) - ExpLSB;789 auto NormalRes = assignSPIRVTy(790 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, ExpMinusOne,791 buildSPIRVConstant(IntTy, MaxExpMinusOne)));792 if (PartialCheck == fcNegNormal)793 NormalRes = MIRBuilder.buildAnd(DstTy, NormalRes, Sign);794 else if (PartialCheck == fcPosNormal) {795 auto PosSign = assignSPIRVTy(MIRBuilder.buildXor(796 DstTy, Sign, buildSPIRVConstant(DstTy, InversionMask)));797 NormalRes = MIRBuilder.buildAnd(DstTy, NormalRes, PosSign);798 }799 appendToRes(std::move(NormalRes));800 }801 802 MIRBuilder.buildCopy(DstReg, Res);803 MI.eraseFromParent();804 return true;805}806