1156 lines · cpp
1//===-- Intrinsics.cpp - Intrinsic Function Handling ------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements functions required for supporting intrinsic functions.10//11//===----------------------------------------------------------------------===//12 13#include "llvm/IR/Intrinsics.h"14#include "llvm/ADT/StringExtras.h"15#include "llvm/ADT/StringTable.h"16#include "llvm/IR/ConstantRange.h"17#include "llvm/IR/Function.h"18#include "llvm/IR/IntrinsicsAArch64.h"19#include "llvm/IR/IntrinsicsAMDGPU.h"20#include "llvm/IR/IntrinsicsARM.h"21#include "llvm/IR/IntrinsicsBPF.h"22#include "llvm/IR/IntrinsicsHexagon.h"23#include "llvm/IR/IntrinsicsLoongArch.h"24#include "llvm/IR/IntrinsicsMips.h"25#include "llvm/IR/IntrinsicsNVPTX.h"26#include "llvm/IR/IntrinsicsPowerPC.h"27#include "llvm/IR/IntrinsicsR600.h"28#include "llvm/IR/IntrinsicsRISCV.h"29#include "llvm/IR/IntrinsicsS390.h"30#include "llvm/IR/IntrinsicsSPIRV.h"31#include "llvm/IR/IntrinsicsVE.h"32#include "llvm/IR/IntrinsicsX86.h"33#include "llvm/IR/IntrinsicsXCore.h"34#include "llvm/IR/Module.h"35#include "llvm/IR/NVVMIntrinsicUtils.h"36#include "llvm/IR/Type.h"37 38using namespace llvm;39 40/// Table of string intrinsic names indexed by enum value.41#define GET_INTRINSIC_NAME_TABLE42#include "llvm/IR/IntrinsicImpl.inc"43#undef GET_INTRINSIC_NAME_TABLE44 45StringRef Intrinsic::getBaseName(ID id) {46 assert(id < num_intrinsics && "Invalid intrinsic ID!");47 return IntrinsicNameTable[IntrinsicNameOffsetTable[id]];48}49 50StringRef Intrinsic::getName(ID id) {51 assert(id < num_intrinsics && "Invalid intrinsic ID!");52 assert(!Intrinsic::isOverloaded(id) &&53 "This version of getName does not support overloading");54 return getBaseName(id);55}56 57/// Returns a stable mangling for the type specified for use in the name58/// mangling scheme used by 'any' types in intrinsic signatures. The mangling59/// of named types is simply their name. Manglings for unnamed types consist60/// of a prefix ('p' for pointers, 'a' for arrays, 'f_' for functions)61/// combined with the mangling of their component types. A vararg function62/// type will have a suffix of 'vararg'. Since function types can contain63/// other function types, we close a function type mangling with suffix 'f'64/// which can't be confused with it's prefix. This ensures we don't have65/// collisions between two unrelated function types. Otherwise, you might66/// parse ffXX as f(fXX) or f(fX)X. (X is a placeholder for any other type.)67/// The HasUnnamedType boolean is set if an unnamed type was encountered,68/// indicating that extra care must be taken to ensure a unique name.69static std::string getMangledTypeStr(Type *Ty, bool &HasUnnamedType) {70 std::string Result;71 if (PointerType *PTyp = dyn_cast<PointerType>(Ty)) {72 Result += "p" + utostr(PTyp->getAddressSpace());73 } else if (ArrayType *ATyp = dyn_cast<ArrayType>(Ty)) {74 Result += "a" + utostr(ATyp->getNumElements()) +75 getMangledTypeStr(ATyp->getElementType(), HasUnnamedType);76 } else if (StructType *STyp = dyn_cast<StructType>(Ty)) {77 if (!STyp->isLiteral()) {78 Result += "s_";79 if (STyp->hasName())80 Result += STyp->getName();81 else82 HasUnnamedType = true;83 } else {84 Result += "sl_";85 for (auto *Elem : STyp->elements())86 Result += getMangledTypeStr(Elem, HasUnnamedType);87 }88 // Ensure nested structs are distinguishable.89 Result += "s";90 } else if (FunctionType *FT = dyn_cast<FunctionType>(Ty)) {91 Result += "f_" + getMangledTypeStr(FT->getReturnType(), HasUnnamedType);92 for (size_t i = 0; i < FT->getNumParams(); i++)93 Result += getMangledTypeStr(FT->getParamType(i), HasUnnamedType);94 if (FT->isVarArg())95 Result += "vararg";96 // Ensure nested function types are distinguishable.97 Result += "f";98 } else if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {99 ElementCount EC = VTy->getElementCount();100 if (EC.isScalable())101 Result += "nx";102 Result += "v" + utostr(EC.getKnownMinValue()) +103 getMangledTypeStr(VTy->getElementType(), HasUnnamedType);104 } else if (TargetExtType *TETy = dyn_cast<TargetExtType>(Ty)) {105 Result += "t";106 Result += TETy->getName();107 for (Type *ParamTy : TETy->type_params())108 Result += "_" + getMangledTypeStr(ParamTy, HasUnnamedType);109 for (unsigned IntParam : TETy->int_params())110 Result += "_" + utostr(IntParam);111 // Ensure nested target extension types are distinguishable.112 Result += "t";113 } else if (Ty) {114 switch (Ty->getTypeID()) {115 default:116 llvm_unreachable("Unhandled type");117 case Type::VoidTyID:118 Result += "isVoid";119 break;120 case Type::MetadataTyID:121 Result += "Metadata";122 break;123 case Type::HalfTyID:124 Result += "f16";125 break;126 case Type::BFloatTyID:127 Result += "bf16";128 break;129 case Type::FloatTyID:130 Result += "f32";131 break;132 case Type::DoubleTyID:133 Result += "f64";134 break;135 case Type::X86_FP80TyID:136 Result += "f80";137 break;138 case Type::FP128TyID:139 Result += "f128";140 break;141 case Type::PPC_FP128TyID:142 Result += "ppcf128";143 break;144 case Type::X86_AMXTyID:145 Result += "x86amx";146 break;147 case Type::IntegerTyID:148 Result += "i" + utostr(cast<IntegerType>(Ty)->getBitWidth());149 break;150 }151 }152 return Result;153}154 155static std::string getIntrinsicNameImpl(Intrinsic::ID Id, ArrayRef<Type *> Tys,156 Module *M, FunctionType *FT,157 bool EarlyModuleCheck) {158 159 assert(Id < Intrinsic::num_intrinsics && "Invalid intrinsic ID!");160 assert((Tys.empty() || Intrinsic::isOverloaded(Id)) &&161 "This version of getName is for overloaded intrinsics only");162 (void)EarlyModuleCheck;163 assert((!EarlyModuleCheck || M ||164 !any_of(Tys, [](Type *T) { return isa<PointerType>(T); })) &&165 "Intrinsic overloading on pointer types need to provide a Module");166 bool HasUnnamedType = false;167 std::string Result(Intrinsic::getBaseName(Id));168 for (Type *Ty : Tys)169 Result += "." + getMangledTypeStr(Ty, HasUnnamedType);170 if (HasUnnamedType) {171 assert(M && "unnamed types need a module");172 if (!FT)173 FT = Intrinsic::getType(M->getContext(), Id, Tys);174 else175 assert((FT == Intrinsic::getType(M->getContext(), Id, Tys)) &&176 "Provided FunctionType must match arguments");177 return M->getUniqueIntrinsicName(Result, Id, FT);178 }179 return Result;180}181 182std::string Intrinsic::getName(ID Id, ArrayRef<Type *> Tys, Module *M,183 FunctionType *FT) {184 assert(M && "We need to have a Module");185 return getIntrinsicNameImpl(Id, Tys, M, FT, true);186}187 188std::string Intrinsic::getNameNoUnnamedTypes(ID Id, ArrayRef<Type *> Tys) {189 return getIntrinsicNameImpl(Id, Tys, nullptr, nullptr, false);190}191 192/// IIT_Info - These are enumerators that describe the entries returned by the193/// getIntrinsicInfoTableEntries function.194///195/// Defined in Intrinsics.td.196enum IIT_Info {197#define GET_INTRINSIC_IITINFO198#include "llvm/IR/IntrinsicImpl.inc"199#undef GET_INTRINSIC_IITINFO200};201 202static void203DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,204 IIT_Info LastInfo,205 SmallVectorImpl<Intrinsic::IITDescriptor> &OutputTable) {206 using namespace Intrinsic;207 208 bool IsScalableVector = (LastInfo == IIT_SCALABLE_VEC);209 210 IIT_Info Info = IIT_Info(Infos[NextElt++]);211 212 switch (Info) {213 case IIT_Done:214 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Void, 0));215 return;216 case IIT_VARARG:217 OutputTable.push_back(IITDescriptor::get(IITDescriptor::VarArg, 0));218 return;219 case IIT_MMX:220 OutputTable.push_back(IITDescriptor::get(IITDescriptor::MMX, 0));221 return;222 case IIT_AMX:223 OutputTable.push_back(IITDescriptor::get(IITDescriptor::AMX, 0));224 return;225 case IIT_TOKEN:226 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Token, 0));227 return;228 case IIT_METADATA:229 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Metadata, 0));230 return;231 case IIT_F16:232 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Half, 0));233 return;234 case IIT_BF16:235 OutputTable.push_back(IITDescriptor::get(IITDescriptor::BFloat, 0));236 return;237 case IIT_F32:238 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Float, 0));239 return;240 case IIT_F64:241 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Double, 0));242 return;243 case IIT_F128:244 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Quad, 0));245 return;246 case IIT_PPCF128:247 OutputTable.push_back(IITDescriptor::get(IITDescriptor::PPCQuad, 0));248 return;249 case IIT_I1:250 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 1));251 return;252 case IIT_I2:253 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 2));254 return;255 case IIT_I4:256 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 4));257 return;258 case IIT_AARCH64_SVCOUNT:259 OutputTable.push_back(IITDescriptor::get(IITDescriptor::AArch64Svcount, 0));260 return;261 case IIT_I8:262 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 8));263 return;264 case IIT_I16:265 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 16));266 return;267 case IIT_I32:268 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 32));269 return;270 case IIT_I64:271 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 64));272 return;273 case IIT_I128:274 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 128));275 return;276 case IIT_V1:277 OutputTable.push_back(IITDescriptor::getVector(1, IsScalableVector));278 DecodeIITType(NextElt, Infos, Info, OutputTable);279 return;280 case IIT_V2:281 OutputTable.push_back(IITDescriptor::getVector(2, IsScalableVector));282 DecodeIITType(NextElt, Infos, Info, OutputTable);283 return;284 case IIT_V3:285 OutputTable.push_back(IITDescriptor::getVector(3, IsScalableVector));286 DecodeIITType(NextElt, Infos, Info, OutputTable);287 return;288 case IIT_V4:289 OutputTable.push_back(IITDescriptor::getVector(4, IsScalableVector));290 DecodeIITType(NextElt, Infos, Info, OutputTable);291 return;292 case IIT_V6:293 OutputTable.push_back(IITDescriptor::getVector(6, IsScalableVector));294 DecodeIITType(NextElt, Infos, Info, OutputTable);295 return;296 case IIT_V8:297 OutputTable.push_back(IITDescriptor::getVector(8, IsScalableVector));298 DecodeIITType(NextElt, Infos, Info, OutputTable);299 return;300 case IIT_V10:301 OutputTable.push_back(IITDescriptor::getVector(10, IsScalableVector));302 DecodeIITType(NextElt, Infos, Info, OutputTable);303 return;304 case IIT_V16:305 OutputTable.push_back(IITDescriptor::getVector(16, IsScalableVector));306 DecodeIITType(NextElt, Infos, Info, OutputTable);307 return;308 case IIT_V32:309 OutputTable.push_back(IITDescriptor::getVector(32, IsScalableVector));310 DecodeIITType(NextElt, Infos, Info, OutputTable);311 return;312 case IIT_V64:313 OutputTable.push_back(IITDescriptor::getVector(64, IsScalableVector));314 DecodeIITType(NextElt, Infos, Info, OutputTable);315 return;316 case IIT_V128:317 OutputTable.push_back(IITDescriptor::getVector(128, IsScalableVector));318 DecodeIITType(NextElt, Infos, Info, OutputTable);319 return;320 case IIT_V256:321 OutputTable.push_back(IITDescriptor::getVector(256, IsScalableVector));322 DecodeIITType(NextElt, Infos, Info, OutputTable);323 return;324 case IIT_V512:325 OutputTable.push_back(IITDescriptor::getVector(512, IsScalableVector));326 DecodeIITType(NextElt, Infos, Info, OutputTable);327 return;328 case IIT_V1024:329 OutputTable.push_back(IITDescriptor::getVector(1024, IsScalableVector));330 DecodeIITType(NextElt, Infos, Info, OutputTable);331 return;332 case IIT_V2048:333 OutputTable.push_back(IITDescriptor::getVector(2048, IsScalableVector));334 DecodeIITType(NextElt, Infos, Info, OutputTable);335 return;336 case IIT_V4096:337 OutputTable.push_back(IITDescriptor::getVector(4096, IsScalableVector));338 DecodeIITType(NextElt, Infos, Info, OutputTable);339 return;340 case IIT_EXTERNREF:341 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Pointer, 10));342 return;343 case IIT_FUNCREF:344 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Pointer, 20));345 return;346 case IIT_PTR:347 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Pointer, 0));348 return;349 case IIT_ANYPTR: // [ANYPTR addrspace]350 OutputTable.push_back(351 IITDescriptor::get(IITDescriptor::Pointer, Infos[NextElt++]));352 return;353 case IIT_ARG: {354 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);355 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Argument, ArgInfo));356 return;357 }358 case IIT_EXTEND_ARG: {359 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);360 OutputTable.push_back(361 IITDescriptor::get(IITDescriptor::ExtendArgument, ArgInfo));362 return;363 }364 case IIT_TRUNC_ARG: {365 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);366 OutputTable.push_back(367 IITDescriptor::get(IITDescriptor::TruncArgument, ArgInfo));368 return;369 }370 case IIT_ONE_NTH_ELTS_VEC_ARG: {371 unsigned short ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);372 unsigned short N = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);373 OutputTable.push_back(374 IITDescriptor::get(IITDescriptor::OneNthEltsVecArgument, N, ArgNo));375 return;376 }377 case IIT_SAME_VEC_WIDTH_ARG: {378 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);379 OutputTable.push_back(380 IITDescriptor::get(IITDescriptor::SameVecWidthArgument, ArgInfo));381 return;382 }383 case IIT_VEC_OF_ANYPTRS_TO_ELT: {384 unsigned short ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);385 unsigned short RefNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);386 OutputTable.push_back(387 IITDescriptor::get(IITDescriptor::VecOfAnyPtrsToElt, ArgNo, RefNo));388 return;389 }390 case IIT_EMPTYSTRUCT:391 OutputTable.push_back(IITDescriptor::get(IITDescriptor::Struct, 0));392 return;393 case IIT_STRUCT: {394 unsigned StructElts = Infos[NextElt++] + 2;395 396 OutputTable.push_back(397 IITDescriptor::get(IITDescriptor::Struct, StructElts));398 399 for (unsigned i = 0; i != StructElts; ++i)400 DecodeIITType(NextElt, Infos, Info, OutputTable);401 return;402 }403 case IIT_SUBDIVIDE2_ARG: {404 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);405 OutputTable.push_back(406 IITDescriptor::get(IITDescriptor::Subdivide2Argument, ArgInfo));407 return;408 }409 case IIT_SUBDIVIDE4_ARG: {410 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);411 OutputTable.push_back(412 IITDescriptor::get(IITDescriptor::Subdivide4Argument, ArgInfo));413 return;414 }415 case IIT_VEC_ELEMENT: {416 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);417 OutputTable.push_back(418 IITDescriptor::get(IITDescriptor::VecElementArgument, ArgInfo));419 return;420 }421 case IIT_SCALABLE_VEC: {422 DecodeIITType(NextElt, Infos, Info, OutputTable);423 return;424 }425 case IIT_VEC_OF_BITCASTS_TO_INT: {426 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);427 OutputTable.push_back(428 IITDescriptor::get(IITDescriptor::VecOfBitcastsToInt, ArgInfo));429 return;430 }431 }432 llvm_unreachable("unhandled");433}434 435#define GET_INTRINSIC_GENERATOR_GLOBAL436#include "llvm/IR/IntrinsicImpl.inc"437#undef GET_INTRINSIC_GENERATOR_GLOBAL438 439void Intrinsic::getIntrinsicInfoTableEntries(440 ID id, SmallVectorImpl<IITDescriptor> &T) {441 static_assert(sizeof(IIT_Table[0]) == 2,442 "Expect 16-bit entries in IIT_Table");443 // Check to see if the intrinsic's type was expressible by the table.444 uint16_t TableVal = IIT_Table[id - 1];445 446 // Decode the TableVal into an array of IITValues.447 SmallVector<unsigned char> IITValues;448 ArrayRef<unsigned char> IITEntries;449 unsigned NextElt = 0;450 if (TableVal >> 15) {451 // This is an offset into the IIT_LongEncodingTable.452 IITEntries = IIT_LongEncodingTable;453 454 // Strip sentinel bit.455 NextElt = TableVal & 0x7fff;456 } else {457 // If the entry was encoded into a single word in the table itself, decode458 // it from an array of nibbles to an array of bytes.459 do {460 IITValues.push_back(TableVal & 0xF);461 TableVal >>= 4;462 } while (TableVal);463 464 IITEntries = IITValues;465 NextElt = 0;466 }467 468 // Okay, decode the table into the output vector of IITDescriptors.469 DecodeIITType(NextElt, IITEntries, IIT_Done, T);470 while (NextElt != IITEntries.size() && IITEntries[NextElt] != 0)471 DecodeIITType(NextElt, IITEntries, IIT_Done, T);472}473 474static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,475 ArrayRef<Type *> Tys, LLVMContext &Context) {476 using namespace Intrinsic;477 478 IITDescriptor D = Infos.front();479 Infos = Infos.slice(1);480 481 switch (D.Kind) {482 case IITDescriptor::Void:483 return Type::getVoidTy(Context);484 case IITDescriptor::VarArg:485 return Type::getVoidTy(Context);486 case IITDescriptor::MMX:487 return llvm::FixedVectorType::get(llvm::IntegerType::get(Context, 64), 1);488 case IITDescriptor::AMX:489 return Type::getX86_AMXTy(Context);490 case IITDescriptor::Token:491 return Type::getTokenTy(Context);492 case IITDescriptor::Metadata:493 return Type::getMetadataTy(Context);494 case IITDescriptor::Half:495 return Type::getHalfTy(Context);496 case IITDescriptor::BFloat:497 return Type::getBFloatTy(Context);498 case IITDescriptor::Float:499 return Type::getFloatTy(Context);500 case IITDescriptor::Double:501 return Type::getDoubleTy(Context);502 case IITDescriptor::Quad:503 return Type::getFP128Ty(Context);504 case IITDescriptor::PPCQuad:505 return Type::getPPC_FP128Ty(Context);506 case IITDescriptor::AArch64Svcount:507 return TargetExtType::get(Context, "aarch64.svcount");508 509 case IITDescriptor::Integer:510 return IntegerType::get(Context, D.Integer_Width);511 case IITDescriptor::Vector:512 return VectorType::get(DecodeFixedType(Infos, Tys, Context),513 D.Vector_Width);514 case IITDescriptor::Pointer:515 return PointerType::get(Context, D.Pointer_AddressSpace);516 case IITDescriptor::Struct: {517 SmallVector<Type *, 8> Elts;518 for (unsigned i = 0, e = D.Struct_NumElements; i != e; ++i)519 Elts.push_back(DecodeFixedType(Infos, Tys, Context));520 return StructType::get(Context, Elts);521 }522 case IITDescriptor::Argument:523 return Tys[D.getArgumentNumber()];524 case IITDescriptor::ExtendArgument: {525 Type *Ty = Tys[D.getArgumentNumber()];526 if (VectorType *VTy = dyn_cast<VectorType>(Ty))527 return VectorType::getExtendedElementVectorType(VTy);528 529 return IntegerType::get(Context, 2 * cast<IntegerType>(Ty)->getBitWidth());530 }531 case IITDescriptor::TruncArgument: {532 Type *Ty = Tys[D.getArgumentNumber()];533 if (VectorType *VTy = dyn_cast<VectorType>(Ty))534 return VectorType::getTruncatedElementVectorType(VTy);535 536 IntegerType *ITy = cast<IntegerType>(Ty);537 assert(ITy->getBitWidth() % 2 == 0);538 return IntegerType::get(Context, ITy->getBitWidth() / 2);539 }540 case IITDescriptor::Subdivide2Argument:541 case IITDescriptor::Subdivide4Argument: {542 Type *Ty = Tys[D.getArgumentNumber()];543 VectorType *VTy = dyn_cast<VectorType>(Ty);544 assert(VTy && "Expected an argument of Vector Type");545 int SubDivs = D.Kind == IITDescriptor::Subdivide2Argument ? 1 : 2;546 return VectorType::getSubdividedVectorType(VTy, SubDivs);547 }548 case IITDescriptor::OneNthEltsVecArgument:549 return VectorType::getOneNthElementsVectorType(550 cast<VectorType>(Tys[D.getRefArgNumber()]), D.getVectorDivisor());551 case IITDescriptor::SameVecWidthArgument: {552 Type *EltTy = DecodeFixedType(Infos, Tys, Context);553 Type *Ty = Tys[D.getArgumentNumber()];554 if (auto *VTy = dyn_cast<VectorType>(Ty))555 return VectorType::get(EltTy, VTy->getElementCount());556 return EltTy;557 }558 case IITDescriptor::VecElementArgument: {559 Type *Ty = Tys[D.getArgumentNumber()];560 if (VectorType *VTy = dyn_cast<VectorType>(Ty))561 return VTy->getElementType();562 llvm_unreachable("Expected an argument of Vector Type");563 }564 case IITDescriptor::VecOfBitcastsToInt: {565 Type *Ty = Tys[D.getArgumentNumber()];566 VectorType *VTy = dyn_cast<VectorType>(Ty);567 assert(VTy && "Expected an argument of Vector Type");568 return VectorType::getInteger(VTy);569 }570 case IITDescriptor::VecOfAnyPtrsToElt:571 // Return the overloaded type (which determines the pointers address space)572 return Tys[D.getOverloadArgNumber()];573 }574 llvm_unreachable("unhandled");575}576 577FunctionType *Intrinsic::getType(LLVMContext &Context, ID id,578 ArrayRef<Type *> Tys) {579 SmallVector<IITDescriptor, 8> Table;580 getIntrinsicInfoTableEntries(id, Table);581 582 ArrayRef<IITDescriptor> TableRef = Table;583 Type *ResultTy = DecodeFixedType(TableRef, Tys, Context);584 585 SmallVector<Type *, 8> ArgTys;586 while (!TableRef.empty())587 ArgTys.push_back(DecodeFixedType(TableRef, Tys, Context));588 589 // DecodeFixedType returns Void for IITDescriptor::Void and590 // IITDescriptor::VarArg If we see void type as the type of the last argument,591 // it is vararg intrinsic592 if (!ArgTys.empty() && ArgTys.back()->isVoidTy()) {593 ArgTys.pop_back();594 return FunctionType::get(ResultTy, ArgTys, true);595 }596 return FunctionType::get(ResultTy, ArgTys, false);597}598 599bool Intrinsic::isOverloaded(ID id) {600#define GET_INTRINSIC_OVERLOAD_TABLE601#include "llvm/IR/IntrinsicImpl.inc"602#undef GET_INTRINSIC_OVERLOAD_TABLE603}604 605bool Intrinsic::hasPrettyPrintedArgs(ID id){606#define GET_INTRINSIC_PRETTY_PRINT_TABLE607#include "llvm/IR/IntrinsicImpl.inc"608#undef GET_INTRINSIC_PRETTY_PRINT_TABLE609}610 611/// Table of per-target intrinsic name tables.612#define GET_INTRINSIC_TARGET_DATA613#include "llvm/IR/IntrinsicImpl.inc"614#undef GET_INTRINSIC_TARGET_DATA615 616bool Intrinsic::isTargetIntrinsic(Intrinsic::ID IID) {617 return IID > TargetInfos[0].Count;618}619 620/// Looks up Name in NameTable via binary search. NameTable must be sorted621/// and all entries must start with "llvm.". If NameTable contains an exact622/// match for Name or a prefix of Name followed by a dot, its index in623/// NameTable is returned. Otherwise, -1 is returned.624static int lookupLLVMIntrinsicByName(ArrayRef<unsigned> NameOffsetTable,625 StringRef Name, StringRef Target = "") {626 assert(Name.starts_with("llvm.") && "Unexpected intrinsic prefix");627 assert(Name.drop_front(5).starts_with(Target) && "Unexpected target");628 629 // Do successive binary searches of the dotted name components. For630 // "llvm.gc.experimental.statepoint.p1i8.p1i32", we will find the range of631 // intrinsics starting with "llvm.gc", then "llvm.gc.experimental", then632 // "llvm.gc.experimental.statepoint", and then we will stop as the range is633 // size 1. During the search, we can skip the prefix that we already know is634 // identical. By using strncmp we consider names with differing suffixes to635 // be part of the equal range.636 size_t CmpEnd = 4; // Skip the "llvm" component.637 if (!Target.empty())638 CmpEnd += 1 + Target.size(); // skip the .target component.639 640 const unsigned *Low = NameOffsetTable.begin();641 const unsigned *High = NameOffsetTable.end();642 const unsigned *LastLow = Low;643 while (CmpEnd < Name.size() && High - Low > 0) {644 size_t CmpStart = CmpEnd;645 CmpEnd = Name.find('.', CmpStart + 1);646 CmpEnd = CmpEnd == StringRef::npos ? Name.size() : CmpEnd;647 auto Cmp = [CmpStart, CmpEnd](auto LHS, auto RHS) {648 // `equal_range` requires the comparison to work with either side being an649 // offset or the value. Detect which kind each side is to set up the650 // compared strings.651 const char *LHSStr;652 if constexpr (std::is_integral_v<decltype(LHS)>)653 LHSStr = IntrinsicNameTable.getCString(LHS);654 else655 LHSStr = LHS;656 657 const char *RHSStr;658 if constexpr (std::is_integral_v<decltype(RHS)>)659 RHSStr = IntrinsicNameTable.getCString(RHS);660 else661 RHSStr = RHS;662 663 return strncmp(LHSStr + CmpStart, RHSStr + CmpStart, CmpEnd - CmpStart) <664 0;665 };666 LastLow = Low;667 std::tie(Low, High) = std::equal_range(Low, High, Name.data(), Cmp);668 }669 if (High - Low > 0)670 LastLow = Low;671 672 if (LastLow == NameOffsetTable.end())673 return -1;674 StringRef NameFound = IntrinsicNameTable[*LastLow];675 if (Name == NameFound ||676 (Name.starts_with(NameFound) && Name[NameFound.size()] == '.'))677 return LastLow - NameOffsetTable.begin();678 return -1;679}680 681/// Find the segment of \c IntrinsicNameOffsetTable for intrinsics with the same682/// target as \c Name, or the generic table if \c Name is not target specific.683///684/// Returns the relevant slice of \c IntrinsicNameOffsetTable and the target685/// name.686static std::pair<ArrayRef<unsigned>, StringRef>687findTargetSubtable(StringRef Name) {688 assert(Name.starts_with("llvm."));689 690 ArrayRef<IntrinsicTargetInfo> Targets(TargetInfos);691 // Drop "llvm." and take the first dotted component. That will be the target692 // if this is target specific.693 StringRef Target = Name.drop_front(5).split('.').first;694 auto It = partition_point(695 Targets, [=](const IntrinsicTargetInfo &TI) { return TI.Name < Target; });696 // We've either found the target or just fall back to the generic set, which697 // is always first.698 const auto &TI = It != Targets.end() && It->Name == Target ? *It : Targets[0];699 return {ArrayRef(&IntrinsicNameOffsetTable[1] + TI.Offset, TI.Count),700 TI.Name};701}702 703/// This does the actual lookup of an intrinsic ID which matches the given704/// function name.705Intrinsic::ID Intrinsic::lookupIntrinsicID(StringRef Name) {706 auto [NameOffsetTable, Target] = findTargetSubtable(Name);707 int Idx = lookupLLVMIntrinsicByName(NameOffsetTable, Name, Target);708 if (Idx == -1)709 return Intrinsic::not_intrinsic;710 711 // Intrinsic IDs correspond to the location in IntrinsicNameTable, but we have712 // an index into a sub-table.713 int Adjust = NameOffsetTable.data() - IntrinsicNameOffsetTable;714 Intrinsic::ID ID = static_cast<Intrinsic::ID>(Idx + Adjust);715 716 // If the intrinsic is not overloaded, require an exact match. If it is717 // overloaded, require either exact or prefix match.718 const auto MatchSize = IntrinsicNameTable[NameOffsetTable[Idx]].size();719 assert(Name.size() >= MatchSize && "Expected either exact or prefix match");720 bool IsExactMatch = Name.size() == MatchSize;721 return IsExactMatch || Intrinsic::isOverloaded(ID) ? ID722 : Intrinsic::not_intrinsic;723}724 725/// This defines the "Intrinsic::getAttributes(ID id)" method.726#define GET_INTRINSIC_ATTRIBUTES727#include "llvm/IR/IntrinsicImpl.inc"728#undef GET_INTRINSIC_ATTRIBUTES729 730Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id,731 ArrayRef<Type *> Tys) {732 // There can never be multiple globals with the same name of different types,733 // because intrinsics must be a specific type.734 auto *FT = getType(M->getContext(), id, Tys);735 Function *F = cast<Function>(736 M->getOrInsertFunction(737 Tys.empty() ? getName(id) : getName(id, Tys, M, FT), FT)738 .getCallee());739 if (F->getFunctionType() == FT)740 return F;741 742 // It's possible that a declaration for this intrinsic already exists with an743 // incorrect signature, if the signature has changed, but this particular744 // declaration has not been auto-upgraded yet. In that case, rename the745 // invalid declaration and insert a new one with the correct signature. The746 // invalid declaration will get upgraded later.747 F->setName(F->getName() + ".invalid");748 return cast<Function>(749 M->getOrInsertFunction(750 Tys.empty() ? getName(id) : getName(id, Tys, M, FT), FT)751 .getCallee());752}753 754Function *Intrinsic::getDeclarationIfExists(const Module *M, ID id) {755 return M->getFunction(getName(id));756}757 758Function *Intrinsic::getDeclarationIfExists(Module *M, ID id,759 ArrayRef<Type *> Tys,760 FunctionType *FT) {761 return M->getFunction(getName(id, Tys, M, FT));762}763 764// This defines the "Intrinsic::getIntrinsicForClangBuiltin()" method.765#define GET_LLVM_INTRINSIC_FOR_CLANG_BUILTIN766#include "llvm/IR/IntrinsicImpl.inc"767#undef GET_LLVM_INTRINSIC_FOR_CLANG_BUILTIN768 769// This defines the "Intrinsic::getIntrinsicForMSBuiltin()" method.770#define GET_LLVM_INTRINSIC_FOR_MS_BUILTIN771#include "llvm/IR/IntrinsicImpl.inc"772#undef GET_LLVM_INTRINSIC_FOR_MS_BUILTIN773 774bool Intrinsic::isConstrainedFPIntrinsic(ID QID) {775 switch (QID) {776#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \777 case Intrinsic::INTRINSIC:778#include "llvm/IR/ConstrainedOps.def"779#undef INSTRUCTION780 return true;781 default:782 return false;783 }784}785 786bool Intrinsic::hasConstrainedFPRoundingModeOperand(Intrinsic::ID QID) {787 switch (QID) {788#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \789 case Intrinsic::INTRINSIC: \790 return ROUND_MODE == 1;791#include "llvm/IR/ConstrainedOps.def"792#undef INSTRUCTION793 default:794 return false;795 }796}797 798using DeferredIntrinsicMatchPair =799 std::pair<Type *, ArrayRef<Intrinsic::IITDescriptor>>;800 801static bool802matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor> &Infos,803 SmallVectorImpl<Type *> &ArgTys,804 SmallVectorImpl<DeferredIntrinsicMatchPair> &DeferredChecks,805 bool IsDeferredCheck) {806 using namespace Intrinsic;807 808 // If we ran out of descriptors, there are too many arguments.809 if (Infos.empty())810 return true;811 812 // Do this before slicing off the 'front' part813 auto InfosRef = Infos;814 auto DeferCheck = [&DeferredChecks, &InfosRef](Type *T) {815 DeferredChecks.emplace_back(T, InfosRef);816 return false;817 };818 819 IITDescriptor D = Infos.front();820 Infos = Infos.slice(1);821 822 switch (D.Kind) {823 case IITDescriptor::Void:824 return !Ty->isVoidTy();825 case IITDescriptor::VarArg:826 return true;827 case IITDescriptor::MMX: {828 FixedVectorType *VT = dyn_cast<FixedVectorType>(Ty);829 return !VT || VT->getNumElements() != 1 ||830 !VT->getElementType()->isIntegerTy(64);831 }832 case IITDescriptor::AMX:833 return !Ty->isX86_AMXTy();834 case IITDescriptor::Token:835 return !Ty->isTokenTy();836 case IITDescriptor::Metadata:837 return !Ty->isMetadataTy();838 case IITDescriptor::Half:839 return !Ty->isHalfTy();840 case IITDescriptor::BFloat:841 return !Ty->isBFloatTy();842 case IITDescriptor::Float:843 return !Ty->isFloatTy();844 case IITDescriptor::Double:845 return !Ty->isDoubleTy();846 case IITDescriptor::Quad:847 return !Ty->isFP128Ty();848 case IITDescriptor::PPCQuad:849 return !Ty->isPPC_FP128Ty();850 case IITDescriptor::Integer:851 return !Ty->isIntegerTy(D.Integer_Width);852 case IITDescriptor::AArch64Svcount:853 return !isa<TargetExtType>(Ty) ||854 cast<TargetExtType>(Ty)->getName() != "aarch64.svcount";855 case IITDescriptor::Vector: {856 VectorType *VT = dyn_cast<VectorType>(Ty);857 return !VT || VT->getElementCount() != D.Vector_Width ||858 matchIntrinsicType(VT->getElementType(), Infos, ArgTys,859 DeferredChecks, IsDeferredCheck);860 }861 case IITDescriptor::Pointer: {862 PointerType *PT = dyn_cast<PointerType>(Ty);863 return !PT || PT->getAddressSpace() != D.Pointer_AddressSpace;864 }865 866 case IITDescriptor::Struct: {867 StructType *ST = dyn_cast<StructType>(Ty);868 if (!ST || !ST->isLiteral() || ST->isPacked() ||869 ST->getNumElements() != D.Struct_NumElements)870 return true;871 872 for (unsigned i = 0, e = D.Struct_NumElements; i != e; ++i)873 if (matchIntrinsicType(ST->getElementType(i), Infos, ArgTys,874 DeferredChecks, IsDeferredCheck))875 return true;876 return false;877 }878 879 case IITDescriptor::Argument:880 // If this is the second occurrence of an argument,881 // verify that the later instance matches the previous instance.882 if (D.getArgumentNumber() < ArgTys.size())883 return Ty != ArgTys[D.getArgumentNumber()];884 885 if (D.getArgumentNumber() > ArgTys.size() ||886 D.getArgumentKind() == IITDescriptor::AK_MatchType)887 return IsDeferredCheck || DeferCheck(Ty);888 889 assert(D.getArgumentNumber() == ArgTys.size() && !IsDeferredCheck &&890 "Table consistency error");891 ArgTys.push_back(Ty);892 893 switch (D.getArgumentKind()) {894 case IITDescriptor::AK_Any:895 return false; // Success896 case IITDescriptor::AK_AnyInteger:897 return !Ty->isIntOrIntVectorTy();898 case IITDescriptor::AK_AnyFloat:899 return !Ty->isFPOrFPVectorTy();900 case IITDescriptor::AK_AnyVector:901 return !isa<VectorType>(Ty);902 case IITDescriptor::AK_AnyPointer:903 return !isa<PointerType>(Ty);904 default:905 break;906 }907 llvm_unreachable("all argument kinds not covered");908 909 case IITDescriptor::ExtendArgument: {910 // If this is a forward reference, defer the check for later.911 if (D.getArgumentNumber() >= ArgTys.size())912 return IsDeferredCheck || DeferCheck(Ty);913 914 Type *NewTy = ArgTys[D.getArgumentNumber()];915 if (VectorType *VTy = dyn_cast<VectorType>(NewTy))916 NewTy = VectorType::getExtendedElementVectorType(VTy);917 else if (IntegerType *ITy = dyn_cast<IntegerType>(NewTy))918 NewTy = IntegerType::get(ITy->getContext(), 2 * ITy->getBitWidth());919 else920 return true;921 922 return Ty != NewTy;923 }924 case IITDescriptor::TruncArgument: {925 // If this is a forward reference, defer the check for later.926 if (D.getArgumentNumber() >= ArgTys.size())927 return IsDeferredCheck || DeferCheck(Ty);928 929 Type *NewTy = ArgTys[D.getArgumentNumber()];930 if (VectorType *VTy = dyn_cast<VectorType>(NewTy))931 NewTy = VectorType::getTruncatedElementVectorType(VTy);932 else if (IntegerType *ITy = dyn_cast<IntegerType>(NewTy))933 NewTy = IntegerType::get(ITy->getContext(), ITy->getBitWidth() / 2);934 else935 return true;936 937 return Ty != NewTy;938 }939 case IITDescriptor::OneNthEltsVecArgument:940 // If this is a forward reference, defer the check for later.941 if (D.getRefArgNumber() >= ArgTys.size())942 return IsDeferredCheck || DeferCheck(Ty);943 return !isa<VectorType>(ArgTys[D.getRefArgNumber()]) ||944 VectorType::getOneNthElementsVectorType(945 cast<VectorType>(ArgTys[D.getRefArgNumber()]),946 D.getVectorDivisor()) != Ty;947 case IITDescriptor::SameVecWidthArgument: {948 if (D.getArgumentNumber() >= ArgTys.size()) {949 // Defer check and subsequent check for the vector element type.950 Infos = Infos.slice(1);951 return IsDeferredCheck || DeferCheck(Ty);952 }953 auto *ReferenceType = dyn_cast<VectorType>(ArgTys[D.getArgumentNumber()]);954 auto *ThisArgType = dyn_cast<VectorType>(Ty);955 // Both must be vectors of the same number of elements or neither.956 if ((ReferenceType != nullptr) != (ThisArgType != nullptr))957 return true;958 Type *EltTy = Ty;959 if (ThisArgType) {960 if (ReferenceType->getElementCount() != ThisArgType->getElementCount())961 return true;962 EltTy = ThisArgType->getElementType();963 }964 return matchIntrinsicType(EltTy, Infos, ArgTys, DeferredChecks,965 IsDeferredCheck);966 }967 case IITDescriptor::VecOfAnyPtrsToElt: {968 unsigned RefArgNumber = D.getRefArgNumber();969 if (RefArgNumber >= ArgTys.size()) {970 if (IsDeferredCheck)971 return true;972 // If forward referencing, already add the pointer-vector type and973 // defer the checks for later.974 ArgTys.push_back(Ty);975 return DeferCheck(Ty);976 }977 978 if (!IsDeferredCheck) {979 assert(D.getOverloadArgNumber() == ArgTys.size() &&980 "Table consistency error");981 ArgTys.push_back(Ty);982 }983 984 // Verify the overloaded type "matches" the Ref type.985 // i.e. Ty is a vector with the same width as Ref.986 // Composed of pointers to the same element type as Ref.987 auto *ReferenceType = dyn_cast<VectorType>(ArgTys[RefArgNumber]);988 auto *ThisArgVecTy = dyn_cast<VectorType>(Ty);989 if (!ThisArgVecTy || !ReferenceType ||990 (ReferenceType->getElementCount() != ThisArgVecTy->getElementCount()))991 return true;992 return !ThisArgVecTy->getElementType()->isPointerTy();993 }994 case IITDescriptor::VecElementArgument: {995 if (D.getArgumentNumber() >= ArgTys.size())996 return IsDeferredCheck ? true : DeferCheck(Ty);997 auto *ReferenceType = dyn_cast<VectorType>(ArgTys[D.getArgumentNumber()]);998 return !ReferenceType || Ty != ReferenceType->getElementType();999 }1000 case IITDescriptor::Subdivide2Argument:1001 case IITDescriptor::Subdivide4Argument: {1002 // If this is a forward reference, defer the check for later.1003 if (D.getArgumentNumber() >= ArgTys.size())1004 return IsDeferredCheck || DeferCheck(Ty);1005 1006 Type *NewTy = ArgTys[D.getArgumentNumber()];1007 if (auto *VTy = dyn_cast<VectorType>(NewTy)) {1008 int SubDivs = D.Kind == IITDescriptor::Subdivide2Argument ? 1 : 2;1009 NewTy = VectorType::getSubdividedVectorType(VTy, SubDivs);1010 return Ty != NewTy;1011 }1012 return true;1013 }1014 case IITDescriptor::VecOfBitcastsToInt: {1015 if (D.getArgumentNumber() >= ArgTys.size())1016 return IsDeferredCheck || DeferCheck(Ty);1017 auto *ReferenceType = dyn_cast<VectorType>(ArgTys[D.getArgumentNumber()]);1018 auto *ThisArgVecTy = dyn_cast<VectorType>(Ty);1019 if (!ThisArgVecTy || !ReferenceType)1020 return true;1021 return ThisArgVecTy != VectorType::getInteger(ReferenceType);1022 }1023 }1024 llvm_unreachable("unhandled");1025}1026 1027Intrinsic::MatchIntrinsicTypesResult1028Intrinsic::matchIntrinsicSignature(FunctionType *FTy,1029 ArrayRef<Intrinsic::IITDescriptor> &Infos,1030 SmallVectorImpl<Type *> &ArgTys) {1031 SmallVector<DeferredIntrinsicMatchPair, 2> DeferredChecks;1032 if (matchIntrinsicType(FTy->getReturnType(), Infos, ArgTys, DeferredChecks,1033 false))1034 return MatchIntrinsicTypes_NoMatchRet;1035 1036 unsigned NumDeferredReturnChecks = DeferredChecks.size();1037 1038 for (auto *Ty : FTy->params())1039 if (matchIntrinsicType(Ty, Infos, ArgTys, DeferredChecks, false))1040 return MatchIntrinsicTypes_NoMatchArg;1041 1042 for (unsigned I = 0, E = DeferredChecks.size(); I != E; ++I) {1043 DeferredIntrinsicMatchPair &Check = DeferredChecks[I];1044 if (matchIntrinsicType(Check.first, Check.second, ArgTys, DeferredChecks,1045 true))1046 return I < NumDeferredReturnChecks ? MatchIntrinsicTypes_NoMatchRet1047 : MatchIntrinsicTypes_NoMatchArg;1048 }1049 1050 return MatchIntrinsicTypes_Match;1051}1052 1053bool Intrinsic::matchIntrinsicVarArg(1054 bool isVarArg, ArrayRef<Intrinsic::IITDescriptor> &Infos) {1055 // If there are no descriptors left, then it can't be a vararg.1056 if (Infos.empty())1057 return isVarArg;1058 1059 // There should be only one descriptor remaining at this point.1060 if (Infos.size() != 1)1061 return true;1062 1063 // Check and verify the descriptor.1064 IITDescriptor D = Infos.front();1065 Infos = Infos.slice(1);1066 if (D.Kind == IITDescriptor::VarArg)1067 return !isVarArg;1068 1069 return true;1070}1071 1072bool Intrinsic::getIntrinsicSignature(Intrinsic::ID ID, FunctionType *FT,1073 SmallVectorImpl<Type *> &ArgTys) {1074 if (!ID)1075 return false;1076 1077 SmallVector<Intrinsic::IITDescriptor, 8> Table;1078 getIntrinsicInfoTableEntries(ID, Table);1079 ArrayRef<Intrinsic::IITDescriptor> TableRef = Table;1080 1081 if (Intrinsic::matchIntrinsicSignature(FT, TableRef, ArgTys) !=1082 Intrinsic::MatchIntrinsicTypesResult::MatchIntrinsicTypes_Match) {1083 return false;1084 }1085 if (Intrinsic::matchIntrinsicVarArg(FT->isVarArg(), TableRef))1086 return false;1087 return true;1088}1089 1090bool Intrinsic::getIntrinsicSignature(Function *F,1091 SmallVectorImpl<Type *> &ArgTys) {1092 return getIntrinsicSignature(F->getIntrinsicID(), F->getFunctionType(),1093 ArgTys);1094}1095 1096std::optional<Function *> Intrinsic::remangleIntrinsicFunction(Function *F) {1097 SmallVector<Type *, 4> ArgTys;1098 if (!getIntrinsicSignature(F, ArgTys))1099 return std::nullopt;1100 1101 Intrinsic::ID ID = F->getIntrinsicID();1102 StringRef Name = F->getName();1103 std::string WantedName =1104 Intrinsic::getName(ID, ArgTys, F->getParent(), F->getFunctionType());1105 if (Name == WantedName)1106 return std::nullopt;1107 1108 Function *NewDecl = [&] {1109 if (auto *ExistingGV = F->getParent()->getNamedValue(WantedName)) {1110 if (auto *ExistingF = dyn_cast<Function>(ExistingGV))1111 if (ExistingF->getFunctionType() == F->getFunctionType())1112 return ExistingF;1113 1114 // The name already exists, but is not a function or has the wrong1115 // prototype. Make place for the new one by renaming the old version.1116 // Either this old version will be removed later on or the module is1117 // invalid and we'll get an error.1118 ExistingGV->setName(WantedName + ".renamed");1119 }1120 return Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ArgTys);1121 }();1122 1123 NewDecl->setCallingConv(F->getCallingConv());1124 assert(NewDecl->getFunctionType() == F->getFunctionType() &&1125 "Shouldn't change the signature");1126 return NewDecl;1127}1128 1129struct InterleaveIntrinsic {1130 Intrinsic::ID Interleave, Deinterleave;1131};1132 1133static InterleaveIntrinsic InterleaveIntrinsics[] = {1134 {Intrinsic::vector_interleave2, Intrinsic::vector_deinterleave2},1135 {Intrinsic::vector_interleave3, Intrinsic::vector_deinterleave3},1136 {Intrinsic::vector_interleave4, Intrinsic::vector_deinterleave4},1137 {Intrinsic::vector_interleave5, Intrinsic::vector_deinterleave5},1138 {Intrinsic::vector_interleave6, Intrinsic::vector_deinterleave6},1139 {Intrinsic::vector_interleave7, Intrinsic::vector_deinterleave7},1140 {Intrinsic::vector_interleave8, Intrinsic::vector_deinterleave8},1141};1142 1143Intrinsic::ID Intrinsic::getInterleaveIntrinsicID(unsigned Factor) {1144 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");1145 return InterleaveIntrinsics[Factor - 2].Interleave;1146}1147 1148Intrinsic::ID Intrinsic::getDeinterleaveIntrinsicID(unsigned Factor) {1149 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");1150 return InterleaveIntrinsics[Factor - 2].Deinterleave;1151}1152 1153#define GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS1154#include "llvm/IR/IntrinsicImpl.inc"1155#undef GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS1156