411 lines · cpp
1//===- RuntimeLibcalls.cpp - Interface for runtime libcalls -----*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/IR/RuntimeLibcalls.h"10#include "llvm/ADT/FloatingPointMode.h"11#include "llvm/ADT/StringTable.h"12#include "llvm/IR/Module.h"13#include "llvm/IR/SystemLibraries.h"14#include "llvm/Support/Debug.h"15#include "llvm/Support/xxhash.h"16#include "llvm/TargetParser/ARMTargetParser.h"17 18#define DEBUG_TYPE "runtime-libcalls-info"19 20using namespace llvm;21using namespace RTLIB;22 23#define GET_RUNTIME_LIBCALLS_INFO24#define GET_INIT_RUNTIME_LIBCALL_NAMES25#define GET_SET_TARGET_RUNTIME_LIBCALL_SETS26#define DEFINE_GET_LOOKUP_LIBCALL_IMPL_NAME27#include "llvm/IR/RuntimeLibcalls.inc"28 29RuntimeLibcallsInfo::RuntimeLibcallsInfo(const Triple &TT,30 ExceptionHandling ExceptionModel,31 FloatABI::ABIType FloatABI,32 EABI EABIVersion, StringRef ABIName,33 VectorLibrary VecLib) {34 // FIXME: The ExceptionModel parameter is to handle the field in35 // TargetOptions. This interface fails to distinguish the forced disable36 // case for targets which support exceptions by default. This should37 // probably be a module flag and removed from TargetOptions.38 if (ExceptionModel == ExceptionHandling::None)39 ExceptionModel = TT.getDefaultExceptionHandling();40 41 initLibcalls(TT, ExceptionModel, FloatABI, EABIVersion, ABIName);42 43 // TODO: Tablegen should generate these sets44 switch (VecLib) {45 case VectorLibrary::SLEEFGNUABI:46 for (RTLIB::LibcallImpl Impl :47 {RTLIB::impl__ZGVnN2vl8_modf, RTLIB::impl__ZGVnN4vl4_modff,48 RTLIB::impl__ZGVsNxvl8_modf, RTLIB::impl__ZGVsNxvl4_modff,49 RTLIB::impl__ZGVnN2vl8l8_sincos, RTLIB::impl__ZGVnN4vl4l4_sincosf,50 RTLIB::impl__ZGVsNxvl8l8_sincos, RTLIB::impl__ZGVsNxvl4l4_sincosf,51 RTLIB::impl__ZGVnN4vl4l4_sincospif, RTLIB::impl__ZGVnN2vl8l8_sincospi,52 RTLIB::impl__ZGVsNxvl4l4_sincospif,53 RTLIB::impl__ZGVsNxvl8l8_sincospi})54 setAvailable(Impl);55 break;56 case VectorLibrary::ArmPL:57 for (RTLIB::LibcallImpl Impl :58 {RTLIB::impl_armpl_vmodfq_f64, RTLIB::impl_armpl_vmodfq_f32,59 RTLIB::impl_armpl_svmodf_f64_x, RTLIB::impl_armpl_svmodf_f32_x,60 RTLIB::impl_armpl_vsincosq_f64, RTLIB::impl_armpl_vsincosq_f32,61 RTLIB::impl_armpl_svsincos_f64_x, RTLIB::impl_armpl_svsincos_f32_x,62 RTLIB::impl_armpl_vsincospiq_f32, RTLIB::impl_armpl_vsincospiq_f64,63 RTLIB::impl_armpl_svsincospi_f32_x,64 RTLIB::impl_armpl_svsincospi_f64_x})65 setAvailable(Impl);66 67 for (RTLIB::LibcallImpl Impl :68 {RTLIB::impl_armpl_vsincosq_f64, RTLIB::impl_armpl_vsincosq_f32})69 setLibcallImplCallingConv(Impl, CallingConv::AArch64_VectorCall);70 71 break;72 default:73 break;74 }75}76 77RuntimeLibcallsInfo::RuntimeLibcallsInfo(const Module &M)78 : RuntimeLibcallsInfo(M.getTargetTriple()) {79 // TODO: Consider module flags80}81 82/// Set default libcall names. If a target wants to opt-out of a libcall it83/// should be placed here.84void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,85 ExceptionHandling ExceptionModel,86 FloatABI::ABIType FloatABI,87 EABI EABIVersion, StringRef ABIName) {88 setTargetRuntimeLibcallSets(TT, ExceptionModel, FloatABI, EABIVersion,89 ABIName);90}91 92LLVM_ATTRIBUTE_ALWAYS_INLINE93iota_range<RTLIB::LibcallImpl>94RuntimeLibcallsInfo::libcallImplNameHit(uint16_t NameOffsetEntry,95 uint16_t StrOffset) {96 int NumAliases = 1;97 for (uint16_t Entry : ArrayRef(RuntimeLibcallNameOffsetTable)98 .drop_front(NameOffsetEntry + 1)) {99 if (Entry != StrOffset)100 break;101 ++NumAliases;102 }103 104 RTLIB::LibcallImpl ImplStart = static_cast<RTLIB::LibcallImpl>(105 &RuntimeLibcallNameOffsetTable[NameOffsetEntry] -106 &RuntimeLibcallNameOffsetTable[0]);107 return enum_seq(ImplStart,108 static_cast<RTLIB::LibcallImpl>(ImplStart + NumAliases));109}110 111bool RuntimeLibcallsInfo::isAAPCS_ABI(const Triple &TT, StringRef ABIName) {112 const ARM::ARMABI TargetABI = ARM::computeTargetABI(TT, ABIName);113 return TargetABI == ARM::ARM_ABI_AAPCS || TargetABI == ARM::ARM_ABI_AAPCS16;114}115 116bool RuntimeLibcallsInfo::darwinHasExp10(const Triple &TT) {117 switch (TT.getOS()) {118 case Triple::MacOSX:119 return !TT.isMacOSXVersionLT(10, 9);120 case Triple::IOS:121 return !TT.isOSVersionLT(7, 0);122 case Triple::DriverKit:123 case Triple::TvOS:124 case Triple::WatchOS:125 case Triple::XROS:126 case Triple::BridgeOS:127 return true;128 default:129 return false;130 }131}132 133/// TODO: There is really no guarantee that sizeof(size_t) is equal to the index134/// size of the default address space. This matches TargetLibraryInfo and should135/// be kept in sync.136static IntegerType *getSizeTType(LLVMContext &Ctx, const DataLayout &DL) {137 return DL.getIndexType(Ctx, /*AddressSpace=*/0);138}139 140std::pair<FunctionType *, AttributeList>141RuntimeLibcallsInfo::getFunctionTy(LLVMContext &Ctx, const Triple &TT,142 const DataLayout &DL,143 RTLIB::LibcallImpl LibcallImpl) const {144 // TODO: NoCallback probably unsafe in general145 static constexpr Attribute::AttrKind CommonFnAttrs[] = {146 Attribute::MustProgress, Attribute::NoCallback, Attribute::NoFree,147 Attribute::NoSync, Attribute::NoUnwind, Attribute::WillReturn};148 static constexpr Attribute::AttrKind MemoryFnAttrs[] = {149 Attribute::MustProgress, Attribute::NoUnwind, Attribute::WillReturn};150 static constexpr Attribute::AttrKind CommonPtrArgAttrs[] = {151 Attribute::NoAlias, Attribute::WriteOnly, Attribute::NonNull};152 153 switch (LibcallImpl) {154 case RTLIB::impl___sincos_stret:155 case RTLIB::impl___sincosf_stret: {156 if (!darwinHasSinCosStret(TT)) // Non-darwin currently unexpected157 return {};158 159 Type *ScalarTy = LibcallImpl == RTLIB::impl___sincosf_stret160 ? Type::getFloatTy(Ctx)161 : Type::getDoubleTy(Ctx);162 163 AttrBuilder FuncAttrBuilder(Ctx);164 for (Attribute::AttrKind Attr : CommonFnAttrs)165 FuncAttrBuilder.addAttribute(Attr);166 167 const bool UseSret =168 TT.isX86_32() || ((TT.isARM() || TT.isThumb()) &&169 ARM::computeTargetABI(TT) == ARM::ARM_ABI_APCS);170 171 FuncAttrBuilder.addMemoryAttr(MemoryEffects::argumentOrErrnoMemOnly(172 UseSret ? ModRefInfo::Mod : ModRefInfo::NoModRef, ModRefInfo::Mod));173 174 AttributeList Attrs;175 Attrs = Attrs.addFnAttributes(Ctx, FuncAttrBuilder);176 177 if (UseSret) {178 AttrBuilder AttrBuilder(Ctx);179 StructType *StructTy = StructType::get(ScalarTy, ScalarTy);180 AttrBuilder.addStructRetAttr(StructTy);181 AttrBuilder.addAlignmentAttr(DL.getABITypeAlign(StructTy));182 FunctionType *FuncTy = FunctionType::get(183 Type::getVoidTy(Ctx), {DL.getAllocaPtrType(Ctx), ScalarTy}, false);184 185 return {FuncTy, Attrs.addParamAttributes(Ctx, 0, AttrBuilder)};186 }187 188 Type *RetTy =189 LibcallImpl == RTLIB::impl___sincosf_stret && TT.isX86_64()190 ? static_cast<Type *>(FixedVectorType::get(ScalarTy, 2))191 : static_cast<Type *>(StructType::get(ScalarTy, ScalarTy));192 193 return {FunctionType::get(RetTy, {ScalarTy}, false), Attrs};194 }195 case RTLIB::impl_malloc:196 case RTLIB::impl_calloc: {197 AttrBuilder FuncAttrBuilder(Ctx);198 for (Attribute::AttrKind Attr : MemoryFnAttrs)199 FuncAttrBuilder.addAttribute(Attr);200 FuncAttrBuilder.addAttribute(Attribute::NoFree);201 202 AllocFnKind AllocKind = AllocFnKind::Alloc;203 if (LibcallImpl == RTLIB::impl_malloc)204 AllocKind |= AllocFnKind::Uninitialized;205 206 // TODO: Set memory attribute207 FuncAttrBuilder.addAllocKindAttr(AllocKind);208 FuncAttrBuilder.addAttribute("alloc-family", "malloc");209 FuncAttrBuilder.addAllocSizeAttr(0, LibcallImpl == RTLIB::impl_malloc210 ? std::nullopt211 : std::make_optional(1));212 213 AttributeList Attrs;214 Attrs = Attrs.addFnAttributes(Ctx, FuncAttrBuilder);215 216 {217 AttrBuilder ArgAttrBuilder(Ctx);218 for (Attribute::AttrKind AK : CommonPtrArgAttrs)219 ArgAttrBuilder.addAttribute(AK);220 221 Attrs = Attrs.addRetAttribute(Ctx, Attribute::NoUndef);222 Attrs = Attrs.addRetAttribute(Ctx, Attribute::NoAlias);223 Attrs = Attrs.addParamAttribute(Ctx, 0, Attribute::NoUndef);224 if (LibcallImpl == RTLIB::impl_calloc)225 Attrs = Attrs.addParamAttribute(Ctx, 1, Attribute::NoUndef);226 }227 228 IntegerType *SizeT = getSizeTType(Ctx, DL);229 PointerType *PtrTy = PointerType::get(Ctx, 0);230 SmallVector<Type *, 2> ArgTys = {SizeT};231 if (LibcallImpl == RTLIB::impl_calloc)232 ArgTys.push_back(SizeT);233 234 return {FunctionType::get(PtrTy, ArgTys, false), Attrs};235 }236 case RTLIB::impl_free: {237 // TODO: Set memory attribute238 AttrBuilder FuncAttrBuilder(Ctx);239 for (Attribute::AttrKind Attr : MemoryFnAttrs)240 FuncAttrBuilder.addAttribute(Attr);241 242 FuncAttrBuilder.addAllocKindAttr(AllocFnKind::Free);243 FuncAttrBuilder.addAttribute("alloc-family", "malloc");244 245 AttributeList Attrs;246 Attrs = Attrs.addFnAttributes(Ctx, FuncAttrBuilder);247 248 {249 AttrBuilder ArgAttrBuilder(Ctx);250 ArgAttrBuilder.addAttribute(Attribute::NoUndef);251 ArgAttrBuilder.addAttribute(Attribute::AllocatedPointer);252 ArgAttrBuilder.addCapturesAttr(CaptureInfo::none());253 Attrs = Attrs.addParamAttributes(Ctx, 0, ArgAttrBuilder);254 }255 256 return {FunctionType::get(Type::getVoidTy(Ctx), {PointerType::get(Ctx, 0)},257 false),258 Attrs};259 }260 case RTLIB::impl_sqrtf:261 case RTLIB::impl_sqrt: {262 AttrBuilder FuncAttrBuilder(Ctx);263 264 for (Attribute::AttrKind Attr : CommonFnAttrs)265 FuncAttrBuilder.addAttribute(Attr);266 FuncAttrBuilder.addMemoryAttr(MemoryEffects::errnoMemOnly(ModRefInfo::Mod));267 268 AttributeList Attrs;269 Attrs = Attrs.addFnAttributes(Ctx, FuncAttrBuilder);270 271 Type *ScalarTy = LibcallImpl == RTLIB::impl_sqrtf ? Type::getFloatTy(Ctx)272 : Type::getDoubleTy(Ctx);273 FunctionType *FuncTy = FunctionType::get(ScalarTy, {ScalarTy}, false);274 275 Attrs = Attrs.addRetAttribute(276 Ctx, Attribute::getWithNoFPClass(Ctx, fcNegInf | fcNegSubnormal |277 fcNegNormal));278 return {FuncTy, Attrs};279 }280 case RTLIB::impl__ZGVnN2vl8_modf:281 case RTLIB::impl__ZGVnN4vl4_modff:282 case RTLIB::impl__ZGVsNxvl8_modf:283 case RTLIB::impl__ZGVsNxvl4_modff:284 case RTLIB::impl_armpl_vmodfq_f64:285 case RTLIB::impl_armpl_vmodfq_f32:286 case RTLIB::impl_armpl_svmodf_f64_x:287 case RTLIB::impl_armpl_svmodf_f32_x: {288 AttrBuilder FuncAttrBuilder(Ctx);289 290 bool IsF32 = LibcallImpl == RTLIB::impl__ZGVnN4vl4_modff ||291 LibcallImpl == RTLIB::impl__ZGVsNxvl4_modff ||292 LibcallImpl == RTLIB::impl_armpl_vmodfq_f32 ||293 LibcallImpl == RTLIB::impl_armpl_svmodf_f32_x;294 295 bool IsScalable = LibcallImpl == RTLIB::impl__ZGVsNxvl8_modf ||296 LibcallImpl == RTLIB::impl__ZGVsNxvl4_modff ||297 LibcallImpl == RTLIB::impl_armpl_svmodf_f64_x ||298 LibcallImpl == RTLIB::impl_armpl_svmodf_f32_x;299 300 Type *ScalarTy = IsF32 ? Type::getFloatTy(Ctx) : Type::getDoubleTy(Ctx);301 unsigned EC = IsF32 ? 4 : 2;302 VectorType *VecTy = VectorType::get(ScalarTy, EC, IsScalable);303 304 for (Attribute::AttrKind Attr : CommonFnAttrs)305 FuncAttrBuilder.addAttribute(Attr);306 FuncAttrBuilder.addMemoryAttr(MemoryEffects::argMemOnly(ModRefInfo::Mod));307 308 AttributeList Attrs;309 Attrs = Attrs.addFnAttributes(Ctx, FuncAttrBuilder);310 311 {312 AttrBuilder ArgAttrBuilder(Ctx);313 for (Attribute::AttrKind AK : CommonPtrArgAttrs)314 ArgAttrBuilder.addAttribute(AK);315 ArgAttrBuilder.addAlignmentAttr(DL.getABITypeAlign(VecTy));316 Attrs = Attrs.addParamAttributes(Ctx, 1, ArgAttrBuilder);317 }318 319 PointerType *PtrTy = PointerType::get(Ctx, 0);320 SmallVector<Type *, 4> ArgTys = {VecTy, PtrTy};321 if (hasVectorMaskArgument(LibcallImpl))322 ArgTys.push_back(VectorType::get(Type::getInt1Ty(Ctx), EC, IsScalable));323 324 return {FunctionType::get(VecTy, ArgTys, false), Attrs};325 }326 case RTLIB::impl__ZGVnN2vl8l8_sincos:327 case RTLIB::impl__ZGVnN4vl4l4_sincosf:328 case RTLIB::impl__ZGVsNxvl8l8_sincos:329 case RTLIB::impl__ZGVsNxvl4l4_sincosf:330 case RTLIB::impl_armpl_vsincosq_f64:331 case RTLIB::impl_armpl_vsincosq_f32:332 case RTLIB::impl_armpl_svsincos_f64_x:333 case RTLIB::impl_armpl_svsincos_f32_x:334 case RTLIB::impl__ZGVnN4vl4l4_sincospif:335 case RTLIB::impl__ZGVnN2vl8l8_sincospi:336 case RTLIB::impl__ZGVsNxvl4l4_sincospif:337 case RTLIB::impl__ZGVsNxvl8l8_sincospi:338 case RTLIB::impl_armpl_vsincospiq_f32:339 case RTLIB::impl_armpl_vsincospiq_f64:340 case RTLIB::impl_armpl_svsincospi_f32_x:341 case RTLIB::impl_armpl_svsincospi_f64_x: {342 AttrBuilder FuncAttrBuilder(Ctx);343 344 bool IsF32 = LibcallImpl == RTLIB::impl__ZGVnN4vl4l4_sincospif ||345 LibcallImpl == RTLIB::impl__ZGVsNxvl4l4_sincospif ||346 LibcallImpl == RTLIB::impl_armpl_vsincospiq_f32 ||347 LibcallImpl == RTLIB::impl_armpl_svsincospi_f32_x ||348 LibcallImpl == RTLIB::impl__ZGVnN4vl4l4_sincosf ||349 LibcallImpl == RTLIB::impl__ZGVsNxvl4l4_sincosf ||350 LibcallImpl == RTLIB::impl_armpl_vsincosq_f32 ||351 LibcallImpl == RTLIB::impl_armpl_svsincos_f32_x;352 353 Type *ScalarTy = IsF32 ? Type::getFloatTy(Ctx) : Type::getDoubleTy(Ctx);354 unsigned EC = IsF32 ? 4 : 2;355 356 bool IsScalable = LibcallImpl == RTLIB::impl__ZGVsNxvl8l8_sincos ||357 LibcallImpl == RTLIB::impl__ZGVsNxvl4l4_sincosf ||358 LibcallImpl == RTLIB::impl_armpl_svsincos_f32_x ||359 LibcallImpl == RTLIB::impl_armpl_svsincos_f64_x ||360 LibcallImpl == RTLIB::impl__ZGVsNxvl4l4_sincospif ||361 LibcallImpl == RTLIB::impl__ZGVsNxvl8l8_sincospi ||362 LibcallImpl == RTLIB::impl_armpl_svsincospi_f32_x ||363 LibcallImpl == RTLIB::impl_armpl_svsincospi_f64_x;364 VectorType *VecTy = VectorType::get(ScalarTy, EC, IsScalable);365 366 for (Attribute::AttrKind Attr : CommonFnAttrs)367 FuncAttrBuilder.addAttribute(Attr);368 FuncAttrBuilder.addMemoryAttr(MemoryEffects::argMemOnly(ModRefInfo::Mod));369 370 AttributeList Attrs;371 Attrs = Attrs.addFnAttributes(Ctx, FuncAttrBuilder);372 373 {374 AttrBuilder ArgAttrBuilder(Ctx);375 for (Attribute::AttrKind AK : CommonPtrArgAttrs)376 ArgAttrBuilder.addAttribute(AK);377 ArgAttrBuilder.addAlignmentAttr(DL.getABITypeAlign(VecTy));378 Attrs = Attrs.addParamAttributes(Ctx, 1, ArgAttrBuilder);379 Attrs = Attrs.addParamAttributes(Ctx, 2, ArgAttrBuilder);380 }381 382 PointerType *PtrTy = PointerType::get(Ctx, 0);383 SmallVector<Type *, 4> ArgTys = {VecTy, PtrTy, PtrTy};384 if (hasVectorMaskArgument(LibcallImpl))385 ArgTys.push_back(VectorType::get(Type::getInt1Ty(Ctx), EC, IsScalable));386 387 return {FunctionType::get(Type::getVoidTy(Ctx), ArgTys, false), Attrs};388 }389 default:390 return {};391 }392 393 return {};394}395 396bool RuntimeLibcallsInfo::hasVectorMaskArgument(RTLIB::LibcallImpl Impl) {397 /// FIXME: This should be generated by tablegen and support the argument at an398 /// arbitrary position399 switch (Impl) {400 case RTLIB::impl_armpl_svmodf_f64_x:401 case RTLIB::impl_armpl_svmodf_f32_x:402 case RTLIB::impl_armpl_svsincos_f32_x:403 case RTLIB::impl_armpl_svsincos_f64_x:404 case RTLIB::impl_armpl_svsincospi_f32_x:405 case RTLIB::impl_armpl_svsincospi_f64_x:406 return true;407 default:408 return false;409 }410}411