143 lines · cpp
1//===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Populates the VFABI attribute with the scalar-to-vector mappings10// from the TargetLibraryInfo.11//12//===----------------------------------------------------------------------===//13 14#include "llvm/Transforms/Utils/InjectTLIMappings.h"15#include "llvm/ADT/Statistic.h"16#include "llvm/Analysis/DemandedBits.h"17#include "llvm/Analysis/GlobalsModRef.h"18#include "llvm/Analysis/TargetLibraryInfo.h"19#include "llvm/Analysis/VectorUtils.h"20#include "llvm/IR/InstIterator.h"21#include "llvm/IR/VFABIDemangler.h"22#include "llvm/Transforms/Utils/ModuleUtils.h"23 24using namespace llvm;25 26#define DEBUG_TYPE "inject-tli-mappings"27 28STATISTIC(NumCallInjected,29 "Number of calls in which the mappings have been injected.");30 31STATISTIC(NumVFDeclAdded,32 "Number of function declarations that have been added.");33STATISTIC(NumCompUsedAdded,34 "Number of `@llvm.compiler.used` operands that have been added.");35 36/// A helper function that adds the vector variant declaration for vectorizing37/// the CallInst \p CI with a vectorization factor of \p VF lanes. For each38/// mapping, TLI provides a VABI prefix, which contains all information required39/// to create vector function declaration.40static void addVariantDeclaration(CallInst &CI, const ElementCount &VF,41 const VecDesc *VD) {42 Module *M = CI.getModule();43 FunctionType *ScalarFTy = CI.getFunctionType();44 45 assert(!ScalarFTy->isVarArg() && "VarArg functions are not supported.");46 47 const std::optional<VFInfo> Info = VFABI::tryDemangleForVFABI(48 VD->getVectorFunctionABIVariantString(), ScalarFTy);49 50 assert(Info && "Failed to demangle vector variant");51 assert(Info->Shape.VF == VF && "Mangled name does not match VF");52 53 const StringRef VFName = VD->getVectorFnName();54 FunctionType *VectorFTy = VFABI::createFunctionType(*Info, ScalarFTy);55 Function *VecFunc =56 Function::Create(VectorFTy, Function::ExternalLinkage, VFName, M);57 VecFunc->copyAttributesFrom(CI.getCalledFunction());58 if (auto CC = VD->getCallingConv())59 VecFunc->setCallingConv(*CC);60 ++NumVFDeclAdded;61 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName62 << "` of type " << *VectorFTy << "\n");63 64 // Make function declaration (without a body) "sticky" in the IR by65 // listing it in the @llvm.compiler.used intrinsic.66 assert(!VecFunc->size() && "VFABI attribute requires `@llvm.compiler.used` "67 "only on declarations.");68 appendToCompilerUsed(*M, {VecFunc});69 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName70 << "` to `@llvm.compiler.used`.\n");71 ++NumCompUsedAdded;72}73 74static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {75 // This is needed to make sure we don't query the TLI for calls to76 // bitcast of function pointers, like `%call = call i32 (i32*, ...)77 // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,78 // as such calls make the `isFunctionVectorizable` raise an79 // exception.80 if (CI.isNoBuiltin() || !CI.getCalledFunction())81 return;82 83 StringRef ScalarName = CI.getCalledFunction()->getName();84 85 // Nothing to be done if the TLI thinks the function is not86 // vectorizable.87 if (!TLI.isFunctionVectorizable(ScalarName))88 return;89 SmallVector<std::string, 8> Mappings;90 VFABI::getVectorVariantNames(CI, Mappings);91 Module *M = CI.getModule();92 const SetVector<StringRef> OriginalSetOfMappings(llvm::from_range, Mappings);93 94 auto AddVariantDecl = [&](const ElementCount &VF, bool Predicate) {95 const VecDesc *VD = TLI.getVectorMappingInfo(ScalarName, VF, Predicate);96 if (VD && !VD->getVectorFnName().empty()) {97 std::string MangledName = VD->getVectorFunctionABIVariantString();98 if (!OriginalSetOfMappings.count(MangledName)) {99 Mappings.push_back(MangledName);100 ++NumCallInjected;101 }102 Function *VariantF = M->getFunction(VD->getVectorFnName());103 if (!VariantF)104 addVariantDeclaration(CI, VF, VD);105 }106 };107 108 // All VFs in the TLI are powers of 2.109 ElementCount WidestFixedVF, WidestScalableVF;110 TLI.getWidestVF(ScalarName, WidestFixedVF, WidestScalableVF);111 112 for (bool Predicated : {false, true}) {113 for (ElementCount VF = ElementCount::getFixed(2);114 ElementCount::isKnownLE(VF, WidestFixedVF); VF *= 2)115 AddVariantDecl(VF, Predicated);116 117 for (ElementCount VF = ElementCount::getScalable(2);118 ElementCount::isKnownLE(VF, WidestScalableVF); VF *= 2)119 AddVariantDecl(VF, Predicated);120 }121 122 VFABI::setVectorVariantNames(&CI, Mappings);123}124 125static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {126 for (auto &I : instructions(F))127 if (auto CI = dyn_cast<CallInst>(&I))128 addMappingsFromTLI(TLI, *CI);129 // Even if the pass adds IR attributes, the analyses are preserved.130 return false;131}132 133////////////////////////////////////////////////////////////////////////////////134// New pass manager implementation.135////////////////////////////////////////////////////////////////////////////////136PreservedAnalyses InjectTLIMappings::run(Function &F,137 FunctionAnalysisManager &AM) {138 const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);139 runImpl(TLI, F);140 // Even if the pass adds IR attributes, the analyses are preserved.141 return PreservedAnalyses::all();142}143