1020 lines · cpp
1//===-- ExpandVariadicsPass.cpp --------------------------------*- C++ -*-=//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This is an optimization pass for variadic functions. If called from codegen,10// it can serve as the implementation of variadic functions for a given target.11//12// The strategy is to turn the ... part of a variadic function into a va_list13// and fix up the call sites. The majority of the pass is target independent.14// The exceptions are the va_list type itself and the rules for where to store15// variables in memory such that va_arg can iterate over them given a va_list.16//17// The majority of the plumbing is splitting the variadic function into a18// single basic block that packs the variadic arguments into a va_list and19// a second function that does the work of the original. That packing is20// exactly what is done by va_start. Further, the transform from ... to va_list21// replaced va_start with an operation to copy a va_list from the new argument,22// which is exactly a va_copy. This is useful for reducing target-dependence.23//24// A va_list instance is a forward iterator, where the primary operation va_arg25// is dereference-then-increment. This interface forces significant convergent26// evolution between target specific implementations. The variation in runtime27// data layout is limited to that representable by the iterator, parameterised28// by the type passed to the va_arg instruction.29//30// Therefore the majority of the target specific subtlety is packing arguments31// into a stack allocated buffer such that a va_list can be initialised with it32// and the va_arg expansion for the target will find the arguments at runtime.33//34// The aggregate effect is to unblock other transforms, most critically the35// general purpose inliner. Known calls to variadic functions become zero cost.36//37// Consistency with clang is primarily tested by emitting va_arg using clang38// then expanding the variadic functions using this pass, followed by trying39// to constant fold the functions to no-ops.40//41// Target specific behaviour is tested in IR - mainly checking that values are42// put into positions in call frames that make sense for that particular target.43//44// There is one "clever" invariant in use. va_start intrinsics that are not45// within a varidic functions are an error in the IR verifier. When this46// transform moves blocks from a variadic function into a fixed arity one, it47// moves va_start intrinsics along with everything else. That means that the48// va_start intrinsics that need to be rewritten to use the trailing argument49// are exactly those that are in non-variadic functions so no further state50// is needed to distinguish those that need to be rewritten.51//52//===----------------------------------------------------------------------===//53 54#include "llvm/Transforms/IPO/ExpandVariadics.h"55#include "llvm/ADT/SmallVector.h"56#include "llvm/IR/IRBuilder.h"57#include "llvm/IR/IntrinsicInst.h"58#include "llvm/IR/Module.h"59#include "llvm/IR/PassManager.h"60#include "llvm/InitializePasses.h"61#include "llvm/Pass.h"62#include "llvm/Support/CommandLine.h"63#include "llvm/TargetParser/Triple.h"64#include "llvm/Transforms/Utils/ModuleUtils.h"65 66#define DEBUG_TYPE "expand-variadics"67 68using namespace llvm;69 70namespace {71 72cl::opt<ExpandVariadicsMode> ExpandVariadicsModeOption(73 DEBUG_TYPE "-override", cl::desc("Override the behaviour of " DEBUG_TYPE),74 cl::init(ExpandVariadicsMode::Unspecified),75 cl::values(clEnumValN(ExpandVariadicsMode::Unspecified, "unspecified",76 "Use the implementation defaults"),77 clEnumValN(ExpandVariadicsMode::Disable, "disable",78 "Disable the pass entirely"),79 clEnumValN(ExpandVariadicsMode::Optimize, "optimize",80 "Optimise without changing ABI"),81 clEnumValN(ExpandVariadicsMode::Lowering, "lowering",82 "Change variadic calling convention")));83 84bool commandLineOverride() {85 return ExpandVariadicsModeOption != ExpandVariadicsMode::Unspecified;86}87 88// Instances of this class encapsulate the target-dependant behaviour as a89// function of triple. Implementing a new ABI is adding a case to the switch90// in create(llvm::Triple) at the end of this file.91// This class may end up instantiated in TargetMachine instances, keeping it92// here for now until enough targets are implemented for the API to evolve.93class VariadicABIInfo {94protected:95 VariadicABIInfo() = default;96 97public:98 static std::unique_ptr<VariadicABIInfo> create(const Triple &T);99 100 // Allow overriding whether the pass runs on a per-target basis101 virtual bool enableForTarget() = 0;102 103 // Whether a valist instance is passed by value or by address104 // I.e. does it need to be alloca'ed and stored into, or can105 // it be passed directly in a SSA register106 virtual bool vaListPassedInSSARegister() = 0;107 108 // The type of a va_list iterator object109 virtual Type *vaListType(LLVMContext &Ctx) = 0;110 111 // The type of a va_list as a function argument as lowered by C112 virtual Type *vaListParameterType(Module &M) = 0;113 114 // Initialize an allocated va_list object to point to an already115 // initialized contiguous memory region.116 // Return the value to pass as the va_list argument117 virtual Value *initializeVaList(Module &M, LLVMContext &Ctx,118 IRBuilder<> &Builder, AllocaInst *VaList,119 Value *Buffer) = 0;120 121 struct VAArgSlotInfo {122 Align DataAlign; // With respect to the call frame123 bool Indirect; // Passed via a pointer124 };125 virtual VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) = 0;126 127 // Targets implemented so far all have the same trivial lowering for these128 bool vaEndIsNop() { return true; }129 bool vaCopyIsMemcpy() { return true; }130 131 virtual ~VariadicABIInfo() = default;132};133 134class ExpandVariadics : public ModulePass {135 136 // The pass construction sets the default to optimize when called from middle137 // end and lowering when called from the backend. The command line variable138 // overrides that. This is useful for testing and debugging. It also allows139 // building an applications with variadic functions wholly removed if one140 // has sufficient control over the dependencies, e.g. a statically linked141 // clang that has no variadic function calls remaining in the binary.142 143public:144 static char ID;145 const ExpandVariadicsMode Mode;146 std::unique_ptr<VariadicABIInfo> ABI;147 148 ExpandVariadics(ExpandVariadicsMode Mode)149 : ModulePass(ID),150 Mode(commandLineOverride() ? ExpandVariadicsModeOption : Mode) {}151 152 StringRef getPassName() const override { return "Expand variadic functions"; }153 154 bool rewriteABI() { return Mode == ExpandVariadicsMode::Lowering; }155 156 bool runOnModule(Module &M) override;157 158 bool runOnFunction(Module &M, IRBuilder<> &Builder, Function *F);159 160 Function *replaceAllUsesWithNewDeclaration(Module &M,161 Function *OriginalFunction);162 163 Function *deriveFixedArityReplacement(Module &M, IRBuilder<> &Builder,164 Function *OriginalFunction);165 166 Function *defineVariadicWrapper(Module &M, IRBuilder<> &Builder,167 Function *VariadicWrapper,168 Function *FixedArityReplacement);169 170 bool expandCall(Module &M, IRBuilder<> &Builder, CallBase *CB, FunctionType *,171 Function *NF);172 173 // The intrinsic functions va_copy and va_end are removed unconditionally.174 // They correspond to a memcpy and a no-op on all implemented targets.175 // The va_start intrinsic is removed from basic blocks that were not created176 // by this pass, some may remain if needed to maintain the external ABI.177 178 template <Intrinsic::ID ID, typename InstructionType>179 bool expandIntrinsicUsers(Module &M, IRBuilder<> &Builder,180 PointerType *IntrinsicArgType) {181 bool Changed = false;182 const DataLayout &DL = M.getDataLayout();183 if (Function *Intrinsic =184 Intrinsic::getDeclarationIfExists(&M, ID, {IntrinsicArgType})) {185 for (User *U : make_early_inc_range(Intrinsic->users()))186 if (auto *I = dyn_cast<InstructionType>(U))187 Changed |= expandVAIntrinsicCall(Builder, DL, I);188 189 if (Intrinsic->use_empty())190 Intrinsic->eraseFromParent();191 }192 return Changed;193 }194 195 bool expandVAIntrinsicUsersWithAddrspace(Module &M, IRBuilder<> &Builder,196 unsigned Addrspace) {197 auto &Ctx = M.getContext();198 PointerType *IntrinsicArgType = PointerType::get(Ctx, Addrspace);199 bool Changed = false;200 201 // expand vastart before vacopy as vastart may introduce a vacopy202 Changed |= expandIntrinsicUsers<Intrinsic::vastart, VAStartInst>(203 M, Builder, IntrinsicArgType);204 Changed |= expandIntrinsicUsers<Intrinsic::vaend, VAEndInst>(205 M, Builder, IntrinsicArgType);206 Changed |= expandIntrinsicUsers<Intrinsic::vacopy, VACopyInst>(207 M, Builder, IntrinsicArgType);208 return Changed;209 }210 211 bool expandVAIntrinsicCall(IRBuilder<> &Builder, const DataLayout &DL,212 VAStartInst *Inst);213 214 bool expandVAIntrinsicCall(IRBuilder<> &, const DataLayout &,215 VAEndInst *Inst);216 217 bool expandVAIntrinsicCall(IRBuilder<> &Builder, const DataLayout &DL,218 VACopyInst *Inst);219 220 FunctionType *inlinableVariadicFunctionType(Module &M, FunctionType *FTy) {221 // The type of "FTy" with the ... removed and a va_list appended222 SmallVector<Type *> ArgTypes(FTy->params());223 ArgTypes.push_back(ABI->vaListParameterType(M));224 return FunctionType::get(FTy->getReturnType(), ArgTypes,225 /*IsVarArgs=*/false);226 }227 228 bool expansionApplicableToFunction(Module &M, Function *F) {229 if (F->isIntrinsic() || !F->isVarArg() ||230 F->hasFnAttribute(Attribute::Naked))231 return false;232 233 if (F->getCallingConv() != CallingConv::C)234 return false;235 236 if (rewriteABI())237 return true;238 239 if (!F->hasExactDefinition())240 return false;241 242 return true;243 }244 245 bool expansionApplicableToFunctionCall(CallBase *CB) {246 if (CallInst *CI = dyn_cast<CallInst>(CB)) {247 if (CI->isMustTailCall()) {248 // Cannot expand musttail calls249 return false;250 }251 252 if (CI->getCallingConv() != CallingConv::C)253 return false;254 255 return true;256 }257 258 if (isa<InvokeInst>(CB)) {259 // Invoke not implemented in initial implementation of pass260 return false;261 }262 263 // Other unimplemented derivative of CallBase264 return false;265 }266 267 class ExpandedCallFrame {268 // Helper for constructing an alloca instance containing the arguments bound269 // to the variadic ... parameter, rearranged to allow indexing through a270 // va_list iterator271 enum { N = 4 };272 SmallVector<Type *, N> FieldTypes;273 enum Tag { Store, Memcpy, Padding };274 SmallVector<std::tuple<Value *, uint64_t, Tag>, N> Source;275 276 template <Tag tag> void append(Type *FieldType, Value *V, uint64_t Bytes) {277 FieldTypes.push_back(FieldType);278 Source.push_back({V, Bytes, tag});279 }280 281 public:282 void store(LLVMContext &Ctx, Type *T, Value *V) { append<Store>(T, V, 0); }283 284 void memcpy(LLVMContext &Ctx, Type *T, Value *V, uint64_t Bytes) {285 append<Memcpy>(T, V, Bytes);286 }287 288 void padding(LLVMContext &Ctx, uint64_t By) {289 append<Padding>(ArrayType::get(Type::getInt8Ty(Ctx), By), nullptr, 0);290 }291 292 size_t size() const { return FieldTypes.size(); }293 bool empty() const { return FieldTypes.empty(); }294 295 StructType *asStruct(LLVMContext &Ctx, StringRef Name) {296 const bool IsPacked = true;297 return StructType::create(Ctx, FieldTypes,298 (Twine(Name) + ".vararg").str(), IsPacked);299 }300 301 void initializeStructAlloca(const DataLayout &DL, IRBuilder<> &Builder,302 AllocaInst *Alloced) {303 304 StructType *VarargsTy = cast<StructType>(Alloced->getAllocatedType());305 306 for (size_t I = 0; I < size(); I++) {307 308 auto [V, bytes, tag] = Source[I];309 310 if (tag == Padding) {311 assert(V == nullptr);312 continue;313 }314 315 auto Dst = Builder.CreateStructGEP(VarargsTy, Alloced, I);316 317 assert(V != nullptr);318 319 if (tag == Store)320 Builder.CreateStore(V, Dst);321 322 if (tag == Memcpy)323 Builder.CreateMemCpy(Dst, {}, V, {}, bytes);324 }325 }326 };327};328 329bool ExpandVariadics::runOnModule(Module &M) {330 bool Changed = false;331 if (Mode == ExpandVariadicsMode::Disable)332 return Changed;333 334 Triple TT(M.getTargetTriple());335 ABI = VariadicABIInfo::create(TT);336 if (!ABI)337 return Changed;338 339 if (!ABI->enableForTarget())340 return Changed;341 342 auto &Ctx = M.getContext();343 const DataLayout &DL = M.getDataLayout();344 IRBuilder<> Builder(Ctx);345 346 // Lowering needs to run on all functions exactly once.347 // Optimize could run on functions containing va_start exactly once.348 for (Function &F : make_early_inc_range(M))349 Changed |= runOnFunction(M, Builder, &F);350 351 // After runOnFunction, all known calls to known variadic functions have been352 // replaced. va_start intrinsics are presently (and invalidly!) only present353 // in functions that used to be variadic and have now been replaced to take a354 // va_list instead. If lowering as opposed to optimising, calls to unknown355 // variadic functions have also been replaced.356 357 {358 // 0 and AllocaAddrSpace are sufficient for the targets implemented so far359 unsigned Addrspace = 0;360 Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace);361 362 Addrspace = DL.getAllocaAddrSpace();363 if (Addrspace != 0)364 Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace);365 }366 367 if (Mode != ExpandVariadicsMode::Lowering)368 return Changed;369 370 for (Function &F : make_early_inc_range(M)) {371 if (F.isDeclaration())372 continue;373 374 // Now need to track down indirect calls. Can't find those375 // by walking uses of variadic functions, need to crawl the instruction376 // stream. Fortunately this is only necessary for the ABI rewrite case.377 for (BasicBlock &BB : F) {378 for (Instruction &I : make_early_inc_range(BB)) {379 if (CallBase *CB = dyn_cast<CallBase>(&I)) {380 if (CB->isIndirectCall()) {381 FunctionType *FTy = CB->getFunctionType();382 if (FTy->isVarArg())383 Changed |= expandCall(M, Builder, CB, FTy, /*NF=*/nullptr);384 }385 }386 }387 }388 }389 390 return Changed;391}392 393bool ExpandVariadics::runOnFunction(Module &M, IRBuilder<> &Builder,394 Function *OriginalFunction) {395 bool Changed = false;396 397 if (!expansionApplicableToFunction(M, OriginalFunction))398 return Changed;399 400 [[maybe_unused]] const bool OriginalFunctionIsDeclaration =401 OriginalFunction->isDeclaration();402 assert(rewriteABI() || !OriginalFunctionIsDeclaration);403 404 // Declare a new function and redirect every use to that new function405 Function *VariadicWrapper =406 replaceAllUsesWithNewDeclaration(M, OriginalFunction);407 assert(VariadicWrapper->isDeclaration());408 assert(OriginalFunction->use_empty());409 410 // Create a new function taking va_list containing the implementation of the411 // original412 Function *FixedArityReplacement =413 deriveFixedArityReplacement(M, Builder, OriginalFunction);414 assert(OriginalFunction->isDeclaration());415 assert(FixedArityReplacement->isDeclaration() ==416 OriginalFunctionIsDeclaration);417 assert(VariadicWrapper->isDeclaration());418 419 // Create a single block forwarding wrapper that turns a ... into a va_list420 [[maybe_unused]] Function *VariadicWrapperDefine =421 defineVariadicWrapper(M, Builder, VariadicWrapper, FixedArityReplacement);422 assert(VariadicWrapperDefine == VariadicWrapper);423 assert(!VariadicWrapper->isDeclaration());424 425 // Add the prof metadata from the original function to the wrapper. Because426 // FixedArityReplacement is the owner of original function's prof metadata427 // after the splice, we need to transfer it to VariadicWrapper.428 VariadicWrapper->setMetadata(429 LLVMContext::MD_prof,430 FixedArityReplacement->getMetadata(LLVMContext::MD_prof));431 432 // We now have:433 // 1. the original function, now as a declaration with no uses434 // 2. a variadic function that unconditionally calls a fixed arity replacement435 // 3. a fixed arity function equivalent to the original function436 437 // Replace known calls to the variadic with calls to the va_list equivalent438 for (User *U : make_early_inc_range(VariadicWrapper->users())) {439 if (CallBase *CB = dyn_cast<CallBase>(U)) {440 Value *CalledOperand = CB->getCalledOperand();441 if (VariadicWrapper == CalledOperand)442 Changed |=443 expandCall(M, Builder, CB, VariadicWrapper->getFunctionType(),444 FixedArityReplacement);445 }446 }447 448 // The original function will be erased.449 // One of the two new functions will become a replacement for the original.450 // When preserving the ABI, the other is an internal implementation detail.451 // When rewriting the ABI, RAUW then the variadic one.452 Function *const ExternallyAccessible =453 rewriteABI() ? FixedArityReplacement : VariadicWrapper;454 Function *const InternalOnly =455 rewriteABI() ? VariadicWrapper : FixedArityReplacement;456 457 // The external function is the replacement for the original458 ExternallyAccessible->setLinkage(OriginalFunction->getLinkage());459 ExternallyAccessible->setVisibility(OriginalFunction->getVisibility());460 ExternallyAccessible->setComdat(OriginalFunction->getComdat());461 ExternallyAccessible->takeName(OriginalFunction);462 463 // Annotate the internal one as internal464 InternalOnly->setVisibility(GlobalValue::DefaultVisibility);465 InternalOnly->setLinkage(GlobalValue::InternalLinkage);466 467 // The original is unused and obsolete468 OriginalFunction->eraseFromParent();469 470 InternalOnly->removeDeadConstantUsers();471 472 if (rewriteABI()) {473 // All known calls to the function have been removed by expandCall474 // Resolve everything else by replaceAllUsesWith475 VariadicWrapper->replaceAllUsesWith(FixedArityReplacement);476 VariadicWrapper->eraseFromParent();477 }478 479 return Changed;480}481 482Function *483ExpandVariadics::replaceAllUsesWithNewDeclaration(Module &M,484 Function *OriginalFunction) {485 auto &Ctx = M.getContext();486 Function &F = *OriginalFunction;487 FunctionType *FTy = F.getFunctionType();488 Function *NF = Function::Create(FTy, F.getLinkage(), F.getAddressSpace());489 490 NF->setName(F.getName() + ".varargs");491 492 F.getParent()->getFunctionList().insert(F.getIterator(), NF);493 494 AttrBuilder ParamAttrs(Ctx);495 AttributeList Attrs = NF->getAttributes();496 Attrs = Attrs.addParamAttributes(Ctx, FTy->getNumParams(), ParamAttrs);497 NF->setAttributes(Attrs);498 499 OriginalFunction->replaceAllUsesWith(NF);500 return NF;501}502 503Function *504ExpandVariadics::deriveFixedArityReplacement(Module &M, IRBuilder<> &Builder,505 Function *OriginalFunction) {506 Function &F = *OriginalFunction;507 // The purpose here is split the variadic function F into two functions508 // One is a variadic function that bundles the passed argument into a va_list509 // and passes it to the second function. The second function does whatever510 // the original F does, except that it takes a va_list instead of the ...511 512 assert(expansionApplicableToFunction(M, &F));513 514 auto &Ctx = M.getContext();515 516 // Returned value isDeclaration() is equal to F.isDeclaration()517 // but that property is not invariant throughout this function518 const bool FunctionIsDefinition = !F.isDeclaration();519 520 FunctionType *FTy = F.getFunctionType();521 SmallVector<Type *> ArgTypes(FTy->params());522 ArgTypes.push_back(ABI->vaListParameterType(M));523 524 FunctionType *NFTy = inlinableVariadicFunctionType(M, FTy);525 Function *NF = Function::Create(NFTy, F.getLinkage(), F.getAddressSpace());526 527 // Note - same attribute handling as DeadArgumentElimination528 NF->copyAttributesFrom(&F);529 NF->setComdat(F.getComdat());530 F.getParent()->getFunctionList().insert(F.getIterator(), NF);531 NF->setName(F.getName() + ".valist");532 533 AttrBuilder ParamAttrs(Ctx);534 535 AttributeList Attrs = NF->getAttributes();536 Attrs = Attrs.addParamAttributes(Ctx, NFTy->getNumParams() - 1, ParamAttrs);537 NF->setAttributes(Attrs);538 539 // Splice the implementation into the new function with minimal changes540 if (FunctionIsDefinition) {541 NF->splice(NF->begin(), &F);542 543 auto NewArg = NF->arg_begin();544 for (Argument &Arg : F.args()) {545 Arg.replaceAllUsesWith(NewArg);546 NewArg->setName(Arg.getName()); // takeName without killing the old one547 ++NewArg;548 }549 NewArg->setName("varargs");550 }551 552 SmallVector<std::pair<unsigned, MDNode *>, 1> MDs;553 F.getAllMetadata(MDs);554 for (auto [KindID, Node] : MDs)555 NF->addMetadata(KindID, *Node);556 F.clearMetadata();557 558 return NF;559}560 561Function *562ExpandVariadics::defineVariadicWrapper(Module &M, IRBuilder<> &Builder,563 Function *VariadicWrapper,564 Function *FixedArityReplacement) {565 auto &Ctx = Builder.getContext();566 const DataLayout &DL = M.getDataLayout();567 assert(VariadicWrapper->isDeclaration());568 Function &F = *VariadicWrapper;569 570 assert(F.isDeclaration());571 Type *VaListTy = ABI->vaListType(Ctx);572 573 auto *BB = BasicBlock::Create(Ctx, "entry", &F);574 Builder.SetInsertPoint(BB);575 576 AllocaInst *VaListInstance =577 Builder.CreateAlloca(VaListTy, nullptr, "va_start");578 579 Builder.CreateLifetimeStart(VaListInstance);580 581 Builder.CreateIntrinsic(Intrinsic::vastart, {DL.getAllocaPtrType(Ctx)},582 {VaListInstance});583 584 SmallVector<Value *> Args(llvm::make_pointer_range(F.args()));585 586 Type *ParameterType = ABI->vaListParameterType(M);587 if (ABI->vaListPassedInSSARegister())588 Args.push_back(Builder.CreateLoad(ParameterType, VaListInstance));589 else590 Args.push_back(Builder.CreateAddrSpaceCast(VaListInstance, ParameterType));591 592 CallInst *Result = Builder.CreateCall(FixedArityReplacement, Args);593 594 Builder.CreateIntrinsic(Intrinsic::vaend, {DL.getAllocaPtrType(Ctx)},595 {VaListInstance});596 Builder.CreateLifetimeEnd(VaListInstance);597 598 if (Result->getType()->isVoidTy())599 Builder.CreateRetVoid();600 else601 Builder.CreateRet(Result);602 603 return VariadicWrapper;604}605 606bool ExpandVariadics::expandCall(Module &M, IRBuilder<> &Builder, CallBase *CB,607 FunctionType *VarargFunctionType,608 Function *NF) {609 bool Changed = false;610 const DataLayout &DL = M.getDataLayout();611 612 if (!expansionApplicableToFunctionCall(CB)) {613 if (rewriteABI())614 report_fatal_error("Cannot lower callbase instruction");615 return Changed;616 }617 618 // This is tricky. The call instruction's function type might not match619 // the type of the caller. When optimising, can leave it unchanged.620 // Webassembly detects that inconsistency and repairs it.621 FunctionType *FuncType = CB->getFunctionType();622 if (FuncType != VarargFunctionType) {623 if (!rewriteABI())624 return Changed;625 FuncType = VarargFunctionType;626 }627 628 auto &Ctx = CB->getContext();629 630 Align MaxFieldAlign(1);631 632 // The strategy is to allocate a call frame containing the variadic633 // arguments laid out such that a target specific va_list can be initialized634 // with it, such that target specific va_arg instructions will correctly635 // iterate over it. This means getting the alignment right and sometimes636 // embedding a pointer to the value instead of embedding the value itself.637 638 Function *CBF = CB->getParent()->getParent();639 640 ExpandedCallFrame Frame;641 642 uint64_t CurrentOffset = 0;643 644 for (unsigned I = FuncType->getNumParams(), E = CB->arg_size(); I < E; ++I) {645 Value *ArgVal = CB->getArgOperand(I);646 const bool IsByVal = CB->paramHasAttr(I, Attribute::ByVal);647 const bool IsByRef = CB->paramHasAttr(I, Attribute::ByRef);648 649 // The type of the value being passed, decoded from byval/byref metadata if650 // required651 Type *const UnderlyingType = IsByVal ? CB->getParamByValType(I)652 : IsByRef ? CB->getParamByRefType(I)653 : ArgVal->getType();654 const uint64_t UnderlyingSize =655 DL.getTypeAllocSize(UnderlyingType).getFixedValue();656 657 // The type to be written into the call frame658 Type *FrameFieldType = UnderlyingType;659 660 // The value to copy from when initialising the frame alloca661 Value *SourceValue = ArgVal;662 663 VariadicABIInfo::VAArgSlotInfo SlotInfo = ABI->slotInfo(DL, UnderlyingType);664 665 if (SlotInfo.Indirect) {666 // The va_arg lowering loads through a pointer. Set up an alloca to aim667 // that pointer at.668 Builder.SetInsertPointPastAllocas(CBF);669 Builder.SetCurrentDebugLocation(CB->getStableDebugLoc());670 Value *CallerCopy =671 Builder.CreateAlloca(UnderlyingType, nullptr, "IndirectAlloca");672 673 Builder.SetInsertPoint(CB);674 if (IsByVal)675 Builder.CreateMemCpy(CallerCopy, {}, ArgVal, {}, UnderlyingSize);676 else677 Builder.CreateStore(ArgVal, CallerCopy);678 679 // Indirection now handled, pass the alloca ptr by value680 FrameFieldType = DL.getAllocaPtrType(Ctx);681 SourceValue = CallerCopy;682 }683 684 // Alignment of the value within the frame685 // This probably needs to be controllable as a function of type686 Align DataAlign = SlotInfo.DataAlign;687 688 MaxFieldAlign = std::max(MaxFieldAlign, DataAlign);689 690 uint64_t DataAlignV = DataAlign.value();691 if (uint64_t Rem = CurrentOffset % DataAlignV) {692 // Inject explicit padding to deal with alignment requirements693 uint64_t Padding = DataAlignV - Rem;694 Frame.padding(Ctx, Padding);695 CurrentOffset += Padding;696 }697 698 if (SlotInfo.Indirect) {699 Frame.store(Ctx, FrameFieldType, SourceValue);700 } else {701 if (IsByVal)702 Frame.memcpy(Ctx, FrameFieldType, SourceValue, UnderlyingSize);703 else704 Frame.store(Ctx, FrameFieldType, SourceValue);705 }706 707 CurrentOffset += DL.getTypeAllocSize(FrameFieldType).getFixedValue();708 }709 710 if (Frame.empty()) {711 // Not passing any arguments, hopefully va_arg won't try to read any712 // Creating a single byte frame containing nothing to point the va_list713 // instance as that is less special-casey in the compiler and probably714 // easier to interpret in a debugger.715 Frame.padding(Ctx, 1);716 }717 718 StructType *VarargsTy = Frame.asStruct(Ctx, CBF->getName());719 720 // The struct instance needs to be at least MaxFieldAlign for the alignment of721 // the fields to be correct at runtime. Use the native stack alignment instead722 // if that's greater as that tends to give better codegen.723 // This is an awkward way to guess whether there is a known stack alignment724 // without hitting an assert in DL.getStackAlignment, 1024 is an arbitrary725 // number likely to be greater than the natural stack alignment.726 Align AllocaAlign = MaxFieldAlign;727 if (MaybeAlign StackAlign = DL.getStackAlignment();728 StackAlign && *StackAlign > AllocaAlign)729 AllocaAlign = *StackAlign;730 731 // Put the alloca to hold the variadic args in the entry basic block.732 Builder.SetInsertPointPastAllocas(CBF);733 734 // SetCurrentDebugLocation when the builder SetInsertPoint method does not735 Builder.SetCurrentDebugLocation(CB->getStableDebugLoc());736 737 // The awkward construction here is to set the alignment on the instance738 AllocaInst *Alloced = Builder.Insert(739 new AllocaInst(VarargsTy, DL.getAllocaAddrSpace(), nullptr, AllocaAlign),740 "vararg_buffer");741 Changed = true;742 assert(Alloced->getAllocatedType() == VarargsTy);743 744 // Initialize the fields in the struct745 Builder.SetInsertPoint(CB);746 Builder.CreateLifetimeStart(Alloced);747 Frame.initializeStructAlloca(DL, Builder, Alloced);748 749 const unsigned NumArgs = FuncType->getNumParams();750 SmallVector<Value *> Args(CB->arg_begin(), CB->arg_begin() + NumArgs);751 752 // Initialize a va_list pointing to that struct and pass it as the last753 // argument754 AllocaInst *VaList = nullptr;755 {756 if (!ABI->vaListPassedInSSARegister()) {757 Type *VaListTy = ABI->vaListType(Ctx);758 Builder.SetInsertPointPastAllocas(CBF);759 Builder.SetCurrentDebugLocation(CB->getStableDebugLoc());760 VaList = Builder.CreateAlloca(VaListTy, nullptr, "va_argument");761 Builder.SetInsertPoint(CB);762 Builder.CreateLifetimeStart(VaList);763 }764 Builder.SetInsertPoint(CB);765 Args.push_back(ABI->initializeVaList(M, Ctx, Builder, VaList, Alloced));766 }767 768 // Attributes excluding any on the vararg arguments769 AttributeList PAL = CB->getAttributes();770 if (!PAL.isEmpty()) {771 SmallVector<AttributeSet, 8> ArgAttrs;772 for (unsigned ArgNo = 0; ArgNo < NumArgs; ArgNo++)773 ArgAttrs.push_back(PAL.getParamAttrs(ArgNo));774 PAL =775 AttributeList::get(Ctx, PAL.getFnAttrs(), PAL.getRetAttrs(), ArgAttrs);776 }777 778 SmallVector<OperandBundleDef, 1> OpBundles;779 CB->getOperandBundlesAsDefs(OpBundles);780 781 CallBase *NewCB = nullptr;782 783 if (CallInst *CI = dyn_cast<CallInst>(CB)) {784 Value *Dst = NF ? NF : CI->getCalledOperand();785 FunctionType *NFTy = inlinableVariadicFunctionType(M, VarargFunctionType);786 787 NewCB = CallInst::Create(NFTy, Dst, Args, OpBundles, "", CI->getIterator());788 789 CallInst::TailCallKind TCK = CI->getTailCallKind();790 assert(TCK != CallInst::TCK_MustTail);791 792 // Can't tail call a function that is being passed a pointer to an alloca793 if (TCK == CallInst::TCK_Tail)794 TCK = CallInst::TCK_None;795 CI->setTailCallKind(TCK);796 797 } else {798 llvm_unreachable("Unreachable when !expansionApplicableToFunctionCall()");799 }800 801 if (VaList)802 Builder.CreateLifetimeEnd(VaList);803 804 Builder.CreateLifetimeEnd(Alloced);805 806 NewCB->setAttributes(PAL);807 NewCB->takeName(CB);808 NewCB->setCallingConv(CB->getCallingConv());809 NewCB->setDebugLoc(DebugLoc());810 811 // DeadArgElim and ArgPromotion copy exactly this metadata812 NewCB->copyMetadata(*CB, {LLVMContext::MD_prof, LLVMContext::MD_dbg});813 814 CB->replaceAllUsesWith(NewCB);815 CB->eraseFromParent();816 return Changed;817}818 819bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &Builder,820 const DataLayout &DL,821 VAStartInst *Inst) {822 // Only removing va_start instructions that are not in variadic functions.823 // Those would be rejected by the IR verifier before this pass.824 // After splicing basic blocks from a variadic function into a fixed arity825 // one the va_start that used to refer to the ... parameter still exist.826 // There are also variadic functions that this pass did not change and827 // va_start instances in the created single block wrapper functions.828 // Replace exactly the instances in non-variadic functions as those are829 // the ones to be fixed up to use the va_list passed as the final argument.830 831 Function *ContainingFunction = Inst->getFunction();832 if (ContainingFunction->isVarArg()) {833 return false;834 }835 836 // The last argument is a vaListParameterType, either a va_list837 // or a pointer to one depending on the target.838 bool PassedByValue = ABI->vaListPassedInSSARegister();839 Argument *PassedVaList =840 ContainingFunction->getArg(ContainingFunction->arg_size() - 1);841 842 // va_start takes a pointer to a va_list, e.g. one on the stack843 Value *VaStartArg = Inst->getArgList();844 845 Builder.SetInsertPoint(Inst);846 847 if (PassedByValue) {848 // The general thing to do is create an alloca, store the va_list argument849 // to it, then create a va_copy. When vaCopyIsMemcpy(), this optimises to a850 // store to the VaStartArg.851 assert(ABI->vaCopyIsMemcpy());852 Builder.CreateStore(PassedVaList, VaStartArg);853 } else {854 855 // Otherwise emit a vacopy to pick up target-specific handling if any856 auto &Ctx = Builder.getContext();857 858 Builder.CreateIntrinsic(Intrinsic::vacopy, {DL.getAllocaPtrType(Ctx)},859 {VaStartArg, PassedVaList});860 }861 862 Inst->eraseFromParent();863 return true;864}865 866bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &, const DataLayout &,867 VAEndInst *Inst) {868 assert(ABI->vaEndIsNop());869 Inst->eraseFromParent();870 return true;871}872 873bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &Builder,874 const DataLayout &DL,875 VACopyInst *Inst) {876 assert(ABI->vaCopyIsMemcpy());877 Builder.SetInsertPoint(Inst);878 879 auto &Ctx = Builder.getContext();880 Type *VaListTy = ABI->vaListType(Ctx);881 uint64_t Size = DL.getTypeAllocSize(VaListTy).getFixedValue();882 883 Builder.CreateMemCpy(Inst->getDest(), {}, Inst->getSrc(), {},884 Builder.getInt32(Size));885 886 Inst->eraseFromParent();887 return true;888}889 890struct Amdgpu final : public VariadicABIInfo {891 892 bool enableForTarget() override { return true; }893 894 bool vaListPassedInSSARegister() override { return true; }895 896 Type *vaListType(LLVMContext &Ctx) override {897 return PointerType::getUnqual(Ctx);898 }899 900 Type *vaListParameterType(Module &M) override {901 return PointerType::getUnqual(M.getContext());902 }903 904 Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,905 AllocaInst * /*va_list*/, Value *Buffer) override {906 // Given Buffer, which is an AllocInst of vararg_buffer907 // need to return something usable as parameter type908 return Builder.CreateAddrSpaceCast(Buffer, vaListParameterType(M));909 }910 911 VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {912 return {Align(4), false};913 }914};915 916struct NVPTX final : public VariadicABIInfo {917 918 bool enableForTarget() override { return true; }919 920 bool vaListPassedInSSARegister() override { return true; }921 922 Type *vaListType(LLVMContext &Ctx) override {923 return PointerType::getUnqual(Ctx);924 }925 926 Type *vaListParameterType(Module &M) override {927 return PointerType::getUnqual(M.getContext());928 }929 930 Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,931 AllocaInst *, Value *Buffer) override {932 return Builder.CreateAddrSpaceCast(Buffer, vaListParameterType(M));933 }934 935 VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {936 // NVPTX expects natural alignment in all cases. The variadic call ABI will937 // handle promoting types to their appropriate size and alignment.938 Align A = DL.getABITypeAlign(Parameter);939 return {A, false};940 }941};942 943struct Wasm final : public VariadicABIInfo {944 945 bool enableForTarget() override {946 // Currently wasm is only used for testing.947 return commandLineOverride();948 }949 950 bool vaListPassedInSSARegister() override { return true; }951 952 Type *vaListType(LLVMContext &Ctx) override {953 return PointerType::getUnqual(Ctx);954 }955 956 Type *vaListParameterType(Module &M) override {957 return PointerType::getUnqual(M.getContext());958 }959 960 Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,961 AllocaInst * /*va_list*/, Value *Buffer) override {962 return Buffer;963 }964 965 VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {966 LLVMContext &Ctx = Parameter->getContext();967 const unsigned MinAlign = 4;968 Align A = DL.getABITypeAlign(Parameter);969 if (A < MinAlign)970 A = Align(MinAlign);971 972 if (auto *S = dyn_cast<StructType>(Parameter)) {973 if (S->getNumElements() > 1) {974 return {DL.getABITypeAlign(PointerType::getUnqual(Ctx)), true};975 }976 }977 978 return {A, false};979 }980};981 982std::unique_ptr<VariadicABIInfo> VariadicABIInfo::create(const Triple &T) {983 switch (T.getArch()) {984 case Triple::r600:985 case Triple::amdgcn: {986 return std::make_unique<Amdgpu>();987 }988 989 case Triple::wasm32: {990 return std::make_unique<Wasm>();991 }992 993 case Triple::nvptx:994 case Triple::nvptx64: {995 return std::make_unique<NVPTX>();996 }997 998 default:999 return {};1000 }1001}1002 1003} // namespace1004 1005char ExpandVariadics::ID = 0;1006 1007INITIALIZE_PASS(ExpandVariadics, DEBUG_TYPE, "Expand variadic functions", false,1008 false)1009 1010ModulePass *llvm::createExpandVariadicsPass(ExpandVariadicsMode M) {1011 return new ExpandVariadics(M);1012}1013 1014PreservedAnalyses ExpandVariadicsPass::run(Module &M, ModuleAnalysisManager &) {1015 return ExpandVariadics(Mode).runOnModule(M) ? PreservedAnalyses::none()1016 : PreservedAnalyses::all();1017}1018 1019ExpandVariadicsPass::ExpandVariadicsPass(ExpandVariadicsMode M) : Mode(M) {}1020