657 lines · cpp
1//===- MemProfInstrumentation.cpp - memory alloc and access instrumentation ==//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file is a part of MemProf. Memory accesses are instrumented10// to increment the access count held in a shadow memory location, or11// alternatively to call into the runtime. Memory intrinsic calls (memmove,12// memcpy, memset) are changed to call the memory profiling runtime version13// instead.14//15//===----------------------------------------------------------------------===//16 17#include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"18#include "llvm/ADT/SmallVector.h"19#include "llvm/ADT/Statistic.h"20#include "llvm/ADT/StringRef.h"21#include "llvm/Analysis/MemoryBuiltins.h"22#include "llvm/Analysis/TargetLibraryInfo.h"23#include "llvm/Analysis/ValueTracking.h"24#include "llvm/IR/Constant.h"25#include "llvm/IR/DataLayout.h"26#include "llvm/IR/DiagnosticInfo.h"27#include "llvm/IR/Function.h"28#include "llvm/IR/GlobalValue.h"29#include "llvm/IR/IRBuilder.h"30#include "llvm/IR/Instruction.h"31#include "llvm/IR/IntrinsicInst.h"32#include "llvm/IR/Module.h"33#include "llvm/IR/Type.h"34#include "llvm/IR/Value.h"35#include "llvm/ProfileData/InstrProf.h"36#include "llvm/ProfileData/MemProf.h"37#include "llvm/Support/CommandLine.h"38#include "llvm/Support/Debug.h"39#include "llvm/TargetParser/Triple.h"40#include "llvm/Transforms/Utils/BasicBlockUtils.h"41#include "llvm/Transforms/Utils/ModuleUtils.h"42 43using namespace llvm;44using namespace llvm::memprof;45 46#define DEBUG_TYPE "memprof"47 48constexpr int LLVM_MEM_PROFILER_VERSION = 1;49 50// Size of memory mapped to a single shadow location.51constexpr uint64_t DefaultMemGranularity = 64;52 53// Size of memory mapped to a single histogram bucket.54constexpr uint64_t HistogramGranularity = 8;55 56// Scale from granularity down to shadow size.57constexpr uint64_t DefaultShadowScale = 3;58 59constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";60constexpr uint64_t MemProfCtorAndDtorPriority = 1;61// On Emscripten, the system needs more than one priorities for constructors.62constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;63constexpr char MemProfInitName[] = "__memprof_init";64constexpr char MemProfVersionCheckNamePrefix[] =65 "__memprof_version_mismatch_check_v";66 67constexpr char MemProfShadowMemoryDynamicAddress[] =68 "__memprof_shadow_memory_dynamic_address";69 70constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";71 72constexpr char MemProfHistogramFlagVar[] = "__memprof_histogram";73 74// Command-line flags.75 76static cl::opt<bool> ClInsertVersionCheck(77 "memprof-guard-against-version-mismatch",78 cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,79 cl::init(true));80 81// This flag may need to be replaced with -f[no-]memprof-reads.82static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",83 cl::desc("instrument read instructions"),84 cl::Hidden, cl::init(true));85 86static cl::opt<bool>87 ClInstrumentWrites("memprof-instrument-writes",88 cl::desc("instrument write instructions"), cl::Hidden,89 cl::init(true));90 91static cl::opt<bool> ClInstrumentAtomics(92 "memprof-instrument-atomics",93 cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,94 cl::init(true));95 96static cl::opt<bool> ClUseCalls(97 "memprof-use-callbacks",98 cl::desc("Use callbacks instead of inline instrumentation sequences."),99 cl::Hidden, cl::init(false));100 101static cl::opt<std::string>102 ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",103 cl::desc("Prefix for memory access callbacks"),104 cl::Hidden, cl::init("__memprof_"));105 106// These flags allow to change the shadow mapping.107// The shadow mapping looks like108// Shadow = ((Mem & mask) >> scale) + offset109 110static cl::opt<int> ClMappingScale("memprof-mapping-scale",111 cl::desc("scale of memprof shadow mapping"),112 cl::Hidden, cl::init(DefaultShadowScale));113 114static cl::opt<int>115 ClMappingGranularity("memprof-mapping-granularity",116 cl::desc("granularity of memprof shadow mapping"),117 cl::Hidden, cl::init(DefaultMemGranularity));118 119static cl::opt<bool> ClStack("memprof-instrument-stack",120 cl::desc("Instrument scalar stack variables"),121 cl::Hidden, cl::init(false));122 123// Debug flags.124 125static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,126 cl::init(0));127 128static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,129 cl::desc("Debug func"));130 131static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),132 cl::Hidden, cl::init(-1));133 134static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),135 cl::Hidden, cl::init(-1));136 137static cl::opt<bool> ClHistogram("memprof-histogram",138 cl::desc("Collect access count histograms"),139 cl::Hidden, cl::init(false));140 141static cl::opt<std::string>142 MemprofRuntimeDefaultOptions("memprof-runtime-default-options",143 cl::desc("The default memprof options"),144 cl::Hidden, cl::init(""));145 146// Instrumentation statistics147STATISTIC(NumInstrumentedReads, "Number of instrumented reads");148STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");149STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");150STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");151 152namespace {153 154/// This struct defines the shadow mapping using the rule:155/// shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.156struct ShadowMapping {157 ShadowMapping() {158 Scale = ClMappingScale;159 Granularity = ClHistogram ? HistogramGranularity : ClMappingGranularity;160 Mask = ~(Granularity - 1);161 }162 163 int Scale;164 int Granularity;165 uint64_t Mask; // Computed as ~(Granularity-1)166};167 168static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {169 return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority170 : MemProfCtorAndDtorPriority;171}172 173struct InterestingMemoryAccess {174 Value *Addr = nullptr;175 bool IsWrite;176 Type *AccessTy;177 Value *MaybeMask = nullptr;178};179 180/// Instrument the code in module to profile memory accesses.181class MemProfiler {182public:183 MemProfiler(Module &M) {184 C = &(M.getContext());185 LongSize = M.getDataLayout().getPointerSizeInBits();186 IntptrTy = Type::getIntNTy(*C, LongSize);187 PtrTy = PointerType::getUnqual(*C);188 }189 190 /// If it is an interesting memory access, populate information191 /// about the access and return a InterestingMemoryAccess struct.192 /// Otherwise return std::nullopt.193 std::optional<InterestingMemoryAccess>194 isInterestingMemoryAccess(Instruction *I) const;195 196 void instrumentMop(Instruction *I, const DataLayout &DL,197 InterestingMemoryAccess &Access);198 void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,199 Value *Addr, bool IsWrite);200 void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,201 Instruction *I, Value *Addr, Type *AccessTy,202 bool IsWrite);203 void instrumentMemIntrinsic(MemIntrinsic *MI);204 Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);205 bool instrumentFunction(Function &F);206 bool maybeInsertMemProfInitAtFunctionEntry(Function &F);207 bool insertDynamicShadowAtFunctionEntry(Function &F);208 209private:210 void initializeCallbacks(Module &M);211 212 LLVMContext *C;213 int LongSize;214 Type *IntptrTy;215 PointerType *PtrTy;216 ShadowMapping Mapping;217 218 // These arrays is indexed by AccessIsWrite219 FunctionCallee MemProfMemoryAccessCallback[2];220 221 FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;222 Value *DynamicShadowOffset = nullptr;223};224 225class ModuleMemProfiler {226public:227 ModuleMemProfiler(Module &M) { TargetTriple = M.getTargetTriple(); }228 229 bool instrumentModule(Module &);230 231private:232 Triple TargetTriple;233 ShadowMapping Mapping;234 Function *MemProfCtorFunction = nullptr;235};236 237} // end anonymous namespace238 239MemProfilerPass::MemProfilerPass() = default;240 241PreservedAnalyses MemProfilerPass::run(Function &F,242 AnalysisManager<Function> &AM) {243 assert((!ClHistogram || ClMappingGranularity == DefaultMemGranularity) &&244 "Memprof with histogram only supports default mapping granularity");245 Module &M = *F.getParent();246 MemProfiler Profiler(M);247 if (Profiler.instrumentFunction(F))248 return PreservedAnalyses::none();249 return PreservedAnalyses::all();250}251 252ModuleMemProfilerPass::ModuleMemProfilerPass() = default;253 254PreservedAnalyses ModuleMemProfilerPass::run(Module &M,255 AnalysisManager<Module> &AM) {256 257 ModuleMemProfiler Profiler(M);258 if (Profiler.instrumentModule(M))259 return PreservedAnalyses::none();260 return PreservedAnalyses::all();261}262 263Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {264 // (Shadow & mask) >> scale265 Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);266 Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);267 // (Shadow >> scale) | offset268 assert(DynamicShadowOffset);269 return IRB.CreateAdd(Shadow, DynamicShadowOffset);270}271 272// Instrument memset/memmove/memcpy273void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {274 IRBuilder<> IRB(MI);275 if (isa<MemTransferInst>(MI)) {276 IRB.CreateCall(isa<MemMoveInst>(MI) ? MemProfMemmove : MemProfMemcpy,277 {MI->getOperand(0), MI->getOperand(1),278 IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});279 } else if (isa<MemSetInst>(MI)) {280 IRB.CreateCall(281 MemProfMemset,282 {MI->getOperand(0),283 IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),284 IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});285 }286 MI->eraseFromParent();287}288 289std::optional<InterestingMemoryAccess>290MemProfiler::isInterestingMemoryAccess(Instruction *I) const {291 // Do not instrument the load fetching the dynamic shadow address.292 if (DynamicShadowOffset == I)293 return std::nullopt;294 295 InterestingMemoryAccess Access;296 297 if (LoadInst *LI = dyn_cast<LoadInst>(I)) {298 if (!ClInstrumentReads)299 return std::nullopt;300 Access.IsWrite = false;301 Access.AccessTy = LI->getType();302 Access.Addr = LI->getPointerOperand();303 } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {304 if (!ClInstrumentWrites)305 return std::nullopt;306 Access.IsWrite = true;307 Access.AccessTy = SI->getValueOperand()->getType();308 Access.Addr = SI->getPointerOperand();309 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {310 if (!ClInstrumentAtomics)311 return std::nullopt;312 Access.IsWrite = true;313 Access.AccessTy = RMW->getValOperand()->getType();314 Access.Addr = RMW->getPointerOperand();315 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {316 if (!ClInstrumentAtomics)317 return std::nullopt;318 Access.IsWrite = true;319 Access.AccessTy = XCHG->getCompareOperand()->getType();320 Access.Addr = XCHG->getPointerOperand();321 } else if (auto *CI = dyn_cast<CallInst>(I)) {322 auto *F = CI->getCalledFunction();323 if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||324 F->getIntrinsicID() == Intrinsic::masked_store)) {325 unsigned OpOffset = 0;326 if (F->getIntrinsicID() == Intrinsic::masked_store) {327 if (!ClInstrumentWrites)328 return std::nullopt;329 // Masked store has an initial operand for the value.330 OpOffset = 1;331 Access.AccessTy = CI->getArgOperand(0)->getType();332 Access.IsWrite = true;333 } else {334 if (!ClInstrumentReads)335 return std::nullopt;336 Access.AccessTy = CI->getType();337 Access.IsWrite = false;338 }339 340 auto *BasePtr = CI->getOperand(0 + OpOffset);341 Access.MaybeMask = CI->getOperand(1 + OpOffset);342 Access.Addr = BasePtr;343 }344 }345 346 if (!Access.Addr)347 return std::nullopt;348 349 // Do not instrument accesses from different address spaces; we cannot deal350 // with them.351 Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());352 if (PtrTy->getPointerAddressSpace() != 0)353 return std::nullopt;354 355 // Ignore swifterror addresses.356 // swifterror memory addresses are mem2reg promoted by instruction357 // selection. As such they cannot have regular uses like an instrumentation358 // function and it makes no sense to track them as memory.359 if (Access.Addr->isSwiftError())360 return std::nullopt;361 362 // Peel off GEPs and BitCasts.363 auto *Addr = Access.Addr->stripInBoundsOffsets();364 365 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {366 // Do not instrument PGO counter updates.367 if (GV->hasSection()) {368 StringRef SectionName = GV->getSection();369 // Check if the global is in the PGO counters section.370 auto OF = I->getModule()->getTargetTriple().getObjectFormat();371 if (SectionName.ends_with(372 getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false)))373 return std::nullopt;374 }375 376 // Do not instrument accesses to LLVM internal variables.377 if (GV->getName().starts_with("__llvm"))378 return std::nullopt;379 }380 381 return Access;382}383 384void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,385 Instruction *I, Value *Addr,386 Type *AccessTy, bool IsWrite) {387 auto *VTy = cast<FixedVectorType>(AccessTy);388 unsigned Num = VTy->getNumElements();389 auto *Zero = ConstantInt::get(IntptrTy, 0);390 for (unsigned Idx = 0; Idx < Num; ++Idx) {391 Value *InstrumentedAddress = nullptr;392 Instruction *InsertBefore = I;393 if (auto *Vector = dyn_cast<ConstantVector>(Mask)) {394 // dyn_cast as we might get UndefValue395 if (auto *Masked = dyn_cast<ConstantInt>(Vector->getOperand(Idx))) {396 if (Masked->isZero())397 // Mask is constant false, so no instrumentation needed.398 continue;399 // If we have a true or undef value, fall through to instrumentAddress.400 // with InsertBefore == I401 }402 } else {403 IRBuilder<> IRB(I);404 Value *MaskElem = IRB.CreateExtractElement(Mask, Idx);405 Instruction *ThenTerm = SplitBlockAndInsertIfThen(MaskElem, I, false);406 InsertBefore = ThenTerm;407 }408 409 IRBuilder<> IRB(InsertBefore);410 InstrumentedAddress =411 IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});412 instrumentAddress(I, InsertBefore, InstrumentedAddress, IsWrite);413 }414}415 416void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,417 InterestingMemoryAccess &Access) {418 // Skip instrumentation of stack accesses unless requested.419 if (!ClStack && isa<AllocaInst>(getUnderlyingObject(Access.Addr))) {420 if (Access.IsWrite)421 ++NumSkippedStackWrites;422 else423 ++NumSkippedStackReads;424 return;425 }426 427 if (Access.IsWrite)428 NumInstrumentedWrites++;429 else430 NumInstrumentedReads++;431 432 if (Access.MaybeMask) {433 instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,434 Access.AccessTy, Access.IsWrite);435 } else {436 // Since the access counts will be accumulated across the entire allocation,437 // we only update the shadow access count for the first location and thus438 // don't need to worry about alignment and type size.439 instrumentAddress(I, I, Access.Addr, Access.IsWrite);440 }441}442 443void MemProfiler::instrumentAddress(Instruction *OrigIns,444 Instruction *InsertBefore, Value *Addr,445 bool IsWrite) {446 IRBuilder<> IRB(InsertBefore);447 Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);448 449 if (ClUseCalls) {450 IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);451 return;452 }453 454 Type *ShadowTy = ClHistogram ? Type::getInt8Ty(*C) : Type::getInt64Ty(*C);455 Type *ShadowPtrTy = PointerType::get(*C, 0);456 457 Value *ShadowPtr = memToShadow(AddrLong, IRB);458 Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);459 Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);460 // If we are profiling with histograms, add overflow protection at 255.461 if (ClHistogram) {462 Value *MaxCount = ConstantInt::get(Type::getInt8Ty(*C), 255);463 Value *Cmp = IRB.CreateICmpULT(ShadowValue, MaxCount);464 Instruction *IncBlock =465 SplitBlockAndInsertIfThen(Cmp, InsertBefore, /*Unreachable=*/false);466 IRB.SetInsertPoint(IncBlock);467 }468 Value *Inc = ConstantInt::get(ShadowTy, 1);469 ShadowValue = IRB.CreateAdd(ShadowValue, Inc);470 IRB.CreateStore(ShadowValue, ShadowAddr);471}472 473// Create the variable for the profile file name.474void createProfileFileNameVar(Module &M) {475 const MDString *MemProfFilename =476 dyn_cast_or_null<MDString>(M.getModuleFlag("MemProfProfileFilename"));477 if (!MemProfFilename)478 return;479 assert(!MemProfFilename->getString().empty() &&480 "Unexpected MemProfProfileFilename metadata with empty string");481 Constant *ProfileNameConst = ConstantDataArray::getString(482 M.getContext(), MemProfFilename->getString(), true);483 GlobalVariable *ProfileNameVar = new GlobalVariable(484 M, ProfileNameConst->getType(), /*isConstant=*/true,485 GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);486 const Triple &TT = M.getTargetTriple();487 if (TT.supportsCOMDAT()) {488 ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);489 ProfileNameVar->setComdat(M.getOrInsertComdat(MemProfFilenameVar));490 }491}492 493// Set MemprofHistogramFlag as a Global variable in IR. This makes it accessible494// to the runtime, changing shadow count behavior.495void createMemprofHistogramFlagVar(Module &M) {496 const StringRef VarName(MemProfHistogramFlagVar);497 Type *IntTy1 = Type::getInt1Ty(M.getContext());498 auto MemprofHistogramFlag = new GlobalVariable(499 M, IntTy1, true, GlobalValue::WeakAnyLinkage,500 Constant::getIntegerValue(IntTy1, APInt(1, ClHistogram)), VarName);501 const Triple &TT = M.getTargetTriple();502 if (TT.supportsCOMDAT()) {503 MemprofHistogramFlag->setLinkage(GlobalValue::ExternalLinkage);504 MemprofHistogramFlag->setComdat(M.getOrInsertComdat(VarName));505 }506 appendToCompilerUsed(M, MemprofHistogramFlag);507}508 509void createMemprofDefaultOptionsVar(Module &M) {510 Constant *OptionsConst = ConstantDataArray::getString(511 M.getContext(), MemprofRuntimeDefaultOptions, /*AddNull=*/true);512 GlobalVariable *OptionsVar =513 new GlobalVariable(M, OptionsConst->getType(), /*isConstant=*/true,514 GlobalValue::WeakAnyLinkage, OptionsConst,515 memprof::getMemprofOptionsSymbolName());516 const Triple &TT = M.getTargetTriple();517 if (TT.supportsCOMDAT()) {518 OptionsVar->setLinkage(GlobalValue::ExternalLinkage);519 OptionsVar->setComdat(M.getOrInsertComdat(OptionsVar->getName()));520 }521}522 523bool ModuleMemProfiler::instrumentModule(Module &M) {524 525 // Create a module constructor.526 std::string MemProfVersion = std::to_string(LLVM_MEM_PROFILER_VERSION);527 std::string VersionCheckName =528 ClInsertVersionCheck ? (MemProfVersionCheckNamePrefix + MemProfVersion)529 : "";530 std::tie(MemProfCtorFunction, std::ignore) =531 createSanitizerCtorAndInitFunctions(M, MemProfModuleCtorName,532 MemProfInitName, /*InitArgTypes=*/{},533 /*InitArgs=*/{}, VersionCheckName);534 535 const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);536 appendToGlobalCtors(M, MemProfCtorFunction, Priority);537 538 createProfileFileNameVar(M);539 540 createMemprofHistogramFlagVar(M);541 542 createMemprofDefaultOptionsVar(M);543 544 return true;545}546 547void MemProfiler::initializeCallbacks(Module &M) {548 IRBuilder<> IRB(*C);549 550 for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {551 const std::string TypeStr = AccessIsWrite ? "store" : "load";552 const std::string HistPrefix = ClHistogram ? "hist_" : "";553 554 SmallVector<Type *, 2> Args1{1, IntptrTy};555 MemProfMemoryAccessCallback[AccessIsWrite] = M.getOrInsertFunction(556 ClMemoryAccessCallbackPrefix + HistPrefix + TypeStr,557 FunctionType::get(IRB.getVoidTy(), Args1, false));558 }559 MemProfMemmove = M.getOrInsertFunction(560 ClMemoryAccessCallbackPrefix + "memmove", PtrTy, PtrTy, PtrTy, IntptrTy);561 MemProfMemcpy = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memcpy",562 PtrTy, PtrTy, PtrTy, IntptrTy);563 MemProfMemset =564 M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memset", PtrTy,565 PtrTy, IRB.getInt32Ty(), IntptrTy);566}567 568bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {569 // For each NSObject descendant having a +load method, this method is invoked570 // by the ObjC runtime before any of the static constructors is called.571 // Therefore we need to instrument such methods with a call to __memprof_init572 // at the beginning in order to initialize our runtime before any access to573 // the shadow memory.574 // We cannot just ignore these methods, because they may call other575 // instrumented functions.576 if (F.getName().contains(" load]")) {577 FunctionCallee MemProfInitFunction =578 declareSanitizerInitFunction(*F.getParent(), MemProfInitName, {});579 IRBuilder<> IRB(&F.front(), F.front().begin());580 IRB.CreateCall(MemProfInitFunction, {});581 return true;582 }583 return false;584}585 586bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {587 IRBuilder<> IRB(&F.front().front());588 Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(589 MemProfShadowMemoryDynamicAddress, IntptrTy);590 if (F.getParent()->getPICLevel() == PICLevel::NotPIC)591 cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);592 DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);593 return true;594}595 596bool MemProfiler::instrumentFunction(Function &F) {597 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)598 return false;599 if (ClDebugFunc == F.getName())600 return false;601 if (F.getName().starts_with("__memprof_"))602 return false;603 604 bool FunctionModified = false;605 606 // If needed, insert __memprof_init.607 // This function needs to be called even if the function body is not608 // instrumented.609 if (maybeInsertMemProfInitAtFunctionEntry(F))610 FunctionModified = true;611 612 LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");613 614 initializeCallbacks(*F.getParent());615 616 SmallVector<Instruction *, 16> ToInstrument;617 618 // Fill the set of memory operations to instrument.619 for (auto &BB : F) {620 for (auto &Inst : BB) {621 if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))622 ToInstrument.push_back(&Inst);623 }624 }625 626 if (ToInstrument.empty()) {627 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified628 << " " << F << "\n");629 630 return FunctionModified;631 }632 633 FunctionModified |= insertDynamicShadowAtFunctionEntry(F);634 635 int NumInstrumented = 0;636 for (auto *Inst : ToInstrument) {637 if (ClDebugMin < 0 || ClDebugMax < 0 ||638 (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {639 std::optional<InterestingMemoryAccess> Access =640 isInterestingMemoryAccess(Inst);641 if (Access)642 instrumentMop(Inst, F.getDataLayout(), *Access);643 else644 instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));645 }646 NumInstrumented++;647 }648 649 if (NumInstrumented > 0)650 FunctionModified = true;651 652 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "653 << F << "\n");654 655 return FunctionModified;656}657