1247 lines · cpp
1//===-- SanitizerCoverage.cpp - coverage instrumentation for sanitizers ---===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Coverage instrumentation done on LLVM IR level, works with Sanitizers.10//11//===----------------------------------------------------------------------===//12 13#include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"14#include "llvm/ADT/ArrayRef.h"15#include "llvm/ADT/SmallVector.h"16#include "llvm/Analysis/GlobalsModRef.h"17#include "llvm/Analysis/PostDominators.h"18#include "llvm/IR/Constant.h"19#include "llvm/IR/Constants.h"20#include "llvm/IR/DataLayout.h"21#include "llvm/IR/Dominators.h"22#include "llvm/IR/EHPersonalities.h"23#include "llvm/IR/Function.h"24#include "llvm/IR/GlobalVariable.h"25#include "llvm/IR/IRBuilder.h"26#include "llvm/IR/IntrinsicInst.h"27#include "llvm/IR/Intrinsics.h"28#include "llvm/IR/LLVMContext.h"29#include "llvm/IR/MDBuilder.h"30#include "llvm/IR/Module.h"31#include "llvm/IR/Type.h"32#include "llvm/IR/ValueSymbolTable.h"33#include "llvm/Support/CommandLine.h"34#include "llvm/Support/SpecialCaseList.h"35#include "llvm/Support/VirtualFileSystem.h"36#include "llvm/TargetParser/Triple.h"37#include "llvm/Transforms/Utils/BasicBlockUtils.h"38#include "llvm/Transforms/Utils/ModuleUtils.h"39 40using namespace llvm;41 42#define DEBUG_TYPE "sancov"43 44const char SanCovTracePCIndirName[] = "__sanitizer_cov_trace_pc_indir";45const char SanCovTracePCName[] = "__sanitizer_cov_trace_pc";46const char SanCovTraceCmp1[] = "__sanitizer_cov_trace_cmp1";47const char SanCovTraceCmp2[] = "__sanitizer_cov_trace_cmp2";48const char SanCovTraceCmp4[] = "__sanitizer_cov_trace_cmp4";49const char SanCovTraceCmp8[] = "__sanitizer_cov_trace_cmp8";50const char SanCovTraceConstCmp1[] = "__sanitizer_cov_trace_const_cmp1";51const char SanCovTraceConstCmp2[] = "__sanitizer_cov_trace_const_cmp2";52const char SanCovTraceConstCmp4[] = "__sanitizer_cov_trace_const_cmp4";53const char SanCovTraceConstCmp8[] = "__sanitizer_cov_trace_const_cmp8";54const char SanCovLoad1[] = "__sanitizer_cov_load1";55const char SanCovLoad2[] = "__sanitizer_cov_load2";56const char SanCovLoad4[] = "__sanitizer_cov_load4";57const char SanCovLoad8[] = "__sanitizer_cov_load8";58const char SanCovLoad16[] = "__sanitizer_cov_load16";59const char SanCovStore1[] = "__sanitizer_cov_store1";60const char SanCovStore2[] = "__sanitizer_cov_store2";61const char SanCovStore4[] = "__sanitizer_cov_store4";62const char SanCovStore8[] = "__sanitizer_cov_store8";63const char SanCovStore16[] = "__sanitizer_cov_store16";64const char SanCovTraceDiv4[] = "__sanitizer_cov_trace_div4";65const char SanCovTraceDiv8[] = "__sanitizer_cov_trace_div8";66const char SanCovTraceGep[] = "__sanitizer_cov_trace_gep";67const char SanCovTraceSwitchName[] = "__sanitizer_cov_trace_switch";68const char SanCovModuleCtorTracePcGuardName[] =69 "sancov.module_ctor_trace_pc_guard";70const char SanCovModuleCtor8bitCountersName[] =71 "sancov.module_ctor_8bit_counters";72const char SanCovModuleCtorBoolFlagName[] = "sancov.module_ctor_bool_flag";73static const uint64_t SanCtorAndDtorPriority = 2;74 75const char SanCovTracePCGuardName[] = "__sanitizer_cov_trace_pc_guard";76const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init";77const char SanCov8bitCountersInitName[] = "__sanitizer_cov_8bit_counters_init";78const char SanCovBoolFlagInitName[] = "__sanitizer_cov_bool_flag_init";79const char SanCovPCsInitName[] = "__sanitizer_cov_pcs_init";80const char SanCovCFsInitName[] = "__sanitizer_cov_cfs_init";81 82const char SanCovGuardsSectionName[] = "sancov_guards";83const char SanCovCountersSectionName[] = "sancov_cntrs";84const char SanCovBoolFlagSectionName[] = "sancov_bools";85const char SanCovPCsSectionName[] = "sancov_pcs";86const char SanCovCFsSectionName[] = "sancov_cfs";87const char SanCovCallbackGateSectionName[] = "sancov_gate";88 89const char SanCovStackDepthCallbackName[] = "__sanitizer_cov_stack_depth";90const char SanCovLowestStackName[] = "__sancov_lowest_stack";91const char SanCovCallbackGateName[] = "__sancov_should_track";92 93static cl::opt<int> ClCoverageLevel(94 "sanitizer-coverage-level",95 cl::desc("Sanitizer Coverage. 0: none, 1: entry block, 2: all blocks, "96 "3: all blocks and critical edges"),97 cl::Hidden);98 99static cl::opt<bool> ClTracePC("sanitizer-coverage-trace-pc",100 cl::desc("Experimental pc tracing"), cl::Hidden);101 102static cl::opt<bool> ClTracePCGuard("sanitizer-coverage-trace-pc-guard",103 cl::desc("pc tracing with a guard"),104 cl::Hidden);105 106// If true, we create a global variable that contains PCs of all instrumented107// BBs, put this global into a named section, and pass this section's bounds108// to __sanitizer_cov_pcs_init.109// This way the coverage instrumentation does not need to acquire the PCs110// at run-time. Works with trace-pc-guard, inline-8bit-counters, and111// inline-bool-flag.112static cl::opt<bool> ClCreatePCTable("sanitizer-coverage-pc-table",113 cl::desc("create a static PC table"),114 cl::Hidden);115 116static cl::opt<bool>117 ClInline8bitCounters("sanitizer-coverage-inline-8bit-counters",118 cl::desc("increments 8-bit counter for every edge"),119 cl::Hidden);120 121static cl::opt<bool>122 ClSancovDropCtors("sanitizer-coverage-drop-ctors",123 cl::desc("do not emit module ctors for global counters"),124 cl::Hidden);125 126static cl::opt<bool>127 ClInlineBoolFlag("sanitizer-coverage-inline-bool-flag",128 cl::desc("sets a boolean flag for every edge"),129 cl::Hidden);130 131static cl::opt<bool>132 ClCMPTracing("sanitizer-coverage-trace-compares",133 cl::desc("Tracing of CMP and similar instructions"),134 cl::Hidden);135 136static cl::opt<bool> ClDIVTracing("sanitizer-coverage-trace-divs",137 cl::desc("Tracing of DIV instructions"),138 cl::Hidden);139 140static cl::opt<bool> ClLoadTracing("sanitizer-coverage-trace-loads",141 cl::desc("Tracing of load instructions"),142 cl::Hidden);143 144static cl::opt<bool> ClStoreTracing("sanitizer-coverage-trace-stores",145 cl::desc("Tracing of store instructions"),146 cl::Hidden);147 148static cl::opt<bool> ClGEPTracing("sanitizer-coverage-trace-geps",149 cl::desc("Tracing of GEP instructions"),150 cl::Hidden);151 152static cl::opt<bool>153 ClPruneBlocks("sanitizer-coverage-prune-blocks",154 cl::desc("Reduce the number of instrumented blocks"),155 cl::Hidden, cl::init(true));156 157static cl::opt<bool> ClStackDepth("sanitizer-coverage-stack-depth",158 cl::desc("max stack depth tracing"),159 cl::Hidden);160 161static cl::opt<int> ClStackDepthCallbackMin(162 "sanitizer-coverage-stack-depth-callback-min",163 cl::desc("max stack depth tracing should use callback and only when "164 "stack depth more than specified"),165 cl::Hidden);166 167static cl::opt<bool>168 ClCollectCF("sanitizer-coverage-control-flow",169 cl::desc("collect control flow for each function"), cl::Hidden);170 171static cl::opt<bool> ClGatedCallbacks(172 "sanitizer-coverage-gated-trace-callbacks",173 cl::desc("Gate the invocation of the tracing callbacks on a global variable"174 ". Currently only supported for trace-pc-guard and trace-cmp."),175 cl::Hidden, cl::init(false));176 177namespace {178 179SanitizerCoverageOptions getOptions(int LegacyCoverageLevel) {180 SanitizerCoverageOptions Res;181 switch (LegacyCoverageLevel) {182 case 0:183 Res.CoverageType = SanitizerCoverageOptions::SCK_None;184 break;185 case 1:186 Res.CoverageType = SanitizerCoverageOptions::SCK_Function;187 break;188 case 2:189 Res.CoverageType = SanitizerCoverageOptions::SCK_BB;190 break;191 case 3:192 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;193 break;194 case 4:195 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;196 Res.IndirectCalls = true;197 break;198 }199 return Res;200}201 202SanitizerCoverageOptions OverrideFromCL(SanitizerCoverageOptions Options) {203 // Sets CoverageType and IndirectCalls.204 SanitizerCoverageOptions CLOpts = getOptions(ClCoverageLevel);205 Options.CoverageType = std::max(Options.CoverageType, CLOpts.CoverageType);206 Options.IndirectCalls |= CLOpts.IndirectCalls;207 Options.TraceCmp |= ClCMPTracing;208 Options.TraceDiv |= ClDIVTracing;209 Options.TraceGep |= ClGEPTracing;210 Options.TracePC |= ClTracePC;211 Options.TracePCGuard |= ClTracePCGuard;212 Options.Inline8bitCounters |= ClInline8bitCounters;213 Options.InlineBoolFlag |= ClInlineBoolFlag;214 Options.PCTable |= ClCreatePCTable;215 Options.NoPrune |= !ClPruneBlocks;216 Options.StackDepth |= ClStackDepth;217 Options.StackDepthCallbackMin = std::max(Options.StackDepthCallbackMin,218 ClStackDepthCallbackMin.getValue());219 Options.TraceLoads |= ClLoadTracing;220 Options.TraceStores |= ClStoreTracing;221 Options.GatedCallbacks |= ClGatedCallbacks;222 if (!Options.TracePCGuard && !Options.TracePC &&223 !Options.Inline8bitCounters && !Options.StackDepth &&224 !Options.InlineBoolFlag && !Options.TraceLoads && !Options.TraceStores)225 Options.TracePCGuard = true; // TracePCGuard is default.226 Options.CollectControlFlow |= ClCollectCF;227 return Options;228}229 230class ModuleSanitizerCoverage {231public:232 using DomTreeCallback = function_ref<const DominatorTree &(Function &F)>;233 using PostDomTreeCallback =234 function_ref<const PostDominatorTree &(Function &F)>;235 236 ModuleSanitizerCoverage(Module &M, DomTreeCallback DTCallback,237 PostDomTreeCallback PDTCallback,238 const SanitizerCoverageOptions &Options,239 const SpecialCaseList *Allowlist,240 const SpecialCaseList *Blocklist)241 : M(M), DTCallback(DTCallback), PDTCallback(PDTCallback),242 Options(Options), Allowlist(Allowlist), Blocklist(Blocklist) {}243 244 bool instrumentModule();245 246private:247 void createFunctionControlFlow(Function &F);248 void instrumentFunction(Function &F);249 void InjectCoverageForIndirectCalls(Function &F,250 ArrayRef<Instruction *> IndirCalls);251 void InjectTraceForCmp(Function &F, ArrayRef<Instruction *> CmpTraceTargets,252 Value *&FunctionGateCmp);253 void InjectTraceForDiv(Function &F,254 ArrayRef<BinaryOperator *> DivTraceTargets);255 void InjectTraceForGep(Function &F,256 ArrayRef<GetElementPtrInst *> GepTraceTargets);257 void InjectTraceForLoadsAndStores(Function &F, ArrayRef<LoadInst *> Loads,258 ArrayRef<StoreInst *> Stores);259 void InjectTraceForSwitch(Function &F,260 ArrayRef<Instruction *> SwitchTraceTargets,261 Value *&FunctionGateCmp);262 bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks,263 Value *&FunctionGateCmp, bool IsLeafFunc);264 GlobalVariable *CreateFunctionLocalArrayInSection(size_t NumElements,265 Function &F, Type *Ty,266 const char *Section);267 GlobalVariable *CreatePCArray(Function &F, ArrayRef<BasicBlock *> AllBlocks);268 void CreateFunctionLocalArrays(Function &F, ArrayRef<BasicBlock *> AllBlocks);269 Instruction *CreateGateBranch(Function &F, Value *&FunctionGateCmp,270 Instruction *I);271 Value *CreateFunctionLocalGateCmp(IRBuilder<> &IRB);272 void InjectCoverageAtBlock(Function &F, BasicBlock &BB, size_t Idx,273 Value *&FunctionGateCmp, bool IsLeafFunc);274 Function *CreateInitCallsForSections(Module &M, const char *CtorName,275 const char *InitFunctionName, Type *Ty,276 const char *Section);277 std::pair<Value *, Value *> CreateSecStartEnd(Module &M, const char *Section,278 Type *Ty);279 280 std::string getSectionName(const std::string &Section) const;281 std::string getSectionStart(const std::string &Section) const;282 std::string getSectionEnd(const std::string &Section) const;283 284 Module &M;285 DomTreeCallback DTCallback;286 PostDomTreeCallback PDTCallback;287 288 FunctionCallee SanCovStackDepthCallback;289 FunctionCallee SanCovTracePCIndir;290 FunctionCallee SanCovTracePC, SanCovTracePCGuard;291 std::array<FunctionCallee, 4> SanCovTraceCmpFunction;292 std::array<FunctionCallee, 4> SanCovTraceConstCmpFunction;293 std::array<FunctionCallee, 5> SanCovLoadFunction;294 std::array<FunctionCallee, 5> SanCovStoreFunction;295 std::array<FunctionCallee, 2> SanCovTraceDivFunction;296 FunctionCallee SanCovTraceGepFunction;297 FunctionCallee SanCovTraceSwitchFunction;298 GlobalVariable *SanCovLowestStack;299 GlobalVariable *SanCovCallbackGate;300 Type *PtrTy, *IntptrTy, *Int64Ty, *Int32Ty, *Int16Ty, *Int8Ty, *Int1Ty;301 Module *CurModule;302 Triple TargetTriple;303 LLVMContext *C;304 const DataLayout *DL;305 306 GlobalVariable *FunctionGuardArray; // for trace-pc-guard.307 GlobalVariable *Function8bitCounterArray; // for inline-8bit-counters.308 GlobalVariable *FunctionBoolArray; // for inline-bool-flag.309 GlobalVariable *FunctionPCsArray; // for pc-table.310 GlobalVariable *FunctionCFsArray; // for control flow table311 SmallVector<GlobalValue *, 20> GlobalsToAppendToUsed;312 SmallVector<GlobalValue *, 20> GlobalsToAppendToCompilerUsed;313 314 SanitizerCoverageOptions Options;315 316 const SpecialCaseList *Allowlist;317 const SpecialCaseList *Blocklist;318};319} // namespace320 321SanitizerCoveragePass::SanitizerCoveragePass(322 SanitizerCoverageOptions Options, IntrusiveRefCntPtr<vfs::FileSystem> VFS,323 const std::vector<std::string> &AllowlistFiles,324 const std::vector<std::string> &BlocklistFiles)325 : Options(std::move(Options)),326 VFS(VFS ? std::move(VFS) : vfs::getRealFileSystem()) {327 if (AllowlistFiles.size() > 0)328 Allowlist = SpecialCaseList::createOrDie(AllowlistFiles, *this->VFS);329 if (BlocklistFiles.size() > 0)330 Blocklist = SpecialCaseList::createOrDie(BlocklistFiles, *this->VFS);331}332 333PreservedAnalyses SanitizerCoveragePass::run(Module &M,334 ModuleAnalysisManager &MAM) {335 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();336 auto DTCallback = [&FAM](Function &F) -> const DominatorTree & {337 return FAM.getResult<DominatorTreeAnalysis>(F);338 };339 auto PDTCallback = [&FAM](Function &F) -> const PostDominatorTree & {340 return FAM.getResult<PostDominatorTreeAnalysis>(F);341 };342 ModuleSanitizerCoverage ModuleSancov(M, DTCallback, PDTCallback,343 OverrideFromCL(Options), Allowlist.get(),344 Blocklist.get());345 if (!ModuleSancov.instrumentModule())346 return PreservedAnalyses::all();347 348 PreservedAnalyses PA = PreservedAnalyses::none();349 // GlobalsAA is considered stateless and does not get invalidated unless350 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers351 // make changes that require GlobalsAA to be invalidated.352 PA.abandon<GlobalsAA>();353 return PA;354}355 356std::pair<Value *, Value *>357ModuleSanitizerCoverage::CreateSecStartEnd(Module &M, const char *Section,358 Type *Ty) {359 // Use ExternalWeak so that if all sections are discarded due to section360 // garbage collection, the linker will not report undefined symbol errors.361 // Windows defines the start/stop symbols in compiler-rt so no need for362 // ExternalWeak.363 GlobalValue::LinkageTypes Linkage = TargetTriple.isOSBinFormatCOFF()364 ? GlobalVariable::ExternalLinkage365 : GlobalVariable::ExternalWeakLinkage;366 GlobalVariable *SecStart = new GlobalVariable(M, Ty, false, Linkage, nullptr,367 getSectionStart(Section));368 SecStart->setVisibility(GlobalValue::HiddenVisibility);369 GlobalVariable *SecEnd = new GlobalVariable(M, Ty, false, Linkage, nullptr,370 getSectionEnd(Section));371 SecEnd->setVisibility(GlobalValue::HiddenVisibility);372 IRBuilder<> IRB(M.getContext());373 if (!TargetTriple.isOSBinFormatCOFF())374 return std::make_pair(SecStart, SecEnd);375 376 // Account for the fact that on windows-msvc __start_* symbols actually377 // point to a uint64_t before the start of the array.378 auto GEP =379 IRB.CreatePtrAdd(SecStart, ConstantInt::get(IntptrTy, sizeof(uint64_t)));380 return std::make_pair(GEP, SecEnd);381}382 383Function *ModuleSanitizerCoverage::CreateInitCallsForSections(384 Module &M, const char *CtorName, const char *InitFunctionName, Type *Ty,385 const char *Section) {386 if (ClSancovDropCtors)387 return nullptr;388 auto SecStartEnd = CreateSecStartEnd(M, Section, Ty);389 auto SecStart = SecStartEnd.first;390 auto SecEnd = SecStartEnd.second;391 Function *CtorFunc;392 std::tie(CtorFunc, std::ignore) = createSanitizerCtorAndInitFunctions(393 M, CtorName, InitFunctionName, {PtrTy, PtrTy}, {SecStart, SecEnd});394 assert(CtorFunc->getName() == CtorName);395 396 if (TargetTriple.supportsCOMDAT()) {397 // Use comdat to dedup CtorFunc.398 CtorFunc->setComdat(M.getOrInsertComdat(CtorName));399 appendToGlobalCtors(M, CtorFunc, SanCtorAndDtorPriority, CtorFunc);400 } else {401 appendToGlobalCtors(M, CtorFunc, SanCtorAndDtorPriority);402 }403 404 if (TargetTriple.isOSBinFormatCOFF()) {405 // In COFF files, if the contructors are set as COMDAT (they are because406 // COFF supports COMDAT) and the linker flag /OPT:REF (strip unreferenced407 // functions and data) is used, the constructors get stripped. To prevent408 // this, give the constructors weak ODR linkage and ensure the linker knows409 // to include the sancov constructor. This way the linker can deduplicate410 // the constructors but always leave one copy.411 CtorFunc->setLinkage(GlobalValue::WeakODRLinkage);412 }413 return CtorFunc;414}415 416bool ModuleSanitizerCoverage::instrumentModule() {417 if (Options.CoverageType == SanitizerCoverageOptions::SCK_None)418 return false;419 if (Allowlist &&420 !Allowlist->inSection("coverage", "src", M.getSourceFileName()))421 return false;422 if (Blocklist &&423 Blocklist->inSection("coverage", "src", M.getSourceFileName()))424 return false;425 C = &(M.getContext());426 DL = &M.getDataLayout();427 CurModule = &M;428 TargetTriple = M.getTargetTriple();429 FunctionGuardArray = nullptr;430 Function8bitCounterArray = nullptr;431 FunctionBoolArray = nullptr;432 FunctionPCsArray = nullptr;433 FunctionCFsArray = nullptr;434 IntptrTy = Type::getIntNTy(*C, DL->getPointerSizeInBits());435 PtrTy = PointerType::getUnqual(*C);436 Type *VoidTy = Type::getVoidTy(*C);437 IRBuilder<> IRB(*C);438 Int64Ty = IRB.getInt64Ty();439 Int32Ty = IRB.getInt32Ty();440 Int16Ty = IRB.getInt16Ty();441 Int8Ty = IRB.getInt8Ty();442 Int1Ty = IRB.getInt1Ty();443 444 SanCovTracePCIndir =445 M.getOrInsertFunction(SanCovTracePCIndirName, VoidTy, IntptrTy);446 // Make sure smaller parameters are zero-extended to i64 if required by the447 // target ABI.448 AttributeList SanCovTraceCmpZeroExtAL;449 SanCovTraceCmpZeroExtAL =450 SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 0, Attribute::ZExt);451 SanCovTraceCmpZeroExtAL =452 SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 1, Attribute::ZExt);453 454 SanCovTraceCmpFunction[0] =455 M.getOrInsertFunction(SanCovTraceCmp1, SanCovTraceCmpZeroExtAL, VoidTy,456 IRB.getInt8Ty(), IRB.getInt8Ty());457 SanCovTraceCmpFunction[1] =458 M.getOrInsertFunction(SanCovTraceCmp2, SanCovTraceCmpZeroExtAL, VoidTy,459 IRB.getInt16Ty(), IRB.getInt16Ty());460 SanCovTraceCmpFunction[2] =461 M.getOrInsertFunction(SanCovTraceCmp4, SanCovTraceCmpZeroExtAL, VoidTy,462 IRB.getInt32Ty(), IRB.getInt32Ty());463 SanCovTraceCmpFunction[3] =464 M.getOrInsertFunction(SanCovTraceCmp8, VoidTy, Int64Ty, Int64Ty);465 466 SanCovTraceConstCmpFunction[0] = M.getOrInsertFunction(467 SanCovTraceConstCmp1, SanCovTraceCmpZeroExtAL, VoidTy, Int8Ty, Int8Ty);468 SanCovTraceConstCmpFunction[1] = M.getOrInsertFunction(469 SanCovTraceConstCmp2, SanCovTraceCmpZeroExtAL, VoidTy, Int16Ty, Int16Ty);470 SanCovTraceConstCmpFunction[2] = M.getOrInsertFunction(471 SanCovTraceConstCmp4, SanCovTraceCmpZeroExtAL, VoidTy, Int32Ty, Int32Ty);472 SanCovTraceConstCmpFunction[3] =473 M.getOrInsertFunction(SanCovTraceConstCmp8, VoidTy, Int64Ty, Int64Ty);474 475 // Loads.476 SanCovLoadFunction[0] = M.getOrInsertFunction(SanCovLoad1, VoidTy, PtrTy);477 SanCovLoadFunction[1] = M.getOrInsertFunction(SanCovLoad2, VoidTy, PtrTy);478 SanCovLoadFunction[2] = M.getOrInsertFunction(SanCovLoad4, VoidTy, PtrTy);479 SanCovLoadFunction[3] = M.getOrInsertFunction(SanCovLoad8, VoidTy, PtrTy);480 SanCovLoadFunction[4] = M.getOrInsertFunction(SanCovLoad16, VoidTy, PtrTy);481 // Stores.482 SanCovStoreFunction[0] = M.getOrInsertFunction(SanCovStore1, VoidTy, PtrTy);483 SanCovStoreFunction[1] = M.getOrInsertFunction(SanCovStore2, VoidTy, PtrTy);484 SanCovStoreFunction[2] = M.getOrInsertFunction(SanCovStore4, VoidTy, PtrTy);485 SanCovStoreFunction[3] = M.getOrInsertFunction(SanCovStore8, VoidTy, PtrTy);486 SanCovStoreFunction[4] = M.getOrInsertFunction(SanCovStore16, VoidTy, PtrTy);487 488 {489 AttributeList AL;490 AL = AL.addParamAttribute(*C, 0, Attribute::ZExt);491 SanCovTraceDivFunction[0] =492 M.getOrInsertFunction(SanCovTraceDiv4, AL, VoidTy, IRB.getInt32Ty());493 }494 SanCovTraceDivFunction[1] =495 M.getOrInsertFunction(SanCovTraceDiv8, VoidTy, Int64Ty);496 SanCovTraceGepFunction =497 M.getOrInsertFunction(SanCovTraceGep, VoidTy, IntptrTy);498 SanCovTraceSwitchFunction =499 M.getOrInsertFunction(SanCovTraceSwitchName, VoidTy, Int64Ty, PtrTy);500 501 SanCovLowestStack = M.getOrInsertGlobal(SanCovLowestStackName, IntptrTy);502 if (SanCovLowestStack->getValueType() != IntptrTy) {503 C->emitError(StringRef("'") + SanCovLowestStackName +504 "' should not be declared by the user");505 return true;506 }507 SanCovLowestStack->setThreadLocalMode(508 GlobalValue::ThreadLocalMode::InitialExecTLSModel);509 if (Options.StackDepth && !SanCovLowestStack->isDeclaration())510 SanCovLowestStack->setInitializer(Constant::getAllOnesValue(IntptrTy));511 512 if (Options.GatedCallbacks) {513 if (!Options.TracePCGuard && !Options.TraceCmp) {514 C->emitError(StringRef("'") + ClGatedCallbacks.ArgStr +515 "' is only supported with trace-pc-guard or trace-cmp");516 return true;517 }518 519 SanCovCallbackGate = cast<GlobalVariable>(520 M.getOrInsertGlobal(SanCovCallbackGateName, Int64Ty));521 SanCovCallbackGate->setSection(522 getSectionName(SanCovCallbackGateSectionName));523 SanCovCallbackGate->setInitializer(Constant::getNullValue(Int64Ty));524 SanCovCallbackGate->setLinkage(GlobalVariable::LinkOnceAnyLinkage);525 SanCovCallbackGate->setVisibility(GlobalVariable::HiddenVisibility);526 appendToCompilerUsed(M, SanCovCallbackGate);527 }528 529 SanCovTracePC = M.getOrInsertFunction(SanCovTracePCName, VoidTy);530 SanCovTracePCGuard =531 M.getOrInsertFunction(SanCovTracePCGuardName, VoidTy, PtrTy);532 533 SanCovStackDepthCallback =534 M.getOrInsertFunction(SanCovStackDepthCallbackName, VoidTy);535 536 for (auto &F : M)537 instrumentFunction(F);538 539 Function *Ctor = nullptr;540 541 if (FunctionGuardArray)542 Ctor = CreateInitCallsForSections(M, SanCovModuleCtorTracePcGuardName,543 SanCovTracePCGuardInitName, Int32Ty,544 SanCovGuardsSectionName);545 if (Function8bitCounterArray)546 Ctor = CreateInitCallsForSections(M, SanCovModuleCtor8bitCountersName,547 SanCov8bitCountersInitName, Int8Ty,548 SanCovCountersSectionName);549 if (FunctionBoolArray) {550 Ctor = CreateInitCallsForSections(M, SanCovModuleCtorBoolFlagName,551 SanCovBoolFlagInitName, Int1Ty,552 SanCovBoolFlagSectionName);553 }554 if (Ctor && Options.PCTable) {555 auto SecStartEnd = CreateSecStartEnd(M, SanCovPCsSectionName, IntptrTy);556 FunctionCallee InitFunction =557 declareSanitizerInitFunction(M, SanCovPCsInitName, {PtrTy, PtrTy});558 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());559 IRBCtor.CreateCall(InitFunction, {SecStartEnd.first, SecStartEnd.second});560 }561 562 if (Ctor && Options.CollectControlFlow) {563 auto SecStartEnd = CreateSecStartEnd(M, SanCovCFsSectionName, IntptrTy);564 FunctionCallee InitFunction =565 declareSanitizerInitFunction(M, SanCovCFsInitName, {PtrTy, PtrTy});566 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());567 IRBCtor.CreateCall(InitFunction, {SecStartEnd.first, SecStartEnd.second});568 }569 570 appendToUsed(M, GlobalsToAppendToUsed);571 appendToCompilerUsed(M, GlobalsToAppendToCompilerUsed);572 return true;573}574 575// True if block has successors and it dominates all of them.576static bool isFullDominator(const BasicBlock *BB, const DominatorTree &DT) {577 if (succ_empty(BB))578 return false;579 580 return llvm::all_of(successors(BB), [&](const BasicBlock *SUCC) {581 return DT.dominates(BB, SUCC);582 });583}584 585// True if block has predecessors and it postdominates all of them.586static bool isFullPostDominator(const BasicBlock *BB,587 const PostDominatorTree &PDT) {588 if (pred_empty(BB))589 return false;590 591 return llvm::all_of(predecessors(BB), [&](const BasicBlock *PRED) {592 return PDT.dominates(BB, PRED);593 });594}595 596static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB,597 const DominatorTree &DT,598 const PostDominatorTree &PDT,599 const SanitizerCoverageOptions &Options) {600 // Don't insert coverage for blocks containing nothing but unreachable: we601 // will never call __sanitizer_cov() for them, so counting them in602 // NumberOfInstrumentedBlocks() might complicate calculation of code coverage603 // percentage. Also, unreachable instructions frequently have no debug604 // locations.605 if (isa<UnreachableInst>(BB->getFirstNonPHIOrDbgOrLifetime()))606 return false;607 608 // Don't insert coverage into blocks without a valid insertion point609 // (catchswitch blocks).610 if (BB->getFirstInsertionPt() == BB->end())611 return false;612 613 if (Options.NoPrune || &F.getEntryBlock() == BB)614 return true;615 616 if (Options.CoverageType == SanitizerCoverageOptions::SCK_Function &&617 &F.getEntryBlock() != BB)618 return false;619 620 // Do not instrument full dominators, or full post-dominators with multiple621 // predecessors.622 return !isFullDominator(BB, DT) &&623 !(isFullPostDominator(BB, PDT) && !BB->getSinglePredecessor());624}625 626// Returns true iff From->To is a backedge.627// A twist here is that we treat From->To as a backedge if628// * To dominates From or629// * To->UniqueSuccessor dominates From630static bool IsBackEdge(BasicBlock *From, BasicBlock *To,631 const DominatorTree &DT) {632 if (DT.dominates(To, From))633 return true;634 if (auto Next = To->getUniqueSuccessor())635 if (DT.dominates(Next, From))636 return true;637 return false;638}639 640// Prunes uninteresting Cmp instrumentation:641// * CMP instructions that feed into loop backedge branch.642//643// Note that Cmp pruning is controlled by the same flag as the644// BB pruning.645static bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree &DT,646 const SanitizerCoverageOptions &Options) {647 if (!Options.NoPrune)648 if (CMP->hasOneUse())649 if (auto BR = dyn_cast<BranchInst>(CMP->user_back()))650 for (BasicBlock *B : BR->successors())651 if (IsBackEdge(BR->getParent(), B, DT))652 return false;653 return true;654}655 656void ModuleSanitizerCoverage::instrumentFunction(Function &F) {657 if (F.empty())658 return;659 if (F.getName().contains(".module_ctor"))660 return; // Should not instrument sanitizer init functions.661 if (F.getName().starts_with("__sanitizer_"))662 return; // Don't instrument __sanitizer_* callbacks.663 // Don't touch available_externally functions, their actual body is elewhere.664 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)665 return;666 // Don't instrument MSVC CRT configuration helpers. They may run before normal667 // initialization.668 if (F.getName() == "__local_stdio_printf_options" ||669 F.getName() == "__local_stdio_scanf_options")670 return;671 if (isa<UnreachableInst>(F.getEntryBlock().getTerminator()))672 return;673 // Don't instrument functions using SEH for now. Splitting basic blocks like674 // we do for coverage breaks WinEHPrepare.675 // FIXME: Remove this when SEH no longer uses landingpad pattern matching.676 if (F.hasPersonalityFn() &&677 isAsynchronousEHPersonality(classifyEHPersonality(F.getPersonalityFn())))678 return;679 if (Allowlist && !Allowlist->inSection("coverage", "fun", F.getName()))680 return;681 if (Blocklist && Blocklist->inSection("coverage", "fun", F.getName()))682 return;683 // Do not apply any instrumentation for naked functions.684 if (F.hasFnAttribute(Attribute::Naked))685 return;686 if (F.hasFnAttribute(Attribute::NoSanitizeCoverage))687 return;688 if (F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))689 return;690 if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge) {691 SplitAllCriticalEdges(692 F, CriticalEdgeSplittingOptions().setIgnoreUnreachableDests());693 }694 SmallVector<Instruction *, 8> IndirCalls;695 SmallVector<BasicBlock *, 16> BlocksToInstrument;696 SmallVector<Instruction *, 8> CmpTraceTargets;697 SmallVector<Instruction *, 8> SwitchTraceTargets;698 SmallVector<BinaryOperator *, 8> DivTraceTargets;699 SmallVector<GetElementPtrInst *, 8> GepTraceTargets;700 SmallVector<LoadInst *, 8> Loads;701 SmallVector<StoreInst *, 8> Stores;702 703 const DominatorTree &DT = DTCallback(F);704 const PostDominatorTree &PDT = PDTCallback(F);705 bool IsLeafFunc = true;706 707 for (auto &BB : F) {708 if (shouldInstrumentBlock(F, &BB, DT, PDT, Options))709 BlocksToInstrument.push_back(&BB);710 for (auto &Inst : BB) {711 if (Options.IndirectCalls) {712 CallBase *CB = dyn_cast<CallBase>(&Inst);713 if (CB && CB->isIndirectCall())714 IndirCalls.push_back(&Inst);715 }716 if (Options.TraceCmp) {717 if (ICmpInst *CMP = dyn_cast<ICmpInst>(&Inst))718 if (IsInterestingCmp(CMP, DT, Options))719 CmpTraceTargets.push_back(&Inst);720 if (isa<SwitchInst>(&Inst))721 SwitchTraceTargets.push_back(&Inst);722 }723 if (Options.TraceDiv)724 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(&Inst))725 if (BO->getOpcode() == Instruction::SDiv ||726 BO->getOpcode() == Instruction::UDiv)727 DivTraceTargets.push_back(BO);728 if (Options.TraceGep)729 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&Inst))730 GepTraceTargets.push_back(GEP);731 if (Options.TraceLoads)732 if (LoadInst *LI = dyn_cast<LoadInst>(&Inst))733 Loads.push_back(LI);734 if (Options.TraceStores)735 if (StoreInst *SI = dyn_cast<StoreInst>(&Inst))736 Stores.push_back(SI);737 if (Options.StackDepth)738 if (isa<InvokeInst>(Inst) ||739 (isa<CallInst>(Inst) && !isa<IntrinsicInst>(Inst)))740 IsLeafFunc = false;741 }742 }743 744 if (Options.CollectControlFlow)745 createFunctionControlFlow(F);746 747 Value *FunctionGateCmp = nullptr;748 InjectCoverage(F, BlocksToInstrument, FunctionGateCmp, IsLeafFunc);749 InjectCoverageForIndirectCalls(F, IndirCalls);750 InjectTraceForCmp(F, CmpTraceTargets, FunctionGateCmp);751 InjectTraceForSwitch(F, SwitchTraceTargets, FunctionGateCmp);752 InjectTraceForDiv(F, DivTraceTargets);753 InjectTraceForGep(F, GepTraceTargets);754 InjectTraceForLoadsAndStores(F, Loads, Stores);755}756 757GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection(758 size_t NumElements, Function &F, Type *Ty, const char *Section) {759 ArrayType *ArrayTy = ArrayType::get(Ty, NumElements);760 auto Array = new GlobalVariable(761 *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage,762 Constant::getNullValue(ArrayTy), "__sancov_gen_");763 764 if (TargetTriple.supportsCOMDAT() &&765 (F.hasComdat() || TargetTriple.isOSBinFormatELF() || !F.isInterposable()))766 if (auto Comdat = getOrCreateFunctionComdat(F, TargetTriple))767 Array->setComdat(Comdat);768 Array->setSection(getSectionName(Section));769 Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedValue()));770 771 // sancov_pcs parallels the other metadata section(s). Optimizers (e.g.772 // GlobalOpt/ConstantMerge) may not discard sancov_pcs and the other773 // section(s) as a unit, so we conservatively retain all unconditionally in774 // the compiler.775 //776 // With comdat (COFF/ELF), the linker can guarantee the associated sections777 // will be retained or discarded as a unit, so llvm.compiler.used is778 // sufficient. Otherwise, conservatively make all of them retained by the779 // linker.780 if (Array->hasComdat())781 GlobalsToAppendToCompilerUsed.push_back(Array);782 else783 GlobalsToAppendToUsed.push_back(Array);784 785 return Array;786}787 788GlobalVariable *789ModuleSanitizerCoverage::CreatePCArray(Function &F,790 ArrayRef<BasicBlock *> AllBlocks) {791 size_t N = AllBlocks.size();792 assert(N);793 SmallVector<Constant *, 32> PCs;794 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());795 for (size_t i = 0; i < N; i++) {796 if (&F.getEntryBlock() == AllBlocks[i]) {797 PCs.push_back((Constant *)IRB.CreatePointerCast(&F, PtrTy));798 PCs.push_back(799 (Constant *)IRB.CreateIntToPtr(ConstantInt::get(IntptrTy, 1), PtrTy));800 } else {801 PCs.push_back((Constant *)IRB.CreatePointerCast(802 BlockAddress::get(AllBlocks[i]), PtrTy));803 PCs.push_back(Constant::getNullValue(PtrTy));804 }805 }806 auto *PCArray =807 CreateFunctionLocalArrayInSection(N * 2, F, PtrTy, SanCovPCsSectionName);808 PCArray->setInitializer(809 ConstantArray::get(ArrayType::get(PtrTy, N * 2), PCs));810 PCArray->setConstant(true);811 812 return PCArray;813}814 815void ModuleSanitizerCoverage::CreateFunctionLocalArrays(816 Function &F, ArrayRef<BasicBlock *> AllBlocks) {817 if (Options.TracePCGuard)818 FunctionGuardArray = CreateFunctionLocalArrayInSection(819 AllBlocks.size(), F, Int32Ty, SanCovGuardsSectionName);820 821 if (Options.Inline8bitCounters)822 Function8bitCounterArray = CreateFunctionLocalArrayInSection(823 AllBlocks.size(), F, Int8Ty, SanCovCountersSectionName);824 if (Options.InlineBoolFlag)825 FunctionBoolArray = CreateFunctionLocalArrayInSection(826 AllBlocks.size(), F, Int1Ty, SanCovBoolFlagSectionName);827 828 if (Options.PCTable)829 FunctionPCsArray = CreatePCArray(F, AllBlocks);830}831 832Value *ModuleSanitizerCoverage::CreateFunctionLocalGateCmp(IRBuilder<> &IRB) {833 auto Load = IRB.CreateLoad(Int64Ty, SanCovCallbackGate);834 Load->setNoSanitizeMetadata();835 auto Cmp = IRB.CreateIsNotNull(Load);836 Cmp->setName("sancov gate cmp");837 return Cmp;838}839 840Instruction *ModuleSanitizerCoverage::CreateGateBranch(Function &F,841 Value *&FunctionGateCmp,842 Instruction *IP) {843 if (!FunctionGateCmp) {844 // Create this in the entry block845 BasicBlock &BB = F.getEntryBlock();846 BasicBlock::iterator IP = BB.getFirstInsertionPt();847 IP = PrepareToSplitEntryBlock(BB, IP);848 IRBuilder<> EntryIRB(&*IP);849 FunctionGateCmp = CreateFunctionLocalGateCmp(EntryIRB);850 }851 // Set the branch weights in order to minimize the price paid when the852 // gate is turned off, allowing the default enablement of this853 // instrumentation with as little of a performance cost as possible854 auto Weights = MDBuilder(*C).createBranchWeights(1, 100000);855 return SplitBlockAndInsertIfThen(FunctionGateCmp, IP, false, Weights);856}857 858bool ModuleSanitizerCoverage::InjectCoverage(Function &F,859 ArrayRef<BasicBlock *> AllBlocks,860 Value *&FunctionGateCmp,861 bool IsLeafFunc) {862 if (AllBlocks.empty())863 return false;864 CreateFunctionLocalArrays(F, AllBlocks);865 for (size_t i = 0, N = AllBlocks.size(); i < N; i++)866 InjectCoverageAtBlock(F, *AllBlocks[i], i, FunctionGateCmp, IsLeafFunc);867 return true;868}869 870// On every indirect call we call a run-time function871// __sanitizer_cov_indir_call* with two parameters:872// - callee address,873// - global cache array that contains CacheSize pointers (zero-initialized).874// The cache is used to speed up recording the caller-callee pairs.875// The address of the caller is passed implicitly via caller PC.876// CacheSize is encoded in the name of the run-time function.877void ModuleSanitizerCoverage::InjectCoverageForIndirectCalls(878 Function &F, ArrayRef<Instruction *> IndirCalls) {879 if (IndirCalls.empty())880 return;881 assert(Options.TracePC || Options.TracePCGuard ||882 Options.Inline8bitCounters || Options.InlineBoolFlag);883 for (auto *I : IndirCalls) {884 InstrumentationIRBuilder IRB(I);885 CallBase &CB = cast<CallBase>(*I);886 Value *Callee = CB.getCalledOperand();887 if (isa<InlineAsm>(Callee))888 continue;889 IRB.CreateCall(SanCovTracePCIndir, IRB.CreatePointerCast(Callee, IntptrTy));890 }891}892 893// For every switch statement we insert a call:894// __sanitizer_cov_trace_switch(CondValue,895// {NumCases, ValueSizeInBits, Case0Value, Case1Value, Case2Value, ... })896 897void ModuleSanitizerCoverage::InjectTraceForSwitch(898 Function &F, ArrayRef<Instruction *> SwitchTraceTargets,899 Value *&FunctionGateCmp) {900 for (auto *I : SwitchTraceTargets) {901 if (SwitchInst *SI = dyn_cast<SwitchInst>(I)) {902 InstrumentationIRBuilder IRB(I);903 SmallVector<Constant *, 16> Initializers;904 Value *Cond = SI->getCondition();905 if (Cond->getType()->getScalarSizeInBits() >906 Int64Ty->getScalarSizeInBits())907 continue;908 Initializers.push_back(ConstantInt::get(Int64Ty, SI->getNumCases()));909 Initializers.push_back(910 ConstantInt::get(Int64Ty, Cond->getType()->getScalarSizeInBits()));911 if (Cond->getType()->getScalarSizeInBits() <912 Int64Ty->getScalarSizeInBits())913 Cond = IRB.CreateIntCast(Cond, Int64Ty, false);914 for (auto It : SI->cases()) {915 ConstantInt *C = It.getCaseValue();916 if (C->getType()->getScalarSizeInBits() < 64)917 C = ConstantInt::get(C->getContext(), C->getValue().zext(64));918 Initializers.push_back(C);919 }920 llvm::sort(drop_begin(Initializers, 2),921 [](const Constant *A, const Constant *B) {922 return cast<ConstantInt>(A)->getLimitedValue() <923 cast<ConstantInt>(B)->getLimitedValue();924 });925 ArrayType *ArrayOfInt64Ty = ArrayType::get(Int64Ty, Initializers.size());926 GlobalVariable *GV = new GlobalVariable(927 *CurModule, ArrayOfInt64Ty, false, GlobalVariable::InternalLinkage,928 ConstantArray::get(ArrayOfInt64Ty, Initializers),929 "__sancov_gen_cov_switch_values");930 if (Options.GatedCallbacks) {931 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, I);932 IRBuilder<> GateIRB(GateBranch);933 GateIRB.CreateCall(SanCovTraceSwitchFunction, {Cond, GV});934 } else {935 IRB.CreateCall(SanCovTraceSwitchFunction, {Cond, GV});936 }937 }938 }939}940 941void ModuleSanitizerCoverage::InjectTraceForDiv(942 Function &, ArrayRef<BinaryOperator *> DivTraceTargets) {943 for (auto *BO : DivTraceTargets) {944 InstrumentationIRBuilder IRB(BO);945 Value *A1 = BO->getOperand(1);946 if (isa<ConstantInt>(A1))947 continue;948 if (!A1->getType()->isIntegerTy())949 continue;950 uint64_t TypeSize = DL->getTypeStoreSizeInBits(A1->getType());951 int CallbackIdx = TypeSize == 32 ? 0 : TypeSize == 64 ? 1 : -1;952 if (CallbackIdx < 0)953 continue;954 auto Ty = Type::getIntNTy(*C, TypeSize);955 IRB.CreateCall(SanCovTraceDivFunction[CallbackIdx],956 {IRB.CreateIntCast(A1, Ty, true)});957 }958}959 960void ModuleSanitizerCoverage::InjectTraceForGep(961 Function &, ArrayRef<GetElementPtrInst *> GepTraceTargets) {962 for (auto *GEP : GepTraceTargets) {963 InstrumentationIRBuilder IRB(GEP);964 for (Use &Idx : GEP->indices())965 if (!isa<ConstantInt>(Idx) && Idx->getType()->isIntegerTy())966 IRB.CreateCall(SanCovTraceGepFunction,967 {IRB.CreateIntCast(Idx, IntptrTy, true)});968 }969}970 971void ModuleSanitizerCoverage::InjectTraceForLoadsAndStores(972 Function &, ArrayRef<LoadInst *> Loads, ArrayRef<StoreInst *> Stores) {973 auto CallbackIdx = [&](Type *ElementTy) -> int {974 uint64_t TypeSize = DL->getTypeStoreSizeInBits(ElementTy);975 return TypeSize == 8 ? 0976 : TypeSize == 16 ? 1977 : TypeSize == 32 ? 2978 : TypeSize == 64 ? 3979 : TypeSize == 128 ? 4980 : -1;981 };982 for (auto *LI : Loads) {983 InstrumentationIRBuilder IRB(LI);984 auto Ptr = LI->getPointerOperand();985 int Idx = CallbackIdx(LI->getType());986 if (Idx < 0)987 continue;988 IRB.CreateCall(SanCovLoadFunction[Idx], Ptr);989 }990 for (auto *SI : Stores) {991 InstrumentationIRBuilder IRB(SI);992 auto Ptr = SI->getPointerOperand();993 int Idx = CallbackIdx(SI->getValueOperand()->getType());994 if (Idx < 0)995 continue;996 IRB.CreateCall(SanCovStoreFunction[Idx], Ptr);997 }998}999 1000void ModuleSanitizerCoverage::InjectTraceForCmp(1001 Function &F, ArrayRef<Instruction *> CmpTraceTargets,1002 Value *&FunctionGateCmp) {1003 for (auto *I : CmpTraceTargets) {1004 if (ICmpInst *ICMP = dyn_cast<ICmpInst>(I)) {1005 InstrumentationIRBuilder IRB(ICMP);1006 Value *A0 = ICMP->getOperand(0);1007 Value *A1 = ICMP->getOperand(1);1008 if (!A0->getType()->isIntegerTy())1009 continue;1010 uint64_t TypeSize = DL->getTypeStoreSizeInBits(A0->getType());1011 int CallbackIdx = TypeSize == 8 ? 01012 : TypeSize == 16 ? 11013 : TypeSize == 32 ? 21014 : TypeSize == 64 ? 31015 : -1;1016 if (CallbackIdx < 0)1017 continue;1018 // __sanitizer_cov_trace_cmp((type_size << 32) | predicate, A0, A1);1019 auto CallbackFunc = SanCovTraceCmpFunction[CallbackIdx];1020 bool FirstIsConst = isa<ConstantInt>(A0);1021 bool SecondIsConst = isa<ConstantInt>(A1);1022 // If both are const, then we don't need such a comparison.1023 if (FirstIsConst && SecondIsConst)1024 continue;1025 // If only one is const, then make it the first callback argument.1026 if (FirstIsConst || SecondIsConst) {1027 CallbackFunc = SanCovTraceConstCmpFunction[CallbackIdx];1028 if (SecondIsConst)1029 std::swap(A0, A1);1030 }1031 1032 auto Ty = Type::getIntNTy(*C, TypeSize);1033 if (Options.GatedCallbacks) {1034 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, I);1035 IRBuilder<> GateIRB(GateBranch);1036 GateIRB.CreateCall(CallbackFunc, {GateIRB.CreateIntCast(A0, Ty, true),1037 GateIRB.CreateIntCast(A1, Ty, true)});1038 } else {1039 IRB.CreateCall(CallbackFunc, {IRB.CreateIntCast(A0, Ty, true),1040 IRB.CreateIntCast(A1, Ty, true)});1041 }1042 }1043 }1044}1045 1046void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,1047 size_t Idx,1048 Value *&FunctionGateCmp,1049 bool IsLeafFunc) {1050 BasicBlock::iterator IP = BB.getFirstInsertionPt();1051 bool IsEntryBB = &BB == &F.getEntryBlock();1052 DebugLoc EntryLoc;1053 if (IsEntryBB) {1054 if (auto SP = F.getSubprogram())1055 EntryLoc = DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP);1056 // Keep static allocas and llvm.localescape calls in the entry block. Even1057 // if we aren't splitting the block, it's nice for allocas to be before1058 // calls.1059 IP = PrepareToSplitEntryBlock(BB, IP);1060 }1061 1062 InstrumentationIRBuilder IRB(&*IP);1063 if (EntryLoc)1064 IRB.SetCurrentDebugLocation(EntryLoc);1065 if (Options.TracePC) {1066 IRB.CreateCall(SanCovTracePC)1067 ->setCannotMerge(); // gets the PC using GET_CALLER_PC.1068 }1069 if (Options.TracePCGuard) {1070 auto GuardPtr = IRB.CreateConstInBoundsGEP2_64(1071 FunctionGuardArray->getValueType(), FunctionGuardArray, 0, Idx);1072 if (Options.GatedCallbacks) {1073 Instruction *I = &*IP;1074 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, I);1075 IRBuilder<> GateIRB(GateBranch);1076 GateIRB.CreateCall(SanCovTracePCGuard, GuardPtr)->setCannotMerge();1077 } else {1078 IRB.CreateCall(SanCovTracePCGuard, GuardPtr)->setCannotMerge();1079 }1080 }1081 if (Options.Inline8bitCounters) {1082 auto CounterPtr = IRB.CreateGEP(1083 Function8bitCounterArray->getValueType(), Function8bitCounterArray,1084 {ConstantInt::get(IntptrTy, 0), ConstantInt::get(IntptrTy, Idx)});1085 auto Load = IRB.CreateLoad(Int8Ty, CounterPtr);1086 auto Inc = IRB.CreateAdd(Load, ConstantInt::get(Int8Ty, 1));1087 auto Store = IRB.CreateStore(Inc, CounterPtr);1088 Load->setNoSanitizeMetadata();1089 Store->setNoSanitizeMetadata();1090 }1091 if (Options.InlineBoolFlag) {1092 auto FlagPtr = IRB.CreateGEP(1093 FunctionBoolArray->getValueType(), FunctionBoolArray,1094 {ConstantInt::get(IntptrTy, 0), ConstantInt::get(IntptrTy, Idx)});1095 auto Load = IRB.CreateLoad(Int1Ty, FlagPtr);1096 auto ThenTerm = SplitBlockAndInsertIfThen(1097 IRB.CreateIsNull(Load), &*IP, false,1098 MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());1099 InstrumentationIRBuilder ThenIRB(ThenTerm);1100 auto Store = ThenIRB.CreateStore(ConstantInt::getTrue(Int1Ty), FlagPtr);1101 if (EntryLoc)1102 Store->setDebugLoc(EntryLoc);1103 Load->setNoSanitizeMetadata();1104 Store->setNoSanitizeMetadata();1105 }1106 if (Options.StackDepth && IsEntryBB && !IsLeafFunc) {1107 Module *M = F.getParent();1108 const DataLayout &DL = M->getDataLayout();1109 1110 if (Options.StackDepthCallbackMin) {1111 // In callback mode, only add call when stack depth reaches minimum.1112 int EstimatedStackSize = 0;1113 // If dynamic alloca found, always add call.1114 bool HasDynamicAlloc = false;1115 // Find an insertion point after last "alloca".1116 llvm::Instruction *InsertBefore = nullptr;1117 1118 // Examine all allocas in the basic block. since we're too early1119 // to have results from Intrinsic::frameaddress, we have to manually1120 // estimate the stack size.1121 for (auto &I : BB) {1122 if (auto *AI = dyn_cast<AllocaInst>(&I)) {1123 // Move potential insertion point past the "alloca".1124 InsertBefore = AI->getNextNode();1125 1126 // Make an estimate on the stack usage.1127 if (AI->isStaticAlloca()) {1128 uint32_t Bytes = DL.getTypeAllocSize(AI->getAllocatedType());1129 if (AI->isArrayAllocation()) {1130 if (const ConstantInt *arraySize =1131 dyn_cast<ConstantInt>(AI->getArraySize())) {1132 Bytes *= arraySize->getZExtValue();1133 } else {1134 HasDynamicAlloc = true;1135 }1136 }1137 EstimatedStackSize += Bytes;1138 } else {1139 HasDynamicAlloc = true;1140 }1141 }1142 }1143 1144 if (HasDynamicAlloc ||1145 EstimatedStackSize >= Options.StackDepthCallbackMin) {1146 if (InsertBefore)1147 IRB.SetInsertPoint(InsertBefore);1148 auto Call = IRB.CreateCall(SanCovStackDepthCallback);1149 if (EntryLoc)1150 Call->setDebugLoc(EntryLoc);1151 Call->setCannotMerge();1152 }1153 } else {1154 // Check stack depth. If it's the deepest so far, record it.1155 auto FrameAddrPtr = IRB.CreateIntrinsic(1156 Intrinsic::frameaddress, IRB.getPtrTy(DL.getAllocaAddrSpace()),1157 {Constant::getNullValue(Int32Ty)});1158 auto FrameAddrInt = IRB.CreatePtrToInt(FrameAddrPtr, IntptrTy);1159 auto LowestStack = IRB.CreateLoad(IntptrTy, SanCovLowestStack);1160 auto IsStackLower = IRB.CreateICmpULT(FrameAddrInt, LowestStack);1161 auto ThenTerm = SplitBlockAndInsertIfThen(1162 IsStackLower, &*IP, false,1163 MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());1164 InstrumentationIRBuilder ThenIRB(ThenTerm);1165 auto Store = ThenIRB.CreateStore(FrameAddrInt, SanCovLowestStack);1166 if (EntryLoc)1167 Store->setDebugLoc(EntryLoc);1168 LowestStack->setNoSanitizeMetadata();1169 Store->setNoSanitizeMetadata();1170 }1171 }1172}1173 1174std::string1175ModuleSanitizerCoverage::getSectionName(const std::string &Section) const {1176 if (TargetTriple.isOSBinFormatCOFF()) {1177 if (Section == SanCovCountersSectionName)1178 return ".SCOV$CM";1179 if (Section == SanCovBoolFlagSectionName)1180 return ".SCOV$BM";1181 if (Section == SanCovPCsSectionName)1182 return ".SCOVP$M";1183 return ".SCOV$GM"; // For SanCovGuardsSectionName.1184 }1185 if (TargetTriple.isOSBinFormatMachO())1186 return "__DATA,__" + Section;1187 return "__" + Section;1188}1189 1190std::string1191ModuleSanitizerCoverage::getSectionStart(const std::string &Section) const {1192 if (TargetTriple.isOSBinFormatMachO())1193 return "\1section$start$__DATA$__" + Section;1194 return "__start___" + Section;1195}1196 1197std::string1198ModuleSanitizerCoverage::getSectionEnd(const std::string &Section) const {1199 if (TargetTriple.isOSBinFormatMachO())1200 return "\1section$end$__DATA$__" + Section;1201 return "__stop___" + Section;1202}1203 1204void ModuleSanitizerCoverage::createFunctionControlFlow(Function &F) {1205 SmallVector<Constant *, 32> CFs;1206 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());1207 1208 for (auto &BB : F) {1209 // blockaddress can not be used on function's entry block.1210 if (&BB == &F.getEntryBlock())1211 CFs.push_back((Constant *)IRB.CreatePointerCast(&F, PtrTy));1212 else1213 CFs.push_back(1214 (Constant *)IRB.CreatePointerCast(BlockAddress::get(&BB), PtrTy));1215 1216 for (auto SuccBB : successors(&BB)) {1217 assert(SuccBB != &F.getEntryBlock());1218 CFs.push_back(1219 (Constant *)IRB.CreatePointerCast(BlockAddress::get(SuccBB), PtrTy));1220 }1221 1222 CFs.push_back((Constant *)Constant::getNullValue(PtrTy));1223 1224 for (auto &Inst : BB) {1225 if (CallBase *CB = dyn_cast<CallBase>(&Inst)) {1226 if (CB->isIndirectCall()) {1227 // TODO(navidem): handle indirect calls, for now mark its existence.1228 CFs.push_back((Constant *)IRB.CreateIntToPtr(1229 ConstantInt::get(IntptrTy, -1), PtrTy));1230 } else {1231 auto CalledF = CB->getCalledFunction();1232 if (CalledF && !CalledF->isIntrinsic())1233 CFs.push_back((Constant *)IRB.CreatePointerCast(CalledF, PtrTy));1234 }1235 }1236 }1237 1238 CFs.push_back((Constant *)Constant::getNullValue(PtrTy));1239 }1240 1241 FunctionCFsArray = CreateFunctionLocalArrayInSection(CFs.size(), F, PtrTy,1242 SanCovCFsSectionName);1243 FunctionCFsArray->setInitializer(1244 ConstantArray::get(ArrayType::get(PtrTy, CFs.size()), CFs));1245 FunctionCFsArray->setConstant(true);1246}1247