1583 lines · cpp
1//===--- CodeGenPGO.cpp - PGO Instrumentation for LLVM CodeGen --*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Instrumentation-based profile-guided optimization10//11//===----------------------------------------------------------------------===//12 13#include "CodeGenPGO.h"14#include "CGDebugInfo.h"15#include "CodeGenFunction.h"16#include "CoverageMappingGen.h"17#include "clang/AST/RecursiveASTVisitor.h"18#include "clang/AST/StmtVisitor.h"19#include "llvm/IR/Intrinsics.h"20#include "llvm/IR/MDBuilder.h"21#include "llvm/Support/CommandLine.h"22#include "llvm/Support/Endian.h"23#include "llvm/Support/MD5.h"24#include <optional>25 26namespace llvm {27extern cl::opt<bool> EnableSingleByteCoverage;28} // namespace llvm29 30static llvm::cl::opt<bool>31 EnableValueProfiling("enable-value-profiling",32 llvm::cl::desc("Enable value profiling"),33 llvm::cl::Hidden, llvm::cl::init(false));34 35using namespace clang;36using namespace CodeGen;37 38void CodeGenPGO::setFuncName(StringRef Name,39 llvm::GlobalValue::LinkageTypes Linkage) {40 llvm::IndexedInstrProfReader *PGOReader = CGM.getPGOReader();41 FuncName = llvm::getPGOFuncName(42 Name, Linkage, CGM.getCodeGenOpts().MainFileName,43 PGOReader ? PGOReader->getVersion() : llvm::IndexedInstrProf::Version);44 45 // If we're generating a profile, create a variable for the name.46 if (CGM.getCodeGenOpts().hasProfileClangInstr())47 FuncNameVar = llvm::createPGOFuncNameVar(CGM.getModule(), Linkage, FuncName);48}49 50void CodeGenPGO::setFuncName(llvm::Function *Fn) {51 setFuncName(Fn->getName(), Fn->getLinkage());52 // Create PGOFuncName meta data.53 llvm::createPGOFuncNameMetadata(*Fn, FuncName);54}55 56/// The version of the PGO hash algorithm.57enum PGOHashVersion : unsigned {58 PGO_HASH_V1,59 PGO_HASH_V2,60 PGO_HASH_V3,61 PGO_HASH_V4,62 63 // Keep this set to the latest hash version.64 PGO_HASH_LATEST = PGO_HASH_V465};66 67namespace {68/// Stable hasher for PGO region counters.69///70/// PGOHash produces a stable hash of a given function's control flow.71///72/// Changing the output of this hash will invalidate all previously generated73/// profiles -- i.e., don't do it.74///75/// \note When this hash does eventually change (years?), we still need to76/// support old hashes. We'll need to pull in the version number from the77/// profile data format and use the matching hash function.78class PGOHash {79 uint64_t Working;80 unsigned Count;81 PGOHashVersion HashVersion;82 llvm::MD5 MD5;83 84 static const int NumBitsPerType = 6;85 static const unsigned NumTypesPerWord = sizeof(uint64_t) * 8 / NumBitsPerType;86 static const unsigned TooBig = 1u << NumBitsPerType;87 88public:89 /// Hash values for AST nodes.90 ///91 /// Distinct values for AST nodes that have region counters attached.92 ///93 /// These values must be stable. All new members must be added at the end,94 /// and no members should be removed. Changing the enumeration value for an95 /// AST node will affect the hash of every function that contains that node.96 enum HashType : unsigned char {97 None = 0,98 LabelStmt = 1,99 WhileStmt,100 DoStmt,101 ForStmt,102 CXXForRangeStmt,103 ObjCForCollectionStmt,104 SwitchStmt,105 CaseStmt,106 DefaultStmt,107 IfStmt,108 CXXTryStmt,109 CXXCatchStmt,110 ConditionalOperator,111 BinaryOperatorLAnd,112 BinaryOperatorLOr,113 BinaryConditionalOperator,114 // The preceding values are available with PGO_HASH_V1.115 116 EndOfScope,117 IfThenBranch,118 IfElseBranch,119 GotoStmt,120 IndirectGotoStmt,121 BreakStmt,122 ContinueStmt,123 ReturnStmt,124 ThrowExpr,125 UnaryOperatorLNot,126 BinaryOperatorLT,127 BinaryOperatorGT,128 BinaryOperatorLE,129 BinaryOperatorGE,130 BinaryOperatorEQ,131 BinaryOperatorNE,132 // The preceding values are available since PGO_HASH_V2.133 134 // Keep this last. It's for the static assert that follows.135 LastHashType136 };137 static_assert(LastHashType <= TooBig, "Too many types in HashType");138 139 PGOHash(PGOHashVersion HashVersion)140 : Working(0), Count(0), HashVersion(HashVersion) {}141 void combine(HashType Type);142 uint64_t finalize();143 PGOHashVersion getHashVersion() const { return HashVersion; }144};145const int PGOHash::NumBitsPerType;146const unsigned PGOHash::NumTypesPerWord;147const unsigned PGOHash::TooBig;148 149/// Get the PGO hash version used in the given indexed profile.150static PGOHashVersion getPGOHashVersion(llvm::IndexedInstrProfReader *PGOReader,151 CodeGenModule &CGM) {152 if (PGOReader->getVersion() <= 4)153 return PGO_HASH_V1;154 if (PGOReader->getVersion() <= 5)155 return PGO_HASH_V2;156 if (PGOReader->getVersion() <= 12)157 return PGO_HASH_V3;158 return PGO_HASH_V4;159}160 161/// A RecursiveASTVisitor that fills a map of statements to PGO counters.162struct MapRegionCounters : public RecursiveASTVisitor<MapRegionCounters> {163 using Base = RecursiveASTVisitor<MapRegionCounters>;164 165 /// The next counter value to assign.166 unsigned NextCounter;167 /// The function hash.168 PGOHash Hash;169 /// The map of statements to counters.170 llvm::DenseMap<const Stmt *, CounterPair> &CounterMap;171 /// The state of MC/DC Coverage in this function.172 MCDC::State &MCDCState;173 /// Maximum number of supported MC/DC conditions in a boolean expression.174 unsigned MCDCMaxCond;175 /// The profile version.176 uint64_t ProfileVersion;177 /// Diagnostics Engine used to report warnings.178 DiagnosticsEngine &Diag;179 180 MapRegionCounters(PGOHashVersion HashVersion, uint64_t ProfileVersion,181 llvm::DenseMap<const Stmt *, CounterPair> &CounterMap,182 MCDC::State &MCDCState, unsigned MCDCMaxCond,183 DiagnosticsEngine &Diag)184 : NextCounter(0), Hash(HashVersion), CounterMap(CounterMap),185 MCDCState(MCDCState), MCDCMaxCond(MCDCMaxCond),186 ProfileVersion(ProfileVersion), Diag(Diag) {}187 188 // Blocks and lambdas are handled as separate functions, so we need not189 // traverse them in the parent context.190 bool TraverseBlockExpr(BlockExpr *BE) { return true; }191 bool TraverseLambdaExpr(LambdaExpr *LE) {192 // Traverse the captures, but not the body.193 for (auto C : zip(LE->captures(), LE->capture_inits()))194 TraverseLambdaCapture(LE, &std::get<0>(C), std::get<1>(C));195 return true;196 }197 bool TraverseCapturedStmt(CapturedStmt *CS) { return true; }198 199 bool VisitDecl(const Decl *D) {200 switch (D->getKind()) {201 default:202 break;203 case Decl::Function:204 case Decl::CXXMethod:205 case Decl::CXXConstructor:206 case Decl::CXXDestructor:207 case Decl::CXXConversion:208 case Decl::ObjCMethod:209 case Decl::Block:210 case Decl::Captured:211 CounterMap[D->getBody()] = NextCounter++;212 break;213 }214 return true;215 }216 217 /// If \p S gets a fresh counter, update the counter mappings. Return the218 /// V1 hash of \p S.219 PGOHash::HashType updateCounterMappings(Stmt *S) {220 auto Type = getHashType(PGO_HASH_V1, S);221 if (Type != PGOHash::None)222 CounterMap[S] = NextCounter++;223 return Type;224 }225 226 /// The following stacks are used with dataTraverseStmtPre() and227 /// dataTraverseStmtPost() to track the depth of nested logical operators in a228 /// boolean expression in a function. The ultimate purpose is to keep track229 /// of the number of leaf-level conditions in the boolean expression so that a230 /// profile bitmap can be allocated based on that number.231 ///232 /// The stacks are also used to find error cases and notify the user. A233 /// standard logical operator nest for a boolean expression could be in a form234 /// similar to this: "x = a && b && c && (d || f)"235 unsigned NumCond = 0;236 bool SplitNestedLogicalOp = false;237 SmallVector<const Stmt *, 16> NonLogOpStack;238 SmallVector<const BinaryOperator *, 16> LogOpStack;239 240 // Hook: dataTraverseStmtPre() is invoked prior to visiting an AST Stmt node.241 bool dataTraverseStmtPre(Stmt *S) {242 /// If MC/DC is not enabled, MCDCMaxCond will be set to 0. Do nothing.243 if (MCDCMaxCond == 0)244 return true;245 246 /// At the top of the logical operator nest, reset the number of conditions,247 /// also forget previously seen split nesting cases.248 if (LogOpStack.empty()) {249 NumCond = 0;250 SplitNestedLogicalOp = false;251 }252 253 if (const Expr *E = dyn_cast<Expr>(S)) {254 const BinaryOperator *BinOp = dyn_cast<BinaryOperator>(E->IgnoreParens());255 if (BinOp && BinOp->isLogicalOp()) {256 /// Check for "split-nested" logical operators. This happens when a new257 /// boolean expression logical-op nest is encountered within an existing258 /// boolean expression, separated by a non-logical operator. For259 /// example, in "x = (a && b && c && foo(d && f))", the "d && f" case260 /// starts a new boolean expression that is separated from the other261 /// conditions by the operator foo(). Split-nested cases are not262 /// supported by MC/DC.263 SplitNestedLogicalOp = SplitNestedLogicalOp || !NonLogOpStack.empty();264 265 LogOpStack.push_back(BinOp);266 return true;267 }268 }269 270 /// Keep track of non-logical operators. These are OK as long as we don't271 /// encounter a new logical operator after seeing one.272 if (!LogOpStack.empty())273 NonLogOpStack.push_back(S);274 275 return true;276 }277 278 // Hook: dataTraverseStmtPost() is invoked by the AST visitor after visiting279 // an AST Stmt node. MC/DC will use it to to signal when the top of a280 // logical operation (boolean expression) nest is encountered.281 bool dataTraverseStmtPost(Stmt *S) {282 /// If MC/DC is not enabled, MCDCMaxCond will be set to 0. Do nothing.283 if (MCDCMaxCond == 0)284 return true;285 286 if (const Expr *E = dyn_cast<Expr>(S)) {287 const BinaryOperator *BinOp = dyn_cast<BinaryOperator>(E->IgnoreParens());288 if (BinOp && BinOp->isLogicalOp()) {289 assert(LogOpStack.back() == BinOp);290 LogOpStack.pop_back();291 292 /// At the top of logical operator nest:293 if (LogOpStack.empty()) {294 /// Was the "split-nested" logical operator case encountered?295 if (SplitNestedLogicalOp) {296 unsigned DiagID = Diag.getCustomDiagID(297 DiagnosticsEngine::Warning,298 "unsupported MC/DC boolean expression; "299 "contains an operation with a nested boolean expression. "300 "Expression will not be covered");301 Diag.Report(S->getBeginLoc(), DiagID);302 return true;303 }304 305 /// Was the maximum number of conditions encountered?306 if (NumCond > MCDCMaxCond) {307 unsigned DiagID = Diag.getCustomDiagID(308 DiagnosticsEngine::Warning,309 "unsupported MC/DC boolean expression; "310 "number of conditions (%0) exceeds max (%1). "311 "Expression will not be covered");312 Diag.Report(S->getBeginLoc(), DiagID) << NumCond << MCDCMaxCond;313 return true;314 }315 316 // Otherwise, allocate the Decision.317 MCDCState.DecisionByStmt[BinOp].BitmapIdx = 0;318 }319 return true;320 }321 }322 323 if (!LogOpStack.empty())324 NonLogOpStack.pop_back();325 326 return true;327 }328 329 /// The RHS of all logical operators gets a fresh counter in order to count330 /// how many times the RHS evaluates to true or false, depending on the331 /// semantics of the operator. This is only valid for ">= v7" of the profile332 /// version so that we facilitate backward compatibility. In addition, in333 /// order to use MC/DC, count the number of total LHS and RHS conditions.334 bool VisitBinaryOperator(BinaryOperator *S) {335 if (S->isLogicalOp()) {336 if (CodeGenFunction::isInstrumentedCondition(S->getLHS()))337 NumCond++;338 339 if (CodeGenFunction::isInstrumentedCondition(S->getRHS())) {340 if (ProfileVersion >= llvm::IndexedInstrProf::Version7)341 CounterMap[S->getRHS()] = NextCounter++;342 343 NumCond++;344 }345 }346 return Base::VisitBinaryOperator(S);347 }348 349 bool VisitConditionalOperator(ConditionalOperator *S) {350 if (llvm::EnableSingleByteCoverage && S->getTrueExpr())351 CounterMap[S->getTrueExpr()] = NextCounter++;352 if (llvm::EnableSingleByteCoverage && S->getFalseExpr())353 CounterMap[S->getFalseExpr()] = NextCounter++;354 return Base::VisitConditionalOperator(S);355 }356 357 /// Include \p S in the function hash.358 bool VisitStmt(Stmt *S) {359 auto Type = updateCounterMappings(S);360 if (Hash.getHashVersion() != PGO_HASH_V1)361 Type = getHashType(Hash.getHashVersion(), S);362 if (Type != PGOHash::None)363 Hash.combine(Type);364 return true;365 }366 367 bool TraverseIfStmt(IfStmt *If) {368 // If we used the V1 hash, use the default traversal.369 if (Hash.getHashVersion() == PGO_HASH_V1)370 return Base::TraverseIfStmt(If);371 372 // When single byte coverage mode is enabled, add a counter to then and373 // else.374 bool NoSingleByteCoverage = !llvm::EnableSingleByteCoverage;375 for (Stmt *CS : If->children()) {376 if (!CS || NoSingleByteCoverage)377 continue;378 if (CS == If->getThen())379 CounterMap[If->getThen()] = NextCounter++;380 else if (CS == If->getElse())381 CounterMap[If->getElse()] = NextCounter++;382 }383 384 // Otherwise, keep track of which branch we're in while traversing.385 VisitStmt(If);386 387 for (Stmt *CS : If->children()) {388 if (!CS)389 continue;390 if (CS == If->getThen())391 Hash.combine(PGOHash::IfThenBranch);392 else if (CS == If->getElse())393 Hash.combine(PGOHash::IfElseBranch);394 TraverseStmt(CS);395 }396 Hash.combine(PGOHash::EndOfScope);397 return true;398 }399 400 bool TraverseWhileStmt(WhileStmt *While) {401 // When single byte coverage mode is enabled, add a counter to condition and402 // body.403 bool NoSingleByteCoverage = !llvm::EnableSingleByteCoverage;404 for (Stmt *CS : While->children()) {405 if (!CS || NoSingleByteCoverage)406 continue;407 if (CS == While->getCond())408 CounterMap[While->getCond()] = NextCounter++;409 else if (CS == While->getBody())410 CounterMap[While->getBody()] = NextCounter++;411 }412 413 Base::TraverseWhileStmt(While);414 if (Hash.getHashVersion() != PGO_HASH_V1)415 Hash.combine(PGOHash::EndOfScope);416 return true;417 }418 419 bool TraverseDoStmt(DoStmt *Do) {420 // When single byte coverage mode is enabled, add a counter to condition and421 // body.422 bool NoSingleByteCoverage = !llvm::EnableSingleByteCoverage;423 for (Stmt *CS : Do->children()) {424 if (!CS || NoSingleByteCoverage)425 continue;426 if (CS == Do->getCond())427 CounterMap[Do->getCond()] = NextCounter++;428 else if (CS == Do->getBody())429 CounterMap[Do->getBody()] = NextCounter++;430 }431 432 Base::TraverseDoStmt(Do);433 if (Hash.getHashVersion() != PGO_HASH_V1)434 Hash.combine(PGOHash::EndOfScope);435 return true;436 }437 438 bool TraverseForStmt(ForStmt *For) {439 // When single byte coverage mode is enabled, add a counter to condition,440 // increment and body.441 bool NoSingleByteCoverage = !llvm::EnableSingleByteCoverage;442 for (Stmt *CS : For->children()) {443 if (!CS || NoSingleByteCoverage)444 continue;445 if (CS == For->getCond())446 CounterMap[For->getCond()] = NextCounter++;447 else if (CS == For->getInc())448 CounterMap[For->getInc()] = NextCounter++;449 else if (CS == For->getBody())450 CounterMap[For->getBody()] = NextCounter++;451 }452 453 Base::TraverseForStmt(For);454 if (Hash.getHashVersion() != PGO_HASH_V1)455 Hash.combine(PGOHash::EndOfScope);456 return true;457 }458 459 bool TraverseCXXForRangeStmt(CXXForRangeStmt *ForRange) {460 // When single byte coverage mode is enabled, add a counter to body.461 bool NoSingleByteCoverage = !llvm::EnableSingleByteCoverage;462 for (Stmt *CS : ForRange->children()) {463 if (!CS || NoSingleByteCoverage)464 continue;465 if (CS == ForRange->getBody())466 CounterMap[ForRange->getBody()] = NextCounter++;467 }468 469 Base::TraverseCXXForRangeStmt(ForRange);470 if (Hash.getHashVersion() != PGO_HASH_V1)471 Hash.combine(PGOHash::EndOfScope);472 return true;473 }474 475// If the statement type \p N is nestable, and its nesting impacts profile476// stability, define a custom traversal which tracks the end of the statement477// in the hash (provided we're not using the V1 hash).478#define DEFINE_NESTABLE_TRAVERSAL(N) \479 bool Traverse##N(N *S) { \480 Base::Traverse##N(S); \481 if (Hash.getHashVersion() != PGO_HASH_V1) \482 Hash.combine(PGOHash::EndOfScope); \483 return true; \484 }485 486 DEFINE_NESTABLE_TRAVERSAL(ObjCForCollectionStmt)487 DEFINE_NESTABLE_TRAVERSAL(CXXTryStmt)488 DEFINE_NESTABLE_TRAVERSAL(CXXCatchStmt)489 490 /// Get version \p HashVersion of the PGO hash for \p S.491 PGOHash::HashType getHashType(PGOHashVersion HashVersion, const Stmt *S) {492 switch (S->getStmtClass()) {493 default:494 break;495 case Stmt::LabelStmtClass:496 return PGOHash::LabelStmt;497 case Stmt::WhileStmtClass:498 return PGOHash::WhileStmt;499 case Stmt::DoStmtClass:500 return PGOHash::DoStmt;501 case Stmt::ForStmtClass:502 return PGOHash::ForStmt;503 case Stmt::CXXForRangeStmtClass:504 return PGOHash::CXXForRangeStmt;505 case Stmt::ObjCForCollectionStmtClass:506 return PGOHash::ObjCForCollectionStmt;507 case Stmt::SwitchStmtClass:508 return PGOHash::SwitchStmt;509 case Stmt::CaseStmtClass:510 return PGOHash::CaseStmt;511 case Stmt::DefaultStmtClass:512 return PGOHash::DefaultStmt;513 case Stmt::IfStmtClass:514 return PGOHash::IfStmt;515 case Stmt::CXXTryStmtClass:516 return PGOHash::CXXTryStmt;517 case Stmt::CXXCatchStmtClass:518 return PGOHash::CXXCatchStmt;519 case Stmt::ConditionalOperatorClass:520 return PGOHash::ConditionalOperator;521 case Stmt::BinaryConditionalOperatorClass:522 return PGOHash::BinaryConditionalOperator;523 case Stmt::BinaryOperatorClass: {524 const BinaryOperator *BO = cast<BinaryOperator>(S);525 if (BO->getOpcode() == BO_LAnd)526 return PGOHash::BinaryOperatorLAnd;527 if (BO->getOpcode() == BO_LOr)528 return PGOHash::BinaryOperatorLOr;529 if (HashVersion >= PGO_HASH_V2) {530 switch (BO->getOpcode()) {531 default:532 break;533 case BO_LT:534 return PGOHash::BinaryOperatorLT;535 case BO_GT:536 return PGOHash::BinaryOperatorGT;537 case BO_LE:538 return PGOHash::BinaryOperatorLE;539 case BO_GE:540 return PGOHash::BinaryOperatorGE;541 case BO_EQ:542 return PGOHash::BinaryOperatorEQ;543 case BO_NE:544 return PGOHash::BinaryOperatorNE;545 }546 }547 break;548 }549 }550 551 if (HashVersion >= PGO_HASH_V2) {552 switch (S->getStmtClass()) {553 default:554 break;555 case Stmt::GotoStmtClass:556 return PGOHash::GotoStmt;557 case Stmt::IndirectGotoStmtClass:558 return PGOHash::IndirectGotoStmt;559 case Stmt::BreakStmtClass:560 return PGOHash::BreakStmt;561 case Stmt::ContinueStmtClass:562 return PGOHash::ContinueStmt;563 case Stmt::ReturnStmtClass:564 return PGOHash::ReturnStmt;565 case Stmt::CXXThrowExprClass:566 return PGOHash::ThrowExpr;567 case Stmt::UnaryOperatorClass: {568 const UnaryOperator *UO = cast<UnaryOperator>(S);569 if (UO->getOpcode() == UO_LNot)570 return PGOHash::UnaryOperatorLNot;571 break;572 }573 }574 }575 576 return PGOHash::None;577 }578};579 580/// A StmtVisitor that propagates the raw counts through the AST and581/// records the count at statements where the value may change.582struct ComputeRegionCounts : public ConstStmtVisitor<ComputeRegionCounts> {583 /// PGO state.584 CodeGenPGO &PGO;585 586 /// A flag that is set when the current count should be recorded on the587 /// next statement, such as at the exit of a loop.588 bool RecordNextStmtCount;589 590 /// The count at the current location in the traversal.591 uint64_t CurrentCount;592 593 /// The map of statements to count values.594 llvm::DenseMap<const Stmt *, uint64_t> &CountMap;595 596 /// BreakContinueStack - Keep counts of breaks and continues inside loops.597 struct BreakContinue {598 uint64_t BreakCount = 0;599 uint64_t ContinueCount = 0;600 BreakContinue() = default;601 };602 SmallVector<BreakContinue, 8> BreakContinueStack;603 604 ComputeRegionCounts(llvm::DenseMap<const Stmt *, uint64_t> &CountMap,605 CodeGenPGO &PGO)606 : PGO(PGO), RecordNextStmtCount(false), CountMap(CountMap) {}607 608 void RecordStmtCount(const Stmt *S) {609 if (RecordNextStmtCount) {610 CountMap[S] = CurrentCount;611 RecordNextStmtCount = false;612 }613 }614 615 /// Set and return the current count.616 uint64_t setCount(uint64_t Count) {617 CurrentCount = Count;618 return Count;619 }620 621 void VisitStmt(const Stmt *S) {622 RecordStmtCount(S);623 for (const Stmt *Child : S->children())624 if (Child)625 this->Visit(Child);626 }627 628 void VisitFunctionDecl(const FunctionDecl *D) {629 // Counter tracks entry to the function body.630 uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody()));631 CountMap[D->getBody()] = BodyCount;632 Visit(D->getBody());633 }634 635 // Skip lambda expressions. We visit these as FunctionDecls when we're636 // generating them and aren't interested in the body when generating a637 // parent context.638 void VisitLambdaExpr(const LambdaExpr *LE) {}639 640 void VisitCapturedDecl(const CapturedDecl *D) {641 // Counter tracks entry to the capture body.642 uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody()));643 CountMap[D->getBody()] = BodyCount;644 Visit(D->getBody());645 }646 647 void VisitObjCMethodDecl(const ObjCMethodDecl *D) {648 // Counter tracks entry to the method body.649 uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody()));650 CountMap[D->getBody()] = BodyCount;651 Visit(D->getBody());652 }653 654 void VisitBlockDecl(const BlockDecl *D) {655 // Counter tracks entry to the block body.656 uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody()));657 CountMap[D->getBody()] = BodyCount;658 Visit(D->getBody());659 }660 661 void VisitReturnStmt(const ReturnStmt *S) {662 RecordStmtCount(S);663 if (S->getRetValue())664 Visit(S->getRetValue());665 CurrentCount = 0;666 RecordNextStmtCount = true;667 }668 669 void VisitCXXThrowExpr(const CXXThrowExpr *E) {670 RecordStmtCount(E);671 if (E->getSubExpr())672 Visit(E->getSubExpr());673 CurrentCount = 0;674 RecordNextStmtCount = true;675 }676 677 void VisitGotoStmt(const GotoStmt *S) {678 RecordStmtCount(S);679 CurrentCount = 0;680 RecordNextStmtCount = true;681 }682 683 void VisitLabelStmt(const LabelStmt *S) {684 RecordNextStmtCount = false;685 // Counter tracks the block following the label.686 uint64_t BlockCount = setCount(PGO.getRegionCount(S));687 CountMap[S] = BlockCount;688 Visit(S->getSubStmt());689 }690 691 void VisitBreakStmt(const BreakStmt *S) {692 RecordStmtCount(S);693 assert(!BreakContinueStack.empty() && "break not in a loop or switch!");694 BreakContinueStack.back().BreakCount += CurrentCount;695 CurrentCount = 0;696 RecordNextStmtCount = true;697 }698 699 void VisitContinueStmt(const ContinueStmt *S) {700 RecordStmtCount(S);701 assert(!BreakContinueStack.empty() && "continue stmt not in a loop!");702 BreakContinueStack.back().ContinueCount += CurrentCount;703 CurrentCount = 0;704 RecordNextStmtCount = true;705 }706 707 void VisitWhileStmt(const WhileStmt *S) {708 RecordStmtCount(S);709 uint64_t ParentCount = CurrentCount;710 711 BreakContinueStack.push_back(BreakContinue());712 // Visit the body region first so the break/continue adjustments can be713 // included when visiting the condition.714 uint64_t BodyCount = setCount(PGO.getRegionCount(S));715 CountMap[S->getBody()] = CurrentCount;716 Visit(S->getBody());717 uint64_t BackedgeCount = CurrentCount;718 719 // ...then go back and propagate counts through the condition. The count720 // at the start of the condition is the sum of the incoming edges,721 // the backedge from the end of the loop body, and the edges from722 // continue statements.723 BreakContinue BC = BreakContinueStack.pop_back_val();724 uint64_t CondCount =725 setCount(ParentCount + BackedgeCount + BC.ContinueCount);726 CountMap[S->getCond()] = CondCount;727 Visit(S->getCond());728 setCount(BC.BreakCount + CondCount - BodyCount);729 RecordNextStmtCount = true;730 }731 732 void VisitDoStmt(const DoStmt *S) {733 RecordStmtCount(S);734 uint64_t LoopCount = PGO.getRegionCount(S);735 736 BreakContinueStack.push_back(BreakContinue());737 // The count doesn't include the fallthrough from the parent scope. Add it.738 uint64_t BodyCount = setCount(LoopCount + CurrentCount);739 CountMap[S->getBody()] = BodyCount;740 Visit(S->getBody());741 uint64_t BackedgeCount = CurrentCount;742 743 BreakContinue BC = BreakContinueStack.pop_back_val();744 // The count at the start of the condition is equal to the count at the745 // end of the body, plus any continues.746 uint64_t CondCount = setCount(BackedgeCount + BC.ContinueCount);747 CountMap[S->getCond()] = CondCount;748 Visit(S->getCond());749 setCount(BC.BreakCount + CondCount - LoopCount);750 RecordNextStmtCount = true;751 }752 753 void VisitForStmt(const ForStmt *S) {754 RecordStmtCount(S);755 if (S->getInit())756 Visit(S->getInit());757 758 uint64_t ParentCount = CurrentCount;759 760 BreakContinueStack.push_back(BreakContinue());761 // Visit the body region first. (This is basically the same as a while762 // loop; see further comments in VisitWhileStmt.)763 uint64_t BodyCount = setCount(PGO.getRegionCount(S));764 CountMap[S->getBody()] = BodyCount;765 Visit(S->getBody());766 uint64_t BackedgeCount = CurrentCount;767 BreakContinue BC = BreakContinueStack.pop_back_val();768 769 // The increment is essentially part of the body but it needs to include770 // the count for all the continue statements.771 if (S->getInc()) {772 uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount);773 CountMap[S->getInc()] = IncCount;774 Visit(S->getInc());775 }776 777 // ...then go back and propagate counts through the condition.778 uint64_t CondCount =779 setCount(ParentCount + BackedgeCount + BC.ContinueCount);780 if (S->getCond()) {781 CountMap[S->getCond()] = CondCount;782 Visit(S->getCond());783 }784 setCount(BC.BreakCount + CondCount - BodyCount);785 RecordNextStmtCount = true;786 }787 788 void VisitCXXForRangeStmt(const CXXForRangeStmt *S) {789 RecordStmtCount(S);790 if (S->getInit())791 Visit(S->getInit());792 Visit(S->getLoopVarStmt());793 Visit(S->getRangeStmt());794 Visit(S->getBeginStmt());795 Visit(S->getEndStmt());796 797 uint64_t ParentCount = CurrentCount;798 BreakContinueStack.push_back(BreakContinue());799 // Visit the body region first. (This is basically the same as a while800 // loop; see further comments in VisitWhileStmt.)801 uint64_t BodyCount = setCount(PGO.getRegionCount(S));802 CountMap[S->getBody()] = BodyCount;803 Visit(S->getBody());804 uint64_t BackedgeCount = CurrentCount;805 BreakContinue BC = BreakContinueStack.pop_back_val();806 807 // The increment is essentially part of the body but it needs to include808 // the count for all the continue statements.809 uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount);810 CountMap[S->getInc()] = IncCount;811 Visit(S->getInc());812 813 // ...then go back and propagate counts through the condition.814 uint64_t CondCount =815 setCount(ParentCount + BackedgeCount + BC.ContinueCount);816 CountMap[S->getCond()] = CondCount;817 Visit(S->getCond());818 setCount(BC.BreakCount + CondCount - BodyCount);819 RecordNextStmtCount = true;820 }821 822 void VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S) {823 RecordStmtCount(S);824 Visit(S->getElement());825 uint64_t ParentCount = CurrentCount;826 BreakContinueStack.push_back(BreakContinue());827 // Counter tracks the body of the loop.828 uint64_t BodyCount = setCount(PGO.getRegionCount(S));829 CountMap[S->getBody()] = BodyCount;830 Visit(S->getBody());831 uint64_t BackedgeCount = CurrentCount;832 BreakContinue BC = BreakContinueStack.pop_back_val();833 834 setCount(BC.BreakCount + ParentCount + BackedgeCount + BC.ContinueCount -835 BodyCount);836 RecordNextStmtCount = true;837 }838 839 void VisitSwitchStmt(const SwitchStmt *S) {840 RecordStmtCount(S);841 if (S->getInit())842 Visit(S->getInit());843 Visit(S->getCond());844 CurrentCount = 0;845 BreakContinueStack.push_back(BreakContinue());846 Visit(S->getBody());847 // If the switch is inside a loop, add the continue counts.848 BreakContinue BC = BreakContinueStack.pop_back_val();849 if (!BreakContinueStack.empty())850 BreakContinueStack.back().ContinueCount += BC.ContinueCount;851 // Counter tracks the exit block of the switch.852 setCount(PGO.getRegionCount(S));853 RecordNextStmtCount = true;854 }855 856 void VisitSwitchCase(const SwitchCase *S) {857 RecordNextStmtCount = false;858 // Counter for this particular case. This counts only jumps from the859 // switch header and does not include fallthrough from the case before860 // this one.861 uint64_t CaseCount = PGO.getRegionCount(S);862 setCount(CurrentCount + CaseCount);863 // We need the count without fallthrough in the mapping, so it's more useful864 // for branch probabilities.865 CountMap[S] = CaseCount;866 RecordNextStmtCount = true;867 Visit(S->getSubStmt());868 }869 870 void VisitIfStmt(const IfStmt *S) {871 RecordStmtCount(S);872 873 if (S->isConsteval()) {874 const Stmt *Stm = S->isNegatedConsteval() ? S->getThen() : S->getElse();875 if (Stm)876 Visit(Stm);877 return;878 }879 880 uint64_t ParentCount = CurrentCount;881 if (S->getInit())882 Visit(S->getInit());883 Visit(S->getCond());884 885 // Counter tracks the "then" part of an if statement. The count for886 // the "else" part, if it exists, will be calculated from this counter.887 uint64_t ThenCount = setCount(PGO.getRegionCount(S));888 CountMap[S->getThen()] = ThenCount;889 Visit(S->getThen());890 uint64_t OutCount = CurrentCount;891 892 uint64_t ElseCount = ParentCount - ThenCount;893 if (S->getElse()) {894 setCount(ElseCount);895 CountMap[S->getElse()] = ElseCount;896 Visit(S->getElse());897 OutCount += CurrentCount;898 } else899 OutCount += ElseCount;900 setCount(OutCount);901 RecordNextStmtCount = true;902 }903 904 void VisitCXXTryStmt(const CXXTryStmt *S) {905 RecordStmtCount(S);906 Visit(S->getTryBlock());907 for (unsigned I = 0, E = S->getNumHandlers(); I < E; ++I)908 Visit(S->getHandler(I));909 // Counter tracks the continuation block of the try statement.910 setCount(PGO.getRegionCount(S));911 RecordNextStmtCount = true;912 }913 914 void VisitCXXCatchStmt(const CXXCatchStmt *S) {915 RecordNextStmtCount = false;916 // Counter tracks the catch statement's handler block.917 uint64_t CatchCount = setCount(PGO.getRegionCount(S));918 CountMap[S] = CatchCount;919 Visit(S->getHandlerBlock());920 }921 922 void VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {923 RecordStmtCount(E);924 uint64_t ParentCount = CurrentCount;925 Visit(E->getCond());926 927 // Counter tracks the "true" part of a conditional operator. The928 // count in the "false" part will be calculated from this counter.929 uint64_t TrueCount = setCount(PGO.getRegionCount(E));930 CountMap[E->getTrueExpr()] = TrueCount;931 Visit(E->getTrueExpr());932 uint64_t OutCount = CurrentCount;933 934 uint64_t FalseCount = setCount(ParentCount - TrueCount);935 CountMap[E->getFalseExpr()] = FalseCount;936 Visit(E->getFalseExpr());937 OutCount += CurrentCount;938 939 setCount(OutCount);940 RecordNextStmtCount = true;941 }942 943 void VisitBinLAnd(const BinaryOperator *E) {944 RecordStmtCount(E);945 uint64_t ParentCount = CurrentCount;946 Visit(E->getLHS());947 // Counter tracks the right hand side of a logical and operator.948 uint64_t RHSCount = setCount(PGO.getRegionCount(E));949 CountMap[E->getRHS()] = RHSCount;950 Visit(E->getRHS());951 setCount(ParentCount + RHSCount - CurrentCount);952 RecordNextStmtCount = true;953 }954 955 void VisitBinLOr(const BinaryOperator *E) {956 RecordStmtCount(E);957 uint64_t ParentCount = CurrentCount;958 Visit(E->getLHS());959 // Counter tracks the right hand side of a logical or operator.960 uint64_t RHSCount = setCount(PGO.getRegionCount(E));961 CountMap[E->getRHS()] = RHSCount;962 Visit(E->getRHS());963 setCount(ParentCount + RHSCount - CurrentCount);964 RecordNextStmtCount = true;965 }966};967} // end anonymous namespace968 969void PGOHash::combine(HashType Type) {970 // Check that we never combine 0 and only have six bits.971 assert(Type && "Hash is invalid: unexpected type 0");972 assert(unsigned(Type) < TooBig && "Hash is invalid: too many types");973 974 // Pass through MD5 if enough work has built up.975 if (Count && Count % NumTypesPerWord == 0) {976 using namespace llvm::support;977 uint64_t Swapped =978 endian::byte_swap<uint64_t>(Working, llvm::endianness::little);979 MD5.update(llvm::ArrayRef((uint8_t *)&Swapped, sizeof(Swapped)));980 Working = 0;981 }982 983 // Accumulate the current type.984 ++Count;985 Working = Working << NumBitsPerType | Type;986}987 988uint64_t PGOHash::finalize() {989 // Use Working as the hash directly if we never used MD5.990 if (Count <= NumTypesPerWord)991 // No need to byte swap here, since none of the math was endian-dependent.992 // This number will be byte-swapped as required on endianness transitions,993 // so we will see the same value on the other side.994 return Working;995 996 // Check for remaining work in Working.997 if (Working) {998 // Keep the buggy behavior from v1 and v2 for backward-compatibility. This999 // is buggy because it converts a uint64_t into an array of uint8_t.1000 if (HashVersion < PGO_HASH_V3) {1001 MD5.update({(uint8_t)Working});1002 } else {1003 using namespace llvm::support;1004 uint64_t Swapped =1005 endian::byte_swap<uint64_t>(Working, llvm::endianness::little);1006 MD5.update(llvm::ArrayRef((uint8_t *)&Swapped, sizeof(Swapped)));1007 }1008 }1009 1010 // Finalize the MD5 and return the hash.1011 llvm::MD5::MD5Result Result;1012 MD5.final(Result);1013 return Result.low();1014}1015 1016void CodeGenPGO::assignRegionCounters(GlobalDecl GD, llvm::Function *Fn) {1017 const Decl *D = GD.getDecl();1018 if (!D->hasBody())1019 return;1020 1021 // Skip CUDA/HIP kernel launch stub functions.1022 if (CGM.getLangOpts().CUDA && !CGM.getLangOpts().CUDAIsDevice &&1023 D->hasAttr<CUDAGlobalAttr>())1024 return;1025 1026 bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr();1027 llvm::IndexedInstrProfReader *PGOReader = CGM.getPGOReader();1028 if (!InstrumentRegions && !PGOReader)1029 return;1030 if (D->isImplicit())1031 return;1032 // Constructors and destructors may be represented by several functions in IR.1033 // If so, instrument only base variant, others are implemented by delegation1034 // to the base one, it would be counted twice otherwise.1035 if (CGM.getTarget().getCXXABI().hasConstructorVariants()) {1036 if (const auto *CCD = dyn_cast<CXXConstructorDecl>(D))1037 if (GD.getCtorType() != Ctor_Base &&1038 CodeGenFunction::IsConstructorDelegationValid(CCD))1039 return;1040 }1041 if (isa<CXXDestructorDecl>(D) && GD.getDtorType() != Dtor_Base)1042 return;1043 1044 CGM.ClearUnusedCoverageMapping(D);1045 if (Fn->hasFnAttribute(llvm::Attribute::NoProfile))1046 return;1047 if (Fn->hasFnAttribute(llvm::Attribute::SkipProfile))1048 return;1049 1050 SourceManager &SM = CGM.getContext().getSourceManager();1051 if (!llvm::coverage::SystemHeadersCoverage &&1052 SM.isInSystemHeader(D->getLocation()))1053 return;1054 1055 setFuncName(Fn);1056 1057 mapRegionCounters(D);1058 if (CGM.getCodeGenOpts().CoverageMapping)1059 emitCounterRegionMapping(D);1060 if (PGOReader) {1061 loadRegionCounts(PGOReader, SM.isInMainFile(D->getLocation()));1062 computeRegionCounts(D);1063 applyFunctionAttributes(PGOReader, Fn);1064 }1065}1066 1067void CodeGenPGO::mapRegionCounters(const Decl *D) {1068 // Use the latest hash version when inserting instrumentation, but use the1069 // version in the indexed profile if we're reading PGO data.1070 PGOHashVersion HashVersion = PGO_HASH_LATEST;1071 uint64_t ProfileVersion = llvm::IndexedInstrProf::Version;1072 if (auto *PGOReader = CGM.getPGOReader()) {1073 HashVersion = getPGOHashVersion(PGOReader, CGM);1074 ProfileVersion = PGOReader->getVersion();1075 }1076 1077 // If MC/DC is enabled, set the MaxConditions to a preset value. Otherwise,1078 // set it to zero. This value impacts the number of conditions accepted in a1079 // given boolean expression, which impacts the size of the bitmap used to1080 // track test vector execution for that boolean expression. Because the1081 // bitmap scales exponentially (2^n) based on the number of conditions seen,1082 // the maximum value is hard-coded at 6 conditions, which is more than enough1083 // for most embedded applications. Setting a maximum value prevents the1084 // bitmap footprint from growing too large without the user's knowledge. In1085 // the future, this value could be adjusted with a command-line option.1086 unsigned MCDCMaxConditions =1087 (CGM.getCodeGenOpts().MCDCCoverage ? CGM.getCodeGenOpts().MCDCMaxConds1088 : 0);1089 1090 RegionCounterMap.reset(new llvm::DenseMap<const Stmt *, CounterPair>);1091 RegionMCDCState.reset(new MCDC::State);1092 MapRegionCounters Walker(HashVersion, ProfileVersion, *RegionCounterMap,1093 *RegionMCDCState, MCDCMaxConditions, CGM.getDiags());1094 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))1095 Walker.TraverseDecl(const_cast<FunctionDecl *>(FD));1096 else if (const ObjCMethodDecl *MD = dyn_cast_or_null<ObjCMethodDecl>(D))1097 Walker.TraverseDecl(const_cast<ObjCMethodDecl *>(MD));1098 else if (const BlockDecl *BD = dyn_cast_or_null<BlockDecl>(D))1099 Walker.TraverseDecl(const_cast<BlockDecl *>(BD));1100 else if (const CapturedDecl *CD = dyn_cast_or_null<CapturedDecl>(D))1101 Walker.TraverseDecl(const_cast<CapturedDecl *>(CD));1102 assert(Walker.NextCounter > 0 && "no entry counter mapped for decl");1103 NumRegionCounters = Walker.NextCounter;1104 FunctionHash = Walker.Hash.finalize();1105 if (HashVersion >= PGO_HASH_V4)1106 FunctionHash &= llvm::NamedInstrProfRecord::FUNC_HASH_MASK;1107}1108 1109bool CodeGenPGO::skipRegionMappingForDecl(const Decl *D) {1110 if (!D->getBody())1111 return true;1112 1113 // Skip host-only functions in the CUDA device compilation and device-only1114 // functions in the host compilation. Just roughly filter them out based on1115 // the function attributes. If there are effectively host-only or device-only1116 // ones, their coverage mapping may still be generated.1117 if (CGM.getLangOpts().CUDA &&1118 ((CGM.getLangOpts().CUDAIsDevice && !D->hasAttr<CUDADeviceAttr>() &&1119 !D->hasAttr<CUDAGlobalAttr>()) ||1120 (!CGM.getLangOpts().CUDAIsDevice &&1121 (D->hasAttr<CUDAGlobalAttr>() ||1122 (!D->hasAttr<CUDAHostAttr>() && D->hasAttr<CUDADeviceAttr>())))))1123 return true;1124 1125 // Don't map the functions in system headers.1126 const auto &SM = CGM.getContext().getSourceManager();1127 auto Loc = D->getBody()->getBeginLoc();1128 return !llvm::coverage::SystemHeadersCoverage && SM.isInSystemHeader(Loc);1129}1130 1131void CodeGenPGO::emitCounterRegionMapping(const Decl *D) {1132 if (skipRegionMappingForDecl(D))1133 return;1134 1135 std::string CoverageMapping;1136 llvm::raw_string_ostream OS(CoverageMapping);1137 RegionMCDCState->BranchByStmt.clear();1138 CoverageMappingGen MappingGen(1139 *CGM.getCoverageMapping(), CGM.getContext().getSourceManager(),1140 CGM.getLangOpts(), RegionCounterMap.get(), RegionMCDCState.get());1141 MappingGen.emitCounterMapping(D, OS);1142 1143 if (CoverageMapping.empty())1144 return;1145 1146 CGM.getCoverageMapping()->addFunctionMappingRecord(1147 FuncNameVar, FuncName, FunctionHash, CoverageMapping);1148}1149 1150void1151CodeGenPGO::emitEmptyCounterMapping(const Decl *D, StringRef Name,1152 llvm::GlobalValue::LinkageTypes Linkage) {1153 if (skipRegionMappingForDecl(D))1154 return;1155 1156 std::string CoverageMapping;1157 llvm::raw_string_ostream OS(CoverageMapping);1158 CoverageMappingGen MappingGen(*CGM.getCoverageMapping(),1159 CGM.getContext().getSourceManager(),1160 CGM.getLangOpts());1161 MappingGen.emitEmptyMapping(D, OS);1162 1163 if (CoverageMapping.empty())1164 return;1165 1166 setFuncName(Name, Linkage);1167 CGM.getCoverageMapping()->addFunctionMappingRecord(1168 FuncNameVar, FuncName, FunctionHash, CoverageMapping, false);1169}1170 1171void CodeGenPGO::computeRegionCounts(const Decl *D) {1172 StmtCountMap.reset(new llvm::DenseMap<const Stmt *, uint64_t>);1173 ComputeRegionCounts Walker(*StmtCountMap, *this);1174 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))1175 Walker.VisitFunctionDecl(FD);1176 else if (const ObjCMethodDecl *MD = dyn_cast_or_null<ObjCMethodDecl>(D))1177 Walker.VisitObjCMethodDecl(MD);1178 else if (const BlockDecl *BD = dyn_cast_or_null<BlockDecl>(D))1179 Walker.VisitBlockDecl(BD);1180 else if (const CapturedDecl *CD = dyn_cast_or_null<CapturedDecl>(D))1181 Walker.VisitCapturedDecl(const_cast<CapturedDecl *>(CD));1182}1183 1184void1185CodeGenPGO::applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader,1186 llvm::Function *Fn) {1187 if (!haveRegionCounts())1188 return;1189 1190 uint64_t FunctionCount = getRegionCount(nullptr);1191 Fn->setEntryCount(FunctionCount);1192}1193 1194std::pair<bool, bool> CodeGenPGO::getIsCounterPair(const Stmt *S) const {1195 if (!RegionCounterMap)1196 return {false, false};1197 1198 auto I = RegionCounterMap->find(S);1199 if (I == RegionCounterMap->end())1200 return {false, false};1201 1202 return {I->second.Executed.hasValue(), I->second.Skipped.hasValue()};1203}1204 1205void CodeGenPGO::emitCounterSetOrIncrement(CGBuilderTy &Builder, const Stmt *S,1206 llvm::Value *StepV) {1207 if (!RegionCounterMap || !Builder.GetInsertBlock())1208 return;1209 1210 unsigned Counter = (*RegionCounterMap)[S].Executed;1211 1212 // Make sure that pointer to global is passed in with zero addrspace1213 // This is relevant during GPU profiling1214 auto *NormalizedFuncNameVarPtr =1215 llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(1216 FuncNameVar, llvm::PointerType::get(CGM.getLLVMContext(), 0));1217 1218 llvm::Value *Args[] = {1219 NormalizedFuncNameVarPtr, Builder.getInt64(FunctionHash),1220 Builder.getInt32(NumRegionCounters), Builder.getInt32(Counter), StepV};1221 1222 if (llvm::EnableSingleByteCoverage)1223 Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::instrprof_cover),1224 ArrayRef(Args, 4));1225 else if (!StepV)1226 Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment),1227 ArrayRef(Args, 4));1228 else1229 Builder.CreateCall(1230 CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment_step), Args);1231}1232 1233bool CodeGenPGO::canEmitMCDCCoverage(const CGBuilderTy &Builder) {1234 return (CGM.getCodeGenOpts().hasProfileClangInstr() &&1235 CGM.getCodeGenOpts().MCDCCoverage && Builder.GetInsertBlock());1236}1237 1238void CodeGenPGO::emitMCDCParameters(CGBuilderTy &Builder) {1239 if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState)1240 return;1241 1242 auto *I8PtrTy = llvm::PointerType::getUnqual(CGM.getLLVMContext());1243 1244 // Emit intrinsic representing MCDC bitmap parameters at function entry.1245 // This is used by the instrumentation pass, but it isn't actually lowered to1246 // anything.1247 llvm::Value *Args[3] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),1248 Builder.getInt64(FunctionHash),1249 Builder.getInt32(RegionMCDCState->BitmapBits)};1250 Builder.CreateCall(1251 CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_parameters), Args);1252}1253 1254void CodeGenPGO::emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder,1255 const Expr *S,1256 Address MCDCCondBitmapAddr,1257 CodeGenFunction &CGF) {1258 if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState)1259 return;1260 1261 S = S->IgnoreParens();1262 1263 auto DecisionStateIter = RegionMCDCState->DecisionByStmt.find(S);1264 if (DecisionStateIter == RegionMCDCState->DecisionByStmt.end())1265 return;1266 1267 // Don't create tvbitmap_update if the record is allocated but excluded.1268 // Or `bitmap |= (1 << 0)` would be wrongly executed to the next bitmap.1269 if (DecisionStateIter->second.Indices.size() == 0)1270 return;1271 1272 // Extract the offset of the global bitmap associated with this expression.1273 unsigned MCDCTestVectorBitmapOffset = DecisionStateIter->second.BitmapIdx;1274 auto *I8PtrTy = llvm::PointerType::getUnqual(CGM.getLLVMContext());1275 1276 // Emit intrinsic responsible for updating the global bitmap corresponding to1277 // a boolean expression. The index being set is based on the value loaded1278 // from a pointer to a dedicated temporary value on the stack that is itself1279 // updated via emitMCDCCondBitmapReset() and emitMCDCCondBitmapUpdate(). The1280 // index represents an executed test vector.1281 llvm::Value *Args[4] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),1282 Builder.getInt64(FunctionHash),1283 Builder.getInt32(MCDCTestVectorBitmapOffset),1284 MCDCCondBitmapAddr.emitRawPointer(CGF)};1285 Builder.CreateCall(1286 CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_tvbitmap_update), Args);1287}1288 1289void CodeGenPGO::emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S,1290 Address MCDCCondBitmapAddr) {1291 if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState)1292 return;1293 1294 S = S->IgnoreParens();1295 1296 if (!RegionMCDCState->DecisionByStmt.contains(S))1297 return;1298 1299 // Emit intrinsic that resets a dedicated temporary value on the stack to 0.1300 Builder.CreateStore(Builder.getInt32(0), MCDCCondBitmapAddr);1301}1302 1303void CodeGenPGO::emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S,1304 Address MCDCCondBitmapAddr,1305 llvm::Value *Val,1306 CodeGenFunction &CGF) {1307 if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState)1308 return;1309 1310 // Even though, for simplicity, parentheses and unary logical-NOT operators1311 // are considered part of their underlying condition for both MC/DC and1312 // branch coverage, the condition IDs themselves are assigned and tracked1313 // using the underlying condition itself. This is done solely for1314 // consistency since parentheses and logical-NOTs are ignored when checking1315 // whether the condition is actually an instrumentable condition. This can1316 // also make debugging a bit easier.1317 S = CodeGenFunction::stripCond(S);1318 1319 auto BranchStateIter = RegionMCDCState->BranchByStmt.find(S);1320 if (BranchStateIter == RegionMCDCState->BranchByStmt.end())1321 return;1322 1323 // Extract the ID of the condition we are setting in the bitmap.1324 const auto &Branch = BranchStateIter->second;1325 assert(Branch.ID >= 0 && "Condition has no ID!");1326 assert(Branch.DecisionStmt);1327 1328 // Cancel the emission if the Decision is erased after the allocation.1329 const auto DecisionIter =1330 RegionMCDCState->DecisionByStmt.find(Branch.DecisionStmt);1331 if (DecisionIter == RegionMCDCState->DecisionByStmt.end())1332 return;1333 1334 const auto &TVIdxs = DecisionIter->second.Indices[Branch.ID];1335 1336 auto *CurTV = Builder.CreateLoad(MCDCCondBitmapAddr,1337 "mcdc." + Twine(Branch.ID + 1) + ".cur");1338 auto *NewTV = Builder.CreateAdd(CurTV, Builder.getInt32(TVIdxs[true]));1339 NewTV = Builder.CreateSelect(1340 Val, NewTV, Builder.CreateAdd(CurTV, Builder.getInt32(TVIdxs[false])));1341 Builder.CreateStore(NewTV, MCDCCondBitmapAddr);1342}1343 1344void CodeGenPGO::setValueProfilingFlag(llvm::Module &M) {1345 if (CGM.getCodeGenOpts().hasProfileClangInstr())1346 M.addModuleFlag(llvm::Module::Warning, "EnableValueProfiling",1347 uint32_t(EnableValueProfiling));1348}1349 1350void CodeGenPGO::setProfileVersion(llvm::Module &M) {1351 if (CGM.getCodeGenOpts().hasProfileClangInstr() &&1352 llvm::EnableSingleByteCoverage) {1353 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));1354 llvm::Type *IntTy64 = llvm::Type::getInt64Ty(M.getContext());1355 uint64_t ProfileVersion =1356 (INSTR_PROF_RAW_VERSION | VARIANT_MASK_BYTE_COVERAGE);1357 1358 auto IRLevelVersionVariable = new llvm::GlobalVariable(1359 M, IntTy64, true, llvm::GlobalValue::WeakAnyLinkage,1360 llvm::Constant::getIntegerValue(IntTy64,1361 llvm::APInt(64, ProfileVersion)),1362 VarName);1363 1364 IRLevelVersionVariable->setVisibility(llvm::GlobalValue::HiddenVisibility);1365 llvm::Triple TT(M.getTargetTriple());1366 if (TT.isGPU())1367 IRLevelVersionVariable->setVisibility(1368 llvm::GlobalValue::ProtectedVisibility);1369 if (TT.supportsCOMDAT()) {1370 IRLevelVersionVariable->setLinkage(llvm::GlobalValue::ExternalLinkage);1371 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName));1372 }1373 IRLevelVersionVariable->setDSOLocal(true);1374 }1375}1376 1377// This method either inserts a call to the profile run-time during1378// instrumentation or puts profile data into metadata for PGO use.1379void CodeGenPGO::valueProfile(CGBuilderTy &Builder, uint32_t ValueKind,1380 llvm::Instruction *ValueSite, llvm::Value *ValuePtr) {1381 1382 if (!EnableValueProfiling)1383 return;1384 1385 if (!ValuePtr || !ValueSite || !Builder.GetInsertBlock())1386 return;1387 1388 if (isa<llvm::Constant>(ValuePtr))1389 return;1390 1391 bool InstrumentValueSites = CGM.getCodeGenOpts().hasProfileClangInstr();1392 if (InstrumentValueSites && RegionCounterMap) {1393 auto BuilderInsertPoint = Builder.saveIP();1394 Builder.SetInsertPoint(ValueSite);1395 llvm::Value *Args[5] = {1396 FuncNameVar,1397 Builder.getInt64(FunctionHash),1398 Builder.CreatePtrToInt(ValuePtr, Builder.getInt64Ty()),1399 Builder.getInt32(ValueKind),1400 Builder.getInt32(NumValueSites[ValueKind]++)1401 };1402 Builder.CreateCall(1403 CGM.getIntrinsic(llvm::Intrinsic::instrprof_value_profile), Args);1404 Builder.restoreIP(BuilderInsertPoint);1405 return;1406 }1407 1408 llvm::IndexedInstrProfReader *PGOReader = CGM.getPGOReader();1409 if (PGOReader && haveRegionCounts()) {1410 // We record the top most called three functions at each call site.1411 // Profile metadata contains "VP" string identifying this metadata1412 // as value profiling data, then a uint32_t value for the value profiling1413 // kind, a uint64_t value for the total number of times the call is1414 // executed, followed by the function hash and execution count (uint64_t)1415 // pairs for each function.1416 if (NumValueSites[ValueKind] >= ProfRecord->getNumValueSites(ValueKind))1417 return;1418 1419 llvm::annotateValueSite(CGM.getModule(), *ValueSite, *ProfRecord,1420 (llvm::InstrProfValueKind)ValueKind,1421 NumValueSites[ValueKind]);1422 1423 NumValueSites[ValueKind]++;1424 }1425}1426 1427void CodeGenPGO::loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader,1428 bool IsInMainFile) {1429 CGM.getPGOStats().addVisited(IsInMainFile);1430 RegionCounts.clear();1431 auto RecordExpected = PGOReader->getInstrProfRecord(FuncName, FunctionHash);1432 if (auto E = RecordExpected.takeError()) {1433 auto IPE = std::get<0>(llvm::InstrProfError::take(std::move(E)));1434 if (IPE == llvm::instrprof_error::unknown_function)1435 CGM.getPGOStats().addMissing(IsInMainFile);1436 else if (IPE == llvm::instrprof_error::hash_mismatch)1437 CGM.getPGOStats().addMismatched(IsInMainFile);1438 else if (IPE == llvm::instrprof_error::malformed)1439 // TODO: Consider a more specific warning for this case.1440 CGM.getPGOStats().addMismatched(IsInMainFile);1441 return;1442 }1443 ProfRecord =1444 std::make_unique<llvm::InstrProfRecord>(std::move(RecordExpected.get()));1445 RegionCounts = ProfRecord->Counts;1446}1447 1448/// Calculate what to divide by to scale weights.1449///1450/// Given the maximum weight, calculate a divisor that will scale all the1451/// weights to strictly less than UINT32_MAX.1452static uint64_t calculateWeightScale(uint64_t MaxWeight) {1453 return MaxWeight < UINT32_MAX ? 1 : MaxWeight / UINT32_MAX + 1;1454}1455 1456/// Scale an individual branch weight (and add 1).1457///1458/// Scale a 64-bit weight down to 32-bits using \c Scale.1459///1460/// According to Laplace's Rule of Succession, it is better to compute the1461/// weight based on the count plus 1, so universally add 1 to the value.1462///1463/// \pre \c Scale was calculated by \a calculateWeightScale() with a weight no1464/// greater than \c Weight.1465static uint32_t scaleBranchWeight(uint64_t Weight, uint64_t Scale) {1466 assert(Scale && "scale by 0?");1467 uint64_t Scaled = Weight / Scale + 1;1468 assert(Scaled <= UINT32_MAX && "overflow 32-bits");1469 return Scaled;1470}1471 1472llvm::MDNode *CodeGenFunction::createProfileWeights(uint64_t TrueCount,1473 uint64_t FalseCount) const {1474 // Check for empty weights.1475 if (!TrueCount && !FalseCount)1476 return nullptr;1477 1478 // Calculate how to scale down to 32-bits.1479 uint64_t Scale = calculateWeightScale(std::max(TrueCount, FalseCount));1480 1481 llvm::MDBuilder MDHelper(CGM.getLLVMContext());1482 return MDHelper.createBranchWeights(scaleBranchWeight(TrueCount, Scale),1483 scaleBranchWeight(FalseCount, Scale));1484}1485 1486llvm::MDNode *1487CodeGenFunction::createProfileWeights(ArrayRef<uint64_t> Weights) const {1488 // We need at least two elements to create meaningful weights.1489 if (Weights.size() < 2)1490 return nullptr;1491 1492 // Check for empty weights.1493 uint64_t MaxWeight = *llvm::max_element(Weights);1494 if (MaxWeight == 0)1495 return nullptr;1496 1497 // Calculate how to scale down to 32-bits.1498 uint64_t Scale = calculateWeightScale(MaxWeight);1499 1500 SmallVector<uint32_t, 16> ScaledWeights;1501 ScaledWeights.reserve(Weights.size());1502 for (uint64_t W : Weights)1503 ScaledWeights.push_back(scaleBranchWeight(W, Scale));1504 1505 llvm::MDBuilder MDHelper(CGM.getLLVMContext());1506 return MDHelper.createBranchWeights(ScaledWeights);1507}1508 1509llvm::MDNode *1510CodeGenFunction::createProfileWeightsForLoop(const Stmt *Cond,1511 uint64_t LoopCount) const {1512 if (!PGO->haveRegionCounts())1513 return nullptr;1514 std::optional<uint64_t> CondCount = PGO->getStmtCount(Cond);1515 if (!CondCount || *CondCount == 0)1516 return nullptr;1517 return createProfileWeights(LoopCount,1518 std::max(*CondCount, LoopCount) - LoopCount);1519}1520 1521void CodeGenFunction::incrementProfileCounter(const Stmt *S,1522 llvm::Value *StepV) {1523 if (CGM.getCodeGenOpts().hasProfileClangInstr() &&1524 !CurFn->hasFnAttribute(llvm::Attribute::NoProfile) &&1525 !CurFn->hasFnAttribute(llvm::Attribute::SkipProfile)) {1526 auto AL = ApplyDebugLocation::CreateArtificial(*this);1527 PGO->emitCounterSetOrIncrement(Builder, S, StepV);1528 }1529 PGO->setCurrentStmt(S);1530}1531 1532std::pair<bool, bool> CodeGenFunction::getIsCounterPair(const Stmt *S) const {1533 return PGO->getIsCounterPair(S);1534}1535void CodeGenFunction::markStmtAsUsed(bool Skipped, const Stmt *S) {1536 PGO->markStmtAsUsed(Skipped, S);1537}1538void CodeGenFunction::markStmtMaybeUsed(const Stmt *S) {1539 PGO->markStmtMaybeUsed(S);1540}1541 1542void CodeGenFunction::maybeCreateMCDCCondBitmap() {1543 if (isMCDCCoverageEnabled()) {1544 PGO->emitMCDCParameters(Builder);1545 MCDCCondBitmapAddr = CreateIRTemp(getContext().UnsignedIntTy, "mcdc.addr");1546 }1547}1548void CodeGenFunction::maybeResetMCDCCondBitmap(const Expr *E) {1549 if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) {1550 PGO->emitMCDCCondBitmapReset(Builder, E, MCDCCondBitmapAddr);1551 PGO->setCurrentStmt(E);1552 }1553}1554void CodeGenFunction::maybeUpdateMCDCTestVectorBitmap(const Expr *E) {1555 if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) {1556 PGO->emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr, *this);1557 PGO->setCurrentStmt(E);1558 }1559}1560 1561void CodeGenFunction::maybeUpdateMCDCCondBitmap(const Expr *E,1562 llvm::Value *Val) {1563 if (isMCDCCoverageEnabled()) {1564 PGO->emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val, *this);1565 PGO->setCurrentStmt(E);1566 }1567}1568 1569uint64_t CodeGenFunction::getProfileCount(const Stmt *S) {1570 return PGO->getStmtCount(S).value_or(0);1571}1572 1573/// Set the profiler's current count.1574void CodeGenFunction::setCurrentProfileCount(uint64_t Count) {1575 PGO->setCurrentRegionCount(Count);1576}1577 1578/// Get the profiler's current count. This is generally the count for the most1579/// recently incremented counter.1580uint64_t CodeGenFunction::getCurrentProfileCount() {1581 return PGO->getCurrentRegionCount();1582}1583