brintos / llvm-project-archived public Read only

1//===--- CGStmt.cpp - Emit LLVM Code from Statements ----------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This contains code to emit Stmt nodes as LLVM code.10//11//===----------------------------------------------------------------------===//12 13#include "CGDebugInfo.h"14#include "CGOpenMPRuntime.h"15#include "CodeGenFunction.h"16#include "CodeGenModule.h"17#include "CodeGenPGO.h"18#include "TargetInfo.h"19#include "clang/AST/Attr.h"20#include "clang/AST/Expr.h"21#include "clang/AST/Stmt.h"22#include "clang/AST/StmtVisitor.h"23#include "clang/Basic/Builtins.h"24#include "clang/Basic/DiagnosticSema.h"25#include "clang/Basic/PrettyStackTrace.h"26#include "clang/Basic/SourceManager.h"27#include "clang/Basic/TargetInfo.h"28#include "llvm/ADT/ArrayRef.h"29#include "llvm/ADT/DenseMap.h"30#include "llvm/ADT/SmallSet.h"31#include "llvm/ADT/StringExtras.h"32#include "llvm/IR/Assumptions.h"33#include "llvm/IR/DataLayout.h"34#include "llvm/IR/InlineAsm.h"35#include "llvm/IR/Intrinsics.h"36#include "llvm/IR/MDBuilder.h"37#include "llvm/Support/SaveAndRestore.h"38#include <optional>39 40using namespace clang;41using namespace CodeGen;42 43//===----------------------------------------------------------------------===//44//                              Statement Emission45//===----------------------------------------------------------------------===//46 47namespace llvm {48extern cl::opt<bool> EnableSingleByteCoverage;49} // namespace llvm50 51void CodeGenFunction::EmitStopPoint(const Stmt *S) {52  if (CGDebugInfo *DI = getDebugInfo()) {53    SourceLocation Loc;54    Loc = S->getBeginLoc();55    DI->EmitLocation(Builder, Loc);56 57    LastStopPoint = Loc;58  }59}60 61void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {62  assert(S && "Null statement?");63  PGO->setCurrentStmt(S);64 65  // These statements have their own debug info handling.66  if (EmitSimpleStmt(S, Attrs))67    return;68 69  // Check if we are generating unreachable code.70  if (!HaveInsertPoint()) {71    // If so, and the statement doesn't contain a label, then we do not need to72    // generate actual code. This is safe because (1) the current point is73    // unreachable, so we don't need to execute the code, and (2) we've already74    // handled the statements which update internal data structures (like the75    // local variable map) which could be used by subsequent statements.76    if (!ContainsLabel(S)) {77      // Verify that any decl statements were handled as simple, they may be in78      // scope of subsequent reachable statements.79      assert(!isa<DeclStmt>(*S) && "Unexpected DeclStmt!");80      PGO->markStmtMaybeUsed(S);81      return;82    }83 84    // Otherwise, make a new block to hold the code.85    EnsureInsertPoint();86  }87 88  // Generate a stoppoint if we are emitting debug info.89  EmitStopPoint(S);90 91  // Ignore all OpenMP directives except for simd if OpenMP with Simd is92  // enabled.93  if (getLangOpts().OpenMP && getLangOpts().OpenMPSimd) {94    if (const auto *D = dyn_cast<OMPExecutableDirective>(S)) {95      EmitSimpleOMPExecutableDirective(*D);96      return;97    }98  }99 100  switch (S->getStmtClass()) {101  case Stmt::NoStmtClass:102  case Stmt::CXXCatchStmtClass:103  case Stmt::SEHExceptStmtClass:104  case Stmt::SEHFinallyStmtClass:105  case Stmt::MSDependentExistsStmtClass:106    llvm_unreachable("invalid statement class to emit generically");107  case Stmt::NullStmtClass:108  case Stmt::CompoundStmtClass:109  case Stmt::DeclStmtClass:110  case Stmt::LabelStmtClass:111  case Stmt::AttributedStmtClass:112  case Stmt::GotoStmtClass:113  case Stmt::BreakStmtClass:114  case Stmt::ContinueStmtClass:115  case Stmt::DefaultStmtClass:116  case Stmt::CaseStmtClass:117  case Stmt::SEHLeaveStmtClass:118  case Stmt::SYCLKernelCallStmtClass:119    llvm_unreachable("should have emitted these statements as simple");120 121#define STMT(Type, Base)122#define ABSTRACT_STMT(Op)123#define EXPR(Type, Base) \124  case Stmt::Type##Class:125#include "clang/AST/StmtNodes.inc"126  {127    // Remember the block we came in on.128    llvm::BasicBlock *incoming = Builder.GetInsertBlock();129    assert(incoming && "expression emission must have an insertion point");130 131    EmitIgnoredExpr(cast<Expr>(S));132 133    llvm::BasicBlock *outgoing = Builder.GetInsertBlock();134    assert(outgoing && "expression emission cleared block!");135 136    // The expression emitters assume (reasonably!) that the insertion137    // point is always set.  To maintain that, the call-emission code138    // for noreturn functions has to enter a new block with no139    // predecessors.  We want to kill that block and mark the current140    // insertion point unreachable in the common case of a call like141    // "exit();".  Since expression emission doesn't otherwise create142    // blocks with no predecessors, we can just test for that.143    // However, we must be careful not to do this to our incoming144    // block, because *statement* emission does sometimes create145    // reachable blocks which will have no predecessors until later in146    // the function.  This occurs with, e.g., labels that are not147    // reachable by fallthrough.148    if (incoming != outgoing && outgoing->use_empty()) {149      outgoing->eraseFromParent();150      Builder.ClearInsertionPoint();151    }152    break;153  }154 155  case Stmt::IndirectGotoStmtClass:156    EmitIndirectGotoStmt(cast<IndirectGotoStmt>(*S)); break;157 158  case Stmt::IfStmtClass:      EmitIfStmt(cast<IfStmt>(*S));              break;159  case Stmt::WhileStmtClass:   EmitWhileStmt(cast<WhileStmt>(*S), Attrs); break;160  case Stmt::DoStmtClass:      EmitDoStmt(cast<DoStmt>(*S), Attrs);       break;161  case Stmt::ForStmtClass:     EmitForStmt(cast<ForStmt>(*S), Attrs);     break;162 163  case Stmt::ReturnStmtClass:  EmitReturnStmt(cast<ReturnStmt>(*S));      break;164 165  case Stmt::SwitchStmtClass:  EmitSwitchStmt(cast<SwitchStmt>(*S));      break;166  case Stmt::GCCAsmStmtClass:  // Intentional fall-through.167  case Stmt::MSAsmStmtClass:   EmitAsmStmt(cast<AsmStmt>(*S));            break;168  case Stmt::CoroutineBodyStmtClass:169    EmitCoroutineBody(cast<CoroutineBodyStmt>(*S));170    break;171  case Stmt::CoreturnStmtClass:172    EmitCoreturnStmt(cast<CoreturnStmt>(*S));173    break;174  case Stmt::CapturedStmtClass: {175    const CapturedStmt *CS = cast<CapturedStmt>(S);176    EmitCapturedStmt(*CS, CS->getCapturedRegionKind());177    }178    break;179  case Stmt::ObjCAtTryStmtClass:180    EmitObjCAtTryStmt(cast<ObjCAtTryStmt>(*S));181    break;182  case Stmt::ObjCAtCatchStmtClass:183    llvm_unreachable(184                    "@catch statements should be handled by EmitObjCAtTryStmt");185  case Stmt::ObjCAtFinallyStmtClass:186    llvm_unreachable(187                  "@finally statements should be handled by EmitObjCAtTryStmt");188  case Stmt::ObjCAtThrowStmtClass:189    EmitObjCAtThrowStmt(cast<ObjCAtThrowStmt>(*S));190    break;191  case Stmt::ObjCAtSynchronizedStmtClass:192    EmitObjCAtSynchronizedStmt(cast<ObjCAtSynchronizedStmt>(*S));193    break;194  case Stmt::ObjCForCollectionStmtClass:195    EmitObjCForCollectionStmt(cast<ObjCForCollectionStmt>(*S));196    break;197  case Stmt::ObjCAutoreleasePoolStmtClass:198    EmitObjCAutoreleasePoolStmt(cast<ObjCAutoreleasePoolStmt>(*S));199    break;200 201  case Stmt::CXXTryStmtClass:202    EmitCXXTryStmt(cast<CXXTryStmt>(*S));203    break;204  case Stmt::CXXForRangeStmtClass:205    EmitCXXForRangeStmt(cast<CXXForRangeStmt>(*S), Attrs);206    break;207  case Stmt::SEHTryStmtClass:208    EmitSEHTryStmt(cast<SEHTryStmt>(*S));209    break;210  case Stmt::OMPMetaDirectiveClass:211    EmitOMPMetaDirective(cast<OMPMetaDirective>(*S));212    break;213  case Stmt::OMPCanonicalLoopClass:214    EmitOMPCanonicalLoop(cast<OMPCanonicalLoop>(S));215    break;216  case Stmt::OMPParallelDirectiveClass:217    EmitOMPParallelDirective(cast<OMPParallelDirective>(*S));218    break;219  case Stmt::OMPSimdDirectiveClass:220    EmitOMPSimdDirective(cast<OMPSimdDirective>(*S));221    break;222  case Stmt::OMPTileDirectiveClass:223    EmitOMPTileDirective(cast<OMPTileDirective>(*S));224    break;225  case Stmt::OMPStripeDirectiveClass:226    EmitOMPStripeDirective(cast<OMPStripeDirective>(*S));227    break;228  case Stmt::OMPUnrollDirectiveClass:229    EmitOMPUnrollDirective(cast<OMPUnrollDirective>(*S));230    break;231  case Stmt::OMPReverseDirectiveClass:232    EmitOMPReverseDirective(cast<OMPReverseDirective>(*S));233    break;234  case Stmt::OMPInterchangeDirectiveClass:235    EmitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*S));236    break;237  case Stmt::OMPFuseDirectiveClass:238    EmitOMPFuseDirective(cast<OMPFuseDirective>(*S));239    break;240  case Stmt::OMPForDirectiveClass:241    EmitOMPForDirective(cast<OMPForDirective>(*S));242    break;243  case Stmt::OMPForSimdDirectiveClass:244    EmitOMPForSimdDirective(cast<OMPForSimdDirective>(*S));245    break;246  case Stmt::OMPSectionsDirectiveClass:247    EmitOMPSectionsDirective(cast<OMPSectionsDirective>(*S));248    break;249  case Stmt::OMPSectionDirectiveClass:250    EmitOMPSectionDirective(cast<OMPSectionDirective>(*S));251    break;252  case Stmt::OMPSingleDirectiveClass:253    EmitOMPSingleDirective(cast<OMPSingleDirective>(*S));254    break;255  case Stmt::OMPMasterDirectiveClass:256    EmitOMPMasterDirective(cast<OMPMasterDirective>(*S));257    break;258  case Stmt::OMPCriticalDirectiveClass:259    EmitOMPCriticalDirective(cast<OMPCriticalDirective>(*S));260    break;261  case Stmt::OMPParallelForDirectiveClass:262    EmitOMPParallelForDirective(cast<OMPParallelForDirective>(*S));263    break;264  case Stmt::OMPParallelForSimdDirectiveClass:265    EmitOMPParallelForSimdDirective(cast<OMPParallelForSimdDirective>(*S));266    break;267  case Stmt::OMPParallelMasterDirectiveClass:268    EmitOMPParallelMasterDirective(cast<OMPParallelMasterDirective>(*S));269    break;270  case Stmt::OMPParallelSectionsDirectiveClass:271    EmitOMPParallelSectionsDirective(cast<OMPParallelSectionsDirective>(*S));272    break;273  case Stmt::OMPTaskDirectiveClass:274    EmitOMPTaskDirective(cast<OMPTaskDirective>(*S));275    break;276  case Stmt::OMPTaskyieldDirectiveClass:277    EmitOMPTaskyieldDirective(cast<OMPTaskyieldDirective>(*S));278    break;279  case Stmt::OMPErrorDirectiveClass:280    EmitOMPErrorDirective(cast<OMPErrorDirective>(*S));281    break;282  case Stmt::OMPBarrierDirectiveClass:283    EmitOMPBarrierDirective(cast<OMPBarrierDirective>(*S));284    break;285  case Stmt::OMPTaskwaitDirectiveClass:286    EmitOMPTaskwaitDirective(cast<OMPTaskwaitDirective>(*S));287    break;288  case Stmt::OMPTaskgroupDirectiveClass:289    EmitOMPTaskgroupDirective(cast<OMPTaskgroupDirective>(*S));290    break;291  case Stmt::OMPFlushDirectiveClass:292    EmitOMPFlushDirective(cast<OMPFlushDirective>(*S));293    break;294  case Stmt::OMPDepobjDirectiveClass:295    EmitOMPDepobjDirective(cast<OMPDepobjDirective>(*S));296    break;297  case Stmt::OMPScanDirectiveClass:298    EmitOMPScanDirective(cast<OMPScanDirective>(*S));299    break;300  case Stmt::OMPOrderedDirectiveClass:301    EmitOMPOrderedDirective(cast<OMPOrderedDirective>(*S));302    break;303  case Stmt::OMPAtomicDirectiveClass:304    EmitOMPAtomicDirective(cast<OMPAtomicDirective>(*S));305    break;306  case Stmt::OMPTargetDirectiveClass:307    EmitOMPTargetDirective(cast<OMPTargetDirective>(*S));308    break;309  case Stmt::OMPTeamsDirectiveClass:310    EmitOMPTeamsDirective(cast<OMPTeamsDirective>(*S));311    break;312  case Stmt::OMPCancellationPointDirectiveClass:313    EmitOMPCancellationPointDirective(cast<OMPCancellationPointDirective>(*S));314    break;315  case Stmt::OMPCancelDirectiveClass:316    EmitOMPCancelDirective(cast<OMPCancelDirective>(*S));317    break;318  case Stmt::OMPTargetDataDirectiveClass:319    EmitOMPTargetDataDirective(cast<OMPTargetDataDirective>(*S));320    break;321  case Stmt::OMPTargetEnterDataDirectiveClass:322    EmitOMPTargetEnterDataDirective(cast<OMPTargetEnterDataDirective>(*S));323    break;324  case Stmt::OMPTargetExitDataDirectiveClass:325    EmitOMPTargetExitDataDirective(cast<OMPTargetExitDataDirective>(*S));326    break;327  case Stmt::OMPTargetParallelDirectiveClass:328    EmitOMPTargetParallelDirective(cast<OMPTargetParallelDirective>(*S));329    break;330  case Stmt::OMPTargetParallelForDirectiveClass:331    EmitOMPTargetParallelForDirective(cast<OMPTargetParallelForDirective>(*S));332    break;333  case Stmt::OMPTaskLoopDirectiveClass:334    EmitOMPTaskLoopDirective(cast<OMPTaskLoopDirective>(*S));335    break;336  case Stmt::OMPTaskLoopSimdDirectiveClass:337    EmitOMPTaskLoopSimdDirective(cast<OMPTaskLoopSimdDirective>(*S));338    break;339  case Stmt::OMPMasterTaskLoopDirectiveClass:340    EmitOMPMasterTaskLoopDirective(cast<OMPMasterTaskLoopDirective>(*S));341    break;342  case Stmt::OMPMaskedTaskLoopDirectiveClass:343    EmitOMPMaskedTaskLoopDirective(cast<OMPMaskedTaskLoopDirective>(*S));344    break;345  case Stmt::OMPMasterTaskLoopSimdDirectiveClass:346    EmitOMPMasterTaskLoopSimdDirective(347        cast<OMPMasterTaskLoopSimdDirective>(*S));348    break;349  case Stmt::OMPMaskedTaskLoopSimdDirectiveClass:350    EmitOMPMaskedTaskLoopSimdDirective(351        cast<OMPMaskedTaskLoopSimdDirective>(*S));352    break;353  case Stmt::OMPParallelMasterTaskLoopDirectiveClass:354    EmitOMPParallelMasterTaskLoopDirective(355        cast<OMPParallelMasterTaskLoopDirective>(*S));356    break;357  case Stmt::OMPParallelMaskedTaskLoopDirectiveClass:358    EmitOMPParallelMaskedTaskLoopDirective(359        cast<OMPParallelMaskedTaskLoopDirective>(*S));360    break;361  case Stmt::OMPParallelMasterTaskLoopSimdDirectiveClass:362    EmitOMPParallelMasterTaskLoopSimdDirective(363        cast<OMPParallelMasterTaskLoopSimdDirective>(*S));364    break;365  case Stmt::OMPParallelMaskedTaskLoopSimdDirectiveClass:366    EmitOMPParallelMaskedTaskLoopSimdDirective(367        cast<OMPParallelMaskedTaskLoopSimdDirective>(*S));368    break;369  case Stmt::OMPDistributeDirectiveClass:370    EmitOMPDistributeDirective(cast<OMPDistributeDirective>(*S));371    break;372  case Stmt::OMPTargetUpdateDirectiveClass:373    EmitOMPTargetUpdateDirective(cast<OMPTargetUpdateDirective>(*S));374    break;375  case Stmt::OMPDistributeParallelForDirectiveClass:376    EmitOMPDistributeParallelForDirective(377        cast<OMPDistributeParallelForDirective>(*S));378    break;379  case Stmt::OMPDistributeParallelForSimdDirectiveClass:380    EmitOMPDistributeParallelForSimdDirective(381        cast<OMPDistributeParallelForSimdDirective>(*S));382    break;383  case Stmt::OMPDistributeSimdDirectiveClass:384    EmitOMPDistributeSimdDirective(cast<OMPDistributeSimdDirective>(*S));385    break;386  case Stmt::OMPTargetParallelForSimdDirectiveClass:387    EmitOMPTargetParallelForSimdDirective(388        cast<OMPTargetParallelForSimdDirective>(*S));389    break;390  case Stmt::OMPTargetSimdDirectiveClass:391    EmitOMPTargetSimdDirective(cast<OMPTargetSimdDirective>(*S));392    break;393  case Stmt::OMPTeamsDistributeDirectiveClass:394    EmitOMPTeamsDistributeDirective(cast<OMPTeamsDistributeDirective>(*S));395    break;396  case Stmt::OMPTeamsDistributeSimdDirectiveClass:397    EmitOMPTeamsDistributeSimdDirective(398        cast<OMPTeamsDistributeSimdDirective>(*S));399    break;400  case Stmt::OMPTeamsDistributeParallelForSimdDirectiveClass:401    EmitOMPTeamsDistributeParallelForSimdDirective(402        cast<OMPTeamsDistributeParallelForSimdDirective>(*S));403    break;404  case Stmt::OMPTeamsDistributeParallelForDirectiveClass:405    EmitOMPTeamsDistributeParallelForDirective(406        cast<OMPTeamsDistributeParallelForDirective>(*S));407    break;408  case Stmt::OMPTargetTeamsDirectiveClass:409    EmitOMPTargetTeamsDirective(cast<OMPTargetTeamsDirective>(*S));410    break;411  case Stmt::OMPTargetTeamsDistributeDirectiveClass:412    EmitOMPTargetTeamsDistributeDirective(413        cast<OMPTargetTeamsDistributeDirective>(*S));414    break;415  case Stmt::OMPTargetTeamsDistributeParallelForDirectiveClass:416    EmitOMPTargetTeamsDistributeParallelForDirective(417        cast<OMPTargetTeamsDistributeParallelForDirective>(*S));418    break;419  case Stmt::OMPTargetTeamsDistributeParallelForSimdDirectiveClass:420    EmitOMPTargetTeamsDistributeParallelForSimdDirective(421        cast<OMPTargetTeamsDistributeParallelForSimdDirective>(*S));422    break;423  case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass:424    EmitOMPTargetTeamsDistributeSimdDirective(425        cast<OMPTargetTeamsDistributeSimdDirective>(*S));426    break;427  case Stmt::OMPInteropDirectiveClass:428    EmitOMPInteropDirective(cast<OMPInteropDirective>(*S));429    break;430  case Stmt::OMPDispatchDirectiveClass:431    CGM.ErrorUnsupported(S, "OpenMP dispatch directive");432    break;433  case Stmt::OMPScopeDirectiveClass:434    EmitOMPScopeDirective(cast<OMPScopeDirective>(*S));435    break;436  case Stmt::OMPMaskedDirectiveClass:437    EmitOMPMaskedDirective(cast<OMPMaskedDirective>(*S));438    break;439  case Stmt::OMPGenericLoopDirectiveClass:440    EmitOMPGenericLoopDirective(cast<OMPGenericLoopDirective>(*S));441    break;442  case Stmt::OMPTeamsGenericLoopDirectiveClass:443    EmitOMPTeamsGenericLoopDirective(cast<OMPTeamsGenericLoopDirective>(*S));444    break;445  case Stmt::OMPTargetTeamsGenericLoopDirectiveClass:446    EmitOMPTargetTeamsGenericLoopDirective(447        cast<OMPTargetTeamsGenericLoopDirective>(*S));448    break;449  case Stmt::OMPParallelGenericLoopDirectiveClass:450    EmitOMPParallelGenericLoopDirective(451        cast<OMPParallelGenericLoopDirective>(*S));452    break;453  case Stmt::OMPTargetParallelGenericLoopDirectiveClass:454    EmitOMPTargetParallelGenericLoopDirective(455        cast<OMPTargetParallelGenericLoopDirective>(*S));456    break;457  case Stmt::OMPParallelMaskedDirectiveClass:458    EmitOMPParallelMaskedDirective(cast<OMPParallelMaskedDirective>(*S));459    break;460  case Stmt::OMPAssumeDirectiveClass:461    EmitOMPAssumeDirective(cast<OMPAssumeDirective>(*S));462    break;463  case Stmt::OpenACCComputeConstructClass:464    EmitOpenACCComputeConstruct(cast<OpenACCComputeConstruct>(*S));465    break;466  case Stmt::OpenACCLoopConstructClass:467    EmitOpenACCLoopConstruct(cast<OpenACCLoopConstruct>(*S));468    break;469  case Stmt::OpenACCCombinedConstructClass:470    EmitOpenACCCombinedConstruct(cast<OpenACCCombinedConstruct>(*S));471    break;472  case Stmt::OpenACCDataConstructClass:473    EmitOpenACCDataConstruct(cast<OpenACCDataConstruct>(*S));474    break;475  case Stmt::OpenACCEnterDataConstructClass:476    EmitOpenACCEnterDataConstruct(cast<OpenACCEnterDataConstruct>(*S));477    break;478  case Stmt::OpenACCExitDataConstructClass:479    EmitOpenACCExitDataConstruct(cast<OpenACCExitDataConstruct>(*S));480    break;481  case Stmt::OpenACCHostDataConstructClass:482    EmitOpenACCHostDataConstruct(cast<OpenACCHostDataConstruct>(*S));483    break;484  case Stmt::OpenACCWaitConstructClass:485    EmitOpenACCWaitConstruct(cast<OpenACCWaitConstruct>(*S));486    break;487  case Stmt::OpenACCInitConstructClass:488    EmitOpenACCInitConstruct(cast<OpenACCInitConstruct>(*S));489    break;490  case Stmt::OpenACCShutdownConstructClass:491    EmitOpenACCShutdownConstruct(cast<OpenACCShutdownConstruct>(*S));492    break;493  case Stmt::OpenACCSetConstructClass:494    EmitOpenACCSetConstruct(cast<OpenACCSetConstruct>(*S));495    break;496  case Stmt::OpenACCUpdateConstructClass:497    EmitOpenACCUpdateConstruct(cast<OpenACCUpdateConstruct>(*S));498    break;499  case Stmt::OpenACCAtomicConstructClass:500    EmitOpenACCAtomicConstruct(cast<OpenACCAtomicConstruct>(*S));501    break;502  case Stmt::OpenACCCacheConstructClass:503    EmitOpenACCCacheConstruct(cast<OpenACCCacheConstruct>(*S));504    break;505  }506}507 508bool CodeGenFunction::EmitSimpleStmt(const Stmt *S,509                                     ArrayRef<const Attr *> Attrs) {510  switch (S->getStmtClass()) {511  default:512    return false;513  case Stmt::NullStmtClass:514    break;515  case Stmt::CompoundStmtClass:516    EmitCompoundStmt(cast<CompoundStmt>(*S));517    break;518  case Stmt::DeclStmtClass:519    EmitDeclStmt(cast<DeclStmt>(*S));520    break;521  case Stmt::LabelStmtClass:522    EmitLabelStmt(cast<LabelStmt>(*S));523    break;524  case Stmt::AttributedStmtClass:525    EmitAttributedStmt(cast<AttributedStmt>(*S));526    break;527  case Stmt::GotoStmtClass:528    EmitGotoStmt(cast<GotoStmt>(*S));529    break;530  case Stmt::BreakStmtClass:531    EmitBreakStmt(cast<BreakStmt>(*S));532    break;533  case Stmt::ContinueStmtClass:534    EmitContinueStmt(cast<ContinueStmt>(*S));535    break;536  case Stmt::DefaultStmtClass:537    EmitDefaultStmt(cast<DefaultStmt>(*S), Attrs);538    break;539  case Stmt::CaseStmtClass:540    EmitCaseStmt(cast<CaseStmt>(*S), Attrs);541    break;542  case Stmt::SEHLeaveStmtClass:543    EmitSEHLeaveStmt(cast<SEHLeaveStmt>(*S));544    break;545  case Stmt::SYCLKernelCallStmtClass:546    // SYCL kernel call statements are generated as wrappers around the body547    // of functions declared with the sycl_kernel_entry_point attribute. Such548    // functions are used to specify how a SYCL kernel (a function object) is549    // to be invoked; the SYCL kernel call statement contains a transformed550    // variation of the function body and is used to generate a SYCL kernel551    // caller function; a function that serves as the device side entry point552    // used to execute the SYCL kernel. The sycl_kernel_entry_point attributed553    // function is invoked by host code in order to trigger emission of the554    // device side SYCL kernel caller function and to generate metadata needed555    // by SYCL run-time library implementations; the function is otherwise556    // intended to have no effect. As such, the function body is not evaluated557    // as part of the invocation during host compilation (and the function558    // should not be called or emitted during device compilation); the SYCL559    // kernel call statement is thus handled as a null statement for the560    // purpose of code generation.561    break;562  }563  return true;564}565 566/// EmitCompoundStmt - Emit a compound statement {..} node.  If GetLast is true,567/// this captures the expression result of the last sub-statement and returns it568/// (for use by the statement expression extension).569Address CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,570                                          AggValueSlot AggSlot) {571  PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),S.getLBracLoc(),572                             "LLVM IR generation of compound statement ('{}')");573 574  // Keep track of the current cleanup stack depth, including debug scopes.575  LexicalScope Scope(*this, S.getSourceRange());576 577  return EmitCompoundStmtWithoutScope(S, GetLast, AggSlot);578}579 580Address581CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S,582                                              bool GetLast,583                                              AggValueSlot AggSlot) {584 585  for (CompoundStmt::const_body_iterator I = S.body_begin(),586                                         E = S.body_end() - GetLast;587       I != E; ++I)588    EmitStmt(*I);589 590  Address RetAlloca = Address::invalid();591  if (GetLast) {592    // We have to special case labels here.  They are statements, but when put593    // at the end of a statement expression, they yield the value of their594    // subexpression.  Handle this by walking through all labels we encounter,595    // emitting them before we evaluate the subexpr.596    // Similar issues arise for attributed statements.597    const Stmt *LastStmt = S.body_back();598    while (!isa<Expr>(LastStmt)) {599      if (const auto *LS = dyn_cast<LabelStmt>(LastStmt)) {600        EmitLabel(LS->getDecl());601        LastStmt = LS->getSubStmt();602      } else if (const auto *AS = dyn_cast<AttributedStmt>(LastStmt)) {603        // FIXME: Update this if we ever have attributes that affect the604        // semantics of an expression.605        LastStmt = AS->getSubStmt();606      } else {607        llvm_unreachable("unknown value statement");608      }609    }610 611    EnsureInsertPoint();612 613    const Expr *E = cast<Expr>(LastStmt);614    QualType ExprTy = E->getType();615    if (hasAggregateEvaluationKind(ExprTy)) {616      EmitAggExpr(E, AggSlot);617    } else {618      // We can't return an RValue here because there might be cleanups at619      // the end of the StmtExpr.  Because of that, we have to emit the result620      // here into a temporary alloca.621      RetAlloca = CreateMemTemp(ExprTy);622      EmitAnyExprToMem(E, RetAlloca, Qualifiers(),623                       /*IsInit*/ false);624    }625  }626 627  return RetAlloca;628}629 630void CodeGenFunction::SimplifyForwardingBlocks(llvm::BasicBlock *BB) {631  llvm::BranchInst *BI = dyn_cast<llvm::BranchInst>(BB->getTerminator());632 633  // If there is a cleanup stack, then we it isn't worth trying to634  // simplify this block (we would need to remove it from the scope map635  // and cleanup entry).636  if (!EHStack.empty())637    return;638 639  // Can only simplify direct branches.640  if (!BI || !BI->isUnconditional())641    return;642 643  // Can only simplify empty blocks.644  if (BI->getIterator() != BB->begin())645    return;646 647  BB->replaceAllUsesWith(BI->getSuccessor(0));648  BI->eraseFromParent();649  BB->eraseFromParent();650}651 652void CodeGenFunction::EmitBlock(llvm::BasicBlock *BB, bool IsFinished) {653  llvm::BasicBlock *CurBB = Builder.GetInsertBlock();654 655  // Fall out of the current block (if necessary).656  EmitBranch(BB);657 658  if (IsFinished && BB->use_empty()) {659    delete BB;660    return;661  }662 663  // Place the block after the current block, if possible, or else at664  // the end of the function.665  if (CurBB && CurBB->getParent())666    CurFn->insert(std::next(CurBB->getIterator()), BB);667  else668    CurFn->insert(CurFn->end(), BB);669  Builder.SetInsertPoint(BB);670}671 672void CodeGenFunction::EmitBranch(llvm::BasicBlock *Target) {673  // Emit a branch from the current block to the target one if this674  // was a real block.  If this was just a fall-through block after a675  // terminator, don't emit it.676  llvm::BasicBlock *CurBB = Builder.GetInsertBlock();677 678  if (!CurBB || CurBB->getTerminator()) {679    // If there is no insert point or the previous block is already680    // terminated, don't touch it.681  } else {682    // Otherwise, create a fall-through branch.683    Builder.CreateBr(Target);684  }685 686  Builder.ClearInsertionPoint();687}688 689void CodeGenFunction::EmitBlockAfterUses(llvm::BasicBlock *block) {690  bool inserted = false;691  for (llvm::User *u : block->users()) {692    if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(u)) {693      CurFn->insert(std::next(insn->getParent()->getIterator()), block);694      inserted = true;695      break;696    }697  }698 699  if (!inserted)700    CurFn->insert(CurFn->end(), block);701 702  Builder.SetInsertPoint(block);703}704 705CodeGenFunction::JumpDest706CodeGenFunction::getJumpDestForLabel(const LabelDecl *D) {707  JumpDest &Dest = LabelMap[D];708  if (Dest.isValid()) return Dest;709 710  // Create, but don't insert, the new block.711  Dest = JumpDest(createBasicBlock(D->getName()),712                  EHScopeStack::stable_iterator::invalid(),713                  NextCleanupDestIndex++);714  return Dest;715}716 717void CodeGenFunction::EmitLabel(const LabelDecl *D) {718  // Add this label to the current lexical scope if we're within any719  // normal cleanups.  Jumps "in" to this label --- when permitted by720  // the language --- may need to be routed around such cleanups.721  if (EHStack.hasNormalCleanups() && CurLexicalScope)722    CurLexicalScope->addLabel(D);723 724  JumpDest &Dest = LabelMap[D];725 726  // If we didn't need a forward reference to this label, just go727  // ahead and create a destination at the current scope.728  if (!Dest.isValid()) {729    Dest = getJumpDestInCurrentScope(D->getName());730 731  // Otherwise, we need to give this label a target depth and remove732  // it from the branch-fixups list.733  } else {734    assert(!Dest.getScopeDepth().isValid() && "already emitted label!");735    Dest.setScopeDepth(EHStack.stable_begin());736    ResolveBranchFixups(Dest.getBlock());737  }738 739  EmitBlock(Dest.getBlock());740 741  // Emit debug info for labels.742  if (CGDebugInfo *DI = getDebugInfo()) {743    if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {744      DI->setLocation(D->getLocation());745      DI->EmitLabel(D, Builder);746    }747  }748 749  incrementProfileCounter(D->getStmt());750}751 752/// Change the cleanup scope of the labels in this lexical scope to753/// match the scope of the enclosing context.754void CodeGenFunction::LexicalScope::rescopeLabels() {755  assert(!Labels.empty());756  EHScopeStack::stable_iterator innermostScope757    = CGF.EHStack.getInnermostNormalCleanup();758 759  // Change the scope depth of all the labels.760  for (const LabelDecl *Label : Labels) {761    assert(CGF.LabelMap.count(Label));762    JumpDest &dest = CGF.LabelMap.find(Label)->second;763    assert(dest.getScopeDepth().isValid());764    assert(innermostScope.encloses(dest.getScopeDepth()));765    dest.setScopeDepth(innermostScope);766  }767 768  // Reparent the labels if the new scope also has cleanups.769  if (innermostScope != EHScopeStack::stable_end() && ParentScope) {770    ParentScope->Labels.append(Labels.begin(), Labels.end());771  }772}773 774 775void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) {776  EmitLabel(S.getDecl());777 778  // IsEHa - emit eha.scope.begin if it's a side entry of a scope779  if (getLangOpts().EHAsynch && S.isSideEntry())780    EmitSehCppScopeBegin();781 782  EmitStmt(S.getSubStmt());783}784 785void CodeGenFunction::EmitAttributedStmt(const AttributedStmt &S) {786  bool nomerge = false;787  bool noinline = false;788  bool alwaysinline = false;789  bool noconvergent = false;790  HLSLControlFlowHintAttr::Spelling flattenOrBranch =791      HLSLControlFlowHintAttr::SpellingNotCalculated;792  const CallExpr *musttail = nullptr;793  const AtomicAttr *AA = nullptr;794 795  for (const auto *A : S.getAttrs()) {796    switch (A->getKind()) {797    default:798      break;799    case attr::NoMerge:800      nomerge = true;801      break;802    case attr::NoInline:803      noinline = true;804      break;805    case attr::AlwaysInline:806      alwaysinline = true;807      break;808    case attr::NoConvergent:809      noconvergent = true;810      break;811    case attr::MustTail: {812      const Stmt *Sub = S.getSubStmt();813      const ReturnStmt *R = cast<ReturnStmt>(Sub);814      musttail = cast<CallExpr>(R->getRetValue()->IgnoreParens());815    } break;816    case attr::CXXAssume: {817      const Expr *Assumption = cast<CXXAssumeAttr>(A)->getAssumption();818      if (getLangOpts().CXXAssumptions && Builder.GetInsertBlock() &&819          !Assumption->HasSideEffects(getContext())) {820        llvm::Value *AssumptionVal = EmitCheckedArgForAssume(Assumption);821        Builder.CreateAssumption(AssumptionVal);822      }823    } break;824    case attr::Atomic:825      AA = cast<AtomicAttr>(A);826      break;827    case attr::HLSLControlFlowHint: {828      flattenOrBranch = cast<HLSLControlFlowHintAttr>(A)->getSemanticSpelling();829    } break;830    }831  }832  SaveAndRestore save_nomerge(InNoMergeAttributedStmt, nomerge);833  SaveAndRestore save_noinline(InNoInlineAttributedStmt, noinline);834  SaveAndRestore save_alwaysinline(InAlwaysInlineAttributedStmt, alwaysinline);835  SaveAndRestore save_noconvergent(InNoConvergentAttributedStmt, noconvergent);836  SaveAndRestore save_musttail(MustTailCall, musttail);837  SaveAndRestore save_flattenOrBranch(HLSLControlFlowAttr, flattenOrBranch);838  CGAtomicOptionsRAII AORAII(CGM, AA);839  EmitStmt(S.getSubStmt(), S.getAttrs());840}841 842void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) {843  // If this code is reachable then emit a stop point (if generating844  // debug info). We have to do this ourselves because we are on the845  // "simple" statement path.846  if (HaveInsertPoint())847    EmitStopPoint(&S);848 849  ApplyAtomGroup Grp(getDebugInfo());850  EmitBranchThroughCleanup(getJumpDestForLabel(S.getLabel()));851}852 853 854void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {855  ApplyAtomGroup Grp(getDebugInfo());856  if (const LabelDecl *Target = S.getConstantTarget()) {857    EmitBranchThroughCleanup(getJumpDestForLabel(Target));858    return;859  }860 861  // Ensure that we have an i8* for our PHI node.862  llvm::Value *V = Builder.CreateBitCast(EmitScalarExpr(S.getTarget()),863                                         Int8PtrTy, "addr");864  llvm::BasicBlock *CurBB = Builder.GetInsertBlock();865 866  // Get the basic block for the indirect goto.867  llvm::BasicBlock *IndGotoBB = GetIndirectGotoBlock();868 869  // The first instruction in the block has to be the PHI for the switch dest,870  // add an entry for this branch.871  cast<llvm::PHINode>(IndGotoBB->begin())->addIncoming(V, CurBB);872 873  EmitBranch(IndGotoBB);874  if (CurBB && CurBB->getTerminator())875    addInstToCurrentSourceAtom(CurBB->getTerminator(), nullptr);876}877 878void CodeGenFunction::EmitIfStmt(const IfStmt &S) {879  const Stmt *Else = S.getElse();880 881  // The else branch of a consteval if statement is always the only branch that882  // can be runtime evaluated.883  if (S.isConsteval()) {884    const Stmt *Executed = S.isNegatedConsteval() ? S.getThen() : Else;885    if (Executed) {886      RunCleanupsScope ExecutedScope(*this);887      EmitStmt(Executed);888    }889    return;890  }891 892  // C99 6.8.4.1: The first substatement is executed if the expression compares893  // unequal to 0.  The condition must be a scalar type.894  LexicalScope ConditionScope(*this, S.getCond()->getSourceRange());895  ApplyDebugLocation DL(*this, S.getCond());896 897  if (S.getInit())898    EmitStmt(S.getInit());899 900  if (S.getConditionVariable())901    EmitDecl(*S.getConditionVariable());902 903  // If the condition constant folds and can be elided, try to avoid emitting904  // the condition and the dead arm of the if/else.905  bool CondConstant;906  if (ConstantFoldsToSimpleInteger(S.getCond(), CondConstant,907                                   S.isConstexpr())) {908    // Figure out which block (then or else) is executed.909    const Stmt *Executed = S.getThen();910    const Stmt *Skipped = Else;911    if (!CondConstant) // Condition false?912      std::swap(Executed, Skipped);913 914    // If the skipped block has no labels in it, just emit the executed block.915    // This avoids emitting dead code and simplifies the CFG substantially.916    if (S.isConstexpr() || !ContainsLabel(Skipped)) {917      if (CondConstant)918        incrementProfileCounter(&S);919      if (Executed) {920        MaybeEmitDeferredVarDeclInit(S.getConditionVariable());921        RunCleanupsScope ExecutedScope(*this);922        EmitStmt(Executed);923      }924      PGO->markStmtMaybeUsed(Skipped);925      return;926    }927  }928 929  // Otherwise, the condition did not fold, or we couldn't elide it.  Just emit930  // the conditional branch.931  llvm::BasicBlock *ThenBlock = createBasicBlock("if.then");932  llvm::BasicBlock *ContBlock = createBasicBlock("if.end");933  llvm::BasicBlock *ElseBlock = ContBlock;934  if (Else)935    ElseBlock = createBasicBlock("if.else");936 937  // Prefer the PGO based weights over the likelihood attribute.938  // When the build isn't optimized the metadata isn't used, so don't generate939  // it.940  // Also, differentiate between disabled PGO and a never executed branch with941  // PGO. Assuming PGO is in use:942  // - we want to ignore the [[likely]] attribute if the branch is never943  // executed,944  // - assuming the profile is poor, preserving the attribute may still be945  // beneficial.946  // As an approximation, preserve the attribute only if both the branch and the947  // parent context were not executed.948  Stmt::Likelihood LH = Stmt::LH_None;949  uint64_t ThenCount = getProfileCount(S.getThen());950  if (!ThenCount && !getCurrentProfileCount() &&951      CGM.getCodeGenOpts().OptimizationLevel)952    LH = Stmt::getLikelihood(S.getThen(), Else);953 954  // When measuring MC/DC, always fully evaluate the condition up front using955  // EvaluateExprAsBool() so that the test vector bitmap can be updated prior to956  // executing the body of the if.then or if.else. This is useful for when957  // there is a 'return' within the body, but this is particularly beneficial958  // when one if-stmt is nested within another if-stmt so that all of the MC/DC959  // updates are kept linear and consistent.960  if (!CGM.getCodeGenOpts().MCDCCoverage) {961    EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, ThenCount, LH,962                         /*ConditionalOp=*/nullptr,963                         /*ConditionalDecl=*/S.getConditionVariable());964  } else {965    llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());966    MaybeEmitDeferredVarDeclInit(S.getConditionVariable());967    Builder.CreateCondBr(BoolCondVal, ThenBlock, ElseBlock);968  }969 970  // Emit the 'then' code.971  EmitBlock(ThenBlock);972  if (llvm::EnableSingleByteCoverage)973    incrementProfileCounter(S.getThen());974  else975    incrementProfileCounter(&S);976  {977    RunCleanupsScope ThenScope(*this);978    EmitStmt(S.getThen());979  }980  EmitBranch(ContBlock);981 982  // Emit the 'else' code if present.983  if (Else) {984    {985      // There is no need to emit line number for an unconditional branch.986      auto NL = ApplyDebugLocation::CreateEmpty(*this);987      EmitBlock(ElseBlock);988    }989    // When single byte coverage mode is enabled, add a counter to else block.990    if (llvm::EnableSingleByteCoverage)991      incrementProfileCounter(Else);992    {993      RunCleanupsScope ElseScope(*this);994      EmitStmt(Else);995    }996    {997      // There is no need to emit line number for an unconditional branch.998      auto NL = ApplyDebugLocation::CreateEmpty(*this);999      EmitBranch(ContBlock);1000    }1001  }1002 1003  // Emit the continuation block for code after the if.1004  EmitBlock(ContBlock, true);1005 1006  // When single byte coverage mode is enabled, add a counter to continuation1007  // block.1008  if (llvm::EnableSingleByteCoverage)1009    incrementProfileCounter(&S);1010}1011 1012bool CodeGenFunction::checkIfLoopMustProgress(const Expr *ControllingExpression,1013                                              bool HasEmptyBody) {1014  if (CGM.getCodeGenOpts().getFiniteLoops() ==1015      CodeGenOptions::FiniteLoopsKind::Never)1016    return false;1017 1018  // Now apply rules for plain C (see  6.8.5.6 in C11).1019  // Loops with constant conditions do not have to make progress in any C1020  // version.1021  // As an extension, we consisider loops whose constant expression1022  // can be constant-folded.1023  Expr::EvalResult Result;1024  bool CondIsConstInt =1025      !ControllingExpression ||1026      (ControllingExpression->EvaluateAsInt(Result, getContext()) &&1027       Result.Val.isInt());1028 1029  bool CondIsTrue = CondIsConstInt && (!ControllingExpression ||1030                                       Result.Val.getInt().getBoolValue());1031 1032  // Loops with non-constant conditions must make progress in C11 and later.1033  if (getLangOpts().C11 && !CondIsConstInt)1034    return true;1035 1036  // [C++26][intro.progress] (DR)1037  // The implementation may assume that any thread will eventually do one of the1038  // following:1039  // [...]1040  // - continue execution of a trivial infinite loop ([stmt.iter.general]).1041  if (CGM.getCodeGenOpts().getFiniteLoops() ==1042          CodeGenOptions::FiniteLoopsKind::Always ||1043      getLangOpts().CPlusPlus11) {1044    if (HasEmptyBody && CondIsTrue) {1045      CurFn->removeFnAttr(llvm::Attribute::MustProgress);1046      return false;1047    }1048    return true;1049  }1050  return false;1051}1052 1053// [C++26][stmt.iter.general] (DR)1054// A trivially empty iteration statement is an iteration statement matching one1055// of the following forms:1056//  - while ( expression ) ;1057//  - while ( expression ) { }1058//  - do ; while ( expression ) ;1059//  - do { } while ( expression ) ;1060//  - for ( init-statement expression(opt); ) ;1061//  - for ( init-statement expression(opt); ) { }1062template <typename LoopStmt> static bool hasEmptyLoopBody(const LoopStmt &S) {1063  if constexpr (std::is_same_v<LoopStmt, ForStmt>) {1064    if (S.getInc())1065      return false;1066  }1067  const Stmt *Body = S.getBody();1068  if (!Body || isa<NullStmt>(Body))1069    return true;1070  if (const CompoundStmt *Compound = dyn_cast<CompoundStmt>(Body))1071    return Compound->body_empty();1072  return false;1073}1074 1075void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,1076                                    ArrayRef<const Attr *> WhileAttrs) {1077  // Emit the header for the loop, which will also become1078  // the continue target.1079  JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond");1080  EmitBlock(LoopHeader.getBlock());1081 1082  if (CGM.shouldEmitConvergenceTokens())1083    ConvergenceTokenStack.push_back(1084        emitConvergenceLoopToken(LoopHeader.getBlock()));1085 1086  // Create an exit block for when the condition fails, which will1087  // also become the break target.1088  JumpDest LoopExit = getJumpDestInCurrentScope("while.end");1089 1090  // Store the blocks to use for break and continue.1091  BreakContinueStack.push_back(BreakContinue(S, LoopExit, LoopHeader));1092 1093  // C++ [stmt.while]p2:1094  //   When the condition of a while statement is a declaration, the1095  //   scope of the variable that is declared extends from its point1096  //   of declaration (3.3.2) to the end of the while statement.1097  //   [...]1098  //   The object created in a condition is destroyed and created1099  //   with each iteration of the loop.1100  RunCleanupsScope ConditionScope(*this);1101 1102  if (S.getConditionVariable())1103    EmitDecl(*S.getConditionVariable());1104 1105  // Evaluate the conditional in the while header.  C99 6.8.5.1: The1106  // evaluation of the controlling expression takes place before each1107  // execution of the loop body.1108  llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());1109 1110  MaybeEmitDeferredVarDeclInit(S.getConditionVariable());1111 1112  // while(1) is common, avoid extra exit blocks.  Be sure1113  // to correctly handle break/continue though.1114  llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal);1115  bool EmitBoolCondBranch = !C || !C->isOne();1116  const SourceRange &R = S.getSourceRange();1117  LoopStack.push(LoopHeader.getBlock(), CGM.getContext(), CGM.getCodeGenOpts(),1118                 WhileAttrs, SourceLocToDebugLoc(R.getBegin()),1119                 SourceLocToDebugLoc(R.getEnd()),1120                 checkIfLoopMustProgress(S.getCond(), hasEmptyLoopBody(S)));1121 1122  // When single byte coverage mode is enabled, add a counter to loop condition.1123  if (llvm::EnableSingleByteCoverage)1124    incrementProfileCounter(S.getCond());1125 1126  // As long as the condition is true, go to the loop body.1127  llvm::BasicBlock *LoopBody = createBasicBlock("while.body");1128  if (EmitBoolCondBranch) {1129    llvm::BasicBlock *ExitBlock = LoopExit.getBlock();1130    if (ConditionScope.requiresCleanups())1131      ExitBlock = createBasicBlock("while.exit");1132    llvm::MDNode *Weights =1133        createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));1134    if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)1135      BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(1136          BoolCondVal, Stmt::getLikelihood(S.getBody()));1137    auto *I = Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, Weights);1138    // Key Instructions: Emit the condition and branch as separate source1139    // location atoms otherwise we may omit a step onto the loop condition in1140    // favour of the `while` keyword.1141    // FIXME: We could have the branch as the backup location for the condition,1142    // which would probably be a better experience. Explore this later.1143    if (auto *CondI = dyn_cast<llvm::Instruction>(BoolCondVal))1144      addInstToNewSourceAtom(CondI, nullptr);1145    addInstToNewSourceAtom(I, nullptr);1146 1147    if (ExitBlock != LoopExit.getBlock()) {1148      EmitBlock(ExitBlock);1149      EmitBranchThroughCleanup(LoopExit);1150    }1151  } else if (const Attr *A = Stmt::getLikelihoodAttr(S.getBody())) {1152    CGM.getDiags().Report(A->getLocation(),1153                          diag::warn_attribute_has_no_effect_on_infinite_loop)1154        << A << A->getRange();1155    CGM.getDiags().Report(1156        S.getWhileLoc(),1157        diag::note_attribute_has_no_effect_on_infinite_loop_here)1158        << SourceRange(S.getWhileLoc(), S.getRParenLoc());1159  }1160 1161  // Emit the loop body.  We have to emit this in a cleanup scope1162  // because it might be a singleton DeclStmt.1163  {1164    RunCleanupsScope BodyScope(*this);1165    EmitBlock(LoopBody);1166    // When single byte coverage mode is enabled, add a counter to the body.1167    if (llvm::EnableSingleByteCoverage)1168      incrementProfileCounter(S.getBody());1169    else1170      incrementProfileCounter(&S);1171    EmitStmt(S.getBody());1172  }1173 1174  BreakContinueStack.pop_back();1175 1176  // Immediately force cleanup.1177  ConditionScope.ForceCleanup();1178 1179  EmitStopPoint(&S);1180  // Branch to the loop header again.1181  EmitBranch(LoopHeader.getBlock());1182 1183  LoopStack.pop();1184 1185  // Emit the exit block.1186  EmitBlock(LoopExit.getBlock(), true);1187 1188  // The LoopHeader typically is just a branch if we skipped emitting1189  // a branch, try to erase it.1190  if (!EmitBoolCondBranch)1191    SimplifyForwardingBlocks(LoopHeader.getBlock());1192 1193  // When single byte coverage mode is enabled, add a counter to continuation1194  // block.1195  if (llvm::EnableSingleByteCoverage)1196    incrementProfileCounter(&S);1197 1198  if (CGM.shouldEmitConvergenceTokens())1199    ConvergenceTokenStack.pop_back();1200}1201 1202void CodeGenFunction::EmitDoStmt(const DoStmt &S,1203                                 ArrayRef<const Attr *> DoAttrs) {1204  JumpDest LoopExit = getJumpDestInCurrentScope("do.end");1205  JumpDest LoopCond = getJumpDestInCurrentScope("do.cond");1206 1207  uint64_t ParentCount = getCurrentProfileCount();1208 1209  // Store the blocks to use for break and continue.1210  BreakContinueStack.push_back(BreakContinue(S, LoopExit, LoopCond));1211 1212  // Emit the body of the loop.1213  llvm::BasicBlock *LoopBody = createBasicBlock("do.body");1214 1215  if (llvm::EnableSingleByteCoverage)1216    EmitBlockWithFallThrough(LoopBody, S.getBody());1217  else1218    EmitBlockWithFallThrough(LoopBody, &S);1219 1220  if (CGM.shouldEmitConvergenceTokens())1221    ConvergenceTokenStack.push_back(emitConvergenceLoopToken(LoopBody));1222 1223  {1224    RunCleanupsScope BodyScope(*this);1225    EmitStmt(S.getBody());1226  }1227 1228  EmitBlock(LoopCond.getBlock());1229  // When single byte coverage mode is enabled, add a counter to loop condition.1230  if (llvm::EnableSingleByteCoverage)1231    incrementProfileCounter(S.getCond());1232 1233  // C99 6.8.5.2: "The evaluation of the controlling expression takes place1234  // after each execution of the loop body."1235 1236  // Evaluate the conditional in the while header.1237  // C99 6.8.5p2/p4: The first substatement is executed if the expression1238  // compares unequal to 0.  The condition must be a scalar type.1239  llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());1240 1241  BreakContinueStack.pop_back();1242 1243  // "do {} while (0)" is common in macros, avoid extra blocks.  Be sure1244  // to correctly handle break/continue though.1245  llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal);1246  bool EmitBoolCondBranch = !C || !C->isZero();1247 1248  const SourceRange &R = S.getSourceRange();1249  LoopStack.push(LoopBody, CGM.getContext(), CGM.getCodeGenOpts(), DoAttrs,1250                 SourceLocToDebugLoc(R.getBegin()),1251                 SourceLocToDebugLoc(R.getEnd()),1252                 checkIfLoopMustProgress(S.getCond(), hasEmptyLoopBody(S)));1253 1254  // As long as the condition is true, iterate the loop.1255  if (EmitBoolCondBranch) {1256    uint64_t BackedgeCount = getProfileCount(S.getBody()) - ParentCount;1257    auto *I = Builder.CreateCondBr(1258        BoolCondVal, LoopBody, LoopExit.getBlock(),1259        createProfileWeightsForLoop(S.getCond(), BackedgeCount));1260 1261    // Key Instructions: Emit the condition and branch as separate source1262    // location atoms otherwise we may omit a step onto the loop condition in1263    // favour of the closing brace.1264    // FIXME: We could have the branch as the backup location for the condition,1265    // which would probably be a better experience (no jumping to the brace).1266    if (auto *CondI = dyn_cast<llvm::Instruction>(BoolCondVal))1267      addInstToNewSourceAtom(CondI, nullptr);1268    addInstToNewSourceAtom(I, nullptr);1269  }1270 1271  LoopStack.pop();1272 1273  // Emit the exit block.1274  EmitBlock(LoopExit.getBlock());1275 1276  // The DoCond block typically is just a branch if we skipped1277  // emitting a branch, try to erase it.1278  if (!EmitBoolCondBranch)1279    SimplifyForwardingBlocks(LoopCond.getBlock());1280 1281  // When single byte coverage mode is enabled, add a counter to continuation1282  // block.1283  if (llvm::EnableSingleByteCoverage)1284    incrementProfileCounter(&S);1285 1286  if (CGM.shouldEmitConvergenceTokens())1287    ConvergenceTokenStack.pop_back();1288}1289 1290void CodeGenFunction::EmitForStmt(const ForStmt &S,1291                                  ArrayRef<const Attr *> ForAttrs) {1292  JumpDest LoopExit = getJumpDestInCurrentScope("for.end");1293 1294  std::optional<LexicalScope> ForScope;1295  if (getLangOpts().C99 || getLangOpts().CPlusPlus)1296    ForScope.emplace(*this, S.getSourceRange());1297 1298  // Evaluate the first part before the loop.1299  if (S.getInit())1300    EmitStmt(S.getInit());1301 1302  // Start the loop with a block that tests the condition.1303  // If there's an increment, the continue scope will be overwritten1304  // later.1305  JumpDest CondDest = getJumpDestInCurrentScope("for.cond");1306  llvm::BasicBlock *CondBlock = CondDest.getBlock();1307  EmitBlock(CondBlock);1308 1309  if (CGM.shouldEmitConvergenceTokens())1310    ConvergenceTokenStack.push_back(emitConvergenceLoopToken(CondBlock));1311 1312  const SourceRange &R = S.getSourceRange();1313  LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(), ForAttrs,1314                 SourceLocToDebugLoc(R.getBegin()),1315                 SourceLocToDebugLoc(R.getEnd()),1316                 checkIfLoopMustProgress(S.getCond(), hasEmptyLoopBody(S)));1317 1318  // Create a cleanup scope for the condition variable cleanups.1319  LexicalScope ConditionScope(*this, S.getSourceRange());1320 1321  // If the for loop doesn't have an increment we can just use the condition as1322  // the continue block. Otherwise, if there is no condition variable, we can1323  // form the continue block now. If there is a condition variable, we can't1324  // form the continue block until after we've emitted the condition, because1325  // the condition is in scope in the increment, but Sema's jump diagnostics1326  // ensure that there are no continues from the condition variable that jump1327  // to the loop increment.1328  JumpDest Continue;1329  if (!S.getInc())1330    Continue = CondDest;1331  else if (!S.getConditionVariable())1332    Continue = getJumpDestInCurrentScope("for.inc");1333  BreakContinueStack.push_back(BreakContinue(S, LoopExit, Continue));1334 1335  if (S.getCond()) {1336    // If the for statement has a condition scope, emit the local variable1337    // declaration.1338    if (S.getConditionVariable()) {1339      EmitDecl(*S.getConditionVariable());1340 1341      // We have entered the condition variable's scope, so we're now able to1342      // jump to the continue block.1343      Continue = S.getInc() ? getJumpDestInCurrentScope("for.inc") : CondDest;1344      BreakContinueStack.back().ContinueBlock = Continue;1345    }1346 1347    // When single byte coverage mode is enabled, add a counter to loop1348    // condition.1349    if (llvm::EnableSingleByteCoverage)1350      incrementProfileCounter(S.getCond());1351 1352    llvm::BasicBlock *ExitBlock = LoopExit.getBlock();1353    // If there are any cleanups between here and the loop-exit scope,1354    // create a block to stage a loop exit along.1355    if (ForScope && ForScope->requiresCleanups())1356      ExitBlock = createBasicBlock("for.cond.cleanup");1357 1358    // As long as the condition is true, iterate the loop.1359    llvm::BasicBlock *ForBody = createBasicBlock("for.body");1360 1361    // C99 6.8.5p2/p4: The first substatement is executed if the expression1362    // compares unequal to 0.  The condition must be a scalar type.1363    llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());1364 1365    MaybeEmitDeferredVarDeclInit(S.getConditionVariable());1366 1367    llvm::MDNode *Weights =1368        createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));1369    if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)1370      BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(1371          BoolCondVal, Stmt::getLikelihood(S.getBody()));1372 1373    auto *I = Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights);1374    // Key Instructions: Emit the condition and branch as separate atoms to1375    // match existing loop stepping behaviour. FIXME: We could have the branch1376    // as the backup location for the condition, which would probably be a1377    // better experience (no jumping to the brace).1378    if (auto *CondI = dyn_cast<llvm::Instruction>(BoolCondVal))1379      addInstToNewSourceAtom(CondI, nullptr);1380    addInstToNewSourceAtom(I, nullptr);1381 1382    if (ExitBlock != LoopExit.getBlock()) {1383      EmitBlock(ExitBlock);1384      EmitBranchThroughCleanup(LoopExit);1385    }1386 1387    EmitBlock(ForBody);1388  } else {1389    // Treat it as a non-zero constant.  Don't even create a new block for the1390    // body, just fall into it.1391  }1392 1393  // When single byte coverage mode is enabled, add a counter to the body.1394  if (llvm::EnableSingleByteCoverage)1395    incrementProfileCounter(S.getBody());1396  else1397    incrementProfileCounter(&S);1398  {1399    // Create a separate cleanup scope for the body, in case it is not1400    // a compound statement.1401    RunCleanupsScope BodyScope(*this);1402    EmitStmt(S.getBody());1403  }1404 1405  // The last block in the loop's body (which unconditionally branches to the1406  // `inc` block if there is one).1407  auto *FinalBodyBB = Builder.GetInsertBlock();1408 1409  // If there is an increment, emit it next.1410  if (S.getInc()) {1411    EmitBlock(Continue.getBlock());1412    EmitStmt(S.getInc());1413    if (llvm::EnableSingleByteCoverage)1414      incrementProfileCounter(S.getInc());1415  }1416 1417  BreakContinueStack.pop_back();1418 1419  ConditionScope.ForceCleanup();1420 1421  EmitStopPoint(&S);1422  EmitBranch(CondBlock);1423 1424  if (ForScope)1425    ForScope->ForceCleanup();1426 1427  LoopStack.pop();1428 1429  // Emit the fall-through block.1430  EmitBlock(LoopExit.getBlock(), true);1431 1432  // When single byte coverage mode is enabled, add a counter to continuation1433  // block.1434  if (llvm::EnableSingleByteCoverage)1435    incrementProfileCounter(&S);1436 1437  if (CGM.shouldEmitConvergenceTokens())1438    ConvergenceTokenStack.pop_back();1439 1440  if (FinalBodyBB) {1441    // Key Instructions: We want the for closing brace to be step-able on to1442    // match existing behaviour.1443    addInstToNewSourceAtom(FinalBodyBB->getTerminator(), nullptr);1444  }1445}1446 1447void1448CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,1449                                     ArrayRef<const Attr *> ForAttrs) {1450  JumpDest LoopExit = getJumpDestInCurrentScope("for.end");1451 1452  LexicalScope ForScope(*this, S.getSourceRange());1453 1454  // Evaluate the first pieces before the loop.1455  if (S.getInit())1456    EmitStmt(S.getInit());1457  EmitStmt(S.getRangeStmt());1458  EmitStmt(S.getBeginStmt());1459  EmitStmt(S.getEndStmt());1460 1461  // Start the loop with a block that tests the condition.1462  // If there's an increment, the continue scope will be overwritten1463  // later.1464  llvm::BasicBlock *CondBlock = createBasicBlock("for.cond");1465  EmitBlock(CondBlock);1466 1467  if (CGM.shouldEmitConvergenceTokens())1468    ConvergenceTokenStack.push_back(emitConvergenceLoopToken(CondBlock));1469 1470  const SourceRange &R = S.getSourceRange();1471  LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(), ForAttrs,1472                 SourceLocToDebugLoc(R.getBegin()),1473                 SourceLocToDebugLoc(R.getEnd()));1474 1475  // If there are any cleanups between here and the loop-exit scope,1476  // create a block to stage a loop exit along.1477  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();1478  if (ForScope.requiresCleanups())1479    ExitBlock = createBasicBlock("for.cond.cleanup");1480 1481  // The loop body, consisting of the specified body and the loop variable.1482  llvm::BasicBlock *ForBody = createBasicBlock("for.body");1483 1484  // The body is executed if the expression, contextually converted1485  // to bool, is true.1486  llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());1487  llvm::MDNode *Weights =1488      createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));1489  if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)1490    BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(1491        BoolCondVal, Stmt::getLikelihood(S.getBody()));1492  auto *I = Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights);1493  // Key Instructions: Emit the condition and branch as separate atoms to1494  // match existing loop stepping behaviour. FIXME: We could have the branch as1495  // the backup location for the condition, which would probably be a better1496  // experience.1497  if (auto *CondI = dyn_cast<llvm::Instruction>(BoolCondVal))1498    addInstToNewSourceAtom(CondI, nullptr);1499  addInstToNewSourceAtom(I, nullptr);1500 1501  if (ExitBlock != LoopExit.getBlock()) {1502    EmitBlock(ExitBlock);1503    EmitBranchThroughCleanup(LoopExit);1504  }1505 1506  EmitBlock(ForBody);1507  if (llvm::EnableSingleByteCoverage)1508    incrementProfileCounter(S.getBody());1509  else1510    incrementProfileCounter(&S);1511 1512  // Create a block for the increment. In case of a 'continue', we jump there.1513  JumpDest Continue = getJumpDestInCurrentScope("for.inc");1514 1515  // Store the blocks to use for break and continue.1516  BreakContinueStack.push_back(BreakContinue(S, LoopExit, Continue));1517 1518  {1519    // Create a separate cleanup scope for the loop variable and body.1520    LexicalScope BodyScope(*this, S.getSourceRange());1521    EmitStmt(S.getLoopVarStmt());1522    EmitStmt(S.getBody());1523  }1524  // The last block in the loop's body (which unconditionally branches to the1525  // `inc` block if there is one).1526  auto *FinalBodyBB = Builder.GetInsertBlock();1527 1528  EmitStopPoint(&S);1529  // If there is an increment, emit it next.1530  EmitBlock(Continue.getBlock());1531  EmitStmt(S.getInc());1532 1533  BreakContinueStack.pop_back();1534 1535  EmitBranch(CondBlock);1536 1537  ForScope.ForceCleanup();1538 1539  LoopStack.pop();1540 1541  // Emit the fall-through block.1542  EmitBlock(LoopExit.getBlock(), true);1543 1544  // When single byte coverage mode is enabled, add a counter to continuation1545  // block.1546  if (llvm::EnableSingleByteCoverage)1547    incrementProfileCounter(&S);1548 1549  if (CGM.shouldEmitConvergenceTokens())1550    ConvergenceTokenStack.pop_back();1551 1552  if (FinalBodyBB) {1553    // We want the for closing brace to be step-able on to match existing1554    // behaviour.1555    addInstToNewSourceAtom(FinalBodyBB->getTerminator(), nullptr);1556  }1557}1558 1559void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {1560  if (RV.isScalar()) {1561    Builder.CreateStore(RV.getScalarVal(), ReturnValue);1562  } else if (RV.isAggregate()) {1563    LValue Dest = MakeAddrLValue(ReturnValue, Ty);1564    LValue Src = MakeAddrLValue(RV.getAggregateAddress(), Ty);1565    EmitAggregateCopy(Dest, Src, Ty, getOverlapForReturnValue());1566  } else {1567    EmitStoreOfComplex(RV.getComplexVal(), MakeAddrLValue(ReturnValue, Ty),1568                       /*init*/ true);1569  }1570  EmitBranchThroughCleanup(ReturnBlock);1571}1572 1573namespace {1574// RAII struct used to save and restore a return statment's result expression.1575struct SaveRetExprRAII {1576  SaveRetExprRAII(const Expr *RetExpr, CodeGenFunction &CGF)1577      : OldRetExpr(CGF.RetExpr), CGF(CGF) {1578    CGF.RetExpr = RetExpr;1579  }1580  ~SaveRetExprRAII() { CGF.RetExpr = OldRetExpr; }1581  const Expr *OldRetExpr;1582  CodeGenFunction &CGF;1583};1584} // namespace1585 1586/// Determine if the given call uses the swiftasync calling convention.1587static bool isSwiftAsyncCallee(const CallExpr *CE) {1588  auto calleeQualType = CE->getCallee()->getType();1589  const FunctionType *calleeType = nullptr;1590  if (calleeQualType->isFunctionPointerType() ||1591      calleeQualType->isFunctionReferenceType() ||1592      calleeQualType->isBlockPointerType() ||1593      calleeQualType->isMemberFunctionPointerType()) {1594    calleeType = calleeQualType->getPointeeType()->castAs<FunctionType>();1595  } else if (auto *ty = dyn_cast<FunctionType>(calleeQualType)) {1596    calleeType = ty;1597  } else if (auto CMCE = dyn_cast<CXXMemberCallExpr>(CE)) {1598    if (auto methodDecl = CMCE->getMethodDecl()) {1599      // getMethodDecl() doesn't handle member pointers at the moment.1600      calleeType = methodDecl->getType()->castAs<FunctionType>();1601    } else {1602      return false;1603    }1604  } else {1605    return false;1606  }1607  return calleeType->getCallConv() == CallingConv::CC_SwiftAsync;1608}1609 1610/// EmitReturnStmt - Note that due to GCC extensions, this can have an operand1611/// if the function returns void, or may be missing one if the function returns1612/// non-void.  Fun stuff :).1613void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {1614  ApplyAtomGroup Grp(getDebugInfo());1615  if (requiresReturnValueCheck()) {1616    llvm::Constant *SLoc = EmitCheckSourceLocation(S.getBeginLoc());1617    auto *SLocPtr =1618        new llvm::GlobalVariable(CGM.getModule(), SLoc->getType(), false,1619                                 llvm::GlobalVariable::PrivateLinkage, SLoc);1620    SLocPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);1621    CGM.getSanitizerMetadata()->disableSanitizerForGlobal(SLocPtr);1622    assert(ReturnLocation.isValid() && "No valid return location");1623    Builder.CreateStore(SLocPtr, ReturnLocation);1624  }1625 1626  // Returning from an outlined SEH helper is UB, and we already warn on it.1627  if (IsOutlinedSEHHelper) {1628    Builder.CreateUnreachable();1629    Builder.ClearInsertionPoint();1630  }1631 1632  // Emit the result value, even if unused, to evaluate the side effects.1633  const Expr *RV = S.getRetValue();1634 1635  // Record the result expression of the return statement. The recorded1636  // expression is used to determine whether a block capture's lifetime should1637  // end at the end of the full expression as opposed to the end of the scope1638  // enclosing the block expression.1639  //1640  // This permits a small, easily-implemented exception to our over-conservative1641  // rules about not jumping to statements following block literals with1642  // non-trivial cleanups.1643  SaveRetExprRAII SaveRetExpr(RV, *this);1644 1645  RunCleanupsScope cleanupScope(*this);1646  if (const auto *EWC = dyn_cast_or_null<ExprWithCleanups>(RV))1647    RV = EWC->getSubExpr();1648 1649  // If we're in a swiftasynccall function, and the return expression is a1650  // call to a swiftasynccall function, mark the call as the musttail call.1651  std::optional<llvm::SaveAndRestore<const CallExpr *>> SaveMustTail;1652  if (RV && CurFnInfo &&1653      CurFnInfo->getASTCallingConvention() == CallingConv::CC_SwiftAsync) {1654    if (auto CE = dyn_cast<CallExpr>(RV)) {1655      if (isSwiftAsyncCallee(CE)) {1656        SaveMustTail.emplace(MustTailCall, CE);1657      }1658    }1659  }1660 1661  // FIXME: Clean this up by using an LValue for ReturnTemp,1662  // EmitStoreThroughLValue, and EmitAnyExpr.1663  // Check if the NRVO candidate was not globalized in OpenMP mode.1664  if (getLangOpts().ElideConstructors && S.getNRVOCandidate() &&1665      S.getNRVOCandidate()->isNRVOVariable() &&1666      (!getLangOpts().OpenMP ||1667       !CGM.getOpenMPRuntime()1668            .getAddressOfLocalVariable(*this, S.getNRVOCandidate())1669            .isValid())) {1670    // Apply the named return value optimization for this return statement,1671    // which means doing nothing: the appropriate result has already been1672    // constructed into the NRVO variable.1673 1674    // If there is an NRVO flag for this variable, set it to 1 into indicate1675    // that the cleanup code should not destroy the variable.1676    if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()])1677      Builder.CreateFlagStore(Builder.getTrue(), NRVOFlag);1678  } else if (!ReturnValue.isValid() || (RV && RV->getType()->isVoidType())) {1679    // Make sure not to return anything, but evaluate the expression1680    // for side effects.1681    if (RV) {1682      EmitAnyExpr(RV);1683    }1684  } else if (!RV) {1685    // Do nothing (return value is left uninitialized)1686  } else if (FnRetTy->isReferenceType()) {1687    // If this function returns a reference, take the address of the expression1688    // rather than the value.1689    RValue Result = EmitReferenceBindingToExpr(RV);1690    auto *I = Builder.CreateStore(Result.getScalarVal(), ReturnValue);1691    addInstToCurrentSourceAtom(I, I->getValueOperand());1692  } else {1693    switch (getEvaluationKind(RV->getType())) {1694    case TEK_Scalar: {1695      llvm::Value *Ret = EmitScalarExpr(RV);1696      if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {1697        EmitStoreOfScalar(Ret, MakeAddrLValue(ReturnValue, RV->getType()),1698                          /*isInit*/ true);1699      } else {1700        auto *I = Builder.CreateStore(Ret, ReturnValue);1701        addInstToCurrentSourceAtom(I, I->getValueOperand());1702      }1703      break;1704    }1705    case TEK_Complex:1706      EmitComplexExprIntoLValue(RV, MakeAddrLValue(ReturnValue, RV->getType()),1707                                /*isInit*/ true);1708      break;1709    case TEK_Aggregate:1710      EmitAggExpr(RV, AggValueSlot::forAddr(1711                          ReturnValue, Qualifiers(),1712                          AggValueSlot::IsDestructed,1713                          AggValueSlot::DoesNotNeedGCBarriers,1714                          AggValueSlot::IsNotAliased,1715                          getOverlapForReturnValue()));1716      break;1717    }1718  }1719 1720  ++NumReturnExprs;1721  if (!RV || RV->isEvaluatable(getContext()))1722    ++NumSimpleReturnExprs;1723 1724  cleanupScope.ForceCleanup();1725  EmitBranchThroughCleanup(ReturnBlock);1726}1727 1728void CodeGenFunction::EmitDeclStmt(const DeclStmt &S) {1729  // As long as debug info is modeled with instructions, we have to ensure we1730  // have a place to insert here and write the stop point here.1731  if (HaveInsertPoint())1732    EmitStopPoint(&S);1733 1734  for (const auto *I : S.decls())1735    EmitDecl(*I, /*EvaluateConditionDecl=*/true);1736}1737 1738auto CodeGenFunction::GetDestForLoopControlStmt(const LoopControlStmt &S)1739    -> const BreakContinue * {1740  if (!S.hasLabelTarget())1741    return &BreakContinueStack.back();1742 1743  const Stmt *LoopOrSwitch = S.getNamedLoopOrSwitch();1744  assert(LoopOrSwitch && "break/continue target not set?");1745  for (const BreakContinue &BC : llvm::reverse(BreakContinueStack))1746    if (BC.LoopOrSwitch == LoopOrSwitch)1747      return &BC;1748 1749  llvm_unreachable("break/continue target not found");1750}1751 1752void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {1753  assert(!BreakContinueStack.empty() && "break stmt not in a loop or switch!");1754 1755  // If this code is reachable then emit a stop point (if generating1756  // debug info). We have to do this ourselves because we are on the1757  // "simple" statement path.1758  if (HaveInsertPoint())1759    EmitStopPoint(&S);1760 1761  ApplyAtomGroup Grp(getDebugInfo());1762  EmitBranchThroughCleanup(GetDestForLoopControlStmt(S)->BreakBlock);1763}1764 1765void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {1766  assert(!BreakContinueStack.empty() && "continue stmt not in a loop!");1767 1768  // If this code is reachable then emit a stop point (if generating1769  // debug info). We have to do this ourselves because we are on the1770  // "simple" statement path.1771  if (HaveInsertPoint())1772    EmitStopPoint(&S);1773 1774  ApplyAtomGroup Grp(getDebugInfo());1775  EmitBranchThroughCleanup(GetDestForLoopControlStmt(S)->ContinueBlock);1776}1777 1778/// EmitCaseStmtRange - If case statement range is not too big then1779/// add multiple cases to switch instruction, one for each value within1780/// the range. If range is too big then emit "if" condition check.1781void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S,1782                                        ArrayRef<const Attr *> Attrs) {1783  assert(S.getRHS() && "Expected RHS value in CaseStmt");1784 1785  llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(getContext());1786  llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(getContext());1787 1788  // Emit the code for this case. We do this first to make sure it is1789  // properly chained from our predecessor before generating the1790  // switch machinery to enter this block.1791  llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");1792  EmitBlockWithFallThrough(CaseDest, &S);1793  EmitStmt(S.getSubStmt());1794 1795  // If range is empty, do nothing.1796  if (LHS.isSigned() ? RHS.slt(LHS) : RHS.ult(LHS))1797    return;1798 1799  Stmt::Likelihood LH = Stmt::getLikelihood(Attrs);1800  llvm::APInt Range = RHS - LHS;1801  // FIXME: parameters such as this should not be hardcoded.1802  if (Range.ult(llvm::APInt(Range.getBitWidth(), 64))) {1803    // Range is small enough to add multiple switch instruction cases.1804    uint64_t Total = getProfileCount(&S);1805    unsigned NCases = Range.getZExtValue() + 1;1806    // We only have one region counter for the entire set of cases here, so we1807    // need to divide the weights evenly between the generated cases, ensuring1808    // that the total weight is preserved. E.g., a weight of 5 over three cases1809    // will be distributed as weights of 2, 2, and 1.1810    uint64_t Weight = Total / NCases, Rem = Total % NCases;1811    for (unsigned I = 0; I != NCases; ++I) {1812      if (SwitchWeights)1813        SwitchWeights->push_back(Weight + (Rem ? 1 : 0));1814      else if (SwitchLikelihood)1815        SwitchLikelihood->push_back(LH);1816 1817      if (Rem)1818        Rem--;1819      SwitchInsn->addCase(Builder.getInt(LHS), CaseDest);1820      ++LHS;1821    }1822    return;1823  }1824 1825  // The range is too big. Emit "if" condition into a new block,1826  // making sure to save and restore the current insertion point.1827  llvm::BasicBlock *RestoreBB = Builder.GetInsertBlock();1828 1829  // Push this test onto the chain of range checks (which terminates1830  // in the default basic block). The switch's default will be changed1831  // to the top of this chain after switch emission is complete.1832  llvm::BasicBlock *FalseDest = CaseRangeBlock;1833  CaseRangeBlock = createBasicBlock("sw.caserange");1834 1835  CurFn->insert(CurFn->end(), CaseRangeBlock);1836  Builder.SetInsertPoint(CaseRangeBlock);1837 1838  // Emit range check.1839  llvm::Value *Diff =1840    Builder.CreateSub(SwitchInsn->getCondition(), Builder.getInt(LHS));1841  llvm::Value *Cond =1842    Builder.CreateICmpULE(Diff, Builder.getInt(Range), "inbounds");1843 1844  llvm::MDNode *Weights = nullptr;1845  if (SwitchWeights) {1846    uint64_t ThisCount = getProfileCount(&S);1847    uint64_t DefaultCount = (*SwitchWeights)[0];1848    Weights = createProfileWeights(ThisCount, DefaultCount);1849 1850    // Since we're chaining the switch default through each large case range, we1851    // need to update the weight for the default, ie, the first case, to include1852    // this case.1853    (*SwitchWeights)[0] += ThisCount;1854  } else if (SwitchLikelihood)1855    Cond = emitCondLikelihoodViaExpectIntrinsic(Cond, LH);1856 1857  Builder.CreateCondBr(Cond, CaseDest, FalseDest, Weights);1858 1859  // Restore the appropriate insertion point.1860  if (RestoreBB)1861    Builder.SetInsertPoint(RestoreBB);1862  else1863    Builder.ClearInsertionPoint();1864}1865 1866void CodeGenFunction::EmitCaseStmt(const CaseStmt &S,1867                                   ArrayRef<const Attr *> Attrs) {1868  // If there is no enclosing switch instance that we're aware of, then this1869  // case statement and its block can be elided.  This situation only happens1870  // when we've constant-folded the switch, are emitting the constant case,1871  // and part of the constant case includes another case statement.  For1872  // instance: switch (4) { case 4: do { case 5: } while (1); }1873  if (!SwitchInsn) {1874    EmitStmt(S.getSubStmt());1875    return;1876  }1877 1878  // Handle case ranges.1879  if (S.getRHS()) {1880    EmitCaseStmtRange(S, Attrs);1881    return;1882  }1883 1884  llvm::ConstantInt *CaseVal =1885    Builder.getInt(S.getLHS()->EvaluateKnownConstInt(getContext()));1886 1887  // Emit debuginfo for the case value if it is an enum value.1888  const ConstantExpr *CE;1889  if (auto ICE = dyn_cast<ImplicitCastExpr>(S.getLHS()))1890    CE = dyn_cast<ConstantExpr>(ICE->getSubExpr());1891  else1892    CE = dyn_cast<ConstantExpr>(S.getLHS());1893  if (CE) {1894    if (auto DE = dyn_cast<DeclRefExpr>(CE->getSubExpr()))1895      if (CGDebugInfo *Dbg = getDebugInfo())1896        if (CGM.getCodeGenOpts().hasReducedDebugInfo())1897          Dbg->EmitGlobalVariable(DE->getDecl(),1898              APValue(llvm::APSInt(CaseVal->getValue())));1899  }1900 1901  if (SwitchLikelihood)1902    SwitchLikelihood->push_back(Stmt::getLikelihood(Attrs));1903 1904  // If the body of the case is just a 'break', try to not emit an empty block.1905  // If we're profiling or we're not optimizing, leave the block in for better1906  // debug and coverage analysis.1907  if (!CGM.getCodeGenOpts().hasProfileClangInstr() &&1908      CGM.getCodeGenOpts().OptimizationLevel > 0 &&1909      isa<BreakStmt>(S.getSubStmt())) {1910    JumpDest Block = BreakContinueStack.back().BreakBlock;1911 1912    // Only do this optimization if there are no cleanups that need emitting.1913    if (isObviouslyBranchWithoutCleanups(Block)) {1914      if (SwitchWeights)1915        SwitchWeights->push_back(getProfileCount(&S));1916      SwitchInsn->addCase(CaseVal, Block.getBlock());1917 1918      // If there was a fallthrough into this case, make sure to redirect it to1919      // the end of the switch as well.1920      if (Builder.GetInsertBlock()) {1921        Builder.CreateBr(Block.getBlock());1922        Builder.ClearInsertionPoint();1923      }1924      return;1925    }1926  }1927 1928  llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");1929  EmitBlockWithFallThrough(CaseDest, &S);1930  if (SwitchWeights)1931    SwitchWeights->push_back(getProfileCount(&S));1932  SwitchInsn->addCase(CaseVal, CaseDest);1933 1934  // Recursively emitting the statement is acceptable, but is not wonderful for1935  // code where we have many case statements nested together, i.e.:1936  //  case 1:1937  //    case 2:1938  //      case 3: etc.1939  // Handling this recursively will create a new block for each case statement1940  // that falls through to the next case which is IR intensive.  It also causes1941  // deep recursion which can run into stack depth limitations.  Handle1942  // sequential non-range case statements specially.1943  //1944  // TODO When the next case has a likelihood attribute the code returns to the1945  // recursive algorithm. Maybe improve this case if it becomes common practice1946  // to use a lot of attributes.1947  const CaseStmt *CurCase = &S;1948  const CaseStmt *NextCase = dyn_cast<CaseStmt>(S.getSubStmt());1949 1950  // Otherwise, iteratively add consecutive cases to this switch stmt.1951  while (NextCase && NextCase->getRHS() == nullptr) {1952    CurCase = NextCase;1953    llvm::ConstantInt *CaseVal =1954      Builder.getInt(CurCase->getLHS()->EvaluateKnownConstInt(getContext()));1955 1956    if (SwitchWeights)1957      SwitchWeights->push_back(getProfileCount(NextCase));1958    if (CGM.getCodeGenOpts().hasProfileClangInstr()) {1959      CaseDest = createBasicBlock("sw.bb");1960      EmitBlockWithFallThrough(CaseDest, CurCase);1961    }1962    // Since this loop is only executed when the CaseStmt has no attributes1963    // use a hard-coded value.1964    if (SwitchLikelihood)1965      SwitchLikelihood->push_back(Stmt::LH_None);1966 1967    SwitchInsn->addCase(CaseVal, CaseDest);1968    NextCase = dyn_cast<CaseStmt>(CurCase->getSubStmt());1969  }1970 1971  // Generate a stop point for debug info if the case statement is1972  // followed by a default statement. A fallthrough case before a1973  // default case gets its own branch target.1974  if (CurCase->getSubStmt()->getStmtClass() == Stmt::DefaultStmtClass)1975    EmitStopPoint(CurCase);1976 1977  // Normal default recursion for non-cases.1978  EmitStmt(CurCase->getSubStmt());1979}1980 1981void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S,1982                                      ArrayRef<const Attr *> Attrs) {1983  // If there is no enclosing switch instance that we're aware of, then this1984  // default statement can be elided. This situation only happens when we've1985  // constant-folded the switch.1986  if (!SwitchInsn) {1987    EmitStmt(S.getSubStmt());1988    return;1989  }1990 1991  llvm::BasicBlock *DefaultBlock = SwitchInsn->getDefaultDest();1992  assert(DefaultBlock->empty() &&1993         "EmitDefaultStmt: Default block already defined?");1994 1995  if (SwitchLikelihood)1996    SwitchLikelihood->front() = Stmt::getLikelihood(Attrs);1997 1998  EmitBlockWithFallThrough(DefaultBlock, &S);1999 2000  EmitStmt(S.getSubStmt());2001}2002 2003/// CollectStatementsForCase - Given the body of a 'switch' statement and a2004/// constant value that is being switched on, see if we can dead code eliminate2005/// the body of the switch to a simple series of statements to emit.  Basically,2006/// on a switch (5) we want to find these statements:2007///    case 5:2008///      printf(...);    <--2009///      ++i;            <--2010///      break;2011///2012/// and add them to the ResultStmts vector.  If it is unsafe to do this2013/// transformation (for example, one of the elided statements contains a label2014/// that might be jumped to), return CSFC_Failure.  If we handled it and 'S'2015/// should include statements after it (e.g. the printf() line is a substmt of2016/// the case) then return CSFC_FallThrough.  If we handled it and found a break2017/// statement, then return CSFC_Success.2018///2019/// If Case is non-null, then we are looking for the specified case, checking2020/// that nothing we jump over contains labels.  If Case is null, then we found2021/// the case and are looking for the break.2022///2023/// If the recursive walk actually finds our Case, then we set FoundCase to2024/// true.2025///2026enum CSFC_Result { CSFC_Failure, CSFC_FallThrough, CSFC_Success };2027static CSFC_Result CollectStatementsForCase(const Stmt *S,2028                                            const SwitchCase *Case,2029                                            bool &FoundCase,2030                              SmallVectorImpl<const Stmt*> &ResultStmts) {2031  // If this is a null statement, just succeed.2032  if (!S)2033    return Case ? CSFC_Success : CSFC_FallThrough;2034 2035  // If this is the switchcase (case 4: or default) that we're looking for, then2036  // we're in business.  Just add the substatement.2037  if (const SwitchCase *SC = dyn_cast<SwitchCase>(S)) {2038    if (S == Case) {2039      FoundCase = true;2040      return CollectStatementsForCase(SC->getSubStmt(), nullptr, FoundCase,2041                                      ResultStmts);2042    }2043 2044    // Otherwise, this is some other case or default statement, just ignore it.2045    return CollectStatementsForCase(SC->getSubStmt(), Case, FoundCase,2046                                    ResultStmts);2047  }2048 2049  // If we are in the live part of the code and we found our break statement,2050  // return a success!2051  if (!Case && isa<BreakStmt>(S))2052    return CSFC_Success;2053 2054  // If this is a switch statement, then it might contain the SwitchCase, the2055  // break, or neither.2056  if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(S)) {2057    // Handle this as two cases: we might be looking for the SwitchCase (if so2058    // the skipped statements must be skippable) or we might already have it.2059    CompoundStmt::const_body_iterator I = CS->body_begin(), E = CS->body_end();2060    bool StartedInLiveCode = FoundCase;2061    unsigned StartSize = ResultStmts.size();2062 2063    // If we've not found the case yet, scan through looking for it.2064    if (Case) {2065      // Keep track of whether we see a skipped declaration.  The code could be2066      // using the declaration even if it is skipped, so we can't optimize out2067      // the decl if the kept statements might refer to it.2068      bool HadSkippedDecl = false;2069 2070      // If we're looking for the case, just see if we can skip each of the2071      // substatements.2072      for (; Case && I != E; ++I) {2073        HadSkippedDecl |= CodeGenFunction::mightAddDeclToScope(*I);2074 2075        switch (CollectStatementsForCase(*I, Case, FoundCase, ResultStmts)) {2076        case CSFC_Failure: return CSFC_Failure;2077        case CSFC_Success:2078          // A successful result means that either 1) that the statement doesn't2079          // have the case and is skippable, or 2) does contain the case value2080          // and also contains the break to exit the switch.  In the later case,2081          // we just verify the rest of the statements are elidable.2082          if (FoundCase) {2083            // If we found the case and skipped declarations, we can't do the2084            // optimization.2085            if (HadSkippedDecl)2086              return CSFC_Failure;2087 2088            for (++I; I != E; ++I)2089              if (CodeGenFunction::ContainsLabel(*I, true))2090                return CSFC_Failure;2091            return CSFC_Success;2092          }2093          break;2094        case CSFC_FallThrough:2095          // If we have a fallthrough condition, then we must have found the2096          // case started to include statements.  Consider the rest of the2097          // statements in the compound statement as candidates for inclusion.2098          assert(FoundCase && "Didn't find case but returned fallthrough?");2099          // We recursively found Case, so we're not looking for it anymore.2100          Case = nullptr;2101 2102          // If we found the case and skipped declarations, we can't do the2103          // optimization.2104          if (HadSkippedDecl)2105            return CSFC_Failure;2106          break;2107        }2108      }2109 2110      if (!FoundCase)2111        return CSFC_Success;2112 2113      assert(!HadSkippedDecl && "fallthrough after skipping decl");2114    }2115 2116    // If we have statements in our range, then we know that the statements are2117    // live and need to be added to the set of statements we're tracking.2118    bool AnyDecls = false;2119    for (; I != E; ++I) {2120      AnyDecls |= CodeGenFunction::mightAddDeclToScope(*I);2121 2122      switch (CollectStatementsForCase(*I, nullptr, FoundCase, ResultStmts)) {2123      case CSFC_Failure: return CSFC_Failure;2124      case CSFC_FallThrough:2125        // A fallthrough result means that the statement was simple and just2126        // included in ResultStmt, keep adding them afterwards.2127        break;2128      case CSFC_Success:2129        // A successful result means that we found the break statement and2130        // stopped statement inclusion.  We just ensure that any leftover stmts2131        // are skippable and return success ourselves.2132        for (++I; I != E; ++I)2133          if (CodeGenFunction::ContainsLabel(*I, true))2134            return CSFC_Failure;2135        return CSFC_Success;2136      }2137    }2138 2139    // If we're about to fall out of a scope without hitting a 'break;', we2140    // can't perform the optimization if there were any decls in that scope2141    // (we'd lose their end-of-lifetime).2142    if (AnyDecls) {2143      // If the entire compound statement was live, there's one more thing we2144      // can try before giving up: emit the whole thing as a single statement.2145      // We can do that unless the statement contains a 'break;'.2146      // FIXME: Such a break must be at the end of a construct within this one.2147      // We could emit this by just ignoring the BreakStmts entirely.2148      if (StartedInLiveCode && !CodeGenFunction::containsBreak(S)) {2149        ResultStmts.resize(StartSize);2150        ResultStmts.push_back(S);2151      } else {2152        return CSFC_Failure;2153      }2154    }2155 2156    return CSFC_FallThrough;2157  }2158 2159  // Okay, this is some other statement that we don't handle explicitly, like a2160  // for statement or increment etc.  If we are skipping over this statement,2161  // just verify it doesn't have labels, which would make it invalid to elide.2162  if (Case) {2163    if (CodeGenFunction::ContainsLabel(S, true))2164      return CSFC_Failure;2165    return CSFC_Success;2166  }2167 2168  // Otherwise, we want to include this statement.  Everything is cool with that2169  // so long as it doesn't contain a break out of the switch we're in.2170  if (CodeGenFunction::containsBreak(S)) return CSFC_Failure;2171 2172  // Otherwise, everything is great.  Include the statement and tell the caller2173  // that we fall through and include the next statement as well.2174  ResultStmts.push_back(S);2175  return CSFC_FallThrough;2176}2177 2178/// FindCaseStatementsForValue - Find the case statement being jumped to and2179/// then invoke CollectStatementsForCase to find the list of statements to emit2180/// for a switch on constant.  See the comment above CollectStatementsForCase2181/// for more details.2182static bool FindCaseStatementsForValue(const SwitchStmt &S,2183                                       const llvm::APSInt &ConstantCondValue,2184                                SmallVectorImpl<const Stmt*> &ResultStmts,2185                                       ASTContext &C,2186                                       const SwitchCase *&ResultCase) {2187  // First step, find the switch case that is being branched to.  We can do this2188  // efficiently by scanning the SwitchCase list.2189  const SwitchCase *Case = S.getSwitchCaseList();2190  const DefaultStmt *DefaultCase = nullptr;2191 2192  for (; Case; Case = Case->getNextSwitchCase()) {2193    // It's either a default or case.  Just remember the default statement in2194    // case we're not jumping to any numbered cases.2195    if (const DefaultStmt *DS = dyn_cast<DefaultStmt>(Case)) {2196      DefaultCase = DS;2197      continue;2198    }2199 2200    // Check to see if this case is the one we're looking for.2201    const CaseStmt *CS = cast<CaseStmt>(Case);2202    // Don't handle case ranges yet.2203    if (CS->getRHS()) return false;2204 2205    // If we found our case, remember it as 'case'.2206    if (CS->getLHS()->EvaluateKnownConstInt(C) == ConstantCondValue)2207      break;2208  }2209 2210  // If we didn't find a matching case, we use a default if it exists, or we2211  // elide the whole switch body!2212  if (!Case) {2213    // It is safe to elide the body of the switch if it doesn't contain labels2214    // etc.  If it is safe, return successfully with an empty ResultStmts list.2215    if (!DefaultCase)2216      return !CodeGenFunction::ContainsLabel(&S);2217    Case = DefaultCase;2218  }2219 2220  // Ok, we know which case is being jumped to, try to collect all the2221  // statements that follow it.  This can fail for a variety of reasons.  Also,2222  // check to see that the recursive walk actually found our case statement.2223  // Insane cases like this can fail to find it in the recursive walk since we2224  // don't handle every stmt kind:2225  // switch (4) {2226  //   while (1) {2227  //     case 4: ...2228  bool FoundCase = false;2229  ResultCase = Case;2230  return CollectStatementsForCase(S.getBody(), Case, FoundCase,2231                                  ResultStmts) != CSFC_Failure &&2232         FoundCase;2233}2234 2235static std::optional<SmallVector<uint64_t, 16>>2236getLikelihoodWeights(ArrayRef<Stmt::Likelihood> Likelihoods) {2237  // Are there enough branches to weight them?2238  if (Likelihoods.size() <= 1)2239    return std::nullopt;2240 2241  uint64_t NumUnlikely = 0;2242  uint64_t NumNone = 0;2243  uint64_t NumLikely = 0;2244  for (const auto LH : Likelihoods) {2245    switch (LH) {2246    case Stmt::LH_Unlikely:2247      ++NumUnlikely;2248      break;2249    case Stmt::LH_None:2250      ++NumNone;2251      break;2252    case Stmt::LH_Likely:2253      ++NumLikely;2254      break;2255    }2256  }2257 2258  // Is there a likelihood attribute used?2259  if (NumUnlikely == 0 && NumLikely == 0)2260    return std::nullopt;2261 2262  // When multiple cases share the same code they can be combined during2263  // optimization. In that case the weights of the branch will be the sum of2264  // the individual weights. Make sure the combined sum of all neutral cases2265  // doesn't exceed the value of a single likely attribute.2266  // The additions both avoid divisions by 0 and make sure the weights of None2267  // don't exceed the weight of Likely.2268  const uint64_t Likely = INT32_MAX / (NumLikely + 2);2269  const uint64_t None = Likely / (NumNone + 1);2270  const uint64_t Unlikely = 0;2271 2272  SmallVector<uint64_t, 16> Result;2273  Result.reserve(Likelihoods.size());2274  for (const auto LH : Likelihoods) {2275    switch (LH) {2276    case Stmt::LH_Unlikely:2277      Result.push_back(Unlikely);2278      break;2279    case Stmt::LH_None:2280      Result.push_back(None);2281      break;2282    case Stmt::LH_Likely:2283      Result.push_back(Likely);2284      break;2285    }2286  }2287 2288  return Result;2289}2290 2291void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {2292  // Handle nested switch statements.2293  llvm::SwitchInst *SavedSwitchInsn = SwitchInsn;2294  SmallVector<uint64_t, 16> *SavedSwitchWeights = SwitchWeights;2295  SmallVector<Stmt::Likelihood, 16> *SavedSwitchLikelihood = SwitchLikelihood;2296  llvm::BasicBlock *SavedCRBlock = CaseRangeBlock;2297 2298  // See if we can constant fold the condition of the switch and therefore only2299  // emit the live case statement (if any) of the switch.2300  llvm::APSInt ConstantCondValue;2301  if (ConstantFoldsToSimpleInteger(S.getCond(), ConstantCondValue)) {2302    SmallVector<const Stmt*, 4> CaseStmts;2303    const SwitchCase *Case = nullptr;2304    if (FindCaseStatementsForValue(S, ConstantCondValue, CaseStmts,2305                                   getContext(), Case)) {2306      if (Case)2307        incrementProfileCounter(Case);2308      RunCleanupsScope ExecutedScope(*this);2309 2310      if (S.getInit())2311        EmitStmt(S.getInit());2312 2313      // Emit the condition variable if needed inside the entire cleanup scope2314      // used by this special case for constant folded switches.2315      if (S.getConditionVariable())2316        EmitDecl(*S.getConditionVariable(), /*EvaluateConditionDecl=*/true);2317 2318      // At this point, we are no longer "within" a switch instance, so2319      // we can temporarily enforce this to ensure that any embedded case2320      // statements are not emitted.2321      SwitchInsn = nullptr;2322 2323      // Okay, we can dead code eliminate everything except this case.  Emit the2324      // specified series of statements and we're good.2325      for (const Stmt *CaseStmt : CaseStmts)2326        EmitStmt(CaseStmt);2327      incrementProfileCounter(&S);2328      PGO->markStmtMaybeUsed(S.getBody());2329 2330      // Now we want to restore the saved switch instance so that nested2331      // switches continue to function properly2332      SwitchInsn = SavedSwitchInsn;2333 2334      return;2335    }2336  }2337 2338  JumpDest SwitchExit = getJumpDestInCurrentScope("sw.epilog");2339 2340  RunCleanupsScope ConditionScope(*this);2341 2342  if (S.getInit())2343    EmitStmt(S.getInit());2344 2345  if (S.getConditionVariable())2346    EmitDecl(*S.getConditionVariable());2347  llvm::Value *CondV = EmitScalarExpr(S.getCond());2348  MaybeEmitDeferredVarDeclInit(S.getConditionVariable());2349 2350  // Create basic block to hold stuff that comes after switch2351  // statement. We also need to create a default block now so that2352  // explicit case ranges tests can have a place to jump to on2353  // failure.2354  llvm::BasicBlock *DefaultBlock = createBasicBlock("sw.default");2355  SwitchInsn = Builder.CreateSwitch(CondV, DefaultBlock);2356  addInstToNewSourceAtom(SwitchInsn, CondV);2357 2358  if (HLSLControlFlowAttr != HLSLControlFlowHintAttr::SpellingNotCalculated) {2359    llvm::MDBuilder MDHelper(CGM.getLLVMContext());2360    llvm::ConstantInt *BranchHintConstant =2361        HLSLControlFlowAttr ==2362                HLSLControlFlowHintAttr::Spelling::Microsoft_branch2363            ? llvm::ConstantInt::get(CGM.Int32Ty, 1)2364            : llvm::ConstantInt::get(CGM.Int32Ty, 2);2365    llvm::Metadata *Vals[] = {MDHelper.createString("hlsl.controlflow.hint"),2366                              MDHelper.createConstant(BranchHintConstant)};2367    SwitchInsn->setMetadata("hlsl.controlflow.hint",2368                            llvm::MDNode::get(CGM.getLLVMContext(), Vals));2369  }2370 2371  if (PGO->haveRegionCounts()) {2372    // Walk the SwitchCase list to find how many there are.2373    uint64_t DefaultCount = 0;2374    unsigned NumCases = 0;2375    for (const SwitchCase *Case = S.getSwitchCaseList();2376         Case;2377         Case = Case->getNextSwitchCase()) {2378      if (isa<DefaultStmt>(Case))2379        DefaultCount = getProfileCount(Case);2380      NumCases += 1;2381    }2382    SwitchWeights = new SmallVector<uint64_t, 16>();2383    SwitchWeights->reserve(NumCases);2384    // The default needs to be first. We store the edge count, so we already2385    // know the right weight.2386    SwitchWeights->push_back(DefaultCount);2387  } else if (CGM.getCodeGenOpts().OptimizationLevel) {2388    SwitchLikelihood = new SmallVector<Stmt::Likelihood, 16>();2389    // Initialize the default case.2390    SwitchLikelihood->push_back(Stmt::LH_None);2391  }2392 2393  CaseRangeBlock = DefaultBlock;2394 2395  // Clear the insertion point to indicate we are in unreachable code.2396  Builder.ClearInsertionPoint();2397 2398  // All break statements jump to NextBlock. If BreakContinueStack is non-empty2399  // then reuse last ContinueBlock.2400  JumpDest OuterContinue;2401  if (!BreakContinueStack.empty())2402    OuterContinue = BreakContinueStack.back().ContinueBlock;2403 2404  BreakContinueStack.push_back(BreakContinue(S, SwitchExit, OuterContinue));2405 2406  // Emit switch body.2407  EmitStmt(S.getBody());2408 2409  BreakContinueStack.pop_back();2410 2411  // Update the default block in case explicit case range tests have2412  // been chained on top.2413  SwitchInsn->setDefaultDest(CaseRangeBlock);2414 2415  // If a default was never emitted:2416  if (!DefaultBlock->getParent()) {2417    // If we have cleanups, emit the default block so that there's a2418    // place to jump through the cleanups from.2419    if (ConditionScope.requiresCleanups()) {2420      EmitBlock(DefaultBlock);2421 2422    // Otherwise, just forward the default block to the switch end.2423    } else {2424      DefaultBlock->replaceAllUsesWith(SwitchExit.getBlock());2425      delete DefaultBlock;2426    }2427  }2428 2429  ConditionScope.ForceCleanup();2430 2431  // Emit continuation.2432  EmitBlock(SwitchExit.getBlock(), true);2433  incrementProfileCounter(&S);2434 2435  // If the switch has a condition wrapped by __builtin_unpredictable,2436  // create metadata that specifies that the switch is unpredictable.2437  // Don't bother if not optimizing because that metadata would not be used.2438  auto *Call = dyn_cast<CallExpr>(S.getCond());2439  if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {2440    auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());2441    if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {2442      llvm::MDBuilder MDHelper(getLLVMContext());2443      SwitchInsn->setMetadata(llvm::LLVMContext::MD_unpredictable,2444                              MDHelper.createUnpredictable());2445    }2446  }2447 2448  if (SwitchWeights) {2449    assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() &&2450           "switch weights do not match switch cases");2451    // If there's only one jump destination there's no sense weighting it.2452    if (SwitchWeights->size() > 1)2453      SwitchInsn->setMetadata(llvm::LLVMContext::MD_prof,2454                              createProfileWeights(*SwitchWeights));2455    delete SwitchWeights;2456  } else if (SwitchLikelihood) {2457    assert(SwitchLikelihood->size() == 1 + SwitchInsn->getNumCases() &&2458           "switch likelihoods do not match switch cases");2459    std::optional<SmallVector<uint64_t, 16>> LHW =2460        getLikelihoodWeights(*SwitchLikelihood);2461    if (LHW) {2462      llvm::MDBuilder MDHelper(CGM.getLLVMContext());2463      SwitchInsn->setMetadata(llvm::LLVMContext::MD_prof,2464                              createProfileWeights(*LHW));2465    }2466    delete SwitchLikelihood;2467  }2468  SwitchInsn = SavedSwitchInsn;2469  SwitchWeights = SavedSwitchWeights;2470  SwitchLikelihood = SavedSwitchLikelihood;2471  CaseRangeBlock = SavedCRBlock;2472}2473 2474/// AddVariableConstraints - Look at AsmExpr and if it is a variable declared2475/// as using a particular register add that as a constraint that will be used2476/// in this asm stmt.2477static std::string2478AddVariableConstraints(const std::string &Constraint, const Expr &AsmExpr,2479                       const TargetInfo &Target, CodeGenModule &CGM,2480                       const AsmStmt &Stmt, const bool EarlyClobber,2481                       std::string *GCCReg = nullptr) {2482  const DeclRefExpr *AsmDeclRef = dyn_cast<DeclRefExpr>(&AsmExpr);2483  if (!AsmDeclRef)2484    return Constraint;2485  const ValueDecl &Value = *AsmDeclRef->getDecl();2486  const VarDecl *Variable = dyn_cast<VarDecl>(&Value);2487  if (!Variable)2488    return Constraint;2489  if (Variable->getStorageClass() != SC_Register)2490    return Constraint;2491  AsmLabelAttr *Attr = Variable->getAttr<AsmLabelAttr>();2492  if (!Attr)2493    return Constraint;2494  StringRef Register = Attr->getLabel();2495  assert(Target.isValidGCCRegisterName(Register));2496  // We're using validateOutputConstraint here because we only care if2497  // this is a register constraint.2498  TargetInfo::ConstraintInfo Info(Constraint, "");2499  if (Target.validateOutputConstraint(Info) &&2500      !Info.allowsRegister()) {2501    CGM.ErrorUnsupported(&Stmt, "__asm__");2502    return Constraint;2503  }2504  // Canonicalize the register here before returning it.2505  Register = Target.getNormalizedGCCRegisterName(Register);2506  if (GCCReg != nullptr)2507    *GCCReg = Register.str();2508  return (EarlyClobber ? "&{" : "{") + Register.str() + "}";2509}2510 2511std::pair<llvm::Value*, llvm::Type *> CodeGenFunction::EmitAsmInputLValue(2512    const TargetInfo::ConstraintInfo &Info, LValue InputValue,2513    QualType InputType, std::string &ConstraintStr, SourceLocation Loc) {2514  if (Info.allowsRegister() || !Info.allowsMemory()) {2515    if (CodeGenFunction::hasScalarEvaluationKind(InputType))2516      return {EmitLoadOfLValue(InputValue, Loc).getScalarVal(), nullptr};2517 2518    llvm::Type *Ty = ConvertType(InputType);2519    uint64_t Size = CGM.getDataLayout().getTypeSizeInBits(Ty);2520    if ((Size <= 64 && llvm::isPowerOf2_64(Size)) ||2521        getTargetHooks().isScalarizableAsmOperand(*this, Ty)) {2522      Ty = llvm::IntegerType::get(getLLVMContext(), Size);2523 2524      return {Builder.CreateLoad(InputValue.getAddress().withElementType(Ty)),2525              nullptr};2526    }2527  }2528 2529  Address Addr = InputValue.getAddress();2530  ConstraintStr += '*';2531  return {InputValue.getPointer(*this), Addr.getElementType()};2532}2533 2534std::pair<llvm::Value *, llvm::Type *>2535CodeGenFunction::EmitAsmInput(const TargetInfo::ConstraintInfo &Info,2536                              const Expr *InputExpr,2537                              std::string &ConstraintStr) {2538  // If this can't be a register or memory, i.e., has to be a constant2539  // (immediate or symbolic), try to emit it as such.2540  if (!Info.allowsRegister() && !Info.allowsMemory()) {2541    if (Info.requiresImmediateConstant()) {2542      Expr::EvalResult EVResult;2543      InputExpr->EvaluateAsRValue(EVResult, getContext(), true);2544 2545      llvm::APSInt IntResult;2546      if (EVResult.Val.toIntegralConstant(IntResult, InputExpr->getType(),2547                                          getContext()))2548        return {llvm::ConstantInt::get(getLLVMContext(), IntResult), nullptr};2549    }2550 2551    Expr::EvalResult Result;2552    if (InputExpr->EvaluateAsInt(Result, getContext()))2553      return {llvm::ConstantInt::get(getLLVMContext(), Result.Val.getInt()),2554              nullptr};2555  }2556 2557  if (Info.allowsRegister() || !Info.allowsMemory())2558    if (CodeGenFunction::hasScalarEvaluationKind(InputExpr->getType()))2559      return {EmitScalarExpr(InputExpr), nullptr};2560  if (InputExpr->getStmtClass() == Expr::CXXThisExprClass)2561    return {EmitScalarExpr(InputExpr), nullptr};2562  InputExpr = InputExpr->IgnoreParenNoopCasts(getContext());2563  LValue Dest = EmitLValue(InputExpr);2564  return EmitAsmInputLValue(Info, Dest, InputExpr->getType(), ConstraintStr,2565                            InputExpr->getExprLoc());2566}2567 2568/// getAsmSrcLocInfo - Return the !srcloc metadata node to attach to an inline2569/// asm call instruction.  The !srcloc MDNode contains a list of constant2570/// integers which are the source locations of the start of each line in the2571/// asm.2572static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,2573                                      CodeGenFunction &CGF) {2574  SmallVector<llvm::Metadata *, 8> Locs;2575  // Add the location of the first line to the MDNode.2576  Locs.push_back(llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(2577      CGF.Int64Ty, Str->getBeginLoc().getRawEncoding())));2578  StringRef StrVal = Str->getString();2579  if (!StrVal.empty()) {2580    const SourceManager &SM = CGF.CGM.getContext().getSourceManager();2581    const LangOptions &LangOpts = CGF.CGM.getLangOpts();2582    unsigned StartToken = 0;2583    unsigned ByteOffset = 0;2584 2585    // Add the location of the start of each subsequent line of the asm to the2586    // MDNode.2587    for (unsigned i = 0, e = StrVal.size() - 1; i != e; ++i) {2588      if (StrVal[i] != '\n') continue;2589      SourceLocation LineLoc = Str->getLocationOfByte(2590          i + 1, SM, LangOpts, CGF.getTarget(), &StartToken, &ByteOffset);2591      Locs.push_back(llvm::ConstantAsMetadata::get(2592          llvm::ConstantInt::get(CGF.Int64Ty, LineLoc.getRawEncoding())));2593    }2594  }2595 2596  return llvm::MDNode::get(CGF.getLLVMContext(), Locs);2597}2598 2599static void UpdateAsmCallInst(llvm::CallBase &Result, bool HasSideEffect,2600                              bool HasUnwindClobber, bool ReadOnly,2601                              bool ReadNone, bool NoMerge, bool NoConvergent,2602                              const AsmStmt &S,2603                              const std::vector<llvm::Type *> &ResultRegTypes,2604                              const std::vector<llvm::Type *> &ArgElemTypes,2605                              CodeGenFunction &CGF,2606                              std::vector<llvm::Value *> &RegResults) {2607  if (!HasUnwindClobber)2608    Result.addFnAttr(llvm::Attribute::NoUnwind);2609 2610  if (NoMerge)2611    Result.addFnAttr(llvm::Attribute::NoMerge);2612  // Attach readnone and readonly attributes.2613  if (!HasSideEffect) {2614    if (ReadNone)2615      Result.setDoesNotAccessMemory();2616    else if (ReadOnly)2617      Result.setOnlyReadsMemory();2618  }2619 2620  // Add elementtype attribute for indirect constraints.2621  for (auto Pair : llvm::enumerate(ArgElemTypes)) {2622    if (Pair.value()) {2623      auto Attr = llvm::Attribute::get(2624          CGF.getLLVMContext(), llvm::Attribute::ElementType, Pair.value());2625      Result.addParamAttr(Pair.index(), Attr);2626    }2627  }2628 2629  // Slap the source location of the inline asm into a !srcloc metadata on the2630  // call.2631  const StringLiteral *SL;2632  if (const auto *gccAsmStmt = dyn_cast<GCCAsmStmt>(&S);2633      gccAsmStmt &&2634      (SL = dyn_cast<StringLiteral>(gccAsmStmt->getAsmStringExpr()))) {2635    Result.setMetadata("srcloc", getAsmSrcLocInfo(SL, CGF));2636  } else {2637    // At least put the line number on MS inline asm blobs and GCC asm constexpr2638    // strings.2639    llvm::Constant *Loc =2640        llvm::ConstantInt::get(CGF.Int64Ty, S.getAsmLoc().getRawEncoding());2641    Result.setMetadata("srcloc",2642                       llvm::MDNode::get(CGF.getLLVMContext(),2643                                         llvm::ConstantAsMetadata::get(Loc)));2644  }2645 2646  // Make inline-asm calls Key for the debug info feature Key Instructions.2647  CGF.addInstToNewSourceAtom(&Result, nullptr);2648 2649  if (!NoConvergent && CGF.getLangOpts().assumeFunctionsAreConvergent())2650    // Conservatively, mark all inline asm blocks in CUDA or OpenCL as2651    // convergent (meaning, they may call an intrinsically convergent op, such2652    // as bar.sync, and so can't have certain optimizations applied around2653    // them) unless it's explicitly marked 'noconvergent'.2654    Result.addFnAttr(llvm::Attribute::Convergent);2655  // Extract all of the register value results from the asm.2656  if (ResultRegTypes.size() == 1) {2657    RegResults.push_back(&Result);2658  } else {2659    for (unsigned i = 0, e = ResultRegTypes.size(); i != e; ++i) {2660      llvm::Value *Tmp = CGF.Builder.CreateExtractValue(&Result, i, "asmresult");2661      RegResults.push_back(Tmp);2662    }2663  }2664}2665 2666static void2667EmitAsmStores(CodeGenFunction &CGF, const AsmStmt &S,2668              const llvm::ArrayRef<llvm::Value *> RegResults,2669              const llvm::ArrayRef<llvm::Type *> ResultRegTypes,2670              const llvm::ArrayRef<llvm::Type *> ResultTruncRegTypes,2671              const llvm::ArrayRef<LValue> ResultRegDests,2672              const llvm::ArrayRef<QualType> ResultRegQualTys,2673              const llvm::BitVector &ResultTypeRequiresCast,2674              const std::vector<std::optional<std::pair<unsigned, unsigned>>>2675                  &ResultBounds) {2676  CGBuilderTy &Builder = CGF.Builder;2677  CodeGenModule &CGM = CGF.CGM;2678  llvm::LLVMContext &CTX = CGF.getLLVMContext();2679 2680  assert(RegResults.size() == ResultRegTypes.size());2681  assert(RegResults.size() == ResultTruncRegTypes.size());2682  assert(RegResults.size() == ResultRegDests.size());2683  // ResultRegDests can be also populated by addReturnRegisterOutputs() above,2684  // in which case its size may grow.2685  assert(ResultTypeRequiresCast.size() <= ResultRegDests.size());2686  assert(ResultBounds.size() <= ResultRegDests.size());2687 2688  for (unsigned i = 0, e = RegResults.size(); i != e; ++i) {2689    llvm::Value *Tmp = RegResults[i];2690    llvm::Type *TruncTy = ResultTruncRegTypes[i];2691 2692    if ((i < ResultBounds.size()) && ResultBounds[i].has_value()) {2693      const auto [LowerBound, UpperBound] = ResultBounds[i].value();2694      // FIXME: Support for nonzero lower bounds not yet implemented.2695      assert(LowerBound == 0 && "Output operand lower bound is not zero.");2696      llvm::Constant *UpperBoundConst =2697          llvm::ConstantInt::get(Tmp->getType(), UpperBound);2698      llvm::Value *IsBooleanValue =2699          Builder.CreateCmp(llvm::CmpInst::ICMP_ULT, Tmp, UpperBoundConst);2700      llvm::Function *FnAssume = CGM.getIntrinsic(llvm::Intrinsic::assume);2701      Builder.CreateCall(FnAssume, IsBooleanValue);2702    }2703 2704    // If the result type of the LLVM IR asm doesn't match the result type of2705    // the expression, do the conversion.2706    if (ResultRegTypes[i] != TruncTy) {2707 2708      // Truncate the integer result to the right size, note that TruncTy can be2709      // a pointer.2710      if (TruncTy->isFloatingPointTy())2711        Tmp = Builder.CreateFPTrunc(Tmp, TruncTy);2712      else if (TruncTy->isPointerTy() && Tmp->getType()->isIntegerTy()) {2713        uint64_t ResSize = CGM.getDataLayout().getTypeSizeInBits(TruncTy);2714        Tmp = Builder.CreateTrunc(2715            Tmp, llvm::IntegerType::get(CTX, (unsigned)ResSize));2716        Tmp = Builder.CreateIntToPtr(Tmp, TruncTy);2717      } else if (Tmp->getType()->isPointerTy() && TruncTy->isIntegerTy()) {2718        uint64_t TmpSize =2719            CGM.getDataLayout().getTypeSizeInBits(Tmp->getType());2720        Tmp = Builder.CreatePtrToInt(2721            Tmp, llvm::IntegerType::get(CTX, (unsigned)TmpSize));2722        Tmp = Builder.CreateTrunc(Tmp, TruncTy);2723      } else if (Tmp->getType()->isIntegerTy() && TruncTy->isIntegerTy()) {2724        Tmp = Builder.CreateZExtOrTrunc(Tmp, TruncTy);2725      } else if (Tmp->getType()->isVectorTy() || TruncTy->isVectorTy()) {2726        Tmp = Builder.CreateBitCast(Tmp, TruncTy);2727      }2728    }2729 2730    ApplyAtomGroup Grp(CGF.getDebugInfo());2731    LValue Dest = ResultRegDests[i];2732    // ResultTypeRequiresCast elements correspond to the first2733    // ResultTypeRequiresCast.size() elements of RegResults.2734    if ((i < ResultTypeRequiresCast.size()) && ResultTypeRequiresCast[i]) {2735      unsigned Size = CGF.getContext().getTypeSize(ResultRegQualTys[i]);2736      Address A = Dest.getAddress().withElementType(ResultRegTypes[i]);2737      if (CGF.getTargetHooks().isScalarizableAsmOperand(CGF, TruncTy)) {2738        llvm::StoreInst *S = Builder.CreateStore(Tmp, A);2739        CGF.addInstToCurrentSourceAtom(S, S->getValueOperand());2740        continue;2741      }2742 2743      QualType Ty =2744          CGF.getContext().getIntTypeForBitwidth(Size, /*Signed=*/false);2745      if (Ty.isNull()) {2746        const Expr *OutExpr = S.getOutputExpr(i);2747        CGM.getDiags().Report(OutExpr->getExprLoc(),2748                              diag::err_store_value_to_reg);2749        return;2750      }2751      Dest = CGF.MakeAddrLValue(A, Ty);2752    }2753    CGF.EmitStoreThroughLValue(RValue::get(Tmp), Dest);2754  }2755}2756 2757static void EmitHipStdParUnsupportedAsm(CodeGenFunction *CGF,2758                                        const AsmStmt &S) {2759  constexpr auto Name = "__ASM__hipstdpar_unsupported";2760 2761  std::string Asm;2762  if (auto GCCAsm = dyn_cast<GCCAsmStmt>(&S))2763    Asm = GCCAsm->getAsmString();2764 2765  auto &Ctx = CGF->CGM.getLLVMContext();2766 2767  auto StrTy = llvm::ConstantDataArray::getString(Ctx, Asm);2768  auto FnTy = llvm::FunctionType::get(llvm::Type::getVoidTy(Ctx),2769                                      {StrTy->getType()}, false);2770  auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, FnTy);2771 2772  CGF->Builder.CreateCall(UBF, {StrTy});2773}2774 2775void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {2776  // Pop all cleanup blocks at the end of the asm statement.2777  CodeGenFunction::RunCleanupsScope Cleanups(*this);2778 2779  // Assemble the final asm string.2780  std::string AsmString = S.generateAsmString(getContext());2781 2782  // Get all the output and input constraints together.2783  SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;2784  SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;2785 2786  bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;2787  bool IsValidTargetAsm = true;2788  for (unsigned i = 0, e = S.getNumOutputs(); i != e && IsValidTargetAsm; i++) {2789    StringRef Name;2790    if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(&S))2791      Name = GAS->getOutputName(i);2792    TargetInfo::ConstraintInfo Info(S.getOutputConstraint(i), Name);2793    bool IsValid = getTarget().validateOutputConstraint(Info); (void)IsValid;2794    if (IsHipStdPar && !IsValid)2795      IsValidTargetAsm = false;2796    else2797      assert(IsValid && "Failed to parse output constraint");2798    OutputConstraintInfos.push_back(Info);2799  }2800 2801  for (unsigned i = 0, e = S.getNumInputs(); i != e && IsValidTargetAsm; i++) {2802    StringRef Name;2803    if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(&S))2804      Name = GAS->getInputName(i);2805    TargetInfo::ConstraintInfo Info(S.getInputConstraint(i), Name);2806    bool IsValid =2807      getTarget().validateInputConstraint(OutputConstraintInfos, Info);2808    if (IsHipStdPar && !IsValid)2809      IsValidTargetAsm = false;2810    else2811      assert(IsValid && "Failed to parse input constraint");2812    InputConstraintInfos.push_back(Info);2813  }2814 2815  if (!IsValidTargetAsm)2816    return EmitHipStdParUnsupportedAsm(this, S);2817 2818  std::string Constraints;2819 2820  std::vector<LValue> ResultRegDests;2821  std::vector<QualType> ResultRegQualTys;2822  std::vector<llvm::Type *> ResultRegTypes;2823  std::vector<llvm::Type *> ResultTruncRegTypes;2824  std::vector<llvm::Type *> ArgTypes;2825  std::vector<llvm::Type *> ArgElemTypes;2826  std::vector<llvm::Value*> Args;2827  llvm::BitVector ResultTypeRequiresCast;2828  std::vector<std::optional<std::pair<unsigned, unsigned>>> ResultBounds;2829 2830  // Keep track of inout constraints.2831  std::string InOutConstraints;2832  std::vector<llvm::Value*> InOutArgs;2833  std::vector<llvm::Type*> InOutArgTypes;2834  std::vector<llvm::Type*> InOutArgElemTypes;2835 2836  // Keep track of out constraints for tied input operand.2837  std::vector<std::string> OutputConstraints;2838 2839  // Keep track of defined physregs.2840  llvm::SmallSet<std::string, 8> PhysRegOutputs;2841 2842  // An inline asm can be marked readonly if it meets the following conditions:2843  //  - it doesn't have any sideeffects2844  //  - it doesn't clobber memory2845  //  - it doesn't return a value by-reference2846  // It can be marked readnone if it doesn't have any input memory constraints2847  // in addition to meeting the conditions listed above.2848  bool ReadOnly = true, ReadNone = true;2849 2850  for (unsigned i = 0, e = S.getNumOutputs(); i != e; i++) {2851    TargetInfo::ConstraintInfo &Info = OutputConstraintInfos[i];2852 2853    // Simplify the output constraint.2854    std::string OutputConstraint(S.getOutputConstraint(i));2855    OutputConstraint = getTarget().simplifyConstraint(2856        StringRef(OutputConstraint).substr(1), &OutputConstraintInfos);2857 2858    const Expr *OutExpr = S.getOutputExpr(i);2859    OutExpr = OutExpr->IgnoreParenNoopCasts(getContext());2860 2861    std::string GCCReg;2862    OutputConstraint = AddVariableConstraints(OutputConstraint, *OutExpr,2863                                              getTarget(), CGM, S,2864                                              Info.earlyClobber(),2865                                              &GCCReg);2866    // Give an error on multiple outputs to same physreg.2867    if (!GCCReg.empty() && !PhysRegOutputs.insert(GCCReg).second)2868      CGM.Error(S.getAsmLoc(), "multiple outputs to hard register: " + GCCReg);2869 2870    OutputConstraints.push_back(OutputConstraint);2871    LValue Dest = EmitLValue(OutExpr);2872    if (!Constraints.empty())2873      Constraints += ',';2874 2875    // If this is a register output, then make the inline asm return it2876    // by-value.  If this is a memory result, return the value by-reference.2877    QualType QTy = OutExpr->getType();2878    const bool IsScalarOrAggregate = hasScalarEvaluationKind(QTy) ||2879                                     hasAggregateEvaluationKind(QTy);2880    if (!Info.allowsMemory() && IsScalarOrAggregate) {2881 2882      Constraints += "=" + OutputConstraint;2883      ResultRegQualTys.push_back(QTy);2884      ResultRegDests.push_back(Dest);2885 2886      ResultBounds.emplace_back(Info.getOutputOperandBounds());2887 2888      llvm::Type *Ty = ConvertTypeForMem(QTy);2889      const bool RequiresCast = Info.allowsRegister() &&2890          (getTargetHooks().isScalarizableAsmOperand(*this, Ty) ||2891           Ty->isAggregateType());2892 2893      ResultTruncRegTypes.push_back(Ty);2894      ResultTypeRequiresCast.push_back(RequiresCast);2895 2896      if (RequiresCast) {2897        unsigned Size = getContext().getTypeSize(QTy);2898        if (Size)2899          Ty = llvm::IntegerType::get(getLLVMContext(), Size);2900        else2901          CGM.Error(OutExpr->getExprLoc(), "output size should not be zero");2902      }2903      ResultRegTypes.push_back(Ty);2904      // If this output is tied to an input, and if the input is larger, then2905      // we need to set the actual result type of the inline asm node to be the2906      // same as the input type.2907      if (Info.hasMatchingInput()) {2908        unsigned InputNo;2909        for (InputNo = 0; InputNo != S.getNumInputs(); ++InputNo) {2910          TargetInfo::ConstraintInfo &Input = InputConstraintInfos[InputNo];2911          if (Input.hasTiedOperand() && Input.getTiedOperand() == i)2912            break;2913        }2914        assert(InputNo != S.getNumInputs() && "Didn't find matching input!");2915 2916        QualType InputTy = S.getInputExpr(InputNo)->getType();2917        QualType OutputType = OutExpr->getType();2918 2919        uint64_t InputSize = getContext().getTypeSize(InputTy);2920        if (getContext().getTypeSize(OutputType) < InputSize) {2921          // Form the asm to return the value as a larger integer or fp type.2922          ResultRegTypes.back() = ConvertType(InputTy);2923        }2924      }2925      if (llvm::Type* AdjTy =2926            getTargetHooks().adjustInlineAsmType(*this, OutputConstraint,2927                                                 ResultRegTypes.back()))2928        ResultRegTypes.back() = AdjTy;2929      else {2930        CGM.getDiags().Report(S.getAsmLoc(),2931                              diag::err_asm_invalid_type_in_input)2932            << OutExpr->getType() << OutputConstraint;2933      }2934 2935      // Update largest vector width for any vector types.2936      if (auto *VT = dyn_cast<llvm::VectorType>(ResultRegTypes.back()))2937        LargestVectorWidth =2938            std::max((uint64_t)LargestVectorWidth,2939                     VT->getPrimitiveSizeInBits().getKnownMinValue());2940    } else {2941      Address DestAddr = Dest.getAddress();2942      // Matrix types in memory are represented by arrays, but accessed through2943      // vector pointers, with the alignment specified on the access operation.2944      // For inline assembly, update pointer arguments to use vector pointers.2945      // Otherwise there will be a mis-match if the matrix is also an2946      // input-argument which is represented as vector.2947      if (isa<MatrixType>(OutExpr->getType().getCanonicalType()))2948        DestAddr = DestAddr.withElementType(ConvertType(OutExpr->getType()));2949 2950      ArgTypes.push_back(DestAddr.getType());2951      ArgElemTypes.push_back(DestAddr.getElementType());2952      Args.push_back(DestAddr.emitRawPointer(*this));2953      Constraints += "=*";2954      Constraints += OutputConstraint;2955      ReadOnly = ReadNone = false;2956    }2957 2958    if (Info.isReadWrite()) {2959      InOutConstraints += ',';2960 2961      const Expr *InputExpr = S.getOutputExpr(i);2962      llvm::Value *Arg;2963      llvm::Type *ArgElemType;2964      std::tie(Arg, ArgElemType) = EmitAsmInputLValue(2965          Info, Dest, InputExpr->getType(), InOutConstraints,2966          InputExpr->getExprLoc());2967 2968      if (llvm::Type* AdjTy =2969          getTargetHooks().adjustInlineAsmType(*this, OutputConstraint,2970                                               Arg->getType()))2971        Arg = Builder.CreateBitCast(Arg, AdjTy);2972 2973      // Update largest vector width for any vector types.2974      if (auto *VT = dyn_cast<llvm::VectorType>(Arg->getType()))2975        LargestVectorWidth =2976            std::max((uint64_t)LargestVectorWidth,2977                     VT->getPrimitiveSizeInBits().getKnownMinValue());2978      // Only tie earlyclobber physregs.2979      if (Info.allowsRegister() && (GCCReg.empty() || Info.earlyClobber()))2980        InOutConstraints += llvm::utostr(i);2981      else2982        InOutConstraints += OutputConstraint;2983 2984      InOutArgTypes.push_back(Arg->getType());2985      InOutArgElemTypes.push_back(ArgElemType);2986      InOutArgs.push_back(Arg);2987    }2988  }2989 2990  // If this is a Microsoft-style asm blob, store the return registers (EAX:EDX)2991  // to the return value slot. Only do this when returning in registers.2992  if (isa<MSAsmStmt>(&S)) {2993    const ABIArgInfo &RetAI = CurFnInfo->getReturnInfo();2994    if (RetAI.isDirect() || RetAI.isExtend()) {2995      // Make a fake lvalue for the return value slot.2996      LValue ReturnSlot = MakeAddrLValueWithoutTBAA(ReturnValue, FnRetTy);2997      CGM.getTargetCodeGenInfo().addReturnRegisterOutputs(2998          *this, ReturnSlot, Constraints, ResultRegTypes, ResultTruncRegTypes,2999          ResultRegDests, AsmString, S.getNumOutputs());3000      SawAsmBlock = true;3001    }3002  }3003 3004  for (unsigned i = 0, e = S.getNumInputs(); i != e; i++) {3005    const Expr *InputExpr = S.getInputExpr(i);3006 3007    TargetInfo::ConstraintInfo &Info = InputConstraintInfos[i];3008 3009    if (Info.allowsMemory())3010      ReadNone = false;3011 3012    if (!Constraints.empty())3013      Constraints += ',';3014 3015    // Simplify the input constraint.3016    std::string InputConstraint(S.getInputConstraint(i));3017    InputConstraint =3018        getTarget().simplifyConstraint(InputConstraint, &OutputConstraintInfos);3019 3020    InputConstraint = AddVariableConstraints(3021        InputConstraint, *InputExpr->IgnoreParenNoopCasts(getContext()),3022        getTarget(), CGM, S, false /* No EarlyClobber */);3023 3024    std::string ReplaceConstraint (InputConstraint);3025    llvm::Value *Arg;3026    llvm::Type *ArgElemType;3027    std::tie(Arg, ArgElemType) = EmitAsmInput(Info, InputExpr, Constraints);3028 3029    // If this input argument is tied to a larger output result, extend the3030    // input to be the same size as the output.  The LLVM backend wants to see3031    // the input and output of a matching constraint be the same size.  Note3032    // that GCC does not define what the top bits are here.  We use zext because3033    // that is usually cheaper, but LLVM IR should really get an anyext someday.3034    if (Info.hasTiedOperand()) {3035      unsigned Output = Info.getTiedOperand();3036      QualType OutputType = S.getOutputExpr(Output)->getType();3037      QualType InputTy = InputExpr->getType();3038 3039      if (getContext().getTypeSize(OutputType) >3040          getContext().getTypeSize(InputTy)) {3041        // Use ptrtoint as appropriate so that we can do our extension.3042        if (isa<llvm::PointerType>(Arg->getType()))3043          Arg = Builder.CreatePtrToInt(Arg, IntPtrTy);3044        llvm::Type *OutputTy = ConvertType(OutputType);3045        if (isa<llvm::IntegerType>(OutputTy))3046          Arg = Builder.CreateZExt(Arg, OutputTy);3047        else if (isa<llvm::PointerType>(OutputTy))3048          Arg = Builder.CreateZExt(Arg, IntPtrTy);3049        else if (OutputTy->isFloatingPointTy())3050          Arg = Builder.CreateFPExt(Arg, OutputTy);3051      }3052      // Deal with the tied operands' constraint code in adjustInlineAsmType.3053      ReplaceConstraint = OutputConstraints[Output];3054    }3055    if (llvm::Type* AdjTy =3056          getTargetHooks().adjustInlineAsmType(*this, ReplaceConstraint,3057                                                   Arg->getType()))3058      Arg = Builder.CreateBitCast(Arg, AdjTy);3059    else3060      CGM.getDiags().Report(S.getAsmLoc(), diag::err_asm_invalid_type_in_input)3061          << InputExpr->getType() << InputConstraint;3062 3063    // Update largest vector width for any vector types.3064    if (auto *VT = dyn_cast<llvm::VectorType>(Arg->getType()))3065      LargestVectorWidth =3066          std::max((uint64_t)LargestVectorWidth,3067                   VT->getPrimitiveSizeInBits().getKnownMinValue());3068 3069    ArgTypes.push_back(Arg->getType());3070    ArgElemTypes.push_back(ArgElemType);3071    Args.push_back(Arg);3072    Constraints += InputConstraint;3073  }3074 3075  // Append the "input" part of inout constraints.3076  for (unsigned i = 0, e = InOutArgs.size(); i != e; i++) {3077    ArgTypes.push_back(InOutArgTypes[i]);3078    ArgElemTypes.push_back(InOutArgElemTypes[i]);3079    Args.push_back(InOutArgs[i]);3080  }3081  Constraints += InOutConstraints;3082 3083  // Labels3084  SmallVector<llvm::BasicBlock *, 16> Transfer;3085  llvm::BasicBlock *Fallthrough = nullptr;3086  bool IsGCCAsmGoto = false;3087  if (const auto *GS = dyn_cast<GCCAsmStmt>(&S)) {3088    IsGCCAsmGoto = GS->isAsmGoto();3089    if (IsGCCAsmGoto) {3090      for (const auto *E : GS->labels()) {3091        JumpDest Dest = getJumpDestForLabel(E->getLabel());3092        Transfer.push_back(Dest.getBlock());3093        if (!Constraints.empty())3094          Constraints += ',';3095        Constraints += "!i";3096      }3097      Fallthrough = createBasicBlock("asm.fallthrough");3098    }3099  }3100 3101  bool HasUnwindClobber = false;3102 3103  // Clobbers3104  for (unsigned i = 0, e = S.getNumClobbers(); i != e; i++) {3105    std::string Clobber = S.getClobber(i);3106 3107    if (Clobber == "memory")3108      ReadOnly = ReadNone = false;3109    else if (Clobber == "unwind") {3110      HasUnwindClobber = true;3111      continue;3112    } else if (Clobber != "cc") {3113      Clobber = getTarget().getNormalizedGCCRegisterName(Clobber);3114      if (CGM.getCodeGenOpts().StackClashProtector &&3115          getTarget().isSPRegName(Clobber)) {3116        CGM.getDiags().Report(S.getAsmLoc(),3117                              diag::warn_stack_clash_protection_inline_asm);3118      }3119    }3120 3121    if (isa<MSAsmStmt>(&S)) {3122      if (Clobber == "eax" || Clobber == "edx") {3123        if (Constraints.find("=&A") != std::string::npos)3124          continue;3125        std::string::size_type position1 =3126            Constraints.find("={" + Clobber + "}");3127        if (position1 != std::string::npos) {3128          Constraints.insert(position1 + 1, "&");3129          continue;3130        }3131        std::string::size_type position2 = Constraints.find("=A");3132        if (position2 != std::string::npos) {3133          Constraints.insert(position2 + 1, "&");3134          continue;3135        }3136      }3137    }3138    if (!Constraints.empty())3139      Constraints += ',';3140 3141    Constraints += "~{";3142    Constraints += Clobber;3143    Constraints += '}';3144  }3145 3146  assert(!(HasUnwindClobber && IsGCCAsmGoto) &&3147         "unwind clobber can't be used with asm goto");3148 3149  // Add machine specific clobbers3150  std::string_view MachineClobbers = getTarget().getClobbers();3151  if (!MachineClobbers.empty()) {3152    if (!Constraints.empty())3153      Constraints += ',';3154    Constraints += MachineClobbers;3155  }3156 3157  llvm::Type *ResultType;3158  if (ResultRegTypes.empty())3159    ResultType = VoidTy;3160  else if (ResultRegTypes.size() == 1)3161    ResultType = ResultRegTypes[0];3162  else3163    ResultType = llvm::StructType::get(getLLVMContext(), ResultRegTypes);3164 3165  llvm::FunctionType *FTy =3166    llvm::FunctionType::get(ResultType, ArgTypes, false);3167 3168  bool HasSideEffect = S.isVolatile() || S.getNumOutputs() == 0;3169 3170  llvm::InlineAsm::AsmDialect GnuAsmDialect =3171      CGM.getCodeGenOpts().getInlineAsmDialect() == CodeGenOptions::IAD_ATT3172          ? llvm::InlineAsm::AD_ATT3173          : llvm::InlineAsm::AD_Intel;3174  llvm::InlineAsm::AsmDialect AsmDialect = isa<MSAsmStmt>(&S) ?3175    llvm::InlineAsm::AD_Intel : GnuAsmDialect;3176 3177  llvm::InlineAsm *IA = llvm::InlineAsm::get(3178      FTy, AsmString, Constraints, HasSideEffect,3179      /* IsAlignStack */ false, AsmDialect, HasUnwindClobber);3180  std::vector<llvm::Value*> RegResults;3181  llvm::CallBrInst *CBR;3182  llvm::DenseMap<llvm::BasicBlock *, SmallVector<llvm::Value *, 4>>3183      CBRRegResults;3184  if (IsGCCAsmGoto) {3185    CBR = Builder.CreateCallBr(IA, Fallthrough, Transfer, Args);3186    EmitBlock(Fallthrough);3187    UpdateAsmCallInst(*CBR, HasSideEffect, /*HasUnwindClobber=*/false, ReadOnly,3188                      ReadNone, InNoMergeAttributedStmt,3189                      InNoConvergentAttributedStmt, S, ResultRegTypes,3190                      ArgElemTypes, *this, RegResults);3191    // Because we are emitting code top to bottom, we don't have enough3192    // information at this point to know precisely whether we have a critical3193    // edge. If we have outputs, split all indirect destinations.3194    if (!RegResults.empty()) {3195      unsigned i = 0;3196      for (llvm::BasicBlock *Dest : CBR->getIndirectDests()) {3197        llvm::Twine SynthName = Dest->getName() + ".split";3198        llvm::BasicBlock *SynthBB = createBasicBlock(SynthName);3199        llvm::IRBuilderBase::InsertPointGuard IPG(Builder);3200        Builder.SetInsertPoint(SynthBB);3201 3202        if (ResultRegTypes.size() == 1) {3203          CBRRegResults[SynthBB].push_back(CBR);3204        } else {3205          for (unsigned j = 0, e = ResultRegTypes.size(); j != e; ++j) {3206            llvm::Value *Tmp = Builder.CreateExtractValue(CBR, j, "asmresult");3207            CBRRegResults[SynthBB].push_back(Tmp);3208          }3209        }3210 3211        EmitBranch(Dest);3212        EmitBlock(SynthBB);3213        CBR->setIndirectDest(i++, SynthBB);3214      }3215    }3216  } else if (HasUnwindClobber) {3217    llvm::CallBase *Result = EmitCallOrInvoke(IA, Args, "");3218    UpdateAsmCallInst(*Result, HasSideEffect, /*HasUnwindClobber=*/true,3219                      ReadOnly, ReadNone, InNoMergeAttributedStmt,3220                      InNoConvergentAttributedStmt, S, ResultRegTypes,3221                      ArgElemTypes, *this, RegResults);3222  } else {3223    llvm::CallInst *Result =3224        Builder.CreateCall(IA, Args, getBundlesForFunclet(IA));3225    UpdateAsmCallInst(*Result, HasSideEffect, /*HasUnwindClobber=*/false,3226                      ReadOnly, ReadNone, InNoMergeAttributedStmt,3227                      InNoConvergentAttributedStmt, S, ResultRegTypes,3228                      ArgElemTypes, *this, RegResults);3229  }3230 3231  EmitAsmStores(*this, S, RegResults, ResultRegTypes, ResultTruncRegTypes,3232                ResultRegDests, ResultRegQualTys, ResultTypeRequiresCast,3233                ResultBounds);3234 3235  // If this is an asm goto with outputs, repeat EmitAsmStores, but with a3236  // different insertion point; one for each indirect destination and with3237  // CBRRegResults rather than RegResults.3238  if (IsGCCAsmGoto && !CBRRegResults.empty()) {3239    for (llvm::BasicBlock *Succ : CBR->getIndirectDests()) {3240      llvm::IRBuilderBase::InsertPointGuard IPG(Builder);3241      Builder.SetInsertPoint(Succ, --(Succ->end()));3242      EmitAsmStores(*this, S, CBRRegResults[Succ], ResultRegTypes,3243                    ResultTruncRegTypes, ResultRegDests, ResultRegQualTys,3244                    ResultTypeRequiresCast, ResultBounds);3245    }3246  }3247}3248 3249LValue CodeGenFunction::InitCapturedStruct(const CapturedStmt &S) {3250  const RecordDecl *RD = S.getCapturedRecordDecl();3251  CanQualType RecordTy = getContext().getCanonicalTagType(RD);3252 3253  // Initialize the captured struct.3254  LValue SlotLV =3255    MakeAddrLValue(CreateMemTemp(RecordTy, "agg.captured"), RecordTy);3256 3257  RecordDecl::field_iterator CurField = RD->field_begin();3258  for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),3259                                                 E = S.capture_init_end();3260       I != E; ++I, ++CurField) {3261    LValue LV = EmitLValueForFieldInitialization(SlotLV, *CurField);3262    if (CurField->hasCapturedVLAType()) {3263      EmitLambdaVLACapture(CurField->getCapturedVLAType(), LV);3264    } else {3265      EmitInitializerForField(*CurField, LV, *I);3266    }3267  }3268 3269  return SlotLV;3270}3271 3272/// Generate an outlined function for the body of a CapturedStmt, store any3273/// captured variables into the captured struct, and call the outlined function.3274llvm::Function *3275CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) {3276  LValue CapStruct = InitCapturedStruct(S);3277 3278  // Emit the CapturedDecl3279  CodeGenFunction CGF(CGM, true);3280  CGCapturedStmtRAII CapInfoRAII(CGF, new CGCapturedStmtInfo(S, K));3281  llvm::Function *F = CGF.GenerateCapturedStmtFunction(S);3282  delete CGF.CapturedStmtInfo;3283 3284  // Emit call to the helper function.3285  EmitCallOrInvoke(F, CapStruct.getPointer(*this));3286 3287  return F;3288}3289 3290Address CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) {3291  LValue CapStruct = InitCapturedStruct(S);3292  return CapStruct.getAddress();3293}3294 3295/// Creates the outlined function for a CapturedStmt.3296llvm::Function *3297CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {3298  assert(CapturedStmtInfo &&3299    "CapturedStmtInfo should be set when generating the captured function");3300  const CapturedDecl *CD = S.getCapturedDecl();3301  const RecordDecl *RD = S.getCapturedRecordDecl();3302  SourceLocation Loc = S.getBeginLoc();3303  assert(CD->hasBody() && "missing CapturedDecl body");3304 3305  // Build the argument list.3306  ASTContext &Ctx = CGM.getContext();3307  FunctionArgList Args;3308  Args.append(CD->param_begin(), CD->param_end());3309 3310  // Create the function declaration.3311  const CGFunctionInfo &FuncInfo =3312    CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);3313  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);3314 3315  llvm::Function *F =3316    llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,3317                           CapturedStmtInfo->getHelperName(), &CGM.getModule());3318  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);3319  if (CD->isNothrow())3320    F->addFnAttr(llvm::Attribute::NoUnwind);3321 3322  // Generate the function.3323  StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, CD->getLocation(),3324                CD->getBody()->getBeginLoc());3325  // Set the context parameter in CapturedStmtInfo.3326  Address DeclPtr = GetAddrOfLocalVar(CD->getContextParam());3327  CapturedStmtInfo->setContextValue(Builder.CreateLoad(DeclPtr));3328 3329  // Initialize variable-length arrays.3330  LValue Base = MakeNaturalAlignRawAddrLValue(3331      CapturedStmtInfo->getContextValue(), Ctx.getCanonicalTagType(RD));3332  for (auto *FD : RD->fields()) {3333    if (FD->hasCapturedVLAType()) {3334      auto *ExprArg =3335          EmitLoadOfLValue(EmitLValueForField(Base, FD), S.getBeginLoc())3336              .getScalarVal();3337      auto VAT = FD->getCapturedVLAType();3338      VLASizeMap[VAT->getSizeExpr()] = ExprArg;3339    }3340  }3341 3342  // If 'this' is captured, load it into CXXThisValue.3343  if (CapturedStmtInfo->isCXXThisExprCaptured()) {3344    FieldDecl *FD = CapturedStmtInfo->getThisFieldDecl();3345    LValue ThisLValue = EmitLValueForField(Base, FD);3346    CXXThisValue = EmitLoadOfLValue(ThisLValue, Loc).getScalarVal();3347  }3348 3349  PGO->assignRegionCounters(GlobalDecl(CD), F);3350  CapturedStmtInfo->EmitBody(*this, CD->getBody());3351  FinishFunction(CD->getBodyRBrace());3352 3353  return F;3354}3355 3356// Returns the first convergence entry/loop/anchor instruction found in |BB|.3357// std::nullptr otherwise.3358static llvm::ConvergenceControlInst *getConvergenceToken(llvm::BasicBlock *BB) {3359  for (auto &I : *BB) {3360    if (auto *CI = dyn_cast<llvm::ConvergenceControlInst>(&I))3361      return CI;3362  }3363  return nullptr;3364}3365 3366llvm::CallBase *3367CodeGenFunction::addConvergenceControlToken(llvm::CallBase *Input) {3368  llvm::ConvergenceControlInst *ParentToken = ConvergenceTokenStack.back();3369  assert(ParentToken);3370 3371  llvm::Value *bundleArgs[] = {ParentToken};3372  llvm::OperandBundleDef OB("convergencectrl", bundleArgs);3373  auto *Output = llvm::CallBase::addOperandBundle(3374      Input, llvm::LLVMContext::OB_convergencectrl, OB, Input->getIterator());3375  Input->replaceAllUsesWith(Output);3376  Input->eraseFromParent();3377  return Output;3378}3379 3380llvm::ConvergenceControlInst *3381CodeGenFunction::emitConvergenceLoopToken(llvm::BasicBlock *BB) {3382  llvm::ConvergenceControlInst *ParentToken = ConvergenceTokenStack.back();3383  assert(ParentToken);3384  return llvm::ConvergenceControlInst::CreateLoop(*BB, ParentToken);3385}3386 3387llvm::ConvergenceControlInst *3388CodeGenFunction::getOrEmitConvergenceEntryToken(llvm::Function *F) {3389  llvm::BasicBlock *BB = &F->getEntryBlock();3390  llvm::ConvergenceControlInst *Token = getConvergenceToken(BB);3391  if (Token)3392    return Token;3393 3394  // Adding a convergence token requires the function to be marked as3395  // convergent.3396  F->setConvergent();3397  return llvm::ConvergenceControlInst::CreateEntry(*BB);3398}3399