brintos / llvm-project-archived public Read only

1//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains the code for emitting atomic operations.10//11//===----------------------------------------------------------------------===//12 13#include "CGCall.h"14#include "CGRecordLayout.h"15#include "CodeGenFunction.h"16#include "CodeGenModule.h"17#include "TargetInfo.h"18#include "clang/AST/ASTContext.h"19#include "clang/Basic/DiagnosticFrontend.h"20#include "clang/CodeGen/CGFunctionInfo.h"21#include "llvm/ADT/DenseMap.h"22#include "llvm/IR/DataLayout.h"23#include "llvm/IR/Intrinsics.h"24 25using namespace clang;26using namespace CodeGen;27 28namespace {29  class AtomicInfo {30    CodeGenFunction &CGF;31    QualType AtomicTy;32    QualType ValueTy;33    uint64_t AtomicSizeInBits;34    uint64_t ValueSizeInBits;35    CharUnits AtomicAlign;36    CharUnits ValueAlign;37    TypeEvaluationKind EvaluationKind;38    bool UseLibcall;39    LValue LVal;40    CGBitFieldInfo BFI;41  public:42    AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)43        : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),44          EvaluationKind(TEK_Scalar), UseLibcall(true) {45      assert(!lvalue.isGlobalReg());46      ASTContext &C = CGF.getContext();47      if (lvalue.isSimple()) {48        AtomicTy = lvalue.getType();49        if (auto *ATy = AtomicTy->getAs<AtomicType>())50          ValueTy = ATy->getValueType();51        else52          ValueTy = AtomicTy;53        EvaluationKind = CGF.getEvaluationKind(ValueTy);54 55        uint64_t ValueAlignInBits;56        uint64_t AtomicAlignInBits;57        TypeInfo ValueTI = C.getTypeInfo(ValueTy);58        ValueSizeInBits = ValueTI.Width;59        ValueAlignInBits = ValueTI.Align;60 61        TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);62        AtomicSizeInBits = AtomicTI.Width;63        AtomicAlignInBits = AtomicTI.Align;64 65        assert(ValueSizeInBits <= AtomicSizeInBits);66        assert(ValueAlignInBits <= AtomicAlignInBits);67 68        AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);69        ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);70        if (lvalue.getAlignment().isZero())71          lvalue.setAlignment(AtomicAlign);72 73        LVal = lvalue;74      } else if (lvalue.isBitField()) {75        ValueTy = lvalue.getType();76        ValueSizeInBits = C.getTypeSize(ValueTy);77        auto &OrigBFI = lvalue.getBitFieldInfo();78        auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());79        AtomicSizeInBits = C.toBits(80            C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)81                .alignTo(lvalue.getAlignment()));82        llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF);83        auto OffsetInChars =84            (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *85            lvalue.getAlignment();86        llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64(87            CGF.Int8Ty, BitFieldPtr, OffsetInChars.getQuantity());88        StoragePtr = CGF.Builder.CreateAddrSpaceCast(89            StoragePtr, CGF.DefaultPtrTy, "atomic_bitfield_base");90        BFI = OrigBFI;91        BFI.Offset = Offset;92        BFI.StorageSize = AtomicSizeInBits;93        BFI.StorageOffset += OffsetInChars;94        llvm::Type *StorageTy = CGF.Builder.getIntNTy(AtomicSizeInBits);95        LVal = LValue::MakeBitfield(96            Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI,97            lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo());98        AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);99        if (AtomicTy.isNull()) {100          llvm::APInt Size(101              /*numBits=*/32,102              C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());103          AtomicTy = C.getConstantArrayType(C.CharTy, Size, nullptr,104                                            ArraySizeModifier::Normal,105                                            /*IndexTypeQuals=*/0);106        }107        AtomicAlign = ValueAlign = lvalue.getAlignment();108      } else if (lvalue.isVectorElt()) {109        ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();110        ValueSizeInBits = C.getTypeSize(ValueTy);111        AtomicTy = lvalue.getType();112        AtomicSizeInBits = C.getTypeSize(AtomicTy);113        AtomicAlign = ValueAlign = lvalue.getAlignment();114        LVal = lvalue;115      } else {116        assert(lvalue.isExtVectorElt());117        ValueTy = lvalue.getType();118        ValueSizeInBits = C.getTypeSize(ValueTy);119        AtomicTy = ValueTy = CGF.getContext().getExtVectorType(120            lvalue.getType(), cast<llvm::FixedVectorType>(121                                  lvalue.getExtVectorAddress().getElementType())122                                  ->getNumElements());123        AtomicSizeInBits = C.getTypeSize(AtomicTy);124        AtomicAlign = ValueAlign = lvalue.getAlignment();125        LVal = lvalue;126      }127      UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(128          AtomicSizeInBits, C.toBits(lvalue.getAlignment()));129    }130 131    QualType getAtomicType() const { return AtomicTy; }132    QualType getValueType() const { return ValueTy; }133    CharUnits getAtomicAlignment() const { return AtomicAlign; }134    uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }135    uint64_t getValueSizeInBits() const { return ValueSizeInBits; }136    TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }137    bool shouldUseLibcall() const { return UseLibcall; }138    const LValue &getAtomicLValue() const { return LVal; }139    llvm::Value *getAtomicPointer() const {140      if (LVal.isSimple())141        return LVal.emitRawPointer(CGF);142      else if (LVal.isBitField())143        return LVal.getRawBitFieldPointer(CGF);144      else if (LVal.isVectorElt())145        return LVal.getRawVectorPointer(CGF);146      assert(LVal.isExtVectorElt());147      return LVal.getRawExtVectorPointer(CGF);148    }149    Address getAtomicAddress() const {150      llvm::Type *ElTy;151      if (LVal.isSimple())152        ElTy = LVal.getAddress().getElementType();153      else if (LVal.isBitField())154        ElTy = LVal.getBitFieldAddress().getElementType();155      else if (LVal.isVectorElt())156        ElTy = LVal.getVectorAddress().getElementType();157      else158        ElTy = LVal.getExtVectorAddress().getElementType();159      return Address(getAtomicPointer(), ElTy, getAtomicAlignment());160    }161 162    Address getAtomicAddressAsAtomicIntPointer() const {163      return castToAtomicIntPointer(getAtomicAddress());164    }165 166    /// Is the atomic size larger than the underlying value type?167    ///168    /// Note that the absence of padding does not mean that atomic169    /// objects are completely interchangeable with non-atomic170    /// objects: we might have promoted the alignment of a type171    /// without making it bigger.172    bool hasPadding() const {173      return (ValueSizeInBits != AtomicSizeInBits);174    }175 176    bool emitMemSetZeroIfNecessary() const;177 178    llvm::Value *getAtomicSizeValue() const {179      CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);180      return CGF.CGM.getSize(size);181    }182 183    /// Cast the given pointer to an integer pointer suitable for atomic184    /// operations if the source.185    Address castToAtomicIntPointer(Address Addr) const;186 187    /// If Addr is compatible with the iN that will be used for an atomic188    /// operation, bitcast it. Otherwise, create a temporary that is suitable189    /// and copy the value across.190    Address convertToAtomicIntPointer(Address Addr) const;191 192    /// Turn an atomic-layout object into an r-value.193    RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,194                                     SourceLocation loc, bool AsValue) const;195 196    llvm::Value *getScalarRValValueOrNull(RValue RVal) const;197 198    /// Converts an rvalue to integer value if needed.199    llvm::Value *convertRValueToInt(RValue RVal, bool CmpXchg = false) const;200 201    RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot,202                                  SourceLocation Loc, bool AsValue,203                                  bool CmpXchg = false) const;204 205    /// Copy an atomic r-value into atomic-layout memory.206    void emitCopyIntoMemory(RValue rvalue) const;207 208    /// Project an l-value down to the value field.209    LValue projectValue() const {210      assert(LVal.isSimple());211      Address addr = getAtomicAddress();212      if (hasPadding())213        addr = CGF.Builder.CreateStructGEP(addr, 0);214 215      return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),216                              LVal.getBaseInfo(), LVal.getTBAAInfo());217    }218 219    /// Emits atomic load.220    /// \returns Loaded value.221    RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,222                          bool AsValue, llvm::AtomicOrdering AO,223                          bool IsVolatile);224 225    /// Emits atomic compare-and-exchange sequence.226    /// \param Expected Expected value.227    /// \param Desired Desired value.228    /// \param Success Atomic ordering for success operation.229    /// \param Failure Atomic ordering for failed operation.230    /// \param IsWeak true if atomic operation is weak, false otherwise.231    /// \returns Pair of values: previous value from storage (value type) and232    /// boolean flag (i1 type) with true if success and false otherwise.233    std::pair<RValue, llvm::Value *>234    EmitAtomicCompareExchange(RValue Expected, RValue Desired,235                              llvm::AtomicOrdering Success =236                                  llvm::AtomicOrdering::SequentiallyConsistent,237                              llvm::AtomicOrdering Failure =238                                  llvm::AtomicOrdering::SequentiallyConsistent,239                              bool IsWeak = false);240 241    /// Emits atomic update.242    /// \param AO Atomic ordering.243    /// \param UpdateOp Update operation for the current lvalue.244    void EmitAtomicUpdate(llvm::AtomicOrdering AO,245                          const llvm::function_ref<RValue(RValue)> &UpdateOp,246                          bool IsVolatile);247    /// Emits atomic update.248    /// \param AO Atomic ordering.249    void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,250                          bool IsVolatile);251 252    /// Materialize an atomic r-value in atomic-layout memory.253    Address materializeRValue(RValue rvalue) const;254 255    /// Creates temp alloca for intermediate operations on atomic value.256    Address CreateTempAlloca() const;257  private:258    bool requiresMemSetZero(llvm::Type *type) const;259 260 261    /// Emits atomic load as a libcall.262    void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,263                               llvm::AtomicOrdering AO, bool IsVolatile);264    /// Emits atomic load as LLVM instruction.265    llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile,266                                  bool CmpXchg = false);267    /// Emits atomic compare-and-exchange op as a libcall.268    llvm::Value *EmitAtomicCompareExchangeLibcall(269        llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,270        llvm::AtomicOrdering Success =271            llvm::AtomicOrdering::SequentiallyConsistent,272        llvm::AtomicOrdering Failure =273            llvm::AtomicOrdering::SequentiallyConsistent);274    /// Emits atomic compare-and-exchange op as LLVM instruction.275    std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(276        llvm::Value *ExpectedVal, llvm::Value *DesiredVal,277        llvm::AtomicOrdering Success =278            llvm::AtomicOrdering::SequentiallyConsistent,279        llvm::AtomicOrdering Failure =280            llvm::AtomicOrdering::SequentiallyConsistent,281        bool IsWeak = false);282    /// Emit atomic update as libcalls.283    void284    EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,285                            const llvm::function_ref<RValue(RValue)> &UpdateOp,286                            bool IsVolatile);287    /// Emit atomic update as LLVM instructions.288    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,289                            const llvm::function_ref<RValue(RValue)> &UpdateOp,290                            bool IsVolatile);291    /// Emit atomic update as libcalls.292    void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,293                                 bool IsVolatile);294    /// Emit atomic update as LLVM instructions.295    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,296                            bool IsVolatile);297  };298}299 300Address AtomicInfo::CreateTempAlloca() const {301  Address TempAlloca = CGF.CreateMemTemp(302      (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy303                                                                : AtomicTy,304      getAtomicAlignment(),305      "atomic-temp");306  // Cast to pointer to value type for bitfields.307  if (LVal.isBitField())308    return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(309        TempAlloca, getAtomicAddress().getType(),310        getAtomicAddress().getElementType());311  return TempAlloca;312}313 314static RValue emitAtomicLibcall(CodeGenFunction &CGF,315                                StringRef fnName,316                                QualType resultType,317                                CallArgList &args) {318  const CGFunctionInfo &fnInfo =319    CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);320  llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);321  llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());322  fnAttrB.addAttribute(llvm::Attribute::NoUnwind);323  fnAttrB.addAttribute(llvm::Attribute::WillReturn);324  llvm::AttributeList fnAttrs = llvm::AttributeList::get(325      CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);326 327  llvm::FunctionCallee fn =328      CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);329  auto callee = CGCallee::forDirect(fn);330  return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);331}332 333/// Does a store of the given IR type modify the full expected width?334static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,335                           uint64_t expectedSize) {336  return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);337}338 339/// Does the atomic type require memsetting to zero before initialization?340///341/// The IR type is provided as a way of making certain queries faster.342bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {343  // If the atomic type has size padding, we definitely need a memset.344  if (hasPadding()) return true;345 346  // Otherwise, do some simple heuristics to try to avoid it:347  switch (getEvaluationKind()) {348  // For scalars and complexes, check whether the store size of the349  // type uses the full size.350  case TEK_Scalar:351    return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);352  case TEK_Complex:353    return !isFullSizeType(CGF.CGM, type->getStructElementType(0),354                           AtomicSizeInBits / 2);355 356  // Padding in structs has an undefined bit pattern.  User beware.357  case TEK_Aggregate:358    return false;359  }360  llvm_unreachable("bad evaluation kind");361}362 363bool AtomicInfo::emitMemSetZeroIfNecessary() const {364  assert(LVal.isSimple());365  Address addr = LVal.getAddress();366  if (!requiresMemSetZero(addr.getElementType()))367    return false;368 369  CGF.Builder.CreateMemSet(370      addr.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.Int8Ty, 0),371      CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),372      LVal.getAlignment().getAsAlign());373  return true;374}375 376static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,377                              Address Dest, Address Ptr, Address Val1,378                              Address Val2, Address ExpectedResult,379                              uint64_t Size, llvm::AtomicOrdering SuccessOrder,380                              llvm::AtomicOrdering FailureOrder,381                              llvm::SyncScope::ID Scope) {382  // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.383  llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);384  llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);385 386  llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(387      Ptr, Expected, Desired, SuccessOrder, FailureOrder, Scope);388  Pair->setVolatile(E->isVolatile());389  Pair->setWeak(IsWeak);390  CGF.getTargetHooks().setTargetAtomicMetadata(CGF, *Pair, E);391 392  // Cmp holds the result of the compare-exchange operation: true on success,393  // false on failure.394  llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);395  llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);396 397  // This basic block is used to hold the store instruction if the operation398  // failed.399  llvm::BasicBlock *StoreExpectedBB =400      CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);401 402  // This basic block is the exit point of the operation, we should end up403  // here regardless of whether or not the operation succeeded.404  llvm::BasicBlock *ContinueBB =405      CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);406 407  // Update Expected if Expected isn't equal to Old, otherwise branch to the408  // exit point.409  CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);410 411  CGF.Builder.SetInsertPoint(StoreExpectedBB);412  // Update the memory at Expected with Old's value.413  llvm::Type *ExpectedType = ExpectedResult.getElementType();414  const llvm::DataLayout &DL = CGF.CGM.getDataLayout();415  uint64_t ExpectedSizeInBytes = DL.getTypeStoreSize(ExpectedType);416 417  if (ExpectedSizeInBytes == Size) {418    // Sizes match: store directly419    auto *I = CGF.Builder.CreateStore(Old, ExpectedResult);420    CGF.addInstToCurrentSourceAtom(I, Old);421  } else {422    // store only the first ExpectedSizeInBytes bytes of Old423    llvm::Type *OldType = Old->getType();424 425    // Allocate temporary storage for Old value426    Address OldTmp =427        CGF.CreateTempAlloca(OldType, Ptr.getAlignment(), "old.tmp");428 429    // Store Old into this temporary430    auto *I = CGF.Builder.CreateStore(Old, OldTmp);431    CGF.addInstToCurrentSourceAtom(I, Old);432 433    // Perform memcpy for first ExpectedSizeInBytes bytes434    CGF.Builder.CreateMemCpy(ExpectedResult, OldTmp, ExpectedSizeInBytes,435                             /*isVolatile=*/false);436  }437 438  // Finally, branch to the exit point.439  CGF.Builder.CreateBr(ContinueBB);440 441  CGF.Builder.SetInsertPoint(ContinueBB);442  // Update the memory at Dest with Cmp's value.443  CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));444}445 446/// Given an ordering required on success, emit all possible cmpxchg447/// instructions to cope with the provided (but possibly only dynamically known)448/// FailureOrder.449static void emitAtomicCmpXchgFailureSet(450    CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr,451    Address Val1, Address Val2, Address ExpectedResult,452    llvm::Value *FailureOrderVal, uint64_t Size,453    llvm::AtomicOrdering SuccessOrder, llvm::SyncScope::ID Scope) {454  llvm::AtomicOrdering FailureOrder;455  if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {456    auto FOS = FO->getSExtValue();457    if (!llvm::isValidAtomicOrderingCABI(FOS))458      FailureOrder = llvm::AtomicOrdering::Monotonic;459    else460      switch ((llvm::AtomicOrderingCABI)FOS) {461      case llvm::AtomicOrderingCABI::relaxed:462      // 31.7.2.18: "The failure argument shall not be memory_order_release463      // nor memory_order_acq_rel". Fallback to monotonic.464      case llvm::AtomicOrderingCABI::release:465      case llvm::AtomicOrderingCABI::acq_rel:466        FailureOrder = llvm::AtomicOrdering::Monotonic;467        break;468      case llvm::AtomicOrderingCABI::consume:469      case llvm::AtomicOrderingCABI::acquire:470        FailureOrder = llvm::AtomicOrdering::Acquire;471        break;472      case llvm::AtomicOrderingCABI::seq_cst:473        FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;474        break;475      }476    // Prior to c++17, "the failure argument shall be no stronger than the477    // success argument". This condition has been lifted and the only478    // precondition is 31.7.2.18. Effectively treat this as a DR and skip479    // language version checks.480    emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult,481                      Size, SuccessOrder, FailureOrder, Scope);482    return;483  }484 485  // Create all the relevant BB's486  auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);487  auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);488  auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);489  auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);490 491  // MonotonicBB is arbitrarily chosen as the default case; in practice, this492  // doesn't matter unless someone is crazy enough to use something that493  // doesn't fold to a constant for the ordering.494  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);495  // Implemented as acquire, since it's the closest in LLVM.496  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),497              AcquireBB);498  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),499              AcquireBB);500  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),501              SeqCstBB);502 503  // Emit all the different atomics504  CGF.Builder.SetInsertPoint(MonotonicBB);505  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult, Size,506                    SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);507  CGF.Builder.CreateBr(ContBB);508 509  CGF.Builder.SetInsertPoint(AcquireBB);510  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult, Size,511                    SuccessOrder, llvm::AtomicOrdering::Acquire, Scope);512  CGF.Builder.CreateBr(ContBB);513 514  CGF.Builder.SetInsertPoint(SeqCstBB);515  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult, Size,516                    SuccessOrder, llvm::AtomicOrdering::SequentiallyConsistent,517                    Scope);518  CGF.Builder.CreateBr(ContBB);519 520  CGF.Builder.SetInsertPoint(ContBB);521}522 523/// Duplicate the atomic min/max operation in conventional IR for the builtin524/// variants that return the new rather than the original value.525static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,526                                         AtomicExpr::AtomicOp Op,527                                         bool IsSigned,528                                         llvm::Value *OldVal,529                                         llvm::Value *RHS) {530  const bool IsFP = OldVal->getType()->isFloatingPointTy();531 532  if (IsFP) {533    llvm::Intrinsic::ID IID = (Op == AtomicExpr::AO__atomic_max_fetch ||534                               Op == AtomicExpr::AO__scoped_atomic_max_fetch)535                                  ? llvm::Intrinsic::maxnum536                                  : llvm::Intrinsic::minnum;537 538    return Builder.CreateBinaryIntrinsic(IID, OldVal, RHS, llvm::FMFSource(),539                                         "newval");540  }541 542  llvm::CmpInst::Predicate Pred;543  switch (Op) {544  default:545    llvm_unreachable("Unexpected min/max operation");546  case AtomicExpr::AO__atomic_max_fetch:547  case AtomicExpr::AO__scoped_atomic_max_fetch:548    Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;549    break;550  case AtomicExpr::AO__atomic_min_fetch:551  case AtomicExpr::AO__scoped_atomic_min_fetch:552    Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;553    break;554  }555  llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");556  return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");557}558 559static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,560                         Address Ptr, Address Val1, Address Val2,561                         Address ExpectedResult, llvm::Value *IsWeak,562                         llvm::Value *FailureOrder, uint64_t Size,563                         llvm::AtomicOrdering Order,564                         llvm::SyncScope::ID Scope) {565  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;566  bool PostOpMinMax = false;567  unsigned PostOp = 0;568 569  switch (E->getOp()) {570  case AtomicExpr::AO__c11_atomic_init:571  case AtomicExpr::AO__opencl_atomic_init:572    llvm_unreachable("Already handled!");573 574  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:575  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:576  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:577    emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,578                                ExpectedResult, FailureOrder, Size, Order,579                                Scope);580    return;581  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:582  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:583  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:584    emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,585                                ExpectedResult, FailureOrder, Size, Order,586                                Scope);587    return;588  case AtomicExpr::AO__atomic_compare_exchange:589  case AtomicExpr::AO__atomic_compare_exchange_n:590  case AtomicExpr::AO__scoped_atomic_compare_exchange:591  case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {592    if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {593      emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,594                                  Val1, Val2, ExpectedResult, FailureOrder,595                                  Size, Order, Scope);596    } else {597      // Create all the relevant BB's598      llvm::BasicBlock *StrongBB =599          CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);600      llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);601      llvm::BasicBlock *ContBB =602          CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);603 604      llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);605      SI->addCase(CGF.Builder.getInt1(false), StrongBB);606 607      CGF.Builder.SetInsertPoint(StrongBB);608      emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,609                                  ExpectedResult, FailureOrder, Size, Order,610                                  Scope);611      CGF.Builder.CreateBr(ContBB);612 613      CGF.Builder.SetInsertPoint(WeakBB);614      emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,615                                  ExpectedResult, FailureOrder, Size, Order,616                                  Scope);617      CGF.Builder.CreateBr(ContBB);618 619      CGF.Builder.SetInsertPoint(ContBB);620    }621    return;622  }623  case AtomicExpr::AO__c11_atomic_load:624  case AtomicExpr::AO__opencl_atomic_load:625  case AtomicExpr::AO__hip_atomic_load:626  case AtomicExpr::AO__atomic_load_n:627  case AtomicExpr::AO__atomic_load:628  case AtomicExpr::AO__scoped_atomic_load_n:629  case AtomicExpr::AO__scoped_atomic_load: {630    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);631    Load->setAtomic(Order, Scope);632    Load->setVolatile(E->isVolatile());633    CGF.maybeAttachRangeForLoad(Load, E->getValueType(), E->getExprLoc());634    auto *I = CGF.Builder.CreateStore(Load, Dest);635    CGF.addInstToCurrentSourceAtom(I, Load);636    return;637  }638 639  case AtomicExpr::AO__c11_atomic_store:640  case AtomicExpr::AO__opencl_atomic_store:641  case AtomicExpr::AO__hip_atomic_store:642  case AtomicExpr::AO__atomic_store:643  case AtomicExpr::AO__atomic_store_n:644  case AtomicExpr::AO__scoped_atomic_store:645  case AtomicExpr::AO__scoped_atomic_store_n: {646    llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);647    llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);648    Store->setAtomic(Order, Scope);649    Store->setVolatile(E->isVolatile());650    CGF.addInstToCurrentSourceAtom(Store, LoadVal1);651    return;652  }653 654  case AtomicExpr::AO__c11_atomic_exchange:655  case AtomicExpr::AO__hip_atomic_exchange:656  case AtomicExpr::AO__opencl_atomic_exchange:657  case AtomicExpr::AO__atomic_exchange_n:658  case AtomicExpr::AO__atomic_exchange:659  case AtomicExpr::AO__scoped_atomic_exchange_n:660  case AtomicExpr::AO__scoped_atomic_exchange:661    Op = llvm::AtomicRMWInst::Xchg;662    break;663 664  case AtomicExpr::AO__atomic_add_fetch:665  case AtomicExpr::AO__scoped_atomic_add_fetch:666    PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd667                                                 : llvm::Instruction::Add;668    [[fallthrough]];669  case AtomicExpr::AO__c11_atomic_fetch_add:670  case AtomicExpr::AO__hip_atomic_fetch_add:671  case AtomicExpr::AO__opencl_atomic_fetch_add:672  case AtomicExpr::AO__atomic_fetch_add:673  case AtomicExpr::AO__scoped_atomic_fetch_add:674    Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd675                                             : llvm::AtomicRMWInst::Add;676    break;677 678  case AtomicExpr::AO__atomic_sub_fetch:679  case AtomicExpr::AO__scoped_atomic_sub_fetch:680    PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub681                                                 : llvm::Instruction::Sub;682    [[fallthrough]];683  case AtomicExpr::AO__c11_atomic_fetch_sub:684  case AtomicExpr::AO__hip_atomic_fetch_sub:685  case AtomicExpr::AO__opencl_atomic_fetch_sub:686  case AtomicExpr::AO__atomic_fetch_sub:687  case AtomicExpr::AO__scoped_atomic_fetch_sub:688    Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub689                                             : llvm::AtomicRMWInst::Sub;690    break;691 692  case AtomicExpr::AO__atomic_min_fetch:693  case AtomicExpr::AO__scoped_atomic_min_fetch:694    PostOpMinMax = true;695    [[fallthrough]];696  case AtomicExpr::AO__c11_atomic_fetch_min:697  case AtomicExpr::AO__hip_atomic_fetch_min:698  case AtomicExpr::AO__opencl_atomic_fetch_min:699  case AtomicExpr::AO__atomic_fetch_min:700  case AtomicExpr::AO__scoped_atomic_fetch_min:701    Op = E->getValueType()->isFloatingType()702             ? llvm::AtomicRMWInst::FMin703             : (E->getValueType()->isSignedIntegerType()704                    ? llvm::AtomicRMWInst::Min705                    : llvm::AtomicRMWInst::UMin);706    break;707 708  case AtomicExpr::AO__atomic_max_fetch:709  case AtomicExpr::AO__scoped_atomic_max_fetch:710    PostOpMinMax = true;711    [[fallthrough]];712  case AtomicExpr::AO__c11_atomic_fetch_max:713  case AtomicExpr::AO__hip_atomic_fetch_max:714  case AtomicExpr::AO__opencl_atomic_fetch_max:715  case AtomicExpr::AO__atomic_fetch_max:716  case AtomicExpr::AO__scoped_atomic_fetch_max:717    Op = E->getValueType()->isFloatingType()718             ? llvm::AtomicRMWInst::FMax719             : (E->getValueType()->isSignedIntegerType()720                    ? llvm::AtomicRMWInst::Max721                    : llvm::AtomicRMWInst::UMax);722    break;723 724  case AtomicExpr::AO__atomic_and_fetch:725  case AtomicExpr::AO__scoped_atomic_and_fetch:726    PostOp = llvm::Instruction::And;727    [[fallthrough]];728  case AtomicExpr::AO__c11_atomic_fetch_and:729  case AtomicExpr::AO__hip_atomic_fetch_and:730  case AtomicExpr::AO__opencl_atomic_fetch_and:731  case AtomicExpr::AO__atomic_fetch_and:732  case AtomicExpr::AO__scoped_atomic_fetch_and:733    Op = llvm::AtomicRMWInst::And;734    break;735 736  case AtomicExpr::AO__atomic_or_fetch:737  case AtomicExpr::AO__scoped_atomic_or_fetch:738    PostOp = llvm::Instruction::Or;739    [[fallthrough]];740  case AtomicExpr::AO__c11_atomic_fetch_or:741  case AtomicExpr::AO__hip_atomic_fetch_or:742  case AtomicExpr::AO__opencl_atomic_fetch_or:743  case AtomicExpr::AO__atomic_fetch_or:744  case AtomicExpr::AO__scoped_atomic_fetch_or:745    Op = llvm::AtomicRMWInst::Or;746    break;747 748  case AtomicExpr::AO__atomic_xor_fetch:749  case AtomicExpr::AO__scoped_atomic_xor_fetch:750    PostOp = llvm::Instruction::Xor;751    [[fallthrough]];752  case AtomicExpr::AO__c11_atomic_fetch_xor:753  case AtomicExpr::AO__hip_atomic_fetch_xor:754  case AtomicExpr::AO__opencl_atomic_fetch_xor:755  case AtomicExpr::AO__atomic_fetch_xor:756  case AtomicExpr::AO__scoped_atomic_fetch_xor:757    Op = llvm::AtomicRMWInst::Xor;758    break;759 760  case AtomicExpr::AO__atomic_nand_fetch:761  case AtomicExpr::AO__scoped_atomic_nand_fetch:762    PostOp = llvm::Instruction::And; // the NOT is special cased below763    [[fallthrough]];764  case AtomicExpr::AO__c11_atomic_fetch_nand:765  case AtomicExpr::AO__atomic_fetch_nand:766  case AtomicExpr::AO__scoped_atomic_fetch_nand:767    Op = llvm::AtomicRMWInst::Nand;768    break;769 770  case AtomicExpr::AO__scoped_atomic_uinc_wrap:771    Op = llvm::AtomicRMWInst::UIncWrap;772    break;773  case AtomicExpr::AO__scoped_atomic_udec_wrap:774    Op = llvm::AtomicRMWInst::UDecWrap;775    break;776 777  case AtomicExpr::AO__atomic_test_and_set: {778    llvm::AtomicRMWInst *RMWI =779        CGF.emitAtomicRMWInst(llvm::AtomicRMWInst::Xchg, Ptr,780                              CGF.Builder.getInt8(1), Order, Scope, E);781    RMWI->setVolatile(E->isVolatile());782    llvm::Value *Result = CGF.EmitToMemory(783        CGF.Builder.CreateIsNotNull(RMWI, "tobool"), E->getType());784    auto *I = CGF.Builder.CreateStore(Result, Dest);785    CGF.addInstToCurrentSourceAtom(I, Result);786    return;787  }788 789  case AtomicExpr::AO__atomic_clear: {790    llvm::StoreInst *Store =791        CGF.Builder.CreateStore(CGF.Builder.getInt8(0), Ptr);792    Store->setAtomic(Order, Scope);793    Store->setVolatile(E->isVolatile());794    CGF.addInstToCurrentSourceAtom(Store, nullptr);795    return;796  }797  }798 799  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);800  llvm::AtomicRMWInst *RMWI =801      CGF.emitAtomicRMWInst(Op, Ptr, LoadVal1, Order, Scope, E);802  RMWI->setVolatile(E->isVolatile());803 804  // For __atomic_*_fetch operations, perform the operation again to805  // determine the value which was written.806  llvm::Value *Result = RMWI;807  if (PostOpMinMax)808    Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),809                                  E->getValueType()->isSignedIntegerType(),810                                  RMWI, LoadVal1);811  else if (PostOp)812    Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,813                                     LoadVal1);814  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch ||815      E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch)816    Result = CGF.Builder.CreateNot(Result);817  auto *I = CGF.Builder.CreateStore(Result, Dest);818  CGF.addInstToCurrentSourceAtom(I, Result);819}820 821// This function emits any expression (scalar, complex, or aggregate)822// into a temporary alloca.823static Address824EmitValToTemp(CodeGenFunction &CGF, Expr *E) {825  Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");826  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),827                       /*Init*/ true);828  return DeclPtr;829}830 831static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,832                         Address Ptr, Address Val1, Address Val2,833                         Address OriginalVal1, llvm::Value *IsWeak,834                         llvm::Value *FailureOrder, uint64_t Size,835                         llvm::AtomicOrdering Order, llvm::Value *Scope) {836  auto ScopeModel = Expr->getScopeModel();837 838  // LLVM atomic instructions always have sync scope. If clang atomic839  // expression has no scope operand, use default LLVM sync scope.840  if (!ScopeModel) {841    llvm::SyncScope::ID SS;842    if (CGF.getLangOpts().OpenCL)843      // OpenCL approach is: "The functions that do not have memory_scope844      // argument have the same semantics as the corresponding functions with845      // the memory_scope argument set to memory_scope_device." See ref.:846      // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions847      SS = CGF.getTargetHooks().getLLVMSyncScopeID(CGF.getLangOpts(),848                                                   SyncScope::OpenCLDevice,849                                                   Order, CGF.getLLVMContext());850    else851      SS = llvm::SyncScope::System;852    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,853                 FailureOrder, Size, Order, SS);854    return;855  }856 857  // Handle constant scope.858  if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {859    auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(860        CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),861        Order, CGF.CGM.getLLVMContext());862    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,863                 FailureOrder, Size, Order, SCID);864    return;865  }866 867  // Handle non-constant scope.868  auto &Builder = CGF.Builder;869  auto Scopes = ScopeModel->getRuntimeValues();870  llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;871  for (auto S : Scopes)872    BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);873 874  llvm::BasicBlock *ContBB =875      CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);876 877  auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);878  // If unsupported sync scope is encountered at run time, assume a fallback879  // sync scope value.880  auto FallBack = ScopeModel->getFallBackValue();881  llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);882  for (auto S : Scopes) {883    auto *B = BB[S];884    if (S != FallBack)885      SI->addCase(Builder.getInt32(S), B);886 887    Builder.SetInsertPoint(B);888    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,889                 FailureOrder, Size, Order,890                 CGF.getTargetHooks().getLLVMSyncScopeID(891                     CGF.CGM.getLangOpts(), ScopeModel->map(S), Order,892                     CGF.getLLVMContext()));893    Builder.CreateBr(ContBB);894  }895 896  Builder.SetInsertPoint(ContBB);897}898 899RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {900  ApplyAtomGroup Grp(getDebugInfo());901 902  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();903  QualType MemTy = AtomicTy;904  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())905    MemTy = AT->getValueType();906  llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;907 908  Address Val1 = Address::invalid();909  Address Val2 = Address::invalid();910  Address Dest = Address::invalid();911  Address Ptr = EmitPointerWithAlignment(E->getPtr());912 913  if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||914      E->getOp() == AtomicExpr::AO__opencl_atomic_init) {915    LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);916    EmitAtomicInit(E->getVal1(), lvalue);917    return RValue::get(nullptr);918  }919 920  auto TInfo = getContext().getTypeInfoInChars(AtomicTy);921  uint64_t Size = TInfo.Width.getQuantity();922  unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();923 924  CharUnits MaxInlineWidth =925      getContext().toCharUnitsFromBits(MaxInlineWidthInBits);926  DiagnosticsEngine &Diags = CGM.getDiags();927  bool Misaligned = !Ptr.getAlignment().isMultipleOf(TInfo.Width);928  bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;929  if (Misaligned) {930    Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)931        << (int)TInfo.Width.getQuantity()932        << (int)Ptr.getAlignment().getQuantity();933  }934  if (Oversized) {935    Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)936        << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();937  }938 939  llvm::Value *Order = EmitScalarExpr(E->getOrder());940  llvm::Value *Scope =941      E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;942  bool ShouldCastToIntPtrTy = true;943 944  switch (E->getOp()) {945  case AtomicExpr::AO__c11_atomic_init:946  case AtomicExpr::AO__opencl_atomic_init:947    llvm_unreachable("Already handled above with EmitAtomicInit!");948 949  case AtomicExpr::AO__atomic_load_n:950  case AtomicExpr::AO__scoped_atomic_load_n:951  case AtomicExpr::AO__c11_atomic_load:952  case AtomicExpr::AO__opencl_atomic_load:953  case AtomicExpr::AO__hip_atomic_load:954  case AtomicExpr::AO__atomic_test_and_set:955  case AtomicExpr::AO__atomic_clear:956    break;957 958  case AtomicExpr::AO__atomic_load:959  case AtomicExpr::AO__scoped_atomic_load:960    Dest = EmitPointerWithAlignment(E->getVal1());961    break;962 963  case AtomicExpr::AO__atomic_store:964  case AtomicExpr::AO__scoped_atomic_store:965    Val1 = EmitPointerWithAlignment(E->getVal1());966    break;967 968  case AtomicExpr::AO__atomic_exchange:969  case AtomicExpr::AO__scoped_atomic_exchange:970    Val1 = EmitPointerWithAlignment(E->getVal1());971    Dest = EmitPointerWithAlignment(E->getVal2());972    break;973 974  case AtomicExpr::AO__atomic_compare_exchange:975  case AtomicExpr::AO__atomic_compare_exchange_n:976  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:977  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:978  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:979  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:980  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:981  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:982  case AtomicExpr::AO__scoped_atomic_compare_exchange:983  case AtomicExpr::AO__scoped_atomic_compare_exchange_n:984    Val1 = EmitPointerWithAlignment(E->getVal1());985    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||986        E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)987      Val2 = EmitPointerWithAlignment(E->getVal2());988    else989      Val2 = EmitValToTemp(*this, E->getVal2());990    OrderFail = EmitScalarExpr(E->getOrderFail());991    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||992        E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||993        E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n ||994        E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)995      IsWeak = EmitScalarExpr(E->getWeak());996    break;997 998  case AtomicExpr::AO__c11_atomic_fetch_add:999  case AtomicExpr::AO__c11_atomic_fetch_sub:1000  case AtomicExpr::AO__hip_atomic_fetch_add:1001  case AtomicExpr::AO__hip_atomic_fetch_sub:1002  case AtomicExpr::AO__opencl_atomic_fetch_add:1003  case AtomicExpr::AO__opencl_atomic_fetch_sub:1004    if (MemTy->isPointerType()) {1005      // For pointer arithmetic, we're required to do a bit of math:1006      // adding 1 to an int* is not the same as adding 1 to a uintptr_t.1007      // ... but only for the C11 builtins. The GNU builtins expect the1008      // user to multiply by sizeof(T).1009      QualType Val1Ty = E->getVal1()->getType();1010      llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());1011      CharUnits PointeeIncAmt =1012          getContext().getTypeSizeInChars(MemTy->getPointeeType());1013      Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));1014      auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");1015      Val1 = Temp;1016      EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));1017      break;1018    }1019    [[fallthrough]];1020  case AtomicExpr::AO__atomic_fetch_add:1021  case AtomicExpr::AO__atomic_fetch_max:1022  case AtomicExpr::AO__atomic_fetch_min:1023  case AtomicExpr::AO__atomic_fetch_sub:1024  case AtomicExpr::AO__atomic_add_fetch:1025  case AtomicExpr::AO__atomic_max_fetch:1026  case AtomicExpr::AO__atomic_min_fetch:1027  case AtomicExpr::AO__atomic_sub_fetch:1028  case AtomicExpr::AO__c11_atomic_fetch_max:1029  case AtomicExpr::AO__c11_atomic_fetch_min:1030  case AtomicExpr::AO__opencl_atomic_fetch_max:1031  case AtomicExpr::AO__opencl_atomic_fetch_min:1032  case AtomicExpr::AO__hip_atomic_fetch_max:1033  case AtomicExpr::AO__hip_atomic_fetch_min:1034  case AtomicExpr::AO__scoped_atomic_fetch_add:1035  case AtomicExpr::AO__scoped_atomic_fetch_max:1036  case AtomicExpr::AO__scoped_atomic_fetch_min:1037  case AtomicExpr::AO__scoped_atomic_fetch_sub:1038  case AtomicExpr::AO__scoped_atomic_add_fetch:1039  case AtomicExpr::AO__scoped_atomic_max_fetch:1040  case AtomicExpr::AO__scoped_atomic_min_fetch:1041  case AtomicExpr::AO__scoped_atomic_sub_fetch:1042    ShouldCastToIntPtrTy = !MemTy->isFloatingType();1043    [[fallthrough]];1044 1045  case AtomicExpr::AO__atomic_fetch_and:1046  case AtomicExpr::AO__atomic_fetch_nand:1047  case AtomicExpr::AO__atomic_fetch_or:1048  case AtomicExpr::AO__atomic_fetch_xor:1049  case AtomicExpr::AO__atomic_and_fetch:1050  case AtomicExpr::AO__atomic_nand_fetch:1051  case AtomicExpr::AO__atomic_or_fetch:1052  case AtomicExpr::AO__atomic_xor_fetch:1053  case AtomicExpr::AO__atomic_store_n:1054  case AtomicExpr::AO__atomic_exchange_n:1055  case AtomicExpr::AO__c11_atomic_fetch_and:1056  case AtomicExpr::AO__c11_atomic_fetch_nand:1057  case AtomicExpr::AO__c11_atomic_fetch_or:1058  case AtomicExpr::AO__c11_atomic_fetch_xor:1059  case AtomicExpr::AO__c11_atomic_store:1060  case AtomicExpr::AO__c11_atomic_exchange:1061  case AtomicExpr::AO__hip_atomic_fetch_and:1062  case AtomicExpr::AO__hip_atomic_fetch_or:1063  case AtomicExpr::AO__hip_atomic_fetch_xor:1064  case AtomicExpr::AO__hip_atomic_store:1065  case AtomicExpr::AO__hip_atomic_exchange:1066  case AtomicExpr::AO__opencl_atomic_fetch_and:1067  case AtomicExpr::AO__opencl_atomic_fetch_or:1068  case AtomicExpr::AO__opencl_atomic_fetch_xor:1069  case AtomicExpr::AO__opencl_atomic_store:1070  case AtomicExpr::AO__opencl_atomic_exchange:1071  case AtomicExpr::AO__scoped_atomic_fetch_and:1072  case AtomicExpr::AO__scoped_atomic_fetch_nand:1073  case AtomicExpr::AO__scoped_atomic_fetch_or:1074  case AtomicExpr::AO__scoped_atomic_fetch_xor:1075  case AtomicExpr::AO__scoped_atomic_and_fetch:1076  case AtomicExpr::AO__scoped_atomic_nand_fetch:1077  case AtomicExpr::AO__scoped_atomic_or_fetch:1078  case AtomicExpr::AO__scoped_atomic_xor_fetch:1079  case AtomicExpr::AO__scoped_atomic_store_n:1080  case AtomicExpr::AO__scoped_atomic_exchange_n:1081  case AtomicExpr::AO__scoped_atomic_uinc_wrap:1082  case AtomicExpr::AO__scoped_atomic_udec_wrap:1083    Val1 = EmitValToTemp(*this, E->getVal1());1084    break;1085  }1086 1087  QualType RValTy = E->getType().getUnqualifiedType();1088 1089  // The inlined atomics only function on iN types, where N is a power of 2. We1090  // need to make sure (via temporaries if necessary) that all incoming values1091  // are compatible.1092  LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);1093  AtomicInfo Atomics(*this, AtomicVal);1094 1095  Address OriginalVal1 = Val1;1096  if (ShouldCastToIntPtrTy) {1097    Ptr = Atomics.castToAtomicIntPointer(Ptr);1098    if (Val1.isValid())1099      Val1 = Atomics.convertToAtomicIntPointer(Val1);1100    if (Val2.isValid())1101      Val2 = Atomics.convertToAtomicIntPointer(Val2);1102  }1103  if (Dest.isValid()) {1104    if (ShouldCastToIntPtrTy)1105      Dest = Atomics.castToAtomicIntPointer(Dest);1106  } else if (E->isCmpXChg())1107    Dest = CreateMemTemp(RValTy, "cmpxchg.bool");1108  else if (!RValTy->isVoidType()) {1109    Dest = Atomics.CreateTempAlloca();1110    if (ShouldCastToIntPtrTy)1111      Dest = Atomics.castToAtomicIntPointer(Dest);1112  }1113 1114  bool PowerOf2Size = (Size & (Size - 1)) == 0;1115  bool UseLibcall = !PowerOf2Size || (Size > 16);1116 1117  // For atomics larger than 16 bytes, emit a libcall from the frontend. This1118  // avoids the overhead of dealing with excessively-large value types in IR.1119  // Non-power-of-2 values also lower to libcall here, as they are not currently1120  // permitted in IR instructions (although that constraint could be relaxed in1121  // the future). For other cases where a libcall is required on a given1122  // platform, we let the backend handle it (this includes handling for all of1123  // the size-optimized libcall variants, which are only valid up to 16 bytes.)1124  //1125  // See: https://llvm.org/docs/Atomics.html#libcalls-atomic1126  if (UseLibcall) {1127    CallArgList Args;1128    // For non-optimized library calls, the size is the first parameter.1129    Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),1130             getContext().getSizeType());1131 1132    // The atomic address is the second parameter.1133    // The OpenCL atomic library functions only accept pointer arguments to1134    // generic address space.1135    auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {1136      if (!E->isOpenCL())1137        return V;1138      auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();1139      if (AS == LangAS::opencl_generic)1140        return V;1141      auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);1142      auto *DestType = llvm::PointerType::get(getLLVMContext(), DestAS);1143 1144      return getTargetHooks().performAddrSpaceCast(*this, V, AS, DestType,1145                                                   false);1146    };1147 1148    Args.add(RValue::get(CastToGenericAddrSpace(Ptr.emitRawPointer(*this),1149                                                E->getPtr()->getType())),1150             getContext().VoidPtrTy);1151 1152    // The next 1-3 parameters are op-dependent.1153    std::string LibCallName;1154    QualType RetTy;1155    bool HaveRetTy = false;1156    switch (E->getOp()) {1157    case AtomicExpr::AO__c11_atomic_init:1158    case AtomicExpr::AO__opencl_atomic_init:1159      llvm_unreachable("Already handled!");1160 1161    // There is only one libcall for compare an exchange, because there is no1162    // optimisation benefit possible from a libcall version of a weak compare1163    // and exchange.1164    // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,1165    //                                void *desired, int success, int failure)1166    case AtomicExpr::AO__atomic_compare_exchange:1167    case AtomicExpr::AO__atomic_compare_exchange_n:1168    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:1169    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:1170    case AtomicExpr::AO__hip_atomic_compare_exchange_weak:1171    case AtomicExpr::AO__hip_atomic_compare_exchange_strong:1172    case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:1173    case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:1174    case AtomicExpr::AO__scoped_atomic_compare_exchange:1175    case AtomicExpr::AO__scoped_atomic_compare_exchange_n:1176      LibCallName = "__atomic_compare_exchange";1177      RetTy = getContext().BoolTy;1178      HaveRetTy = true;1179      Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this),1180                                                  E->getVal1()->getType())),1181               getContext().VoidPtrTy);1182      Args.add(RValue::get(CastToGenericAddrSpace(Val2.emitRawPointer(*this),1183                                                  E->getVal2()->getType())),1184               getContext().VoidPtrTy);1185      Args.add(RValue::get(Order), getContext().IntTy);1186      Order = OrderFail;1187      break;1188    // void __atomic_exchange(size_t size, void *mem, void *val, void *return,1189    //                        int order)1190    case AtomicExpr::AO__atomic_exchange:1191    case AtomicExpr::AO__atomic_exchange_n:1192    case AtomicExpr::AO__c11_atomic_exchange:1193    case AtomicExpr::AO__hip_atomic_exchange:1194    case AtomicExpr::AO__opencl_atomic_exchange:1195    case AtomicExpr::AO__scoped_atomic_exchange:1196    case AtomicExpr::AO__scoped_atomic_exchange_n:1197      LibCallName = "__atomic_exchange";1198      Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this),1199                                                  E->getVal1()->getType())),1200               getContext().VoidPtrTy);1201      break;1202    // void __atomic_store(size_t size, void *mem, void *val, int order)1203    case AtomicExpr::AO__atomic_store:1204    case AtomicExpr::AO__atomic_store_n:1205    case AtomicExpr::AO__c11_atomic_store:1206    case AtomicExpr::AO__hip_atomic_store:1207    case AtomicExpr::AO__opencl_atomic_store:1208    case AtomicExpr::AO__scoped_atomic_store:1209    case AtomicExpr::AO__scoped_atomic_store_n:1210      LibCallName = "__atomic_store";1211      RetTy = getContext().VoidTy;1212      HaveRetTy = true;1213      Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this),1214                                                  E->getVal1()->getType())),1215               getContext().VoidPtrTy);1216      break;1217    // void __atomic_load(size_t size, void *mem, void *return, int order)1218    case AtomicExpr::AO__atomic_load:1219    case AtomicExpr::AO__atomic_load_n:1220    case AtomicExpr::AO__c11_atomic_load:1221    case AtomicExpr::AO__hip_atomic_load:1222    case AtomicExpr::AO__opencl_atomic_load:1223    case AtomicExpr::AO__scoped_atomic_load:1224    case AtomicExpr::AO__scoped_atomic_load_n:1225      LibCallName = "__atomic_load";1226      break;1227    case AtomicExpr::AO__atomic_add_fetch:1228    case AtomicExpr::AO__scoped_atomic_add_fetch:1229    case AtomicExpr::AO__atomic_fetch_add:1230    case AtomicExpr::AO__c11_atomic_fetch_add:1231    case AtomicExpr::AO__hip_atomic_fetch_add:1232    case AtomicExpr::AO__opencl_atomic_fetch_add:1233    case AtomicExpr::AO__scoped_atomic_fetch_add:1234    case AtomicExpr::AO__atomic_and_fetch:1235    case AtomicExpr::AO__scoped_atomic_and_fetch:1236    case AtomicExpr::AO__atomic_fetch_and:1237    case AtomicExpr::AO__c11_atomic_fetch_and:1238    case AtomicExpr::AO__hip_atomic_fetch_and:1239    case AtomicExpr::AO__opencl_atomic_fetch_and:1240    case AtomicExpr::AO__scoped_atomic_fetch_and:1241    case AtomicExpr::AO__atomic_or_fetch:1242    case AtomicExpr::AO__scoped_atomic_or_fetch:1243    case AtomicExpr::AO__atomic_fetch_or:1244    case AtomicExpr::AO__c11_atomic_fetch_or:1245    case AtomicExpr::AO__hip_atomic_fetch_or:1246    case AtomicExpr::AO__opencl_atomic_fetch_or:1247    case AtomicExpr::AO__scoped_atomic_fetch_or:1248    case AtomicExpr::AO__atomic_sub_fetch:1249    case AtomicExpr::AO__scoped_atomic_sub_fetch:1250    case AtomicExpr::AO__atomic_fetch_sub:1251    case AtomicExpr::AO__c11_atomic_fetch_sub:1252    case AtomicExpr::AO__hip_atomic_fetch_sub:1253    case AtomicExpr::AO__opencl_atomic_fetch_sub:1254    case AtomicExpr::AO__scoped_atomic_fetch_sub:1255    case AtomicExpr::AO__atomic_xor_fetch:1256    case AtomicExpr::AO__scoped_atomic_xor_fetch:1257    case AtomicExpr::AO__atomic_fetch_xor:1258    case AtomicExpr::AO__c11_atomic_fetch_xor:1259    case AtomicExpr::AO__hip_atomic_fetch_xor:1260    case AtomicExpr::AO__opencl_atomic_fetch_xor:1261    case AtomicExpr::AO__scoped_atomic_fetch_xor:1262    case AtomicExpr::AO__atomic_nand_fetch:1263    case AtomicExpr::AO__atomic_fetch_nand:1264    case AtomicExpr::AO__c11_atomic_fetch_nand:1265    case AtomicExpr::AO__scoped_atomic_fetch_nand:1266    case AtomicExpr::AO__scoped_atomic_nand_fetch:1267    case AtomicExpr::AO__atomic_min_fetch:1268    case AtomicExpr::AO__atomic_fetch_min:1269    case AtomicExpr::AO__c11_atomic_fetch_min:1270    case AtomicExpr::AO__hip_atomic_fetch_min:1271    case AtomicExpr::AO__opencl_atomic_fetch_min:1272    case AtomicExpr::AO__scoped_atomic_fetch_min:1273    case AtomicExpr::AO__scoped_atomic_min_fetch:1274    case AtomicExpr::AO__atomic_max_fetch:1275    case AtomicExpr::AO__atomic_fetch_max:1276    case AtomicExpr::AO__c11_atomic_fetch_max:1277    case AtomicExpr::AO__hip_atomic_fetch_max:1278    case AtomicExpr::AO__opencl_atomic_fetch_max:1279    case AtomicExpr::AO__scoped_atomic_fetch_max:1280    case AtomicExpr::AO__scoped_atomic_max_fetch:1281    case AtomicExpr::AO__scoped_atomic_uinc_wrap:1282    case AtomicExpr::AO__scoped_atomic_udec_wrap:1283    case AtomicExpr::AO__atomic_test_and_set:1284    case AtomicExpr::AO__atomic_clear:1285      llvm_unreachable("Integral atomic operations always become atomicrmw!");1286    }1287 1288    if (E->isOpenCL()) {1289      LibCallName =1290          std::string("__opencl") + StringRef(LibCallName).drop_front(1).str();1291    }1292    // By default, assume we return a value of the atomic type.1293    if (!HaveRetTy) {1294      // Value is returned through parameter before the order.1295      RetTy = getContext().VoidTy;1296      Args.add(RValue::get(1297                   CastToGenericAddrSpace(Dest.emitRawPointer(*this), RetTy)),1298               getContext().VoidPtrTy);1299    }1300    // Order is always the last parameter.1301    Args.add(RValue::get(Order),1302             getContext().IntTy);1303    if (E->isOpenCL())1304      Args.add(RValue::get(Scope), getContext().IntTy);1305 1306    RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);1307    // The value is returned directly from the libcall.1308    if (E->isCmpXChg())1309      return Res;1310 1311    if (RValTy->isVoidType())1312      return RValue::get(nullptr);1313 1314    return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)),1315                               RValTy, E->getExprLoc());1316  }1317 1318  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||1319                 E->getOp() == AtomicExpr::AO__opencl_atomic_store ||1320                 E->getOp() == AtomicExpr::AO__hip_atomic_store ||1321                 E->getOp() == AtomicExpr::AO__atomic_store ||1322                 E->getOp() == AtomicExpr::AO__atomic_store_n ||1323                 E->getOp() == AtomicExpr::AO__scoped_atomic_store ||1324                 E->getOp() == AtomicExpr::AO__scoped_atomic_store_n ||1325                 E->getOp() == AtomicExpr::AO__atomic_clear;1326  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||1327                E->getOp() == AtomicExpr::AO__opencl_atomic_load ||1328                E->getOp() == AtomicExpr::AO__hip_atomic_load ||1329                E->getOp() == AtomicExpr::AO__atomic_load ||1330                E->getOp() == AtomicExpr::AO__atomic_load_n ||1331                E->getOp() == AtomicExpr::AO__scoped_atomic_load ||1332                E->getOp() == AtomicExpr::AO__scoped_atomic_load_n;1333 1334  if (isa<llvm::ConstantInt>(Order)) {1335    auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();1336    // We should not ever get to a case where the ordering isn't a valid C ABI1337    // value, but it's hard to enforce that in general.1338    if (llvm::isValidAtomicOrderingCABI(ord))1339      switch ((llvm::AtomicOrderingCABI)ord) {1340      case llvm::AtomicOrderingCABI::relaxed:1341        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,1342                     OrderFail, Size, llvm::AtomicOrdering::Monotonic, Scope);1343        break;1344      case llvm::AtomicOrderingCABI::consume:1345      case llvm::AtomicOrderingCABI::acquire:1346        if (IsStore)1347          break; // Avoid crashing on code with undefined behavior1348        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,1349                     OrderFail, Size, llvm::AtomicOrdering::Acquire, Scope);1350        break;1351      case llvm::AtomicOrderingCABI::release:1352        if (IsLoad)1353          break; // Avoid crashing on code with undefined behavior1354        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,1355                     OrderFail, Size, llvm::AtomicOrdering::Release, Scope);1356        break;1357      case llvm::AtomicOrderingCABI::acq_rel:1358        if (IsLoad || IsStore)1359          break; // Avoid crashing on code with undefined behavior1360        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,1361                     OrderFail, Size, llvm::AtomicOrdering::AcquireRelease,1362                     Scope);1363        break;1364      case llvm::AtomicOrderingCABI::seq_cst:1365        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,1366                     OrderFail, Size,1367                     llvm::AtomicOrdering::SequentiallyConsistent, Scope);1368        break;1369      }1370    if (RValTy->isVoidType())1371      return RValue::get(nullptr);1372 1373    return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)),1374                               RValTy, E->getExprLoc());1375  }1376 1377  // Long case, when Order isn't obviously constant.1378 1379  // Create all the relevant BB's1380  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,1381                   *ReleaseBB = nullptr, *AcqRelBB = nullptr,1382                   *SeqCstBB = nullptr;1383  MonotonicBB = createBasicBlock("monotonic", CurFn);1384  if (!IsStore)1385    AcquireBB = createBasicBlock("acquire", CurFn);1386  if (!IsLoad)1387    ReleaseBB = createBasicBlock("release", CurFn);1388  if (!IsLoad && !IsStore)1389    AcqRelBB = createBasicBlock("acqrel", CurFn);1390  SeqCstBB = createBasicBlock("seqcst", CurFn);1391  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);1392 1393  // Create the switch for the split1394  // MonotonicBB is arbitrarily chosen as the default case; in practice, this1395  // doesn't matter unless someone is crazy enough to use something that1396  // doesn't fold to a constant for the ordering.1397  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);1398  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);1399 1400  // Emit all the different atomics1401  Builder.SetInsertPoint(MonotonicBB);1402  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, OrderFail,1403               Size, llvm::AtomicOrdering::Monotonic, Scope);1404  Builder.CreateBr(ContBB);1405  if (!IsStore) {1406    Builder.SetInsertPoint(AcquireBB);1407    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,1408                 OrderFail, Size, llvm::AtomicOrdering::Acquire, Scope);1409    Builder.CreateBr(ContBB);1410    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),1411                AcquireBB);1412    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),1413                AcquireBB);1414  }1415  if (!IsLoad) {1416    Builder.SetInsertPoint(ReleaseBB);1417    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,1418                 OrderFail, Size, llvm::AtomicOrdering::Release, Scope);1419    Builder.CreateBr(ContBB);1420    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),1421                ReleaseBB);1422  }1423  if (!IsLoad && !IsStore) {1424    Builder.SetInsertPoint(AcqRelBB);1425    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,1426                 OrderFail, Size, llvm::AtomicOrdering::AcquireRelease, Scope);1427    Builder.CreateBr(ContBB);1428    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),1429                AcqRelBB);1430  }1431  Builder.SetInsertPoint(SeqCstBB);1432  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, OrderFail,1433               Size, llvm::AtomicOrdering::SequentiallyConsistent, Scope);1434  Builder.CreateBr(ContBB);1435  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),1436              SeqCstBB);1437 1438  // Cleanup and return1439  Builder.SetInsertPoint(ContBB);1440  if (RValTy->isVoidType())1441    return RValue::get(nullptr);1442 1443  assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());1444  return convertTempToRValue(Dest.withElementType(ConvertTypeForMem(RValTy)),1445                             RValTy, E->getExprLoc());1446}1447 1448Address AtomicInfo::castToAtomicIntPointer(Address addr) const {1449  llvm::IntegerType *ty =1450    llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);1451  return addr.withElementType(ty);1452}1453 1454Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {1455  llvm::Type *Ty = Addr.getElementType();1456  uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);1457  if (SourceSizeInBits != AtomicSizeInBits) {1458    Address Tmp = CreateTempAlloca();1459    CGF.Builder.CreateMemSet(1460        Tmp.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.Int8Ty, 0),1461        CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),1462        Tmp.getAlignment().getAsAlign());1463 1464    CGF.Builder.CreateMemCpy(Tmp, Addr,1465                             std::min(AtomicSizeInBits, SourceSizeInBits) / 8);1466    Addr = Tmp;1467  }1468 1469  return castToAtomicIntPointer(Addr);1470}1471 1472RValue AtomicInfo::convertAtomicTempToRValue(Address addr,1473                                             AggValueSlot resultSlot,1474                                             SourceLocation loc,1475                                             bool asValue) const {1476  if (LVal.isSimple()) {1477    if (EvaluationKind == TEK_Aggregate)1478      return resultSlot.asRValue();1479 1480    // Drill into the padding structure if we have one.1481    if (hasPadding())1482      addr = CGF.Builder.CreateStructGEP(addr, 0);1483 1484    // Otherwise, just convert the temporary to an r-value using the1485    // normal conversion routine.1486    return CGF.convertTempToRValue(addr, getValueType(), loc);1487  }1488  if (!asValue)1489    // Get RValue from temp memory as atomic for non-simple lvalues1490    return RValue::get(CGF.Builder.CreateLoad(addr));1491  if (LVal.isBitField())1492    return CGF.EmitLoadOfBitfieldLValue(1493        LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),1494                             LVal.getBaseInfo(), TBAAAccessInfo()), loc);1495  if (LVal.isVectorElt())1496    return CGF.EmitLoadOfLValue(1497        LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),1498                              LVal.getBaseInfo(), TBAAAccessInfo()), loc);1499  assert(LVal.isExtVectorElt());1500  return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(1501      addr, LVal.getExtVectorElts(), LVal.getType(),1502      LVal.getBaseInfo(), TBAAAccessInfo()));1503}1504 1505/// Return true if \param ValTy is a type that should be casted to integer1506/// around the atomic memory operation. If \param CmpXchg is true, then the1507/// cast of a floating point type is made as that instruction can not have1508/// floating point operands.  TODO: Allow compare-and-exchange and FP - see1509/// comment in AtomicExpandPass.cpp.1510static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) {1511  if (ValTy->isFloatingPointTy())1512    return ValTy->isX86_FP80Ty() || CmpXchg;1513  return !ValTy->isIntegerTy() && !ValTy->isPointerTy();1514}1515 1516RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val,1517                                          AggValueSlot ResultSlot,1518                                          SourceLocation Loc, bool AsValue,1519                                          bool CmpXchg) const {1520  // Try not to in some easy cases.1521  assert((Val->getType()->isIntegerTy() || Val->getType()->isPointerTy() ||1522          Val->getType()->isIEEELikeFPTy()) &&1523         "Expected integer, pointer or floating point value when converting "1524         "result.");1525  if (getEvaluationKind() == TEK_Scalar &&1526      (((!LVal.isBitField() ||1527         LVal.getBitFieldInfo().Size == ValueSizeInBits) &&1528        !hasPadding()) ||1529       !AsValue)) {1530    auto *ValTy = AsValue1531                      ? CGF.ConvertTypeForMem(ValueTy)1532                      : getAtomicAddress().getElementType();1533    if (!shouldCastToInt(ValTy, CmpXchg)) {1534      assert((!ValTy->isIntegerTy() || Val->getType() == ValTy) &&1535             "Different integer types.");1536      return RValue::get(CGF.EmitFromMemory(Val, ValueTy));1537    }1538    if (llvm::CastInst::isBitCastable(Val->getType(), ValTy))1539      return RValue::get(CGF.Builder.CreateBitCast(Val, ValTy));1540  }1541 1542  // Create a temporary.  This needs to be big enough to hold the1543  // atomic integer.1544  Address Temp = Address::invalid();1545  bool TempIsVolatile = false;1546  if (AsValue && getEvaluationKind() == TEK_Aggregate) {1547    assert(!ResultSlot.isIgnored());1548    Temp = ResultSlot.getAddress();1549    TempIsVolatile = ResultSlot.isVolatile();1550  } else {1551    Temp = CreateTempAlloca();1552  }1553 1554  // Slam the integer into the temporary.1555  Address CastTemp = castToAtomicIntPointer(Temp);1556  CGF.Builder.CreateStore(Val, CastTemp)->setVolatile(TempIsVolatile);1557 1558  return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);1559}1560 1561void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,1562                                       llvm::AtomicOrdering AO, bool) {1563  // void __atomic_load(size_t size, void *mem, void *return, int order);1564  CallArgList Args;1565  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());1566  Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy);1567  Args.add(RValue::get(AddForLoaded), CGF.getContext().VoidPtrTy);1568  Args.add(1569      RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),1570      CGF.getContext().IntTy);1571  emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);1572}1573 1574llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,1575                                          bool IsVolatile, bool CmpXchg) {1576  // Okay, we're doing this natively.1577  Address Addr = getAtomicAddress();1578  if (shouldCastToInt(Addr.getElementType(), CmpXchg))1579    Addr = castToAtomicIntPointer(Addr);1580  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");1581  Load->setAtomic(AO);1582 1583  // Other decoration.1584  if (IsVolatile)1585    Load->setVolatile(true);1586  CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());1587  return Load;1588}1589 1590/// An LValue is a candidate for having its loads and stores be made atomic if1591/// we are operating under /volatile:ms *and* the LValue itself is volatile and1592/// performing such an operation can be performed without a libcall.1593bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {1594  if (!CGM.getLangOpts().MSVolatile) return false;1595  AtomicInfo AI(*this, LV);1596  bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());1597  // An atomic is inline if we don't need to use a libcall.1598  bool AtomicIsInline = !AI.shouldUseLibcall();1599  // MSVC doesn't seem to do this for types wider than a pointer.1600  if (getContext().getTypeSize(LV.getType()) >1601      getContext().getTypeSize(getContext().getIntPtrType()))1602    return false;1603  return IsVolatile && AtomicIsInline;1604}1605 1606RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,1607                                       AggValueSlot Slot) {1608  llvm::AtomicOrdering AO;1609  bool IsVolatile = LV.isVolatileQualified();1610  if (LV.getType()->isAtomicType()) {1611    AO = llvm::AtomicOrdering::SequentiallyConsistent;1612  } else {1613    AO = llvm::AtomicOrdering::Acquire;1614    IsVolatile = true;1615  }1616  return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);1617}1618 1619RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,1620                                  bool AsValue, llvm::AtomicOrdering AO,1621                                  bool IsVolatile) {1622  // Check whether we should use a library call.1623  if (shouldUseLibcall()) {1624    Address TempAddr = Address::invalid();1625    if (LVal.isSimple() && !ResultSlot.isIgnored()) {1626      assert(getEvaluationKind() == TEK_Aggregate);1627      TempAddr = ResultSlot.getAddress();1628    } else1629      TempAddr = CreateTempAlloca();1630 1631    EmitAtomicLoadLibcall(TempAddr.emitRawPointer(CGF), AO, IsVolatile);1632 1633    // Okay, turn that back into the original value or whole atomic (for1634    // non-simple lvalues) type.1635    return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);1636  }1637 1638  // Okay, we're doing this natively.1639  auto *Load = EmitAtomicLoadOp(AO, IsVolatile);1640 1641  // If we're ignoring an aggregate return, don't do anything.1642  if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())1643    return RValue::getAggregate(Address::invalid(), false);1644 1645  // Okay, turn that back into the original value or atomic (for non-simple1646  // lvalues) type.1647  return ConvertToValueOrAtomic(Load, ResultSlot, Loc, AsValue);1648}1649 1650/// Emit a load from an l-value of atomic type.  Note that the r-value1651/// we produce is an r-value of the atomic *value* type.1652RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,1653                                       llvm::AtomicOrdering AO, bool IsVolatile,1654                                       AggValueSlot resultSlot) {1655  AtomicInfo Atomics(*this, src);1656  return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,1657                                IsVolatile);1658}1659 1660/// Copy an r-value into memory as part of storing to an atomic type.1661/// This needs to create a bit-pattern suitable for atomic operations.1662void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {1663  assert(LVal.isSimple());1664  // If we have an r-value, the rvalue should be of the atomic type,1665  // which means that the caller is responsible for having zeroed1666  // any padding.  Just do an aggregate copy of that type.1667  if (rvalue.isAggregate()) {1668    LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());1669    LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),1670                                    getAtomicType());1671    bool IsVolatile = rvalue.isVolatileQualified() ||1672                      LVal.isVolatileQualified();1673    CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),1674                          AggValueSlot::DoesNotOverlap, IsVolatile);1675    return;1676  }1677 1678  // Okay, otherwise we're copying stuff.1679 1680  // Zero out the buffer if necessary.1681  emitMemSetZeroIfNecessary();1682 1683  // Drill past the padding if present.1684  LValue TempLVal = projectValue();1685 1686  // Okay, store the rvalue in.1687  if (rvalue.isScalar()) {1688    CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);1689  } else {1690    CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);1691  }1692}1693 1694 1695/// Materialize an r-value into memory for the purposes of storing it1696/// to an atomic type.1697Address AtomicInfo::materializeRValue(RValue rvalue) const {1698  // Aggregate r-values are already in memory, and EmitAtomicStore1699  // requires them to be values of the atomic type.1700  if (rvalue.isAggregate())1701    return rvalue.getAggregateAddress();1702 1703  // Otherwise, make a temporary and materialize into it.1704  LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());1705  AtomicInfo Atomics(CGF, TempLV);1706  Atomics.emitCopyIntoMemory(rvalue);1707  return TempLV.getAddress();1708}1709 1710llvm::Value *AtomicInfo::getScalarRValValueOrNull(RValue RVal) const {1711  if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple()))1712    return RVal.getScalarVal();1713  return nullptr;1714}1715 1716llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CmpXchg) const {1717  // If we've got a scalar value of the right size, try to avoid going1718  // through memory. Floats get casted if needed by AtomicExpandPass.1719  if (llvm::Value *Value = getScalarRValValueOrNull(RVal)) {1720    if (!shouldCastToInt(Value->getType(), CmpXchg))1721      return CGF.EmitToMemory(Value, ValueTy);1722    else {1723      llvm::IntegerType *InputIntTy = llvm::IntegerType::get(1724          CGF.getLLVMContext(),1725          LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());1726      if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))1727        return CGF.Builder.CreateBitCast(Value, InputIntTy);1728    }1729  }1730  // Otherwise, we need to go through memory.1731  // Put the r-value in memory.1732  Address Addr = materializeRValue(RVal);1733 1734  // Cast the temporary to the atomic int type and pull a value out.1735  Addr = castToAtomicIntPointer(Addr);1736  return CGF.Builder.CreateLoad(Addr);1737}1738 1739std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(1740    llvm::Value *ExpectedVal, llvm::Value *DesiredVal,1741    llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {1742  // Do the atomic store.1743  Address Addr = getAtomicAddressAsAtomicIntPointer();1744  auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,1745                                               Success, Failure);1746  // Other decoration.1747  Inst->setVolatile(LVal.isVolatileQualified());1748  Inst->setWeak(IsWeak);1749 1750  // Okay, turn that back into the original value type.1751  auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);1752  auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);1753  return std::make_pair(PreviousVal, SuccessFailureVal);1754}1755 1756llvm::Value *1757AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,1758                                             llvm::Value *DesiredAddr,1759                                             llvm::AtomicOrdering Success,1760                                             llvm::AtomicOrdering Failure) {1761  // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,1762  // void *desired, int success, int failure);1763  CallArgList Args;1764  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());1765  Args.add(RValue::get(getAtomicPointer()), CGF.getContext().VoidPtrTy);1766  Args.add(RValue::get(ExpectedAddr), CGF.getContext().VoidPtrTy);1767  Args.add(RValue::get(DesiredAddr), CGF.getContext().VoidPtrTy);1768  Args.add(RValue::get(1769               llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),1770           CGF.getContext().IntTy);1771  Args.add(RValue::get(1772               llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),1773           CGF.getContext().IntTy);1774  auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",1775                                              CGF.getContext().BoolTy, Args);1776 1777  return SuccessFailureRVal.getScalarVal();1778}1779 1780std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(1781    RValue Expected, RValue Desired, llvm::AtomicOrdering Success,1782    llvm::AtomicOrdering Failure, bool IsWeak) {1783  // Check whether we should use a library call.1784  if (shouldUseLibcall()) {1785    // Produce a source address.1786    Address ExpectedAddr = materializeRValue(Expected);1787    llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);1788    llvm::Value *DesiredPtr = materializeRValue(Desired).emitRawPointer(CGF);1789    auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr,1790                                                 Success, Failure);1791    return std::make_pair(1792        convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),1793                                  SourceLocation(), /*AsValue=*/false),1794        Res);1795  }1796 1797  // If we've got a scalar value of the right size, try to avoid going1798  // through memory.1799  auto *ExpectedVal = convertRValueToInt(Expected, /*CmpXchg=*/true);1800  auto *DesiredVal = convertRValueToInt(Desired, /*CmpXchg=*/true);1801  auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,1802                                         Failure, IsWeak);1803  return std::make_pair(1804      ConvertToValueOrAtomic(Res.first, AggValueSlot::ignored(),1805                             SourceLocation(), /*AsValue=*/false,1806                             /*CmpXchg=*/true),1807      Res.second);1808}1809 1810static void1811EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,1812                      const llvm::function_ref<RValue(RValue)> &UpdateOp,1813                      Address DesiredAddr) {1814  RValue UpRVal;1815  LValue AtomicLVal = Atomics.getAtomicLValue();1816  LValue DesiredLVal;1817  if (AtomicLVal.isSimple()) {1818    UpRVal = OldRVal;1819    DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());1820  } else {1821    // Build new lvalue for temp address.1822    Address Ptr = Atomics.materializeRValue(OldRVal);1823    LValue UpdateLVal;1824    if (AtomicLVal.isBitField()) {1825      UpdateLVal =1826          LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),1827                               AtomicLVal.getType(),1828                               AtomicLVal.getBaseInfo(),1829                               AtomicLVal.getTBAAInfo());1830      DesiredLVal =1831          LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),1832                               AtomicLVal.getType(), AtomicLVal.getBaseInfo(),1833                               AtomicLVal.getTBAAInfo());1834    } else if (AtomicLVal.isVectorElt()) {1835      UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),1836                                         AtomicLVal.getType(),1837                                         AtomicLVal.getBaseInfo(),1838                                         AtomicLVal.getTBAAInfo());1839      DesiredLVal = LValue::MakeVectorElt(1840          DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),1841          AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());1842    } else {1843      assert(AtomicLVal.isExtVectorElt());1844      UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),1845                                            AtomicLVal.getType(),1846                                            AtomicLVal.getBaseInfo(),1847                                            AtomicLVal.getTBAAInfo());1848      DesiredLVal = LValue::MakeExtVectorElt(1849          DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),1850          AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());1851    }1852    UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());1853  }1854  // Store new value in the corresponding memory area.1855  RValue NewRVal = UpdateOp(UpRVal);1856  if (NewRVal.isScalar()) {1857    CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);1858  } else {1859    assert(NewRVal.isComplex());1860    CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,1861                           /*isInit=*/false);1862  }1863}1864 1865void AtomicInfo::EmitAtomicUpdateLibcall(1866    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,1867    bool IsVolatile) {1868  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);1869 1870  Address ExpectedAddr = CreateTempAlloca();1871 1872  EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);1873  auto *ContBB = CGF.createBasicBlock("atomic_cont");1874  auto *ExitBB = CGF.createBasicBlock("atomic_exit");1875  CGF.EmitBlock(ContBB);1876  Address DesiredAddr = CreateTempAlloca();1877  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||1878      requiresMemSetZero(getAtomicAddress().getElementType())) {1879    auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);1880    CGF.Builder.CreateStore(OldVal, DesiredAddr);1881  }1882  auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,1883                                           AggValueSlot::ignored(),1884                                           SourceLocation(), /*AsValue=*/false);1885  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);1886  llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);1887  llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);1888  auto *Res =1889      EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure);1890  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);1891  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);1892}1893 1894void AtomicInfo::EmitAtomicUpdateOp(1895    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,1896    bool IsVolatile) {1897  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);1898 1899  // Do the atomic load.1900  auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile, /*CmpXchg=*/true);1901  // For non-simple lvalues perform compare-and-swap procedure.1902  auto *ContBB = CGF.createBasicBlock("atomic_cont");1903  auto *ExitBB = CGF.createBasicBlock("atomic_exit");1904  auto *CurBB = CGF.Builder.GetInsertBlock();1905  CGF.EmitBlock(ContBB);1906  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),1907                                             /*NumReservedValues=*/2);1908  PHI->addIncoming(OldVal, CurBB);1909  Address NewAtomicAddr = CreateTempAlloca();1910  Address NewAtomicIntAddr =1911      shouldCastToInt(NewAtomicAddr.getElementType(), /*CmpXchg=*/true)1912          ? castToAtomicIntPointer(NewAtomicAddr)1913          : NewAtomicAddr;1914 1915  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||1916      requiresMemSetZero(getAtomicAddress().getElementType())) {1917    CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);1918  }1919  auto OldRVal = ConvertToValueOrAtomic(PHI, AggValueSlot::ignored(),1920                                        SourceLocation(), /*AsValue=*/false,1921                                        /*CmpXchg=*/true);1922  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);1923  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);1924  // Try to write new value using cmpxchg operation.1925  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);1926  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());1927  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);1928  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);1929}1930 1931static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,1932                                  RValue UpdateRVal, Address DesiredAddr) {1933  LValue AtomicLVal = Atomics.getAtomicLValue();1934  LValue DesiredLVal;1935  // Build new lvalue for temp address.1936  if (AtomicLVal.isBitField()) {1937    DesiredLVal =1938        LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),1939                             AtomicLVal.getType(), AtomicLVal.getBaseInfo(),1940                             AtomicLVal.getTBAAInfo());1941  } else if (AtomicLVal.isVectorElt()) {1942    DesiredLVal =1943        LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),1944                              AtomicLVal.getType(), AtomicLVal.getBaseInfo(),1945                              AtomicLVal.getTBAAInfo());1946  } else {1947    assert(AtomicLVal.isExtVectorElt());1948    DesiredLVal = LValue::MakeExtVectorElt(1949        DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),1950        AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());1951  }1952  // Store new value in the corresponding memory area.1953  assert(UpdateRVal.isScalar());1954  CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);1955}1956 1957void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,1958                                         RValue UpdateRVal, bool IsVolatile) {1959  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);1960 1961  Address ExpectedAddr = CreateTempAlloca();1962 1963  EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);1964  auto *ContBB = CGF.createBasicBlock("atomic_cont");1965  auto *ExitBB = CGF.createBasicBlock("atomic_exit");1966  CGF.EmitBlock(ContBB);1967  Address DesiredAddr = CreateTempAlloca();1968  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||1969      requiresMemSetZero(getAtomicAddress().getElementType())) {1970    auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);1971    CGF.Builder.CreateStore(OldVal, DesiredAddr);1972  }1973  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);1974  llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);1975  llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);1976  auto *Res =1977      EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure);1978  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);1979  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);1980}1981 1982void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,1983                                    bool IsVolatile) {1984  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);1985 1986  // Do the atomic load.1987  auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile, /*CmpXchg=*/true);1988  // For non-simple lvalues perform compare-and-swap procedure.1989  auto *ContBB = CGF.createBasicBlock("atomic_cont");1990  auto *ExitBB = CGF.createBasicBlock("atomic_exit");1991  auto *CurBB = CGF.Builder.GetInsertBlock();1992  CGF.EmitBlock(ContBB);1993  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),1994                                             /*NumReservedValues=*/2);1995  PHI->addIncoming(OldVal, CurBB);1996  Address NewAtomicAddr = CreateTempAlloca();1997  Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr);1998  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||1999      requiresMemSetZero(getAtomicAddress().getElementType())) {2000    CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);2001  }2002  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);2003  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);2004  // Try to write new value using cmpxchg operation.2005  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);2006  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());2007  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);2008  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);2009}2010 2011void AtomicInfo::EmitAtomicUpdate(2012    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,2013    bool IsVolatile) {2014  if (shouldUseLibcall()) {2015    EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);2016  } else {2017    EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);2018  }2019}2020 2021void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,2022                                  bool IsVolatile) {2023  if (shouldUseLibcall()) {2024    EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);2025  } else {2026    EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);2027  }2028}2029 2030void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,2031                                      bool isInit) {2032  bool IsVolatile = lvalue.isVolatileQualified();2033  llvm::AtomicOrdering AO;2034  if (lvalue.getType()->isAtomicType()) {2035    AO = llvm::AtomicOrdering::SequentiallyConsistent;2036  } else {2037    AO = llvm::AtomicOrdering::Release;2038    IsVolatile = true;2039  }2040  return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);2041}2042 2043/// Emit a store to an l-value of atomic type.2044///2045/// Note that the r-value is expected to be an r-value *of the atomic2046/// type*; this means that for aggregate r-values, it should include2047/// storage for any padding that was necessary.2048void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,2049                                      llvm::AtomicOrdering AO, bool IsVolatile,2050                                      bool isInit) {2051  // If this is an aggregate r-value, it should agree in type except2052  // maybe for address-space qualification.2053  assert(!rvalue.isAggregate() ||2054         rvalue.getAggregateAddress().getElementType() ==2055             dest.getAddress().getElementType());2056 2057  AtomicInfo atomics(*this, dest);2058  LValue LVal = atomics.getAtomicLValue();2059 2060  // If this is an initialization, just put the value there normally.2061  if (LVal.isSimple()) {2062    if (isInit) {2063      atomics.emitCopyIntoMemory(rvalue);2064      return;2065    }2066 2067    // Check whether we should use a library call.2068    if (atomics.shouldUseLibcall()) {2069      // Produce a source address.2070      Address srcAddr = atomics.materializeRValue(rvalue);2071 2072      // void __atomic_store(size_t size, void *mem, void *val, int order)2073      CallArgList args;2074      args.add(RValue::get(atomics.getAtomicSizeValue()),2075               getContext().getSizeType());2076      args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy);2077      args.add(RValue::get(srcAddr.emitRawPointer(*this)),2078               getContext().VoidPtrTy);2079      args.add(2080          RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),2081          getContext().IntTy);2082      emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);2083      return;2084    }2085 2086    // Okay, we're doing this natively.2087    llvm::Value *ValToStore = atomics.convertRValueToInt(rvalue);2088 2089    // Do the atomic store.2090    Address Addr = atomics.getAtomicAddress();2091    if (llvm::Value *Value = atomics.getScalarRValValueOrNull(rvalue))2092      if (shouldCastToInt(Value->getType(), /*CmpXchg=*/false)) {2093        Addr = atomics.castToAtomicIntPointer(Addr);2094        ValToStore = Builder.CreateIntCast(ValToStore, Addr.getElementType(),2095                                           /*isSigned=*/false);2096      }2097    llvm::StoreInst *store = Builder.CreateStore(ValToStore, Addr);2098 2099    if (AO == llvm::AtomicOrdering::Acquire)2100      AO = llvm::AtomicOrdering::Monotonic;2101    else if (AO == llvm::AtomicOrdering::AcquireRelease)2102      AO = llvm::AtomicOrdering::Release;2103    // Initializations don't need to be atomic.2104    if (!isInit)2105      store->setAtomic(AO);2106 2107    // Other decoration.2108    if (IsVolatile)2109      store->setVolatile(true);2110    CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());2111    return;2112  }2113 2114  // Emit simple atomic update operation.2115  atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);2116}2117 2118/// Emit a compare-and-exchange op for atomic type.2119///2120std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(2121    LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,2122    llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,2123    AggValueSlot Slot) {2124  // If this is an aggregate r-value, it should agree in type except2125  // maybe for address-space qualification.2126  assert(!Expected.isAggregate() ||2127         Expected.getAggregateAddress().getElementType() ==2128             Obj.getAddress().getElementType());2129  assert(!Desired.isAggregate() ||2130         Desired.getAggregateAddress().getElementType() ==2131             Obj.getAddress().getElementType());2132  AtomicInfo Atomics(*this, Obj);2133 2134  return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,2135                                           IsWeak);2136}2137 2138llvm::AtomicRMWInst *2139CodeGenFunction::emitAtomicRMWInst(llvm::AtomicRMWInst::BinOp Op, Address Addr,2140                                   llvm::Value *Val, llvm::AtomicOrdering Order,2141                                   llvm::SyncScope::ID SSID,2142                                   const AtomicExpr *AE) {2143  llvm::AtomicRMWInst *RMW =2144      Builder.CreateAtomicRMW(Op, Addr, Val, Order, SSID);2145  getTargetHooks().setTargetAtomicMetadata(*this, *RMW, AE);2146  return RMW;2147}2148 2149void CodeGenFunction::EmitAtomicUpdate(2150    LValue LVal, llvm::AtomicOrdering AO,2151    const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {2152  AtomicInfo Atomics(*this, LVal);2153  Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);2154}2155 2156void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {2157  AtomicInfo atomics(*this, dest);2158 2159  switch (atomics.getEvaluationKind()) {2160  case TEK_Scalar: {2161    llvm::Value *value = EmitScalarExpr(init);2162    atomics.emitCopyIntoMemory(RValue::get(value));2163    return;2164  }2165 2166  case TEK_Complex: {2167    ComplexPairTy value = EmitComplexExpr(init);2168    atomics.emitCopyIntoMemory(RValue::getComplex(value));2169    return;2170  }2171 2172  case TEK_Aggregate: {2173    // Fix up the destination if the initializer isn't an expression2174    // of atomic type.2175    bool Zeroed = false;2176    if (!init->getType()->isAtomicType()) {2177      Zeroed = atomics.emitMemSetZeroIfNecessary();2178      dest = atomics.projectValue();2179    }2180 2181    // Evaluate the expression directly into the destination.2182    AggValueSlot slot = AggValueSlot::forLValue(2183        dest, AggValueSlot::IsNotDestructed,2184        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,2185        AggValueSlot::DoesNotOverlap,2186        Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);2187 2188    EmitAggExpr(init, slot);2189    return;2190  }2191  }2192  llvm_unreachable("bad evaluation kind");2193}2194