195 lines · cpp
1//===--- Atomic.cpp - Codegen of atomic operations ------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/Frontend/Atomic/Atomic.h"10#include "llvm/IR/DerivedTypes.h"11#include "llvm/IR/IRBuilder.h"12#include <utility>13 14using namespace llvm;15 16bool AtomicInfo::shouldCastToInt(Type *ValTy, bool CmpXchg) {17 if (ValTy->isFloatingPointTy())18 return ValTy->isX86_FP80Ty() || CmpXchg;19 return !ValTy->isIntegerTy() && !ValTy->isPointerTy();20}21 22Value *AtomicInfo::EmitAtomicLoadOp(AtomicOrdering AO, bool IsVolatile,23 bool CmpXchg) {24 Value *Ptr = getAtomicPointer();25 Type *AtomicTy = Ty;26 if (shouldCastToInt(Ty, CmpXchg))27 AtomicTy = IntegerType::get(getLLVMContext(), AtomicSizeInBits);28 LoadInst *Load =29 Builder->CreateAlignedLoad(AtomicTy, Ptr, AtomicAlign, "atomic-load");30 Load->setAtomic(AO);31 if (IsVolatile)32 Load->setVolatile(true);33 decorateWithTBAA(Load);34 return Load;35}36 37CallInst *AtomicInfo::EmitAtomicLibcall(StringRef fnName, Type *ResultType,38 ArrayRef<Value *> Args) {39 LLVMContext &ctx = Builder->getContext();40 SmallVector<Type *, 6> ArgTys;41 for (Value *Arg : Args)42 ArgTys.push_back(Arg->getType());43 FunctionType *FnType = FunctionType::get(ResultType, ArgTys, false);44 Module *M = Builder->GetInsertBlock()->getModule();45 46 // TODO: Use llvm::TargetLowering for Libcall ABI47 AttrBuilder fnAttrBuilder(ctx);48 fnAttrBuilder.addAttribute(Attribute::NoUnwind);49 fnAttrBuilder.addAttribute(Attribute::WillReturn);50 AttributeList fnAttrs =51 AttributeList::get(ctx, AttributeList::FunctionIndex, fnAttrBuilder);52 FunctionCallee LibcallFn = M->getOrInsertFunction(fnName, FnType, fnAttrs);53 CallInst *Call = Builder->CreateCall(LibcallFn, Args);54 return Call;55}56 57std::pair<Value *, Value *> AtomicInfo::EmitAtomicCompareExchangeLibcall(58 Value *ExpectedVal, Value *DesiredVal, AtomicOrdering Success,59 AtomicOrdering Failure) {60 LLVMContext &ctx = getLLVMContext();61 62 // __atomic_compare_exchange's expected and desired are passed by pointers63 // FIXME: types64 65 // TODO: Get from llvm::TargetMachine / clang::TargetInfo66 // if clang shares this codegen in future67 constexpr uint64_t IntBits = 32;68 69 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,70 // void *desired, int success, int failure);71 72 Value *Args[6] = {73 getAtomicSizeValue(),74 getAtomicPointer(),75 ExpectedVal,76 DesiredVal,77 Constant::getIntegerValue(IntegerType::get(ctx, IntBits),78 APInt(IntBits, static_cast<uint64_t>(Success),79 /*signed=*/true)),80 Constant::getIntegerValue(IntegerType::get(ctx, IntBits),81 APInt(IntBits, static_cast<uint64_t>(Failure),82 /*signed=*/true)),83 };84 auto Result = EmitAtomicLibcall("__atomic_compare_exchange",85 IntegerType::getInt1Ty(ctx), Args);86 return std::make_pair(ExpectedVal, Result);87}88 89std::pair<Value *, Value *> AtomicInfo::EmitAtomicCompareExchangeOp(90 Value *ExpectedVal, Value *DesiredVal, AtomicOrdering Success,91 AtomicOrdering Failure, bool IsVolatile, bool IsWeak) {92 // Do the atomic store.93 Value *Addr = getAtomicAddressAsAtomicIntPointer();94 auto *Inst = Builder->CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,95 getAtomicAlignment(), Success,96 Failure, SyncScope::System);97 98 // Other decoration.99 Inst->setVolatile(IsVolatile);100 Inst->setWeak(IsWeak);101 auto *PreviousVal = Builder->CreateExtractValue(Inst, /*Idxs=*/0);102 auto *SuccessFailureVal = Builder->CreateExtractValue(Inst, /*Idxs=*/1);103 return std::make_pair(PreviousVal, SuccessFailureVal);104}105 106std::pair<LoadInst *, AllocaInst *>107AtomicInfo::EmitAtomicLoadLibcall(AtomicOrdering AO) {108 LLVMContext &Ctx = getLLVMContext();109 Type *SizedIntTy = Type::getIntNTy(Ctx, getAtomicSizeInBits());110 Type *ResultTy;111 SmallVector<Value *, 6> Args;112 AttributeList Attr;113 Module *M = Builder->GetInsertBlock()->getModule();114 const DataLayout &DL = M->getDataLayout();115 Args.push_back(116 ConstantInt::get(DL.getIntPtrType(Ctx), this->getAtomicSizeInBits() / 8));117 118 Value *PtrVal = getAtomicPointer();119 PtrVal = Builder->CreateAddrSpaceCast(PtrVal, PointerType::getUnqual(Ctx));120 Args.push_back(PtrVal);121 122 auto CurrentIP = Builder->saveIP();123 Builder->restoreIP(AllocaIP);124 AllocaInst *AllocaResult =125 CreateAlloca(Ty, getAtomicPointer()->getName() + "atomic.temp.load");126 Builder->restoreIP(CurrentIP);127 const Align AllocaAlignment = DL.getPrefTypeAlign(SizedIntTy);128 AllocaResult->setAlignment(AllocaAlignment);129 Args.push_back(AllocaResult);130 Constant *OrderingVal =131 ConstantInt::get(Type::getInt32Ty(Ctx), (int)toCABI(AO));132 Args.push_back(OrderingVal);133 134 ResultTy = Type::getVoidTy(Ctx);135 SmallVector<Type *, 6> ArgTys;136 for (Value *Arg : Args)137 ArgTys.push_back(Arg->getType());138 FunctionType *FnType = FunctionType::get(ResultTy, ArgTys, false);139 FunctionCallee LibcallFn =140 M->getOrInsertFunction("__atomic_load", FnType, Attr);141 CallInst *Call = Builder->CreateCall(LibcallFn, Args);142 Call->setAttributes(Attr);143 return std::make_pair(144 Builder->CreateAlignedLoad(Ty, AllocaResult, AllocaAlignment),145 AllocaResult);146}147 148void AtomicInfo::EmitAtomicStoreLibcall(AtomicOrdering AO, Value *Source) {149 LLVMContext &Ctx = getLLVMContext();150 SmallVector<Value *, 6> Args;151 AttributeList Attr;152 Module *M = Builder->GetInsertBlock()->getModule();153 const DataLayout &DL = M->getDataLayout();154 Args.push_back(155 ConstantInt::get(DL.getIntPtrType(Ctx), this->getAtomicSizeInBits() / 8));156 157 Value *PtrVal = getAtomicPointer();158 PtrVal = Builder->CreateAddrSpaceCast(PtrVal, PointerType::getUnqual(Ctx));159 Args.push_back(PtrVal);160 161 auto CurrentIP = Builder->saveIP();162 Builder->restoreIP(AllocaIP);163 Value *SourceAlloca = Builder->CreateAlloca(Source->getType());164 Builder->restoreIP(CurrentIP);165 Builder->CreateStore(Source, SourceAlloca);166 SourceAlloca = Builder->CreatePointerBitCastOrAddrSpaceCast(167 SourceAlloca, Builder->getPtrTy());168 Args.push_back(SourceAlloca);169 170 Constant *OrderingVal =171 ConstantInt::get(Type::getInt32Ty(Ctx), (int)toCABI(AO));172 Args.push_back(OrderingVal);173 174 SmallVector<Type *, 6> ArgTys;175 for (Value *Arg : Args)176 ArgTys.push_back(Arg->getType());177 FunctionType *FnType = FunctionType::get(Type::getVoidTy(Ctx), ArgTys, false);178 FunctionCallee LibcallFn =179 M->getOrInsertFunction("__atomic_store", FnType, Attr);180 CallInst *Call = Builder->CreateCall(LibcallFn, Args);181 Call->setAttributes(Attr);182}183 184std::pair<Value *, Value *> AtomicInfo::EmitAtomicCompareExchange(185 Value *ExpectedVal, Value *DesiredVal, AtomicOrdering Success,186 AtomicOrdering Failure, bool IsVolatile, bool IsWeak) {187 if (shouldUseLibcall())188 return EmitAtomicCompareExchangeLibcall(ExpectedVal, DesiredVal, Success,189 Failure);190 191 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,192 Failure, IsVolatile, IsWeak);193 return Res;194}195