140 lines · cpp
1//===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This pass lowers atomic intrinsics to non-atomic form for use in a known10// non-preemptible environment.11//12//===----------------------------------------------------------------------===//13 14#include "llvm/Transforms/Utils/LowerAtomic.h"15#include "llvm/IR/Function.h"16#include "llvm/IR/IRBuilder.h"17 18using namespace llvm;19 20#define DEBUG_TYPE "loweratomic"21 22bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {23 IRBuilder<> Builder(CXI);24 Value *Ptr = CXI->getPointerOperand();25 Value *Cmp = CXI->getCompareOperand();26 Value *Val = CXI->getNewValOperand();27 28 auto [Orig, Equal] =29 buildCmpXchgValue(Builder, Ptr, Cmp, Val, CXI->getAlign());30 31 Value *Res =32 Builder.CreateInsertValue(PoisonValue::get(CXI->getType()), Orig, 0);33 Res = Builder.CreateInsertValue(Res, Equal, 1);34 35 CXI->replaceAllUsesWith(Res);36 CXI->eraseFromParent();37 return true;38}39 40std::pair<Value *, Value *> llvm::buildCmpXchgValue(IRBuilderBase &Builder,41 Value *Ptr, Value *Cmp,42 Value *Val,43 Align Alignment) {44 LoadInst *Orig = Builder.CreateAlignedLoad(Val->getType(), Ptr, Alignment);45 Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);46 Value *Res = Builder.CreateSelect(Equal, Val, Orig);47 Builder.CreateAlignedStore(Res, Ptr, Alignment);48 49 return {Orig, Equal};50}51 52Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op,53 IRBuilderBase &Builder, Value *Loaded,54 Value *Val) {55 Value *NewVal;56 switch (Op) {57 case AtomicRMWInst::Xchg:58 return Val;59 case AtomicRMWInst::Add:60 return Builder.CreateAdd(Loaded, Val, "new");61 case AtomicRMWInst::Sub:62 return Builder.CreateSub(Loaded, Val, "new");63 case AtomicRMWInst::And:64 return Builder.CreateAnd(Loaded, Val, "new");65 case AtomicRMWInst::Nand:66 return Builder.CreateNot(Builder.CreateAnd(Loaded, Val), "new");67 case AtomicRMWInst::Or:68 return Builder.CreateOr(Loaded, Val, "new");69 case AtomicRMWInst::Xor:70 return Builder.CreateXor(Loaded, Val, "new");71 case AtomicRMWInst::Max:72 NewVal = Builder.CreateICmpSGT(Loaded, Val);73 return Builder.CreateSelect(NewVal, Loaded, Val, "new");74 case AtomicRMWInst::Min:75 NewVal = Builder.CreateICmpSLE(Loaded, Val);76 return Builder.CreateSelect(NewVal, Loaded, Val, "new");77 case AtomicRMWInst::UMax:78 NewVal = Builder.CreateICmpUGT(Loaded, Val);79 return Builder.CreateSelect(NewVal, Loaded, Val, "new");80 case AtomicRMWInst::UMin:81 NewVal = Builder.CreateICmpULE(Loaded, Val);82 return Builder.CreateSelect(NewVal, Loaded, Val, "new");83 case AtomicRMWInst::FAdd:84 return Builder.CreateFAdd(Loaded, Val, "new");85 case AtomicRMWInst::FSub:86 return Builder.CreateFSub(Loaded, Val, "new");87 case AtomicRMWInst::FMax:88 return Builder.CreateMaxNum(Loaded, Val);89 case AtomicRMWInst::FMin:90 return Builder.CreateMinNum(Loaded, Val);91 case AtomicRMWInst::FMaximum:92 return Builder.CreateMaximum(Loaded, Val);93 case AtomicRMWInst::FMinimum:94 return Builder.CreateMinimum(Loaded, Val);95 case AtomicRMWInst::UIncWrap: {96 Constant *One = ConstantInt::get(Loaded->getType(), 1);97 Value *Inc = Builder.CreateAdd(Loaded, One);98 Value *Cmp = Builder.CreateICmpUGE(Loaded, Val);99 Constant *Zero = ConstantInt::get(Loaded->getType(), 0);100 return Builder.CreateSelect(Cmp, Zero, Inc, "new");101 }102 case AtomicRMWInst::UDecWrap: {103 Constant *Zero = ConstantInt::get(Loaded->getType(), 0);104 Constant *One = ConstantInt::get(Loaded->getType(), 1);105 106 Value *Dec = Builder.CreateSub(Loaded, One);107 Value *CmpEq0 = Builder.CreateICmpEQ(Loaded, Zero);108 Value *CmpOldGtVal = Builder.CreateICmpUGT(Loaded, Val);109 Value *Or = Builder.CreateOr(CmpEq0, CmpOldGtVal);110 return Builder.CreateSelect(Or, Val, Dec, "new");111 }112 case AtomicRMWInst::USubCond: {113 Value *Cmp = Builder.CreateICmpUGE(Loaded, Val);114 Value *Sub = Builder.CreateSub(Loaded, Val);115 return Builder.CreateSelect(Cmp, Sub, Loaded, "new");116 }117 case AtomicRMWInst::USubSat:118 return Builder.CreateIntrinsic(Intrinsic::usub_sat, Loaded->getType(),119 {Loaded, Val}, nullptr, "new");120 default:121 llvm_unreachable("Unknown atomic op");122 }123}124 125bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) {126 IRBuilder<> Builder(RMWI);127 Builder.setIsFPConstrained(128 RMWI->getFunction()->hasFnAttribute(Attribute::StrictFP));129 130 Value *Ptr = RMWI->getPointerOperand();131 Value *Val = RMWI->getValOperand();132 133 LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);134 Value *Res = buildAtomicRMWValue(RMWI->getOperation(), Builder, Orig, Val);135 Builder.CreateStore(Res, Ptr);136 RMWI->replaceAllUsesWith(Orig);137 RMWI->eraseFromParent();138 return true;139}140