140 lines · cpp
1//===- InstCombineAtomicRMW.cpp -------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the visit functions for atomic rmw instructions.10//11//===----------------------------------------------------------------------===//12 13#include "InstCombineInternal.h"14#include "llvm/IR/Instructions.h"15 16using namespace llvm;17 18/// Return true if and only if the given instruction does not modify the memory19/// location referenced. Note that an idemptent atomicrmw may still have20/// ordering effects on nearby instructions, or be volatile.21/// TODO: Common w/ the version in AtomicExpandPass, and change the term used.22/// Idemptotent is confusing in this context.23static bool isIdempotentRMW(AtomicRMWInst &RMWI) {24 if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))25 switch(RMWI.getOperation()) {26 case AtomicRMWInst::FAdd: // -0.027 return CF->isZero() && CF->isNegative();28 case AtomicRMWInst::FSub: // +0.029 return CF->isZero() && !CF->isNegative();30 default:31 return false;32 };33 34 auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());35 if(!C)36 return false;37 38 switch(RMWI.getOperation()) {39 case AtomicRMWInst::Add:40 case AtomicRMWInst::Sub:41 case AtomicRMWInst::Or:42 case AtomicRMWInst::Xor:43 return C->isZero();44 case AtomicRMWInst::And:45 return C->isMinusOne();46 case AtomicRMWInst::Min:47 return C->isMaxValue(true);48 case AtomicRMWInst::Max:49 return C->isMinValue(true);50 case AtomicRMWInst::UMin:51 return C->isMaxValue(false);52 case AtomicRMWInst::UMax:53 return C->isMinValue(false);54 default:55 return false;56 }57}58 59/// Return true if the given instruction always produces a value in memory60/// equivalent to its value operand.61static bool isSaturating(AtomicRMWInst &RMWI) {62 if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))63 switch (RMWI.getOperation()) {64 case AtomicRMWInst::FMax:65 // maxnum(x, +inf) -> +inf66 return !CF->isNegative() && CF->isInfinity();67 case AtomicRMWInst::FMin:68 // minnum(x, -inf) -> +inf69 return CF->isNegative() && CF->isInfinity();70 case AtomicRMWInst::FAdd:71 case AtomicRMWInst::FSub:72 return CF->isNaN();73 default:74 return false;75 };76 77 auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());78 if(!C)79 return false;80 81 switch(RMWI.getOperation()) {82 default:83 return false;84 case AtomicRMWInst::Xchg:85 return true;86 case AtomicRMWInst::Or:87 return C->isAllOnesValue();88 case AtomicRMWInst::And:89 return C->isZero();90 case AtomicRMWInst::Min:91 return C->isMinValue(true);92 case AtomicRMWInst::Max:93 return C->isMaxValue(true);94 case AtomicRMWInst::UMin:95 return C->isMinValue(false);96 case AtomicRMWInst::UMax:97 return C->isMaxValue(false);98 };99}100 101Instruction *InstCombinerImpl::visitAtomicRMWInst(AtomicRMWInst &RMWI) {102 103 // Volatile RMWs perform a load and a store, we cannot replace this by just a104 // load or just a store. We chose not to canonicalize out of general paranoia105 // about user expectations around volatile.106 if (RMWI.isVolatile())107 return nullptr;108 109 // Any atomicrmw op which produces a known result in memory can be110 // replaced w/an atomicrmw xchg.111 if (isSaturating(RMWI) &&112 RMWI.getOperation() != AtomicRMWInst::Xchg) {113 RMWI.setOperation(AtomicRMWInst::Xchg);114 return &RMWI;115 }116 117 assert(RMWI.getOrdering() != AtomicOrdering::NotAtomic &&118 RMWI.getOrdering() != AtomicOrdering::Unordered &&119 "AtomicRMWs don't make sense with Unordered or NotAtomic");120 121 if (!isIdempotentRMW(RMWI))122 return nullptr;123 124 // We chose to canonicalize all idempotent operations to an single125 // operation code and constant. This makes it easier for the rest of the126 // optimizer to match easily. The choices of or w/0 and fadd w/-0.0 are127 // arbitrary.128 if (RMWI.getType()->isIntegerTy() &&129 RMWI.getOperation() != AtomicRMWInst::Or) {130 RMWI.setOperation(AtomicRMWInst::Or);131 return replaceOperand(RMWI, 1, ConstantInt::get(RMWI.getType(), 0));132 } else if (RMWI.getType()->isFloatingPointTy() &&133 RMWI.getOperation() != AtomicRMWInst::FAdd) {134 RMWI.setOperation(AtomicRMWInst::FAdd);135 return replaceOperand(RMWI, 1, ConstantFP::getNegativeZero(RMWI.getType()));136 }137 138 return nullptr;139}140