1064 lines · cpp
1//===--- CIRGenAtomic.cpp - Emit CIR for atomic operations ----------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains the code for emitting atomic operations.10//11//===----------------------------------------------------------------------===//12 13#include "CIRGenFunction.h"14#include "clang/CIR/MissingFeatures.h"15 16using namespace clang;17using namespace clang::CIRGen;18using namespace cir;19 20namespace {21class AtomicInfo {22 CIRGenFunction &cgf;23 QualType atomicTy;24 QualType valueTy;25 uint64_t atomicSizeInBits = 0;26 uint64_t valueSizeInBits = 0;27 CharUnits atomicAlign;28 CharUnits valueAlign;29 TypeEvaluationKind evaluationKind = cir::TEK_Scalar;30 bool useLibCall = true;31 LValue lvalue;32 mlir::Location loc;33 34public:35 AtomicInfo(CIRGenFunction &cgf, LValue &lvalue, mlir::Location loc)36 : cgf(cgf), loc(loc) {37 assert(!lvalue.isGlobalReg());38 ASTContext &ctx = cgf.getContext();39 if (lvalue.isSimple()) {40 atomicTy = lvalue.getType();41 if (auto *ty = atomicTy->getAs<AtomicType>())42 valueTy = ty->getValueType();43 else44 valueTy = atomicTy;45 evaluationKind = cgf.getEvaluationKind(valueTy);46 47 TypeInfo valueTypeInfo = ctx.getTypeInfo(valueTy);48 TypeInfo atomicTypeInfo = ctx.getTypeInfo(atomicTy);49 uint64_t valueAlignInBits = valueTypeInfo.Align;50 uint64_t atomicAlignInBits = atomicTypeInfo.Align;51 valueSizeInBits = valueTypeInfo.Width;52 atomicSizeInBits = atomicTypeInfo.Width;53 assert(valueSizeInBits <= atomicSizeInBits);54 assert(valueAlignInBits <= atomicAlignInBits);55 56 atomicAlign = ctx.toCharUnitsFromBits(atomicAlignInBits);57 valueAlign = ctx.toCharUnitsFromBits(valueAlignInBits);58 if (lvalue.getAlignment().isZero())59 lvalue.setAlignment(atomicAlign);60 61 this->lvalue = lvalue;62 } else {63 assert(!cir::MissingFeatures::atomicInfo());64 cgf.cgm.errorNYI(loc, "AtomicInfo: non-simple lvalue");65 }66 useLibCall = !ctx.getTargetInfo().hasBuiltinAtomic(67 atomicSizeInBits, ctx.toBits(lvalue.getAlignment()));68 }69 70 QualType getValueType() const { return valueTy; }71 CharUnits getAtomicAlignment() const { return atomicAlign; }72 TypeEvaluationKind getEvaluationKind() const { return evaluationKind; }73 mlir::Value getAtomicPointer() const {74 if (lvalue.isSimple())75 return lvalue.getPointer();76 assert(!cir::MissingFeatures::atomicInfoGetAtomicPointer());77 return nullptr;78 }79 bool shouldUseLibCall() const { return useLibCall; }80 const LValue &getAtomicLValue() const { return lvalue; }81 Address getAtomicAddress() const {82 mlir::Type elemTy;83 if (lvalue.isSimple()) {84 elemTy = lvalue.getAddress().getElementType();85 } else {86 assert(!cir::MissingFeatures::atomicInfoGetAtomicAddress());87 cgf.cgm.errorNYI(loc, "AtomicInfo::getAtomicAddress: non-simple lvalue");88 }89 return Address(getAtomicPointer(), elemTy, getAtomicAlignment());90 }91 92 /// Is the atomic size larger than the underlying value type?93 ///94 /// Note that the absence of padding does not mean that atomic95 /// objects are completely interchangeable with non-atomic96 /// objects: we might have promoted the alignment of a type97 /// without making it bigger.98 bool hasPadding() const { return (valueSizeInBits != atomicSizeInBits); }99 100 bool emitMemSetZeroIfNecessary() const;101 102 mlir::Value getScalarRValValueOrNull(RValue rvalue) const;103 104 /// Cast the given pointer to an integer pointer suitable for atomic105 /// operations on the source.106 Address castToAtomicIntPointer(Address addr) const;107 108 /// If addr is compatible with the iN that will be used for an atomic109 /// operation, bitcast it. Otherwise, create a temporary that is suitable and110 /// copy the value across.111 Address convertToAtomicIntPointer(Address addr) const;112 113 /// Converts a rvalue to integer value.114 mlir::Value convertRValueToInt(RValue rvalue, bool cmpxchg = false) const;115 116 /// Copy an atomic r-value into atomic-layout memory.117 void emitCopyIntoMemory(RValue rvalue) const;118 119 /// Project an l-value down to the value field.120 LValue projectValue() const {121 assert(lvalue.isSimple());122 Address addr = getAtomicAddress();123 if (hasPadding()) {124 cgf.cgm.errorNYI(loc, "AtomicInfo::projectValue: padding");125 }126 127 assert(!cir::MissingFeatures::opTBAA());128 return LValue::makeAddr(addr, getValueType(), lvalue.getBaseInfo());129 }130 131 /// Creates temp alloca for intermediate operations on atomic value.132 Address createTempAlloca() const;133 134private:135 bool requiresMemSetZero(mlir::Type ty) const;136};137} // namespace138 139// This function emits any expression (scalar, complex, or aggregate)140// into a temporary alloca.141static Address emitValToTemp(CIRGenFunction &cgf, Expr *e) {142 Address declPtr = cgf.createMemTemp(143 e->getType(), cgf.getLoc(e->getSourceRange()), ".atomictmp");144 cgf.emitAnyExprToMem(e, declPtr, e->getType().getQualifiers(),145 /*Init*/ true);146 return declPtr;147}148 149/// Does a store of the given IR type modify the full expected width?150static bool isFullSizeType(CIRGenModule &cgm, mlir::Type ty,151 uint64_t expectedSize) {152 return cgm.getDataLayout().getTypeStoreSize(ty) * 8 == expectedSize;153}154 155/// Does the atomic type require memsetting to zero before initialization?156///157/// The IR type is provided as a way of making certain queries faster.158bool AtomicInfo::requiresMemSetZero(mlir::Type ty) const {159 // If the atomic type has size padding, we definitely need a memset.160 if (hasPadding())161 return true;162 163 // Otherwise, do some simple heuristics to try to avoid it:164 switch (getEvaluationKind()) {165 // For scalars and complexes, check whether the store size of the166 // type uses the full size.167 case cir::TEK_Scalar:168 return !isFullSizeType(cgf.cgm, ty, atomicSizeInBits);169 case cir::TEK_Complex:170 return !isFullSizeType(cgf.cgm,171 mlir::cast<cir::ComplexType>(ty).getElementType(),172 atomicSizeInBits / 2);173 // Padding in structs has an undefined bit pattern. User beware.174 case cir::TEK_Aggregate:175 return false;176 }177 llvm_unreachable("bad evaluation kind");178}179 180Address AtomicInfo::convertToAtomicIntPointer(Address addr) const {181 mlir::Type ty = addr.getElementType();182 uint64_t sourceSizeInBits = cgf.cgm.getDataLayout().getTypeSizeInBits(ty);183 if (sourceSizeInBits != atomicSizeInBits) {184 cgf.cgm.errorNYI(185 loc,186 "AtomicInfo::convertToAtomicIntPointer: convert through temp alloca");187 }188 189 return castToAtomicIntPointer(addr);190}191 192Address AtomicInfo::createTempAlloca() const {193 Address tempAlloca = cgf.createMemTemp(194 (lvalue.isBitField() && valueSizeInBits > atomicSizeInBits) ? valueTy195 : atomicTy,196 getAtomicAlignment(), loc, "atomic-temp");197 198 // Cast to pointer to value type for bitfields.199 if (lvalue.isBitField()) {200 cgf.cgm.errorNYI(loc, "AtomicInfo::createTempAlloca: bitfield lvalue");201 }202 203 return tempAlloca;204}205 206mlir::Value AtomicInfo::getScalarRValValueOrNull(RValue rvalue) const {207 if (rvalue.isScalar() && (!hasPadding() || !lvalue.isSimple()))208 return rvalue.getValue();209 return nullptr;210}211 212Address AtomicInfo::castToAtomicIntPointer(Address addr) const {213 auto intTy = mlir::dyn_cast<cir::IntType>(addr.getElementType());214 // Don't bother with int casts if the integer size is the same.215 if (intTy && intTy.getWidth() == atomicSizeInBits)216 return addr;217 auto ty = cgf.getBuilder().getUIntNTy(atomicSizeInBits);218 return addr.withElementType(cgf.getBuilder(), ty);219}220 221bool AtomicInfo::emitMemSetZeroIfNecessary() const {222 assert(lvalue.isSimple());223 Address addr = lvalue.getAddress();224 if (!requiresMemSetZero(addr.getElementType()))225 return false;226 227 cgf.cgm.errorNYI(loc,228 "AtomicInfo::emitMemSetZeroIfNecaessary: emit memset zero");229 return false;230}231 232/// Return true if \param valueTy is a type that should be casted to integer233/// around the atomic memory operation. If \param cmpxchg is true, then the234/// cast of a floating point type is made as that instruction can not have235/// floating point operands. TODO: Allow compare-and-exchange and FP - see236/// comment in CIRGenAtomicExpandPass.cpp.237static bool shouldCastToInt(mlir::Type valueTy, bool cmpxchg) {238 if (cir::isAnyFloatingPointType(valueTy))239 return isa<cir::FP80Type>(valueTy) || cmpxchg;240 return !isa<cir::IntType>(valueTy) && !isa<cir::PointerType>(valueTy);241}242 243mlir::Value AtomicInfo::convertRValueToInt(RValue rvalue, bool cmpxchg) const {244 // If we've got a scalar value of the right size, try to avoid going245 // through memory. Floats get casted if needed by AtomicExpandPass.246 if (mlir::Value value = getScalarRValValueOrNull(rvalue)) {247 if (!shouldCastToInt(value.getType(), cmpxchg))248 return cgf.emitToMemory(value, valueTy);249 250 cgf.cgm.errorNYI(251 loc, "AtomicInfo::convertRValueToInt: cast scalar rvalue to int");252 return nullptr;253 }254 255 cgf.cgm.errorNYI(256 loc, "AtomicInfo::convertRValueToInt: cast non-scalar rvalue to int");257 return nullptr;258}259 260/// Copy an r-value into memory as part of storing to an atomic type.261/// This needs to create a bit-pattern suitable for atomic operations.262void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {263 assert(lvalue.isSimple());264 265 // If we have an r-value, the rvalue should be of the atomic type,266 // which means that the caller is responsible for having zeroed267 // any padding. Just do an aggregate copy of that type.268 if (rvalue.isAggregate()) {269 cgf.cgm.errorNYI("copying aggregate into atomic lvalue");270 return;271 }272 273 // Okay, otherwise we're copying stuff.274 275 // Zero out the buffer if necessary.276 emitMemSetZeroIfNecessary();277 278 // Drill past the padding if present.279 LValue tempLValue = projectValue();280 281 // Okay, store the rvalue in.282 if (rvalue.isScalar()) {283 cgf.emitStoreOfScalar(rvalue.getValue(), tempLValue, /*isInit=*/true);284 } else {285 cgf.cgm.errorNYI("copying complex into atomic lvalue");286 }287}288 289static void emitMemOrderDefaultCaseLabel(CIRGenBuilderTy &builder,290 mlir::Location loc) {291 mlir::ArrayAttr ordersAttr = builder.getArrayAttr({});292 mlir::OpBuilder::InsertPoint insertPoint;293 cir::CaseOp::create(builder, loc, ordersAttr, cir::CaseOpKind::Default,294 insertPoint);295 builder.restoreInsertionPoint(insertPoint);296}297 298// Create a "case" operation with the given list of orders as its values. Also299// create the region that will hold the body of the switch-case label.300static void emitMemOrderCaseLabel(CIRGenBuilderTy &builder, mlir::Location loc,301 mlir::Type orderType,302 llvm::ArrayRef<cir::MemOrder> orders) {303 llvm::SmallVector<mlir::Attribute, 2> orderAttrs;304 for (cir::MemOrder order : orders)305 orderAttrs.push_back(cir::IntAttr::get(orderType, static_cast<int>(order)));306 mlir::ArrayAttr ordersAttr = builder.getArrayAttr(orderAttrs);307 308 mlir::OpBuilder::InsertPoint insertPoint;309 cir::CaseOp::create(builder, loc, ordersAttr, cir::CaseOpKind::Anyof,310 insertPoint);311 builder.restoreInsertionPoint(insertPoint);312}313 314static void emitAtomicCmpXchg(CIRGenFunction &cgf, AtomicExpr *e, bool isWeak,315 Address dest, Address ptr, Address val1,316 Address val2, uint64_t size,317 cir::MemOrder successOrder,318 cir::MemOrder failureOrder) {319 mlir::Location loc = cgf.getLoc(e->getSourceRange());320 321 CIRGenBuilderTy &builder = cgf.getBuilder();322 mlir::Value expected = builder.createLoad(loc, val1);323 mlir::Value desired = builder.createLoad(loc, val2);324 325 auto cmpxchg = cir::AtomicCmpXchgOp::create(326 builder, loc, expected.getType(), builder.getBoolTy(), ptr.getPointer(),327 expected, desired,328 cir::MemOrderAttr::get(&cgf.getMLIRContext(), successOrder),329 cir::MemOrderAttr::get(&cgf.getMLIRContext(), failureOrder),330 builder.getI64IntegerAttr(ptr.getAlignment().getAsAlign().value()));331 332 cmpxchg.setIsVolatile(e->isVolatile());333 cmpxchg.setWeak(isWeak);334 335 mlir::Value failed = builder.createNot(cmpxchg.getSuccess());336 cir::IfOp::create(builder, loc, failed, /*withElseRegion=*/false,337 [&](mlir::OpBuilder &, mlir::Location) {338 auto ptrTy = mlir::cast<cir::PointerType>(339 val1.getPointer().getType());340 if (val1.getElementType() != ptrTy.getPointee()) {341 val1 = val1.withPointer(builder.createPtrBitcast(342 val1.getPointer(), val1.getElementType()));343 }344 builder.createStore(loc, cmpxchg.getOld(), val1);345 builder.createYield(loc);346 });347 348 // Update the memory at Dest with Success's value.349 cgf.emitStoreOfScalar(cmpxchg.getSuccess(),350 cgf.makeAddrLValue(dest, e->getType()),351 /*isInit=*/false);352}353 354static void emitAtomicCmpXchgFailureSet(CIRGenFunction &cgf, AtomicExpr *e,355 bool isWeak, Address dest, Address ptr,356 Address val1, Address val2,357 Expr *failureOrderExpr, uint64_t size,358 cir::MemOrder successOrder) {359 Expr::EvalResult failureOrderEval;360 if (failureOrderExpr->EvaluateAsInt(failureOrderEval, cgf.getContext())) {361 uint64_t failureOrderInt = failureOrderEval.Val.getInt().getZExtValue();362 363 cir::MemOrder failureOrder;364 if (!cir::isValidCIRAtomicOrderingCABI(failureOrderInt)) {365 failureOrder = cir::MemOrder::Relaxed;366 } else {367 switch ((cir::MemOrder)failureOrderInt) {368 case cir::MemOrder::Relaxed:369 // 31.7.2.18: "The failure argument shall not be memory_order_release370 // nor memory_order_acq_rel". Fallback to monotonic.371 case cir::MemOrder::Release:372 case cir::MemOrder::AcquireRelease:373 failureOrder = cir::MemOrder::Relaxed;374 break;375 case cir::MemOrder::Consume:376 case cir::MemOrder::Acquire:377 failureOrder = cir::MemOrder::Acquire;378 break;379 case cir::MemOrder::SequentiallyConsistent:380 failureOrder = cir::MemOrder::SequentiallyConsistent;381 break;382 }383 }384 385 // Prior to c++17, "the failure argument shall be no stronger than the386 // success argument". This condition has been lifted and the only387 // precondition is 31.7.2.18. Effectively treat this as a DR and skip388 // language version checks.389 emitAtomicCmpXchg(cgf, e, isWeak, dest, ptr, val1, val2, size, successOrder,390 failureOrder);391 return;392 }393 394 assert(!cir::MissingFeatures::atomicExpr());395 cgf.cgm.errorNYI(e->getSourceRange(),396 "emitAtomicCmpXchgFailureSet: non-constant failure order");397}398 399static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,400 Address ptr, Address val1, Address val2,401 Expr *isWeakExpr, Expr *failureOrderExpr, int64_t size,402 cir::MemOrder order) {403 std::unique_ptr<AtomicScopeModel> scopeModel = expr->getScopeModel();404 if (scopeModel) {405 assert(!cir::MissingFeatures::atomicScope());406 cgf.cgm.errorNYI(expr->getSourceRange(), "emitAtomicOp: atomic scope");407 return;408 }409 410 assert(!cir::MissingFeatures::atomicSyncScopeID());411 llvm::StringRef opName;412 413 CIRGenBuilderTy &builder = cgf.getBuilder();414 mlir::Location loc = cgf.getLoc(expr->getSourceRange());415 auto orderAttr = cir::MemOrderAttr::get(builder.getContext(), order);416 cir::AtomicFetchKindAttr fetchAttr;417 bool fetchFirst = true;418 419 switch (expr->getOp()) {420 case AtomicExpr::AO__c11_atomic_init:421 llvm_unreachable("already handled!");422 423 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:424 emitAtomicCmpXchgFailureSet(cgf, expr, /*isWeak=*/false, dest, ptr, val1,425 val2, failureOrderExpr, size, order);426 return;427 428 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:429 emitAtomicCmpXchgFailureSet(cgf, expr, /*isWeak=*/true, dest, ptr, val1,430 val2, failureOrderExpr, size, order);431 return;432 433 case AtomicExpr::AO__atomic_compare_exchange:434 case AtomicExpr::AO__atomic_compare_exchange_n: {435 bool isWeak = false;436 if (isWeakExpr->EvaluateAsBooleanCondition(isWeak, cgf.getContext())) {437 emitAtomicCmpXchgFailureSet(cgf, expr, isWeak, dest, ptr, val1, val2,438 failureOrderExpr, size, order);439 } else {440 assert(!cir::MissingFeatures::atomicExpr());441 cgf.cgm.errorNYI(expr->getSourceRange(),442 "emitAtomicOp: non-constant isWeak");443 }444 return;445 }446 447 case AtomicExpr::AO__c11_atomic_load:448 case AtomicExpr::AO__atomic_load_n:449 case AtomicExpr::AO__atomic_load: {450 cir::LoadOp load =451 builder.createLoad(loc, ptr, /*isVolatile=*/expr->isVolatile());452 453 assert(!cir::MissingFeatures::atomicSyncScopeID());454 455 load->setAttr("mem_order", orderAttr);456 457 builder.createStore(loc, load->getResult(0), dest);458 return;459 }460 461 case AtomicExpr::AO__c11_atomic_store:462 case AtomicExpr::AO__atomic_store_n:463 case AtomicExpr::AO__atomic_store: {464 cir::LoadOp loadVal1 = builder.createLoad(loc, val1);465 466 assert(!cir::MissingFeatures::atomicSyncScopeID());467 468 builder.createStore(loc, loadVal1, ptr, expr->isVolatile(),469 /*align=*/mlir::IntegerAttr{}, orderAttr);470 return;471 }472 473 case AtomicExpr::AO__c11_atomic_exchange:474 case AtomicExpr::AO__atomic_exchange_n:475 case AtomicExpr::AO__atomic_exchange:476 opName = cir::AtomicXchgOp::getOperationName();477 break;478 479 case AtomicExpr::AO__atomic_add_fetch:480 fetchFirst = false;481 [[fallthrough]];482 case AtomicExpr::AO__c11_atomic_fetch_add:483 case AtomicExpr::AO__atomic_fetch_add:484 opName = cir::AtomicFetchOp::getOperationName();485 fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),486 cir::AtomicFetchKind::Add);487 break;488 489 case AtomicExpr::AO__atomic_sub_fetch:490 fetchFirst = false;491 [[fallthrough]];492 case AtomicExpr::AO__c11_atomic_fetch_sub:493 case AtomicExpr::AO__atomic_fetch_sub:494 opName = cir::AtomicFetchOp::getOperationName();495 fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),496 cir::AtomicFetchKind::Sub);497 break;498 499 case AtomicExpr::AO__atomic_min_fetch:500 fetchFirst = false;501 [[fallthrough]];502 case AtomicExpr::AO__c11_atomic_fetch_min:503 case AtomicExpr::AO__atomic_fetch_min:504 opName = cir::AtomicFetchOp::getOperationName();505 fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),506 cir::AtomicFetchKind::Min);507 break;508 509 case AtomicExpr::AO__atomic_max_fetch:510 fetchFirst = false;511 [[fallthrough]];512 case AtomicExpr::AO__c11_atomic_fetch_max:513 case AtomicExpr::AO__atomic_fetch_max:514 opName = cir::AtomicFetchOp::getOperationName();515 fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),516 cir::AtomicFetchKind::Max);517 break;518 519 case AtomicExpr::AO__atomic_and_fetch:520 fetchFirst = false;521 [[fallthrough]];522 case AtomicExpr::AO__c11_atomic_fetch_and:523 case AtomicExpr::AO__atomic_fetch_and:524 opName = cir::AtomicFetchOp::getOperationName();525 fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),526 cir::AtomicFetchKind::And);527 break;528 529 case AtomicExpr::AO__atomic_or_fetch:530 fetchFirst = false;531 [[fallthrough]];532 case AtomicExpr::AO__c11_atomic_fetch_or:533 case AtomicExpr::AO__atomic_fetch_or:534 opName = cir::AtomicFetchOp::getOperationName();535 fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),536 cir::AtomicFetchKind::Or);537 break;538 539 case AtomicExpr::AO__atomic_xor_fetch:540 fetchFirst = false;541 [[fallthrough]];542 case AtomicExpr::AO__c11_atomic_fetch_xor:543 case AtomicExpr::AO__atomic_fetch_xor:544 opName = cir::AtomicFetchOp::getOperationName();545 fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),546 cir::AtomicFetchKind::Xor);547 break;548 549 case AtomicExpr::AO__atomic_nand_fetch:550 fetchFirst = false;551 [[fallthrough]];552 case AtomicExpr::AO__c11_atomic_fetch_nand:553 case AtomicExpr::AO__atomic_fetch_nand:554 opName = cir::AtomicFetchOp::getOperationName();555 fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),556 cir::AtomicFetchKind::Nand);557 break;558 559 case AtomicExpr::AO__atomic_test_and_set: {560 auto op = cir::AtomicTestAndSetOp::create(561 builder, loc, ptr.getPointer(), order,562 builder.getI64IntegerAttr(ptr.getAlignment().getQuantity()),563 expr->isVolatile());564 builder.createStore(loc, op, dest);565 return;566 }567 568 case AtomicExpr::AO__atomic_clear: {569 cir::AtomicClearOp::create(570 builder, loc, ptr.getPointer(), order,571 builder.getI64IntegerAttr(ptr.getAlignment().getQuantity()),572 expr->isVolatile());573 return;574 }575 576 case AtomicExpr::AO__opencl_atomic_init:577 578 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:579 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:580 581 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:582 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:583 584 case AtomicExpr::AO__scoped_atomic_compare_exchange:585 case AtomicExpr::AO__scoped_atomic_compare_exchange_n:586 587 case AtomicExpr::AO__opencl_atomic_load:588 case AtomicExpr::AO__hip_atomic_load:589 case AtomicExpr::AO__scoped_atomic_load_n:590 case AtomicExpr::AO__scoped_atomic_load:591 592 case AtomicExpr::AO__opencl_atomic_store:593 case AtomicExpr::AO__hip_atomic_store:594 case AtomicExpr::AO__scoped_atomic_store:595 case AtomicExpr::AO__scoped_atomic_store_n:596 597 case AtomicExpr::AO__hip_atomic_exchange:598 case AtomicExpr::AO__opencl_atomic_exchange:599 case AtomicExpr::AO__scoped_atomic_exchange_n:600 case AtomicExpr::AO__scoped_atomic_exchange:601 602 case AtomicExpr::AO__scoped_atomic_add_fetch:603 604 case AtomicExpr::AO__hip_atomic_fetch_add:605 case AtomicExpr::AO__opencl_atomic_fetch_add:606 case AtomicExpr::AO__scoped_atomic_fetch_add:607 608 case AtomicExpr::AO__scoped_atomic_sub_fetch:609 610 case AtomicExpr::AO__hip_atomic_fetch_sub:611 case AtomicExpr::AO__opencl_atomic_fetch_sub:612 case AtomicExpr::AO__scoped_atomic_fetch_sub:613 614 case AtomicExpr::AO__scoped_atomic_min_fetch:615 616 case AtomicExpr::AO__hip_atomic_fetch_min:617 case AtomicExpr::AO__opencl_atomic_fetch_min:618 case AtomicExpr::AO__scoped_atomic_fetch_min:619 620 case AtomicExpr::AO__scoped_atomic_max_fetch:621 622 case AtomicExpr::AO__hip_atomic_fetch_max:623 case AtomicExpr::AO__opencl_atomic_fetch_max:624 case AtomicExpr::AO__scoped_atomic_fetch_max:625 626 case AtomicExpr::AO__scoped_atomic_and_fetch:627 628 case AtomicExpr::AO__hip_atomic_fetch_and:629 case AtomicExpr::AO__opencl_atomic_fetch_and:630 case AtomicExpr::AO__scoped_atomic_fetch_and:631 632 case AtomicExpr::AO__scoped_atomic_or_fetch:633 634 case AtomicExpr::AO__hip_atomic_fetch_or:635 case AtomicExpr::AO__opencl_atomic_fetch_or:636 case AtomicExpr::AO__scoped_atomic_fetch_or:637 638 case AtomicExpr::AO__scoped_atomic_xor_fetch:639 640 case AtomicExpr::AO__hip_atomic_fetch_xor:641 case AtomicExpr::AO__opencl_atomic_fetch_xor:642 case AtomicExpr::AO__scoped_atomic_fetch_xor:643 644 case AtomicExpr::AO__scoped_atomic_nand_fetch:645 646 case AtomicExpr::AO__scoped_atomic_fetch_nand:647 648 case AtomicExpr::AO__scoped_atomic_uinc_wrap:649 case AtomicExpr::AO__scoped_atomic_udec_wrap:650 cgf.cgm.errorNYI(expr->getSourceRange(), "emitAtomicOp: expr op NYI");651 return;652 }653 654 assert(!opName.empty() && "expected operation name to build");655 mlir::Value loadVal1 = builder.createLoad(loc, val1);656 657 SmallVector<mlir::Value> atomicOperands = {ptr.getPointer(), loadVal1};658 SmallVector<mlir::Type> atomicResTys = {loadVal1.getType()};659 mlir::Operation *rmwOp = builder.create(loc, builder.getStringAttr(opName),660 atomicOperands, atomicResTys);661 662 if (fetchAttr)663 rmwOp->setAttr("binop", fetchAttr);664 rmwOp->setAttr("mem_order", orderAttr);665 if (expr->isVolatile())666 rmwOp->setAttr("is_volatile", builder.getUnitAttr());667 if (fetchFirst && opName == cir::AtomicFetchOp::getOperationName())668 rmwOp->setAttr("fetch_first", builder.getUnitAttr());669 670 mlir::Value result = rmwOp->getResult(0);671 builder.createStore(loc, result, dest);672}673 674static bool isMemOrderValid(uint64_t order, bool isStore, bool isLoad) {675 if (!cir::isValidCIRAtomicOrderingCABI(order))676 return false;677 auto memOrder = static_cast<cir::MemOrder>(order);678 if (isStore)679 return memOrder != cir::MemOrder::Consume &&680 memOrder != cir::MemOrder::Acquire &&681 memOrder != cir::MemOrder::AcquireRelease;682 if (isLoad)683 return memOrder != cir::MemOrder::Release &&684 memOrder != cir::MemOrder::AcquireRelease;685 return true;686}687 688static void emitAtomicExprWithDynamicMemOrder(689 CIRGenFunction &cgf, mlir::Value order, AtomicExpr *e, Address dest,690 Address ptr, Address val1, Address val2, Expr *isWeakExpr,691 Expr *orderFailExpr, uint64_t size, bool isStore, bool isLoad) {692 // The memory order is not known at compile-time. The atomic operations693 // can't handle runtime memory orders; the memory order must be hard coded.694 // Generate a "switch" statement that converts a runtime value into a695 // compile-time value.696 CIRGenBuilderTy &builder = cgf.getBuilder();697 cir::SwitchOp::create(698 builder, order.getLoc(), order,699 [&](mlir::OpBuilder &, mlir::Location loc, mlir::OperationState &) {700 mlir::Block *switchBlock = builder.getBlock();701 702 auto emitMemOrderCase = [&](llvm::ArrayRef<cir::MemOrder> caseOrders,703 cir::MemOrder actualOrder) {704 if (caseOrders.empty())705 emitMemOrderDefaultCaseLabel(builder, loc);706 else707 emitMemOrderCaseLabel(builder, loc, order.getType(), caseOrders);708 emitAtomicOp(cgf, e, dest, ptr, val1, val2, isWeakExpr, orderFailExpr,709 size, actualOrder);710 builder.createBreak(loc);711 builder.setInsertionPointToEnd(switchBlock);712 };713 714 // default:715 // Use memory_order_relaxed for relaxed operations and for any memory716 // order value that is not supported. There is no good way to report717 // an unsupported memory order at runtime, hence the fallback to718 // memory_order_relaxed.719 emitMemOrderCase(/*caseOrders=*/{}, cir::MemOrder::Relaxed);720 721 if (!isStore) {722 // case consume:723 // case acquire:724 // memory_order_consume is not implemented; it is always treated725 // like memory_order_acquire. These memory orders are not valid for726 // write-only operations.727 emitMemOrderCase({cir::MemOrder::Consume, cir::MemOrder::Acquire},728 cir::MemOrder::Acquire);729 }730 731 if (!isLoad) {732 // case release:733 // memory_order_release is not valid for read-only operations.734 emitMemOrderCase({cir::MemOrder::Release}, cir::MemOrder::Release);735 }736 737 if (!isLoad && !isStore) {738 // case acq_rel:739 // memory_order_acq_rel is only valid for read-write operations.740 emitMemOrderCase({cir::MemOrder::AcquireRelease},741 cir::MemOrder::AcquireRelease);742 }743 744 // case seq_cst:745 emitMemOrderCase({cir::MemOrder::SequentiallyConsistent},746 cir::MemOrder::SequentiallyConsistent);747 748 builder.createYield(loc);749 });750}751 752RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {753 QualType atomicTy = e->getPtr()->getType()->getPointeeType();754 QualType memTy = atomicTy;755 if (const auto *ty = atomicTy->getAs<AtomicType>())756 memTy = ty->getValueType();757 758 Expr *isWeakExpr = nullptr;759 Expr *orderFailExpr = nullptr;760 761 Address val1 = Address::invalid();762 Address val2 = Address::invalid();763 Address dest = Address::invalid();764 Address ptr = emitPointerWithAlignment(e->getPtr());765 766 assert(!cir::MissingFeatures::openCL());767 if (e->getOp() == AtomicExpr::AO__c11_atomic_init) {768 LValue lvalue = makeAddrLValue(ptr, atomicTy);769 emitAtomicInit(e->getVal1(), lvalue);770 return RValue::get(nullptr);771 }772 773 TypeInfoChars typeInfo = getContext().getTypeInfoInChars(atomicTy);774 uint64_t size = typeInfo.Width.getQuantity();775 776 Expr::EvalResult orderConst;777 mlir::Value order;778 if (!e->getOrder()->EvaluateAsInt(orderConst, getContext()))779 order = emitScalarExpr(e->getOrder());780 781 bool shouldCastToIntPtrTy = true;782 783 switch (e->getOp()) {784 default:785 cgm.errorNYI(e->getSourceRange(), "atomic op NYI");786 return RValue::get(nullptr);787 788 case AtomicExpr::AO__c11_atomic_init:789 llvm_unreachable("already handled above with emitAtomicInit");790 791 case AtomicExpr::AO__atomic_load_n:792 case AtomicExpr::AO__c11_atomic_load:793 case AtomicExpr::AO__atomic_test_and_set:794 case AtomicExpr::AO__atomic_clear:795 break;796 797 case AtomicExpr::AO__atomic_load:798 dest = emitPointerWithAlignment(e->getVal1());799 break;800 801 case AtomicExpr::AO__atomic_store:802 val1 = emitPointerWithAlignment(e->getVal1());803 break;804 805 case AtomicExpr::AO__atomic_exchange:806 val1 = emitPointerWithAlignment(e->getVal1());807 dest = emitPointerWithAlignment(e->getVal2());808 break;809 810 case AtomicExpr::AO__atomic_compare_exchange:811 case AtomicExpr::AO__atomic_compare_exchange_n:812 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:813 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:814 val1 = emitPointerWithAlignment(e->getVal1());815 if (e->getOp() == AtomicExpr::AO__atomic_compare_exchange ||816 e->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)817 val2 = emitPointerWithAlignment(e->getVal2());818 else819 val2 = emitValToTemp(*this, e->getVal2());820 orderFailExpr = e->getOrderFail();821 if (e->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||822 e->getOp() == AtomicExpr::AO__atomic_compare_exchange ||823 e->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n ||824 e->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)825 isWeakExpr = e->getWeak();826 break;827 828 case AtomicExpr::AO__c11_atomic_fetch_add:829 case AtomicExpr::AO__c11_atomic_fetch_sub:830 if (memTy->isPointerType()) {831 cgm.errorNYI(e->getSourceRange(),832 "atomic fetch-and-add and fetch-and-sub for pointers");833 return RValue::get(nullptr);834 }835 [[fallthrough]];836 case AtomicExpr::AO__atomic_fetch_add:837 case AtomicExpr::AO__atomic_fetch_max:838 case AtomicExpr::AO__atomic_fetch_min:839 case AtomicExpr::AO__atomic_fetch_sub:840 case AtomicExpr::AO__atomic_add_fetch:841 case AtomicExpr::AO__atomic_max_fetch:842 case AtomicExpr::AO__atomic_min_fetch:843 case AtomicExpr::AO__atomic_sub_fetch:844 case AtomicExpr::AO__c11_atomic_fetch_max:845 case AtomicExpr::AO__c11_atomic_fetch_min:846 shouldCastToIntPtrTy = !memTy->isFloatingType();847 [[fallthrough]];848 849 case AtomicExpr::AO__atomic_fetch_and:850 case AtomicExpr::AO__atomic_fetch_nand:851 case AtomicExpr::AO__atomic_fetch_or:852 case AtomicExpr::AO__atomic_fetch_xor:853 case AtomicExpr::AO__atomic_and_fetch:854 case AtomicExpr::AO__atomic_nand_fetch:855 case AtomicExpr::AO__atomic_or_fetch:856 case AtomicExpr::AO__atomic_xor_fetch:857 case AtomicExpr::AO__atomic_exchange_n:858 case AtomicExpr::AO__atomic_store_n:859 case AtomicExpr::AO__c11_atomic_fetch_and:860 case AtomicExpr::AO__c11_atomic_fetch_nand:861 case AtomicExpr::AO__c11_atomic_fetch_or:862 case AtomicExpr::AO__c11_atomic_fetch_xor:863 case AtomicExpr::AO__c11_atomic_exchange:864 case AtomicExpr::AO__c11_atomic_store:865 val1 = emitValToTemp(*this, e->getVal1());866 break;867 }868 869 QualType resultTy = e->getType().getUnqualifiedType();870 871 // The inlined atomics only function on iN types, where N is a power of 2. We872 // need to make sure (via temporaries if necessary) that all incoming values873 // are compatible.874 LValue atomicValue = makeAddrLValue(ptr, atomicTy);875 AtomicInfo atomics(*this, atomicValue, getLoc(e->getSourceRange()));876 877 if (shouldCastToIntPtrTy) {878 ptr = atomics.castToAtomicIntPointer(ptr);879 if (val1.isValid())880 val1 = atomics.convertToAtomicIntPointer(val1);881 }882 if (dest.isValid()) {883 if (shouldCastToIntPtrTy)884 dest = atomics.castToAtomicIntPointer(dest);885 } else if (e->isCmpXChg()) {886 dest = createMemTemp(resultTy, getLoc(e->getSourceRange()), "cmpxchg.bool");887 } else if (e->getOp() == AtomicExpr::AO__atomic_test_and_set) {888 dest = createMemTemp(resultTy, getLoc(e->getSourceRange()),889 "test_and_set.bool");890 } else if (!resultTy->isVoidType()) {891 dest = atomics.createTempAlloca();892 if (shouldCastToIntPtrTy)893 dest = atomics.castToAtomicIntPointer(dest);894 }895 896 bool powerOf2Size = (size & (size - 1)) == 0;897 bool useLibCall = !powerOf2Size || (size > 16);898 899 // For atomics larger than 16 bytes, emit a libcall from the frontend. This900 // avoids the overhead of dealing with excessively-large value types in IR.901 // Non-power-of-2 values also lower to libcall here, as they are not currently902 // permitted in IR instructions (although that constraint could be relaxed in903 // the future). For other cases where a libcall is required on a given904 // platform, we let the backend handle it (this includes handling for all of905 // the size-optimized libcall variants, which are only valid up to 16 bytes.)906 //907 // See: https://llvm.org/docs/Atomics.html#libcalls-atomic908 if (useLibCall) {909 assert(!cir::MissingFeatures::atomicUseLibCall());910 cgm.errorNYI(e->getSourceRange(), "emitAtomicExpr: emit atomic lib call");911 return RValue::get(nullptr);912 }913 914 bool isStore = e->getOp() == AtomicExpr::AO__c11_atomic_store ||915 e->getOp() == AtomicExpr::AO__opencl_atomic_store ||916 e->getOp() == AtomicExpr::AO__hip_atomic_store ||917 e->getOp() == AtomicExpr::AO__atomic_store ||918 e->getOp() == AtomicExpr::AO__atomic_store_n ||919 e->getOp() == AtomicExpr::AO__scoped_atomic_store ||920 e->getOp() == AtomicExpr::AO__scoped_atomic_store_n ||921 e->getOp() == AtomicExpr::AO__atomic_clear;922 bool isLoad = e->getOp() == AtomicExpr::AO__c11_atomic_load ||923 e->getOp() == AtomicExpr::AO__opencl_atomic_load ||924 e->getOp() == AtomicExpr::AO__hip_atomic_load ||925 e->getOp() == AtomicExpr::AO__atomic_load ||926 e->getOp() == AtomicExpr::AO__atomic_load_n ||927 e->getOp() == AtomicExpr::AO__scoped_atomic_load ||928 e->getOp() == AtomicExpr::AO__scoped_atomic_load_n;929 930 if (!order) {931 // We have evaluated the memory order as an integer constant in orderConst.932 // We should not ever get to a case where the ordering isn't a valid CABI933 // value, but it's hard to enforce that in general.934 uint64_t ord = orderConst.Val.getInt().getZExtValue();935 if (isMemOrderValid(ord, isStore, isLoad))936 emitAtomicOp(*this, e, dest, ptr, val1, val2, isWeakExpr, orderFailExpr,937 size, static_cast<cir::MemOrder>(ord));938 } else {939 emitAtomicExprWithDynamicMemOrder(*this, order, e, dest, ptr, val1, val2,940 isWeakExpr, orderFailExpr, size, isStore,941 isLoad);942 }943 944 if (resultTy->isVoidType())945 return RValue::get(nullptr);946 947 return convertTempToRValue(948 dest.withElementType(builder, convertTypeForMem(resultTy)), resultTy,949 e->getExprLoc());950}951 952void CIRGenFunction::emitAtomicStore(RValue rvalue, LValue dest, bool isInit) {953 bool isVolatile = dest.isVolatileQualified();954 auto order = cir::MemOrder::SequentiallyConsistent;955 if (!dest.getType()->isAtomicType()) {956 assert(!cir::MissingFeatures::atomicMicrosoftVolatile());957 }958 return emitAtomicStore(rvalue, dest, order, isVolatile, isInit);959}960 961/// Emit a store to an l-value of atomic type.962///963/// Note that the r-value is expected to be an r-value of the atomic type; this964/// means that for aggregate r-values, it should include storage for any padding965/// that was necessary.966void CIRGenFunction::emitAtomicStore(RValue rvalue, LValue dest,967 cir::MemOrder order, bool isVolatile,968 bool isInit) {969 // If this is an aggregate r-value, it should agree in type except970 // maybe for address-space qualification.971 mlir::Location loc = dest.getPointer().getLoc();972 assert(!rvalue.isAggregate() ||973 rvalue.getAggregateAddress().getElementType() ==974 dest.getAddress().getElementType());975 976 AtomicInfo atomics(*this, dest, loc);977 LValue lvalue = atomics.getAtomicLValue();978 979 if (lvalue.isSimple()) {980 // If this is an initialization, just put the value there normally.981 if (isInit) {982 atomics.emitCopyIntoMemory(rvalue);983 return;984 }985 986 // Check whether we should use a library call.987 if (atomics.shouldUseLibCall()) {988 assert(!cir::MissingFeatures::atomicUseLibCall());989 cgm.errorNYI(loc, "emitAtomicStore: atomic store with library call");990 return;991 }992 993 // Okay, we're doing this natively.994 mlir::Value valueToStore = atomics.convertRValueToInt(rvalue);995 996 // Do the atomic store.997 Address addr = atomics.getAtomicAddress();998 if (mlir::Value value = atomics.getScalarRValValueOrNull(rvalue)) {999 if (shouldCastToInt(value.getType(), /*CmpXchg=*/false)) {1000 addr = atomics.castToAtomicIntPointer(addr);1001 valueToStore =1002 builder.createIntCast(valueToStore, addr.getElementType());1003 }1004 }1005 cir::StoreOp store = builder.createStore(loc, valueToStore, addr);1006 1007 // Initializations don't need to be atomic.1008 if (!isInit) {1009 assert(!cir::MissingFeatures::atomicOpenMP());1010 store.setMemOrder(order);1011 }1012 1013 // Other decoration.1014 if (isVolatile)1015 store.setIsVolatile(true);1016 1017 assert(!cir::MissingFeatures::opLoadStoreTbaa());1018 return;1019 }1020 1021 cgm.errorNYI(loc, "emitAtomicStore: non-simple atomic lvalue");1022 assert(!cir::MissingFeatures::opLoadStoreAtomic());1023}1024 1025void CIRGenFunction::emitAtomicInit(Expr *init, LValue dest) {1026 AtomicInfo atomics(*this, dest, getLoc(init->getSourceRange()));1027 1028 switch (atomics.getEvaluationKind()) {1029 case cir::TEK_Scalar: {1030 mlir::Value value = emitScalarExpr(init);1031 atomics.emitCopyIntoMemory(RValue::get(value));1032 return;1033 }1034 1035 case cir::TEK_Complex: {1036 mlir::Value value = emitComplexExpr(init);1037 atomics.emitCopyIntoMemory(RValue::get(value));1038 return;1039 }1040 1041 case cir::TEK_Aggregate: {1042 // Fix up the destination if the initializer isn't an expression1043 // of atomic type.1044 bool zeroed = false;1045 if (!init->getType()->isAtomicType()) {1046 zeroed = atomics.emitMemSetZeroIfNecessary();1047 dest = atomics.projectValue();1048 }1049 1050 // Evaluate the expression directly into the destination.1051 assert(!cir::MissingFeatures::aggValueSlotGC());1052 AggValueSlot slot = AggValueSlot::forLValue(1053 dest, AggValueSlot::IsNotDestructed, AggValueSlot::IsNotAliased,1054 AggValueSlot::DoesNotOverlap,1055 zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);1056 1057 emitAggExpr(init, slot);1058 return;1059 }1060 }1061 1062 llvm_unreachable("bad evaluation kind");1063}1064