1465 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This contains code to emit Builtin calls as CIR or a function call to be10// later resolved.11//12//===----------------------------------------------------------------------===//13 14#include "CIRGenCall.h"15#include "CIRGenFunction.h"16#include "CIRGenModule.h"17#include "CIRGenValue.h"18#include "mlir/IR/BuiltinAttributes.h"19#include "mlir/IR/Value.h"20#include "mlir/Support/LLVM.h"21#include "clang/AST/DeclBase.h"22#include "clang/AST/Expr.h"23#include "clang/AST/GlobalDecl.h"24#include "clang/Basic/Builtins.h"25#include "clang/Basic/OperatorKinds.h"26#include "clang/CIR/Dialect/IR/CIRTypes.h"27#include "clang/CIR/MissingFeatures.h"28#include "llvm/Support/ErrorHandling.h"29 30using namespace clang;31using namespace clang::CIRGen;32using namespace llvm;33 34static RValue emitLibraryCall(CIRGenFunction &cgf, const FunctionDecl *fd,35 const CallExpr *e, mlir::Operation *calleeValue) {36 CIRGenCallee callee = CIRGenCallee::forDirect(calleeValue, GlobalDecl(fd));37 return cgf.emitCall(e->getCallee()->getType(), callee, e, ReturnValueSlot());38}39 40template <typename Op>41static RValue emitBuiltinBitOp(CIRGenFunction &cgf, const CallExpr *e,42 bool poisonZero = false) {43 assert(!cir::MissingFeatures::builtinCheckKind());44 45 mlir::Value arg = cgf.emitScalarExpr(e->getArg(0));46 CIRGenBuilderTy &builder = cgf.getBuilder();47 48 Op op;49 if constexpr (std::is_same_v<Op, cir::BitClzOp> ||50 std::is_same_v<Op, cir::BitCtzOp>)51 op = Op::create(builder, cgf.getLoc(e->getSourceRange()), arg, poisonZero);52 else53 op = Op::create(builder, cgf.getLoc(e->getSourceRange()), arg);54 55 mlir::Value result = op.getResult();56 mlir::Type exprTy = cgf.convertType(e->getType());57 if (exprTy != result.getType())58 result = builder.createIntCast(result, exprTy);59 60 return RValue::get(result);61}62 63namespace {64struct WidthAndSignedness {65 unsigned width;66 bool isSigned;67};68} // namespace69 70static WidthAndSignedness71getIntegerWidthAndSignedness(const clang::ASTContext &astContext,72 const clang::QualType type) {73 assert(type->isIntegerType() && "Given type is not an integer.");74 unsigned width = type->isBooleanType() ? 175 : type->isBitIntType() ? astContext.getIntWidth(type)76 : astContext.getTypeInfo(type).Width;77 bool isSigned = type->isSignedIntegerType();78 return {width, isSigned};79}80 81// Given one or more integer types, this function produces an integer type that82// encompasses them: any value in one of the given types could be expressed in83// the encompassing type.84static struct WidthAndSignedness85EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> types) {86 assert(types.size() > 0 && "Empty list of types.");87 88 // If any of the given types is signed, we must return a signed type.89 bool isSigned = llvm::any_of(types, [](const auto &t) { return t.isSigned; });90 91 // The encompassing type must have a width greater than or equal to the width92 // of the specified types. Additionally, if the encompassing type is signed,93 // its width must be strictly greater than the width of any unsigned types94 // given.95 unsigned width = 0;96 for (const auto &type : types)97 width = std::max(width, type.width + (isSigned && !type.isSigned));98 99 return {width, isSigned};100}101 102RValue CIRGenFunction::emitRotate(const CallExpr *e, bool isRotateLeft) {103 mlir::Value input = emitScalarExpr(e->getArg(0));104 mlir::Value amount = emitScalarExpr(e->getArg(1));105 106 // TODO(cir): MSVC flavor bit rotate builtins use different types for input107 // and amount, but cir.rotate requires them to have the same type. Cast amount108 // to the type of input when necessary.109 assert(!cir::MissingFeatures::msvcBuiltins());110 111 auto r = cir::RotateOp::create(builder, getLoc(e->getSourceRange()), input,112 amount, isRotateLeft);113 return RValue::get(r);114}115 116template <class Operation>117static RValue emitUnaryMaybeConstrainedFPBuiltin(CIRGenFunction &cgf,118 const CallExpr &e) {119 mlir::Value arg = cgf.emitScalarExpr(e.getArg(0));120 121 assert(!cir::MissingFeatures::cgFPOptionsRAII());122 assert(!cir::MissingFeatures::fpConstraints());123 124 auto call =125 Operation::create(cgf.getBuilder(), arg.getLoc(), arg.getType(), arg);126 return RValue::get(call->getResult(0));127}128 129template <class Operation>130static RValue emitUnaryFPBuiltin(CIRGenFunction &cgf, const CallExpr &e) {131 mlir::Value arg = cgf.emitScalarExpr(e.getArg(0));132 auto call =133 Operation::create(cgf.getBuilder(), arg.getLoc(), arg.getType(), arg);134 return RValue::get(call->getResult(0));135}136 137static RValue errorBuiltinNYI(CIRGenFunction &cgf, const CallExpr *e,138 unsigned builtinID) {139 140 if (cgf.getContext().BuiltinInfo.isLibFunction(builtinID)) {141 cgf.cgm.errorNYI(142 e->getSourceRange(),143 std::string("unimplemented X86 library function builtin call: ") +144 cgf.getContext().BuiltinInfo.getName(builtinID));145 } else {146 cgf.cgm.errorNYI(e->getSourceRange(),147 std::string("unimplemented X86 builtin call: ") +148 cgf.getContext().BuiltinInfo.getName(builtinID));149 }150 151 return cgf.getUndefRValue(e->getType());152}153 154static RValue emitBuiltinAlloca(CIRGenFunction &cgf, const CallExpr *e,155 unsigned builtinID) {156 assert(builtinID == Builtin::BI__builtin_alloca ||157 builtinID == Builtin::BI__builtin_alloca_uninitialized ||158 builtinID == Builtin::BIalloca || builtinID == Builtin::BI_alloca);159 160 // Get alloca size input161 mlir::Value size = cgf.emitScalarExpr(e->getArg(0));162 163 // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.164 const TargetInfo &ti = cgf.getContext().getTargetInfo();165 const CharUnits suitableAlignmentInBytes =166 cgf.getContext().toCharUnitsFromBits(ti.getSuitableAlign());167 168 // Emit the alloca op with type `u8 *` to match the semantics of169 // `llvm.alloca`. We later bitcast the type to `void *` to match the170 // semantics of C/C++171 // FIXME(cir): It may make sense to allow AllocaOp of type `u8` to return a172 // pointer of type `void *`. This will require a change to the allocaOp173 // verifier.174 CIRGenBuilderTy &builder = cgf.getBuilder();175 mlir::Value allocaAddr = builder.createAlloca(176 cgf.getLoc(e->getSourceRange()), builder.getUInt8PtrTy(),177 builder.getUInt8Ty(), "bi_alloca", suitableAlignmentInBytes, size);178 179 // Initialize the allocated buffer if required.180 if (builtinID != Builtin::BI__builtin_alloca_uninitialized) {181 // Initialize the alloca with the given size and alignment according to182 // the lang opts. Only the trivial non-initialization is supported for183 // now.184 185 switch (cgf.getLangOpts().getTrivialAutoVarInit()) {186 case LangOptions::TrivialAutoVarInitKind::Uninitialized:187 // Nothing to initialize.188 break;189 case LangOptions::TrivialAutoVarInitKind::Zero:190 case LangOptions::TrivialAutoVarInitKind::Pattern:191 cgf.cgm.errorNYI("trivial auto var init");192 break;193 }194 }195 196 // An alloca will always return a pointer to the alloca (stack) address197 // space. This address space need not be the same as the AST / Language198 // default (e.g. in C / C++ auto vars are in the generic address space). At199 // the AST level this is handled within CreateTempAlloca et al., but for the200 // builtin / dynamic alloca we have to handle it here.201 202 if (!cir::isMatchingAddressSpace(203 cgf.getCIRAllocaAddressSpace(),204 e->getType()->getPointeeType().getAddressSpace())) {205 cgf.cgm.errorNYI(e->getSourceRange(),206 "Non-default address space for alloca");207 }208 209 // Bitcast the alloca to the expected type.210 return RValue::get(builder.createBitcast(211 allocaAddr, builder.getVoidPtrTy(cgf.getCIRAllocaAddressSpace())));212}213 214RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,215 const CallExpr *e,216 ReturnValueSlot returnValue) {217 mlir::Location loc = getLoc(e->getSourceRange());218 219 // See if we can constant fold this builtin. If so, don't emit it at all.220 // TODO: Extend this handling to all builtin calls that we can constant-fold.221 Expr::EvalResult result;222 if (e->isPRValue() && e->EvaluateAsRValue(result, cgm.getASTContext()) &&223 !result.hasSideEffects()) {224 if (result.Val.isInt())225 return RValue::get(builder.getConstInt(loc, result.Val.getInt()));226 if (result.Val.isFloat()) {227 // Note: we are using result type of CallExpr to determine the type of228 // the constant. Classic codegen uses the result value to determine the229 // type. We feel it should be Ok to use expression type because it is230 // hard to imagine a builtin function evaluates to a value that231 // over/underflows its own defined type.232 mlir::Type type = convertType(e->getType());233 return RValue::get(builder.getConstFP(loc, type, result.Val.getFloat()));234 }235 }236 237 const FunctionDecl *fd = gd.getDecl()->getAsFunction();238 239 assert(!cir::MissingFeatures::builtinCallF128());240 241 // If the builtin has been declared explicitly with an assembler label,242 // disable the specialized emitting below. Ideally we should communicate the243 // rename in IR, or at least avoid generating the intrinsic calls that are244 // likely to get lowered to the renamed library functions.245 unsigned builtinIDIfNoAsmLabel = fd->hasAttr<AsmLabelAttr>() ? 0 : builtinID;246 247 assert(!cir::MissingFeatures::builtinCallMathErrno());248 assert(!cir::MissingFeatures::builtinCall());249 250 switch (builtinIDIfNoAsmLabel) {251 default:252 break;253 254 // C stdarg builtins.255 case Builtin::BI__builtin_stdarg_start:256 case Builtin::BI__builtin_va_start:257 case Builtin::BI__va_start: {258 mlir::Value vaList = builtinID == Builtin::BI__va_start259 ? emitScalarExpr(e->getArg(0))260 : emitVAListRef(e->getArg(0)).getPointer();261 mlir::Value count = emitScalarExpr(e->getArg(1));262 emitVAStart(vaList, count);263 return {};264 }265 266 case Builtin::BI__builtin_va_end:267 emitVAEnd(emitVAListRef(e->getArg(0)).getPointer());268 return {};269 270 case Builtin::BIcos:271 case Builtin::BIcosf:272 case Builtin::BIcosl:273 case Builtin::BI__builtin_cos:274 case Builtin::BI__builtin_cosf:275 case Builtin::BI__builtin_cosf16:276 case Builtin::BI__builtin_cosl:277 case Builtin::BI__builtin_cosf128:278 assert(!cir::MissingFeatures::fastMathFlags());279 return emitUnaryMaybeConstrainedFPBuiltin<cir::CosOp>(*this, *e);280 281 case Builtin::BIceil:282 case Builtin::BIceilf:283 case Builtin::BIceill:284 case Builtin::BI__builtin_ceil:285 case Builtin::BI__builtin_ceilf:286 case Builtin::BI__builtin_ceilf16:287 case Builtin::BI__builtin_ceill:288 case Builtin::BI__builtin_ceilf128:289 assert(!cir::MissingFeatures::fastMathFlags());290 return emitUnaryMaybeConstrainedFPBuiltin<cir::CeilOp>(*this, *e);291 292 case Builtin::BIexp:293 case Builtin::BIexpf:294 case Builtin::BIexpl:295 case Builtin::BI__builtin_exp:296 case Builtin::BI__builtin_expf:297 case Builtin::BI__builtin_expf16:298 case Builtin::BI__builtin_expl:299 case Builtin::BI__builtin_expf128:300 assert(!cir::MissingFeatures::fastMathFlags());301 return emitUnaryMaybeConstrainedFPBuiltin<cir::ExpOp>(*this, *e);302 303 case Builtin::BIexp2:304 case Builtin::BIexp2f:305 case Builtin::BIexp2l:306 case Builtin::BI__builtin_exp2:307 case Builtin::BI__builtin_exp2f:308 case Builtin::BI__builtin_exp2f16:309 case Builtin::BI__builtin_exp2l:310 case Builtin::BI__builtin_exp2f128:311 assert(!cir::MissingFeatures::fastMathFlags());312 return emitUnaryMaybeConstrainedFPBuiltin<cir::Exp2Op>(*this, *e);313 314 case Builtin::BIfabs:315 case Builtin::BIfabsf:316 case Builtin::BIfabsl:317 case Builtin::BI__builtin_fabs:318 case Builtin::BI__builtin_fabsf:319 case Builtin::BI__builtin_fabsf16:320 case Builtin::BI__builtin_fabsl:321 case Builtin::BI__builtin_fabsf128:322 return emitUnaryMaybeConstrainedFPBuiltin<cir::FAbsOp>(*this, *e);323 324 case Builtin::BI__assume:325 case Builtin::BI__builtin_assume: {326 if (e->getArg(0)->HasSideEffects(getContext()))327 return RValue::get(nullptr);328 329 mlir::Value argValue = emitCheckedArgForAssume(e->getArg(0));330 cir::AssumeOp::create(builder, loc, argValue);331 return RValue::get(nullptr);332 }333 334 case Builtin::BI__builtin_assume_separate_storage: {335 mlir::Value value0 = emitScalarExpr(e->getArg(0));336 mlir::Value value1 = emitScalarExpr(e->getArg(1));337 cir::AssumeSepStorageOp::create(builder, loc, value0, value1);338 return RValue::get(nullptr);339 }340 341 case Builtin::BI__builtin_assume_aligned: {342 const Expr *ptrExpr = e->getArg(0);343 mlir::Value ptrValue = emitScalarExpr(ptrExpr);344 mlir::Value offsetValue =345 (e->getNumArgs() > 2) ? emitScalarExpr(e->getArg(2)) : nullptr;346 347 std::optional<llvm::APSInt> alignment =348 e->getArg(1)->getIntegerConstantExpr(getContext());349 assert(alignment.has_value() &&350 "the second argument to __builtin_assume_aligned must be an "351 "integral constant expression");352 353 mlir::Value result =354 emitAlignmentAssumption(ptrValue, ptrExpr, ptrExpr->getExprLoc(),355 alignment->getSExtValue(), offsetValue);356 return RValue::get(result);357 }358 359 case Builtin::BI__builtin_complex: {360 mlir::Value real = emitScalarExpr(e->getArg(0));361 mlir::Value imag = emitScalarExpr(e->getArg(1));362 mlir::Value complex = builder.createComplexCreate(loc, real, imag);363 return RValue::getComplex(complex);364 }365 366 case Builtin::BI__builtin_creal:367 case Builtin::BI__builtin_crealf:368 case Builtin::BI__builtin_creall:369 case Builtin::BIcreal:370 case Builtin::BIcrealf:371 case Builtin::BIcreall: {372 mlir::Value complex = emitComplexExpr(e->getArg(0));373 mlir::Value real = builder.createComplexReal(loc, complex);374 return RValue::get(real);375 }376 377 case Builtin::BI__builtin_cimag:378 case Builtin::BI__builtin_cimagf:379 case Builtin::BI__builtin_cimagl:380 case Builtin::BIcimag:381 case Builtin::BIcimagf:382 case Builtin::BIcimagl: {383 mlir::Value complex = emitComplexExpr(e->getArg(0));384 mlir::Value imag = builder.createComplexImag(loc, complex);385 return RValue::get(imag);386 }387 388 case Builtin::BI__builtin_conj:389 case Builtin::BI__builtin_conjf:390 case Builtin::BI__builtin_conjl:391 case Builtin::BIconj:392 case Builtin::BIconjf:393 case Builtin::BIconjl: {394 mlir::Value complex = emitComplexExpr(e->getArg(0));395 mlir::Value conj = builder.createUnaryOp(getLoc(e->getExprLoc()),396 cir::UnaryOpKind::Not, complex);397 return RValue::getComplex(conj);398 }399 400 case Builtin::BI__builtin_clrsb:401 case Builtin::BI__builtin_clrsbl:402 case Builtin::BI__builtin_clrsbll:403 return emitBuiltinBitOp<cir::BitClrsbOp>(*this, e);404 405 case Builtin::BI__builtin_ctzs:406 case Builtin::BI__builtin_ctz:407 case Builtin::BI__builtin_ctzl:408 case Builtin::BI__builtin_ctzll:409 case Builtin::BI__builtin_ctzg:410 assert(!cir::MissingFeatures::builtinCheckKind());411 return emitBuiltinBitOp<cir::BitCtzOp>(*this, e, /*poisonZero=*/true);412 413 case Builtin::BI__builtin_clzs:414 case Builtin::BI__builtin_clz:415 case Builtin::BI__builtin_clzl:416 case Builtin::BI__builtin_clzll:417 case Builtin::BI__builtin_clzg:418 assert(!cir::MissingFeatures::builtinCheckKind());419 return emitBuiltinBitOp<cir::BitClzOp>(*this, e, /*poisonZero=*/true);420 421 case Builtin::BI__builtin_ffs:422 case Builtin::BI__builtin_ffsl:423 case Builtin::BI__builtin_ffsll:424 return emitBuiltinBitOp<cir::BitFfsOp>(*this, e);425 426 case Builtin::BI__builtin_parity:427 case Builtin::BI__builtin_parityl:428 case Builtin::BI__builtin_parityll:429 return emitBuiltinBitOp<cir::BitParityOp>(*this, e);430 431 case Builtin::BI__lzcnt16:432 case Builtin::BI__lzcnt:433 case Builtin::BI__lzcnt64:434 assert(!cir::MissingFeatures::builtinCheckKind());435 return emitBuiltinBitOp<cir::BitClzOp>(*this, e, /*poisonZero=*/false);436 437 case Builtin::BI__popcnt16:438 case Builtin::BI__popcnt:439 case Builtin::BI__popcnt64:440 case Builtin::BI__builtin_popcount:441 case Builtin::BI__builtin_popcountl:442 case Builtin::BI__builtin_popcountll:443 case Builtin::BI__builtin_popcountg:444 return emitBuiltinBitOp<cir::BitPopcountOp>(*this, e);445 446 case Builtin::BI__builtin_expect:447 case Builtin::BI__builtin_expect_with_probability: {448 mlir::Value argValue = emitScalarExpr(e->getArg(0));449 mlir::Value expectedValue = emitScalarExpr(e->getArg(1));450 451 mlir::FloatAttr probAttr;452 if (builtinIDIfNoAsmLabel == Builtin::BI__builtin_expect_with_probability) {453 llvm::APFloat probability(0.0);454 const Expr *probArg = e->getArg(2);455 [[maybe_unused]] bool evalSucceeded =456 probArg->EvaluateAsFloat(probability, cgm.getASTContext());457 assert(evalSucceeded &&458 "probability should be able to evaluate as float");459 bool loseInfo = false; // ignored460 probability.convert(llvm::APFloat::IEEEdouble(),461 llvm::RoundingMode::Dynamic, &loseInfo);462 probAttr = mlir::FloatAttr::get(mlir::Float64Type::get(&getMLIRContext()),463 probability);464 }465 466 auto result = cir::ExpectOp::create(builder, loc, argValue.getType(),467 argValue, expectedValue, probAttr);468 return RValue::get(result);469 }470 471 case Builtin::BI__builtin_bswap16:472 case Builtin::BI__builtin_bswap32:473 case Builtin::BI__builtin_bswap64:474 case Builtin::BI_byteswap_ushort:475 case Builtin::BI_byteswap_ulong:476 case Builtin::BI_byteswap_uint64: {477 mlir::Value arg = emitScalarExpr(e->getArg(0));478 return RValue::get(cir::ByteSwapOp::create(builder, loc, arg));479 }480 481 case Builtin::BI__builtin_bitreverse8:482 case Builtin::BI__builtin_bitreverse16:483 case Builtin::BI__builtin_bitreverse32:484 case Builtin::BI__builtin_bitreverse64: {485 mlir::Value arg = emitScalarExpr(e->getArg(0));486 return RValue::get(cir::BitReverseOp::create(builder, loc, arg));487 }488 489 case Builtin::BI__builtin_rotateleft8:490 case Builtin::BI__builtin_rotateleft16:491 case Builtin::BI__builtin_rotateleft32:492 case Builtin::BI__builtin_rotateleft64:493 return emitRotate(e, /*isRotateLeft=*/true);494 495 case Builtin::BI__builtin_rotateright8:496 case Builtin::BI__builtin_rotateright16:497 case Builtin::BI__builtin_rotateright32:498 case Builtin::BI__builtin_rotateright64:499 return emitRotate(e, /*isRotateLeft=*/false);500 501 case Builtin::BI__builtin_coro_id:502 case Builtin::BI__builtin_coro_promise:503 case Builtin::BI__builtin_coro_resume:504 case Builtin::BI__builtin_coro_noop:505 case Builtin::BI__builtin_coro_destroy:506 case Builtin::BI__builtin_coro_done:507 case Builtin::BI__builtin_coro_alloc:508 case Builtin::BI__builtin_coro_begin:509 case Builtin::BI__builtin_coro_end:510 case Builtin::BI__builtin_coro_suspend:511 case Builtin::BI__builtin_coro_align:512 cgm.errorNYI(e->getSourceRange(), "BI__builtin_coro_id like NYI");513 return getUndefRValue(e->getType());514 515 case Builtin::BI__builtin_coro_frame: {516 return emitCoroutineFrame();517 }518 case Builtin::BI__builtin_coro_free:519 case Builtin::BI__builtin_coro_size: {520 GlobalDecl gd{fd};521 mlir::Type ty = cgm.getTypes().getFunctionType(522 cgm.getTypes().arrangeGlobalDeclaration(gd));523 const auto *nd = cast<NamedDecl>(gd.getDecl());524 cir::FuncOp fnOp =525 cgm.getOrCreateCIRFunction(nd->getName(), ty, gd, /*ForVTable=*/false);526 fnOp.setBuiltin(true);527 return emitCall(e->getCallee()->getType(), CIRGenCallee::forDirect(fnOp), e,528 returnValue);529 }530 case Builtin::BI__builtin_dynamic_object_size:531 case Builtin::BI__builtin_object_size: {532 unsigned type =533 e->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue();534 auto resType = mlir::cast<cir::IntType>(convertType(e->getType()));535 536 // We pass this builtin onto the optimizer so that it can figure out the537 // object size in more complex cases.538 bool isDynamic = builtinID == Builtin::BI__builtin_dynamic_object_size;539 return RValue::get(emitBuiltinObjectSize(e->getArg(0), type, resType,540 /*EmittedE=*/nullptr, isDynamic));541 }542 543 case Builtin::BI__builtin_prefetch: {544 auto evaluateOperandAsInt = [&](const Expr *arg) {545 Expr::EvalResult res;546 [[maybe_unused]] bool evalSucceed =547 arg->EvaluateAsInt(res, cgm.getASTContext());548 assert(evalSucceed && "expression should be able to evaluate as int");549 return res.Val.getInt().getZExtValue();550 };551 552 bool isWrite = false;553 if (e->getNumArgs() > 1)554 isWrite = evaluateOperandAsInt(e->getArg(1));555 556 int locality = 3;557 if (e->getNumArgs() > 2)558 locality = evaluateOperandAsInt(e->getArg(2));559 560 mlir::Value address = emitScalarExpr(e->getArg(0));561 cir::PrefetchOp::create(builder, loc, address, locality, isWrite);562 return RValue::get(nullptr);563 }564 case Builtin::BI__builtin_readcyclecounter:565 case Builtin::BI__builtin_readsteadycounter:566 case Builtin::BI__builtin___clear_cache:567 return errorBuiltinNYI(*this, e, builtinID);568 case Builtin::BI__builtin_trap:569 emitTrap(loc, /*createNewBlock=*/true);570 return RValue::getIgnored();571 case Builtin::BI__builtin_verbose_trap:572 case Builtin::BI__debugbreak:573 return errorBuiltinNYI(*this, e, builtinID);574 case Builtin::BI__builtin_unreachable:575 emitUnreachable(e->getExprLoc(), /*createNewBlock=*/true);576 return RValue::getIgnored();577 case Builtin::BI__builtin_powi:578 case Builtin::BI__builtin_powif:579 case Builtin::BI__builtin_powil:580 case Builtin::BI__builtin_frexpl:581 case Builtin::BI__builtin_frexp:582 case Builtin::BI__builtin_frexpf:583 case Builtin::BI__builtin_frexpf128:584 case Builtin::BI__builtin_frexpf16:585 case Builtin::BImodf:586 case Builtin::BImodff:587 case Builtin::BImodfl:588 case Builtin::BI__builtin_modf:589 case Builtin::BI__builtin_modff:590 case Builtin::BI__builtin_modfl:591 case Builtin::BI__builtin_isgreater:592 case Builtin::BI__builtin_isgreaterequal:593 case Builtin::BI__builtin_isless:594 case Builtin::BI__builtin_islessequal:595 case Builtin::BI__builtin_islessgreater:596 case Builtin::BI__builtin_isunordered:597 // From https://clang.llvm.org/docs/LanguageExtensions.html#builtin-isfpclass598 //599 // The `__builtin_isfpclass()` builtin is a generalization of functions600 // isnan, isinf, isfinite and some others defined by the C standard. It tests601 // if the floating-point value, specified by the first argument, falls into602 // any of data classes, specified by the second argument.603 case Builtin::BI__builtin_isnan: {604 assert(!cir::MissingFeatures::cgFPOptionsRAII());605 mlir::Value v = emitScalarExpr(e->getArg(0));606 assert(!cir::MissingFeatures::fpConstraints());607 mlir::Location loc = getLoc(e->getBeginLoc());608 return RValue::get(builder.createBoolToInt(609 builder.createIsFPClass(loc, v, cir::FPClassTest::Nan),610 convertType(e->getType())));611 }612 613 case Builtin::BI__builtin_issignaling: {614 assert(!cir::MissingFeatures::cgFPOptionsRAII());615 mlir::Value v = emitScalarExpr(e->getArg(0));616 mlir::Location loc = getLoc(e->getBeginLoc());617 return RValue::get(builder.createBoolToInt(618 builder.createIsFPClass(loc, v, cir::FPClassTest::SignalingNaN),619 convertType(e->getType())));620 }621 622 case Builtin::BI__builtin_isinf: {623 assert(!cir::MissingFeatures::cgFPOptionsRAII());624 mlir::Value v = emitScalarExpr(e->getArg(0));625 assert(!cir::MissingFeatures::fpConstraints());626 mlir::Location loc = getLoc(e->getBeginLoc());627 return RValue::get(builder.createBoolToInt(628 builder.createIsFPClass(loc, v, cir::FPClassTest::Infinity),629 convertType(e->getType())));630 }631 case Builtin::BIfinite:632 case Builtin::BI__finite:633 case Builtin::BIfinitef:634 case Builtin::BI__finitef:635 case Builtin::BIfinitel:636 case Builtin::BI__finitel:637 case Builtin::BI__builtin_isfinite: {638 assert(!cir::MissingFeatures::cgFPOptionsRAII());639 mlir::Value v = emitScalarExpr(e->getArg(0));640 assert(!cir::MissingFeatures::fpConstraints());641 mlir::Location loc = getLoc(e->getBeginLoc());642 return RValue::get(builder.createBoolToInt(643 builder.createIsFPClass(loc, v, cir::FPClassTest::Finite),644 convertType(e->getType())));645 }646 647 case Builtin::BI__builtin_isnormal: {648 assert(!cir::MissingFeatures::cgFPOptionsRAII());649 mlir::Value v = emitScalarExpr(e->getArg(0));650 mlir::Location loc = getLoc(e->getBeginLoc());651 return RValue::get(builder.createBoolToInt(652 builder.createIsFPClass(loc, v, cir::FPClassTest::Normal),653 convertType(e->getType())));654 }655 656 case Builtin::BI__builtin_issubnormal: {657 assert(!cir::MissingFeatures::cgFPOptionsRAII());658 mlir::Value v = emitScalarExpr(e->getArg(0));659 mlir::Location loc = getLoc(e->getBeginLoc());660 return RValue::get(builder.createBoolToInt(661 builder.createIsFPClass(loc, v, cir::FPClassTest::Subnormal),662 convertType(e->getType())));663 }664 665 case Builtin::BI__builtin_iszero: {666 assert(!cir::MissingFeatures::cgFPOptionsRAII());667 mlir::Value v = emitScalarExpr(e->getArg(0));668 mlir::Location loc = getLoc(e->getBeginLoc());669 return RValue::get(builder.createBoolToInt(670 builder.createIsFPClass(loc, v, cir::FPClassTest::Zero),671 convertType(e->getType())));672 }673 case Builtin::BI__builtin_isfpclass: {674 Expr::EvalResult result;675 if (!e->getArg(1)->EvaluateAsInt(result, cgm.getASTContext()))676 break;677 678 assert(!cir::MissingFeatures::cgFPOptionsRAII());679 mlir::Value v = emitScalarExpr(e->getArg(0));680 uint64_t test = result.Val.getInt().getLimitedValue();681 mlir::Location loc = getLoc(e->getBeginLoc());682 //683 return RValue::get(builder.createBoolToInt(684 builder.createIsFPClass(loc, v, cir::FPClassTest(test)),685 convertType(e->getType())));686 }687 case Builtin::BI__builtin_nondeterministic_value:688 case Builtin::BI__builtin_elementwise_abs:689 return errorBuiltinNYI(*this, e, builtinID);690 case Builtin::BI__builtin_elementwise_acos:691 return emitUnaryFPBuiltin<cir::ACosOp>(*this, *e);692 case Builtin::BI__builtin_elementwise_asin:693 return emitUnaryFPBuiltin<cir::ASinOp>(*this, *e);694 case Builtin::BI__builtin_elementwise_atan:695 return emitUnaryFPBuiltin<cir::ATanOp>(*this, *e);696 case Builtin::BI__builtin_elementwise_atan2:697 case Builtin::BI__builtin_elementwise_ceil:698 case Builtin::BI__builtin_elementwise_exp:699 case Builtin::BI__builtin_elementwise_exp2:700 case Builtin::BI__builtin_elementwise_exp10:701 case Builtin::BI__builtin_elementwise_ldexp:702 case Builtin::BI__builtin_elementwise_log:703 case Builtin::BI__builtin_elementwise_log2:704 case Builtin::BI__builtin_elementwise_log10:705 case Builtin::BI__builtin_elementwise_pow:706 case Builtin::BI__builtin_elementwise_bitreverse:707 return errorBuiltinNYI(*this, e, builtinID);708 case Builtin::BI__builtin_elementwise_cos:709 return emitUnaryFPBuiltin<cir::CosOp>(*this, *e);710 case Builtin::BI__builtin_elementwise_cosh:711 case Builtin::BI__builtin_elementwise_floor:712 case Builtin::BI__builtin_elementwise_popcount:713 case Builtin::BI__builtin_elementwise_roundeven:714 case Builtin::BI__builtin_elementwise_round:715 case Builtin::BI__builtin_elementwise_rint:716 case Builtin::BI__builtin_elementwise_nearbyint:717 case Builtin::BI__builtin_elementwise_sin:718 case Builtin::BI__builtin_elementwise_sinh:719 case Builtin::BI__builtin_elementwise_tan:720 case Builtin::BI__builtin_elementwise_tanh:721 case Builtin::BI__builtin_elementwise_trunc:722 case Builtin::BI__builtin_elementwise_canonicalize:723 case Builtin::BI__builtin_elementwise_copysign:724 case Builtin::BI__builtin_elementwise_fma:725 case Builtin::BI__builtin_elementwise_fshl:726 case Builtin::BI__builtin_elementwise_fshr:727 case Builtin::BI__builtin_elementwise_add_sat:728 case Builtin::BI__builtin_elementwise_sub_sat:729 case Builtin::BI__builtin_elementwise_max:730 case Builtin::BI__builtin_elementwise_min:731 case Builtin::BI__builtin_elementwise_maxnum:732 case Builtin::BI__builtin_elementwise_minnum:733 case Builtin::BI__builtin_elementwise_maximum:734 case Builtin::BI__builtin_elementwise_minimum:735 case Builtin::BI__builtin_elementwise_maximumnum:736 case Builtin::BI__builtin_elementwise_minimumnum:737 case Builtin::BI__builtin_reduce_max:738 case Builtin::BI__builtin_reduce_min:739 case Builtin::BI__builtin_reduce_add:740 case Builtin::BI__builtin_reduce_mul:741 case Builtin::BI__builtin_reduce_xor:742 case Builtin::BI__builtin_reduce_or:743 case Builtin::BI__builtin_reduce_and:744 case Builtin::BI__builtin_reduce_maximum:745 case Builtin::BI__builtin_reduce_minimum:746 case Builtin::BI__builtin_matrix_transpose:747 case Builtin::BI__builtin_matrix_column_major_load:748 case Builtin::BI__builtin_matrix_column_major_store:749 case Builtin::BI__builtin_masked_load:750 case Builtin::BI__builtin_masked_expand_load:751 case Builtin::BI__builtin_masked_gather:752 case Builtin::BI__builtin_masked_store:753 case Builtin::BI__builtin_masked_compress_store:754 case Builtin::BI__builtin_masked_scatter:755 case Builtin::BI__builtin_isinf_sign:756 case Builtin::BI__builtin_flt_rounds:757 case Builtin::BI__builtin_set_flt_rounds:758 case Builtin::BI__builtin_fpclassify:759 return errorBuiltinNYI(*this, e, builtinID);760 case Builtin::BIalloca:761 case Builtin::BI_alloca:762 case Builtin::BI__builtin_alloca_uninitialized:763 case Builtin::BI__builtin_alloca:764 return emitBuiltinAlloca(*this, e, builtinID);765 case Builtin::BI__builtin_alloca_with_align_uninitialized:766 case Builtin::BI__builtin_alloca_with_align:767 case Builtin::BI__builtin_infer_alloc_token:768 case Builtin::BIbzero:769 case Builtin::BI__builtin_bzero:770 case Builtin::BIbcopy:771 case Builtin::BI__builtin_bcopy:772 return errorBuiltinNYI(*this, e, builtinID);773 case Builtin::BImemcpy:774 case Builtin::BI__builtin_memcpy:775 case Builtin::BImempcpy:776 case Builtin::BI__builtin_mempcpy:777 case Builtin::BI__builtin_memcpy_inline:778 case Builtin::BI__builtin_char_memchr:779 case Builtin::BI__builtin___memcpy_chk:780 case Builtin::BI__builtin_objc_memmove_collectable:781 case Builtin::BI__builtin___memmove_chk:782 case Builtin::BI__builtin_trivially_relocate:783 case Builtin::BImemmove:784 case Builtin::BI__builtin_memmove:785 case Builtin::BImemset:786 case Builtin::BI__builtin_memset:787 case Builtin::BI__builtin_memset_inline:788 case Builtin::BI__builtin___memset_chk:789 case Builtin::BI__builtin_wmemchr:790 case Builtin::BI__builtin_wmemcmp:791 break; // Handled as library calls below.792 case Builtin::BI__builtin_dwarf_cfa:793 return errorBuiltinNYI(*this, e, builtinID);794 case Builtin::BI__builtin_return_address:795 case Builtin::BI_ReturnAddress:796 case Builtin::BI__builtin_frame_address: {797 mlir::Location loc = getLoc(e->getExprLoc());798 llvm::APSInt level = e->getArg(0)->EvaluateKnownConstInt(getContext());799 if (builtinID == Builtin::BI__builtin_return_address) {800 return RValue::get(cir::ReturnAddrOp::create(801 builder, loc,802 builder.getConstAPInt(loc, builder.getUInt32Ty(), level)));803 }804 return RValue::get(cir::FrameAddrOp::create(805 builder, loc,806 builder.getConstAPInt(loc, builder.getUInt32Ty(), level)));807 }808 case Builtin::BI__builtin_extract_return_addr:809 case Builtin::BI__builtin_frob_return_addr:810 case Builtin::BI__builtin_dwarf_sp_column:811 case Builtin::BI__builtin_init_dwarf_reg_size_table:812 case Builtin::BI__builtin_eh_return:813 case Builtin::BI__builtin_unwind_init:814 case Builtin::BI__builtin_extend_pointer:815 case Builtin::BI__builtin_setjmp:816 case Builtin::BI__builtin_longjmp:817 case Builtin::BI__builtin_launder:818 case Builtin::BI__sync_fetch_and_add:819 case Builtin::BI__sync_fetch_and_sub:820 case Builtin::BI__sync_fetch_and_or:821 case Builtin::BI__sync_fetch_and_and:822 case Builtin::BI__sync_fetch_and_xor:823 case Builtin::BI__sync_fetch_and_nand:824 case Builtin::BI__sync_add_and_fetch:825 case Builtin::BI__sync_sub_and_fetch:826 case Builtin::BI__sync_and_and_fetch:827 case Builtin::BI__sync_or_and_fetch:828 case Builtin::BI__sync_xor_and_fetch:829 case Builtin::BI__sync_nand_and_fetch:830 case Builtin::BI__sync_val_compare_and_swap:831 case Builtin::BI__sync_bool_compare_and_swap:832 case Builtin::BI__sync_lock_test_and_set:833 case Builtin::BI__sync_lock_release:834 case Builtin::BI__sync_swap:835 case Builtin::BI__sync_fetch_and_add_1:836 case Builtin::BI__sync_fetch_and_add_2:837 case Builtin::BI__sync_fetch_and_add_4:838 case Builtin::BI__sync_fetch_and_add_8:839 case Builtin::BI__sync_fetch_and_add_16:840 case Builtin::BI__sync_fetch_and_sub_1:841 case Builtin::BI__sync_fetch_and_sub_2:842 case Builtin::BI__sync_fetch_and_sub_4:843 case Builtin::BI__sync_fetch_and_sub_8:844 case Builtin::BI__sync_fetch_and_sub_16:845 case Builtin::BI__sync_fetch_and_or_1:846 case Builtin::BI__sync_fetch_and_or_2:847 case Builtin::BI__sync_fetch_and_or_4:848 case Builtin::BI__sync_fetch_and_or_8:849 case Builtin::BI__sync_fetch_and_or_16:850 case Builtin::BI__sync_fetch_and_and_1:851 case Builtin::BI__sync_fetch_and_and_2:852 case Builtin::BI__sync_fetch_and_and_4:853 case Builtin::BI__sync_fetch_and_and_8:854 case Builtin::BI__sync_fetch_and_and_16:855 case Builtin::BI__sync_fetch_and_xor_1:856 case Builtin::BI__sync_fetch_and_xor_2:857 case Builtin::BI__sync_fetch_and_xor_4:858 case Builtin::BI__sync_fetch_and_xor_8:859 case Builtin::BI__sync_fetch_and_xor_16:860 case Builtin::BI__sync_fetch_and_nand_1:861 case Builtin::BI__sync_fetch_and_nand_2:862 case Builtin::BI__sync_fetch_and_nand_4:863 case Builtin::BI__sync_fetch_and_nand_8:864 case Builtin::BI__sync_fetch_and_nand_16:865 case Builtin::BI__sync_fetch_and_min:866 case Builtin::BI__sync_fetch_and_max:867 case Builtin::BI__sync_fetch_and_umin:868 case Builtin::BI__sync_fetch_and_umax:869 case Builtin::BI__sync_add_and_fetch_1:870 case Builtin::BI__sync_add_and_fetch_2:871 case Builtin::BI__sync_add_and_fetch_4:872 case Builtin::BI__sync_add_and_fetch_8:873 case Builtin::BI__sync_add_and_fetch_16:874 case Builtin::BI__sync_sub_and_fetch_1:875 case Builtin::BI__sync_sub_and_fetch_2:876 case Builtin::BI__sync_sub_and_fetch_4:877 case Builtin::BI__sync_sub_and_fetch_8:878 case Builtin::BI__sync_sub_and_fetch_16:879 case Builtin::BI__sync_and_and_fetch_1:880 case Builtin::BI__sync_and_and_fetch_2:881 case Builtin::BI__sync_and_and_fetch_4:882 case Builtin::BI__sync_and_and_fetch_8:883 case Builtin::BI__sync_and_and_fetch_16:884 case Builtin::BI__sync_or_and_fetch_1:885 case Builtin::BI__sync_or_and_fetch_2:886 case Builtin::BI__sync_or_and_fetch_4:887 case Builtin::BI__sync_or_and_fetch_8:888 case Builtin::BI__sync_or_and_fetch_16:889 case Builtin::BI__sync_xor_and_fetch_1:890 case Builtin::BI__sync_xor_and_fetch_2:891 case Builtin::BI__sync_xor_and_fetch_4:892 case Builtin::BI__sync_xor_and_fetch_8:893 case Builtin::BI__sync_xor_and_fetch_16:894 case Builtin::BI__sync_nand_and_fetch_1:895 case Builtin::BI__sync_nand_and_fetch_2:896 case Builtin::BI__sync_nand_and_fetch_4:897 case Builtin::BI__sync_nand_and_fetch_8:898 case Builtin::BI__sync_nand_and_fetch_16:899 case Builtin::BI__sync_val_compare_and_swap_1:900 case Builtin::BI__sync_val_compare_and_swap_2:901 case Builtin::BI__sync_val_compare_and_swap_4:902 case Builtin::BI__sync_val_compare_and_swap_8:903 case Builtin::BI__sync_val_compare_and_swap_16:904 case Builtin::BI__sync_bool_compare_and_swap_1:905 case Builtin::BI__sync_bool_compare_and_swap_2:906 case Builtin::BI__sync_bool_compare_and_swap_4:907 case Builtin::BI__sync_bool_compare_and_swap_8:908 case Builtin::BI__sync_bool_compare_and_swap_16:909 case Builtin::BI__sync_swap_1:910 case Builtin::BI__sync_swap_2:911 case Builtin::BI__sync_swap_4:912 case Builtin::BI__sync_swap_8:913 case Builtin::BI__sync_swap_16:914 case Builtin::BI__sync_lock_test_and_set_1:915 case Builtin::BI__sync_lock_test_and_set_2:916 case Builtin::BI__sync_lock_test_and_set_4:917 case Builtin::BI__sync_lock_test_and_set_8:918 case Builtin::BI__sync_lock_test_and_set_16:919 case Builtin::BI__sync_lock_release_1:920 case Builtin::BI__sync_lock_release_2:921 case Builtin::BI__sync_lock_release_4:922 case Builtin::BI__sync_lock_release_8:923 case Builtin::BI__sync_lock_release_16:924 case Builtin::BI__sync_synchronize:925 case Builtin::BI__builtin_nontemporal_load:926 case Builtin::BI__builtin_nontemporal_store:927 case Builtin::BI__c11_atomic_is_lock_free:928 case Builtin::BI__atomic_is_lock_free:929 case Builtin::BI__atomic_test_and_set:930 case Builtin::BI__atomic_clear:931 case Builtin::BI__atomic_thread_fence:932 case Builtin::BI__atomic_signal_fence:933 case Builtin::BI__c11_atomic_thread_fence:934 case Builtin::BI__c11_atomic_signal_fence:935 case Builtin::BI__scoped_atomic_thread_fence:936 case Builtin::BI__builtin_signbit:937 case Builtin::BI__builtin_signbitf:938 case Builtin::BI__builtin_signbitl:939 case Builtin::BI__warn_memset_zero_len:940 case Builtin::BI__annotation:941 case Builtin::BI__builtin_annotation:942 case Builtin::BI__builtin_addcb:943 case Builtin::BI__builtin_addcs:944 case Builtin::BI__builtin_addc:945 case Builtin::BI__builtin_addcl:946 case Builtin::BI__builtin_addcll:947 case Builtin::BI__builtin_subcb:948 case Builtin::BI__builtin_subcs:949 case Builtin::BI__builtin_subc:950 case Builtin::BI__builtin_subcl:951 case Builtin::BI__builtin_subcll:952 return errorBuiltinNYI(*this, e, builtinID);953 954 case Builtin::BI__builtin_add_overflow:955 case Builtin::BI__builtin_sub_overflow:956 case Builtin::BI__builtin_mul_overflow: {957 const clang::Expr *leftArg = e->getArg(0);958 const clang::Expr *rightArg = e->getArg(1);959 const clang::Expr *resultArg = e->getArg(2);960 961 clang::QualType resultQTy =962 resultArg->getType()->castAs<clang::PointerType>()->getPointeeType();963 964 WidthAndSignedness leftInfo =965 getIntegerWidthAndSignedness(cgm.getASTContext(), leftArg->getType());966 WidthAndSignedness rightInfo =967 getIntegerWidthAndSignedness(cgm.getASTContext(), rightArg->getType());968 WidthAndSignedness resultInfo =969 getIntegerWidthAndSignedness(cgm.getASTContext(), resultQTy);970 971 // Note we compute the encompassing type with the consideration to the972 // result type, so later in LLVM lowering we don't get redundant integral973 // extension casts.974 WidthAndSignedness encompassingInfo =975 EncompassingIntegerType({leftInfo, rightInfo, resultInfo});976 977 auto encompassingCIRTy = cir::IntType::get(978 &getMLIRContext(), encompassingInfo.width, encompassingInfo.isSigned);979 auto resultCIRTy = mlir::cast<cir::IntType>(cgm.convertType(resultQTy));980 981 mlir::Value left = emitScalarExpr(leftArg);982 mlir::Value right = emitScalarExpr(rightArg);983 Address resultPtr = emitPointerWithAlignment(resultArg);984 985 // Extend each operand to the encompassing type, if necessary.986 if (left.getType() != encompassingCIRTy)987 left =988 builder.createCast(cir::CastKind::integral, left, encompassingCIRTy);989 if (right.getType() != encompassingCIRTy)990 right =991 builder.createCast(cir::CastKind::integral, right, encompassingCIRTy);992 993 // Perform the operation on the extended values.994 cir::BinOpOverflowKind opKind;995 switch (builtinID) {996 default:997 llvm_unreachable("Unknown overflow builtin id.");998 case Builtin::BI__builtin_add_overflow:999 opKind = cir::BinOpOverflowKind::Add;1000 break;1001 case Builtin::BI__builtin_sub_overflow:1002 opKind = cir::BinOpOverflowKind::Sub;1003 break;1004 case Builtin::BI__builtin_mul_overflow:1005 opKind = cir::BinOpOverflowKind::Mul;1006 break;1007 }1008 1009 mlir::Location loc = getLoc(e->getSourceRange());1010 auto arithOp = cir::BinOpOverflowOp::create(builder, loc, resultCIRTy,1011 opKind, left, right);1012 1013 // Here is a slight difference from the original clang CodeGen:1014 // - In the original clang CodeGen, the checked arithmetic result is1015 // first computed as a value of the encompassing type, and then it is1016 // truncated to the actual result type with a second overflow checking.1017 // - In CIRGen, the checked arithmetic operation directly produce the1018 // checked arithmetic result in its expected type.1019 //1020 // So we don't need a truncation and a second overflow checking here.1021 1022 // Finally, store the result using the pointer.1023 bool isVolatile =1024 resultArg->getType()->getPointeeType().isVolatileQualified();1025 builder.createStore(loc, emitToMemory(arithOp.getResult(), resultQTy),1026 resultPtr, isVolatile);1027 1028 return RValue::get(arithOp.getOverflow());1029 }1030 1031 case Builtin::BI__builtin_uadd_overflow:1032 case Builtin::BI__builtin_uaddl_overflow:1033 case Builtin::BI__builtin_uaddll_overflow:1034 case Builtin::BI__builtin_usub_overflow:1035 case Builtin::BI__builtin_usubl_overflow:1036 case Builtin::BI__builtin_usubll_overflow:1037 case Builtin::BI__builtin_umul_overflow:1038 case Builtin::BI__builtin_umull_overflow:1039 case Builtin::BI__builtin_umulll_overflow:1040 case Builtin::BI__builtin_sadd_overflow:1041 case Builtin::BI__builtin_saddl_overflow:1042 case Builtin::BI__builtin_saddll_overflow:1043 case Builtin::BI__builtin_ssub_overflow:1044 case Builtin::BI__builtin_ssubl_overflow:1045 case Builtin::BI__builtin_ssubll_overflow:1046 case Builtin::BI__builtin_smul_overflow:1047 case Builtin::BI__builtin_smull_overflow:1048 case Builtin::BI__builtin_smulll_overflow: {1049 // Scalarize our inputs.1050 mlir::Value x = emitScalarExpr(e->getArg(0));1051 mlir::Value y = emitScalarExpr(e->getArg(1));1052 1053 const clang::Expr *resultArg = e->getArg(2);1054 Address resultPtr = emitPointerWithAlignment(resultArg);1055 1056 // Decide which of the arithmetic operation we are lowering to:1057 cir::BinOpOverflowKind arithKind;1058 switch (builtinID) {1059 default:1060 llvm_unreachable("Unknown overflow builtin id.");1061 case Builtin::BI__builtin_uadd_overflow:1062 case Builtin::BI__builtin_uaddl_overflow:1063 case Builtin::BI__builtin_uaddll_overflow:1064 case Builtin::BI__builtin_sadd_overflow:1065 case Builtin::BI__builtin_saddl_overflow:1066 case Builtin::BI__builtin_saddll_overflow:1067 arithKind = cir::BinOpOverflowKind::Add;1068 break;1069 case Builtin::BI__builtin_usub_overflow:1070 case Builtin::BI__builtin_usubl_overflow:1071 case Builtin::BI__builtin_usubll_overflow:1072 case Builtin::BI__builtin_ssub_overflow:1073 case Builtin::BI__builtin_ssubl_overflow:1074 case Builtin::BI__builtin_ssubll_overflow:1075 arithKind = cir::BinOpOverflowKind::Sub;1076 break;1077 case Builtin::BI__builtin_umul_overflow:1078 case Builtin::BI__builtin_umull_overflow:1079 case Builtin::BI__builtin_umulll_overflow:1080 case Builtin::BI__builtin_smul_overflow:1081 case Builtin::BI__builtin_smull_overflow:1082 case Builtin::BI__builtin_smulll_overflow:1083 arithKind = cir::BinOpOverflowKind::Mul;1084 break;1085 }1086 1087 clang::QualType resultQTy =1088 resultArg->getType()->castAs<clang::PointerType>()->getPointeeType();1089 auto resultCIRTy = mlir::cast<cir::IntType>(cgm.convertType(resultQTy));1090 1091 mlir::Location loc = getLoc(e->getSourceRange());1092 cir::BinOpOverflowOp arithOp = cir::BinOpOverflowOp::create(1093 builder, loc, resultCIRTy, arithKind, x, y);1094 1095 bool isVolatile =1096 resultArg->getType()->getPointeeType().isVolatileQualified();1097 builder.createStore(loc, emitToMemory(arithOp.getResult(), resultQTy),1098 resultPtr, isVolatile);1099 1100 return RValue::get(arithOp.getOverflow());1101 }1102 1103 case Builtin::BIaddressof:1104 case Builtin::BI__addressof:1105 case Builtin::BI__builtin_addressof:1106 case Builtin::BI__builtin_function_start:1107 return errorBuiltinNYI(*this, e, builtinID);1108 case Builtin::BI__builtin_operator_new:1109 return emitNewOrDeleteBuiltinCall(1110 e->getCallee()->getType()->castAs<FunctionProtoType>(), e, OO_New);1111 case Builtin::BI__builtin_operator_delete:1112 emitNewOrDeleteBuiltinCall(1113 e->getCallee()->getType()->castAs<FunctionProtoType>(), e, OO_Delete);1114 return RValue::get(nullptr);1115 case Builtin::BI__builtin_is_aligned:1116 case Builtin::BI__builtin_align_up:1117 case Builtin::BI__builtin_align_down:1118 case Builtin::BI__noop:1119 case Builtin::BI__builtin_call_with_static_chain:1120 case Builtin::BI_InterlockedExchange8:1121 case Builtin::BI_InterlockedExchange16:1122 case Builtin::BI_InterlockedExchange:1123 case Builtin::BI_InterlockedExchangePointer:1124 case Builtin::BI_InterlockedCompareExchangePointer:1125 case Builtin::BI_InterlockedCompareExchangePointer_nf:1126 case Builtin::BI_InterlockedCompareExchange8:1127 case Builtin::BI_InterlockedCompareExchange16:1128 case Builtin::BI_InterlockedCompareExchange:1129 case Builtin::BI_InterlockedCompareExchange64:1130 case Builtin::BI_InterlockedIncrement16:1131 case Builtin::BI_InterlockedIncrement:1132 case Builtin::BI_InterlockedDecrement16:1133 case Builtin::BI_InterlockedDecrement:1134 case Builtin::BI_InterlockedAnd8:1135 case Builtin::BI_InterlockedAnd16:1136 case Builtin::BI_InterlockedAnd:1137 case Builtin::BI_InterlockedExchangeAdd8:1138 case Builtin::BI_InterlockedExchangeAdd16:1139 case Builtin::BI_InterlockedExchangeAdd:1140 case Builtin::BI_InterlockedExchangeSub8:1141 case Builtin::BI_InterlockedExchangeSub16:1142 case Builtin::BI_InterlockedExchangeSub:1143 case Builtin::BI_InterlockedOr8:1144 case Builtin::BI_InterlockedOr16:1145 case Builtin::BI_InterlockedOr:1146 case Builtin::BI_InterlockedXor8:1147 case Builtin::BI_InterlockedXor16:1148 case Builtin::BI_InterlockedXor:1149 case Builtin::BI_bittest64:1150 case Builtin::BI_bittest:1151 case Builtin::BI_bittestandcomplement64:1152 case Builtin::BI_bittestandcomplement:1153 case Builtin::BI_bittestandreset64:1154 case Builtin::BI_bittestandreset:1155 case Builtin::BI_bittestandset64:1156 case Builtin::BI_bittestandset:1157 case Builtin::BI_interlockedbittestandreset:1158 case Builtin::BI_interlockedbittestandreset64:1159 case Builtin::BI_interlockedbittestandreset64_acq:1160 case Builtin::BI_interlockedbittestandreset64_rel:1161 case Builtin::BI_interlockedbittestandreset64_nf:1162 case Builtin::BI_interlockedbittestandset64:1163 case Builtin::BI_interlockedbittestandset64_acq:1164 case Builtin::BI_interlockedbittestandset64_rel:1165 case Builtin::BI_interlockedbittestandset64_nf:1166 case Builtin::BI_interlockedbittestandset:1167 case Builtin::BI_interlockedbittestandset_acq:1168 case Builtin::BI_interlockedbittestandset_rel:1169 case Builtin::BI_interlockedbittestandset_nf:1170 case Builtin::BI_interlockedbittestandreset_acq:1171 case Builtin::BI_interlockedbittestandreset_rel:1172 case Builtin::BI_interlockedbittestandreset_nf:1173 case Builtin::BI__iso_volatile_load8:1174 case Builtin::BI__iso_volatile_load16:1175 case Builtin::BI__iso_volatile_load32:1176 case Builtin::BI__iso_volatile_load64:1177 case Builtin::BI__iso_volatile_store8:1178 case Builtin::BI__iso_volatile_store16:1179 case Builtin::BI__iso_volatile_store32:1180 case Builtin::BI__iso_volatile_store64:1181 case Builtin::BI__builtin_ptrauth_sign_constant:1182 case Builtin::BI__builtin_ptrauth_auth:1183 case Builtin::BI__builtin_ptrauth_auth_and_resign:1184 case Builtin::BI__builtin_ptrauth_blend_discriminator:1185 case Builtin::BI__builtin_ptrauth_sign_generic_data:1186 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:1187 case Builtin::BI__builtin_ptrauth_strip:1188 case Builtin::BI__builtin_get_vtable_pointer:1189 case Builtin::BI__exception_code:1190 case Builtin::BI_exception_code:1191 case Builtin::BI__exception_info:1192 case Builtin::BI_exception_info:1193 case Builtin::BI__abnormal_termination:1194 case Builtin::BI_abnormal_termination:1195 case Builtin::BI_setjmpex:1196 case Builtin::BI_setjmp:1197 case Builtin::BImove:1198 case Builtin::BImove_if_noexcept:1199 case Builtin::BIforward:1200 case Builtin::BIforward_like:1201 case Builtin::BIas_const:1202 case Builtin::BI__GetExceptionInfo:1203 case Builtin::BI__fastfail:1204 case Builtin::BIread_pipe:1205 case Builtin::BIwrite_pipe:1206 case Builtin::BIreserve_read_pipe:1207 case Builtin::BIreserve_write_pipe:1208 case Builtin::BIwork_group_reserve_read_pipe:1209 case Builtin::BIwork_group_reserve_write_pipe:1210 case Builtin::BIsub_group_reserve_read_pipe:1211 case Builtin::BIsub_group_reserve_write_pipe:1212 case Builtin::BIcommit_read_pipe:1213 case Builtin::BIcommit_write_pipe:1214 case Builtin::BIwork_group_commit_read_pipe:1215 case Builtin::BIwork_group_commit_write_pipe:1216 case Builtin::BIsub_group_commit_read_pipe:1217 case Builtin::BIsub_group_commit_write_pipe:1218 case Builtin::BIget_pipe_num_packets:1219 case Builtin::BIget_pipe_max_packets:1220 case Builtin::BIto_global:1221 case Builtin::BIto_local:1222 case Builtin::BIto_private:1223 case Builtin::BIenqueue_kernel:1224 case Builtin::BIget_kernel_work_group_size:1225 case Builtin::BIget_kernel_preferred_work_group_size_multiple:1226 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:1227 case Builtin::BIget_kernel_sub_group_count_for_ndrange:1228 case Builtin::BI__builtin_store_half:1229 case Builtin::BI__builtin_store_halff:1230 case Builtin::BI__builtin_load_half:1231 case Builtin::BI__builtin_load_halff:1232 return errorBuiltinNYI(*this, e, builtinID);1233 case Builtin::BI__builtin_printf:1234 case Builtin::BIprintf:1235 break;1236 case Builtin::BI__builtin_canonicalize:1237 case Builtin::BI__builtin_canonicalizef:1238 case Builtin::BI__builtin_canonicalizef16:1239 case Builtin::BI__builtin_canonicalizel:1240 case Builtin::BI__builtin_thread_pointer:1241 case Builtin::BI__builtin_os_log_format:1242 case Builtin::BI__xray_customevent:1243 case Builtin::BI__xray_typedevent:1244 case Builtin::BI__builtin_ms_va_start:1245 case Builtin::BI__builtin_ms_va_end:1246 case Builtin::BI__builtin_ms_va_copy:1247 case Builtin::BI__builtin_get_device_side_mangled_name:1248 return errorBuiltinNYI(*this, e, builtinID);1249 }1250 1251 // If this is an alias for a lib function (e.g. __builtin_sin), emit1252 // the call using the normal call path, but using the unmangled1253 // version of the function name.1254 if (getContext().BuiltinInfo.isLibFunction(builtinID))1255 return emitLibraryCall(*this, fd, e,1256 cgm.getBuiltinLibFunction(fd, builtinID));1257 1258 // Some target-specific builtins can have aggregate return values, e.g.1259 // __builtin_arm_mve_vld2q_u32. So if the result is an aggregate, force1260 // returnValue to be non-null, so that the target-specific emission code can1261 // always just emit into it.1262 cir::TypeEvaluationKind evalKind = getEvaluationKind(e->getType());1263 if (evalKind == cir::TEK_Aggregate && returnValue.isNull()) {1264 cgm.errorNYI(e->getSourceRange(), "aggregate return value from builtin");1265 return getUndefRValue(e->getType());1266 }1267 1268 // Now see if we can emit a target-specific builtin.1269 if (mlir::Value v = emitTargetBuiltinExpr(builtinID, e, returnValue)) {1270 switch (evalKind) {1271 case cir::TEK_Scalar:1272 if (mlir::isa<cir::VoidType>(v.getType()))1273 return RValue::get(nullptr);1274 return RValue::get(v);1275 case cir::TEK_Aggregate:1276 cgm.errorNYI(e->getSourceRange(), "aggregate return value from builtin");1277 return getUndefRValue(e->getType());1278 case cir::TEK_Complex:1279 llvm_unreachable("No current target builtin returns complex");1280 }1281 llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");1282 }1283 1284 cgm.errorNYI(e->getSourceRange(),1285 std::string("unimplemented builtin call: ") +1286 getContext().BuiltinInfo.getName(builtinID));1287 return getUndefRValue(e->getType());1288}1289 1290static mlir::Value emitTargetArchBuiltinExpr(CIRGenFunction *cgf,1291 unsigned builtinID,1292 const CallExpr *e,1293 ReturnValueSlot &returnValue,1294 llvm::Triple::ArchType arch) {1295 // When compiling in HipStdPar mode we have to be conservative in rejecting1296 // target specific features in the FE, and defer the possible error to the1297 // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is1298 // referenced by an accelerator executable function, we emit an error.1299 // Returning nullptr here leads to the builtin being handled in1300 // EmitStdParUnsupportedBuiltin.1301 if (cgf->getLangOpts().HIPStdPar && cgf->getLangOpts().CUDAIsDevice &&1302 arch != cgf->getTarget().getTriple().getArch())1303 return {};1304 1305 switch (arch) {1306 case llvm::Triple::arm:1307 case llvm::Triple::armeb:1308 case llvm::Triple::thumb:1309 case llvm::Triple::thumbeb:1310 case llvm::Triple::aarch64:1311 case llvm::Triple::aarch64_32:1312 case llvm::Triple::aarch64_be:1313 case llvm::Triple::bpfeb:1314 case llvm::Triple::bpfel:1315 // These are actually NYI, but that will be reported by emitBuiltinExpr.1316 // At this point, we don't even know that the builtin is target-specific.1317 return nullptr;1318 1319 case llvm::Triple::x86:1320 case llvm::Triple::x86_64:1321 return cgf->emitX86BuiltinExpr(builtinID, e);1322 1323 case llvm::Triple::ppc:1324 case llvm::Triple::ppcle:1325 case llvm::Triple::ppc64:1326 case llvm::Triple::ppc64le:1327 case llvm::Triple::r600:1328 case llvm::Triple::amdgcn:1329 case llvm::Triple::systemz:1330 case llvm::Triple::nvptx:1331 case llvm::Triple::nvptx64:1332 case llvm::Triple::wasm32:1333 case llvm::Triple::wasm64:1334 case llvm::Triple::hexagon:1335 case llvm::Triple::riscv32:1336 case llvm::Triple::riscv64:1337 // These are actually NYI, but that will be reported by emitBuiltinExpr.1338 // At this point, we don't even know that the builtin is target-specific.1339 return {};1340 default:1341 return {};1342 }1343}1344 1345mlir::Value1346CIRGenFunction::emitTargetBuiltinExpr(unsigned builtinID, const CallExpr *e,1347 ReturnValueSlot &returnValue) {1348 if (getContext().BuiltinInfo.isAuxBuiltinID(builtinID)) {1349 assert(getContext().getAuxTargetInfo() && "Missing aux target info");1350 return emitTargetArchBuiltinExpr(1351 this, getContext().BuiltinInfo.getAuxBuiltinID(builtinID), e,1352 returnValue, getContext().getAuxTargetInfo()->getTriple().getArch());1353 }1354 1355 return emitTargetArchBuiltinExpr(this, builtinID, e, returnValue,1356 getTarget().getTriple().getArch());1357}1358 1359mlir::Value CIRGenFunction::emitScalarOrConstFoldImmArg(1360 const unsigned iceArguments, const unsigned idx, const Expr *argExpr) {1361 mlir::Value arg = {};1362 if ((iceArguments & (1 << idx)) == 0) {1363 arg = emitScalarExpr(argExpr);1364 } else {1365 // If this is required to be a constant, constant fold it so that we1366 // know that the generated intrinsic gets a ConstantInt.1367 const std::optional<llvm::APSInt> result =1368 argExpr->getIntegerConstantExpr(getContext());1369 assert(result && "Expected argument to be a constant");1370 arg = builder.getConstInt(getLoc(argExpr->getSourceRange()), *result);1371 }1372 return arg;1373}1374 1375/// Given a builtin id for a function like "__builtin_fabsf", return a Function*1376/// for "fabsf".1377cir::FuncOp CIRGenModule::getBuiltinLibFunction(const FunctionDecl *fd,1378 unsigned builtinID) {1379 assert(astContext.BuiltinInfo.isLibFunction(builtinID));1380 1381 // Get the name, skip over the __builtin_ prefix (if necessary). We may have1382 // to build this up so provide a small stack buffer to handle the vast1383 // majority of names.1384 llvm::SmallString<64> name;1385 1386 assert(!cir::MissingFeatures::asmLabelAttr());1387 name = astContext.BuiltinInfo.getName(builtinID).substr(10);1388 1389 GlobalDecl d(fd);1390 mlir::Type type = convertType(fd->getType());1391 return getOrCreateCIRFunction(name, type, d, /*forVTable=*/false);1392}1393 1394mlir::Value CIRGenFunction::emitCheckedArgForAssume(const Expr *e) {1395 mlir::Value argValue = evaluateExprAsBool(e);1396 if (!sanOpts.has(SanitizerKind::Builtin))1397 return argValue;1398 1399 assert(!cir::MissingFeatures::sanitizers());1400 cgm.errorNYI(e->getSourceRange(),1401 "emitCheckedArgForAssume: sanitizers are NYI");1402 return {};1403}1404 1405void CIRGenFunction::emitVAStart(mlir::Value vaList, mlir::Value count) {1406 // LLVM codegen casts to *i8, no real gain on doing this for CIRGen this1407 // early, defer to LLVM lowering.1408 cir::VAStartOp::create(builder, vaList.getLoc(), vaList, count);1409}1410 1411void CIRGenFunction::emitVAEnd(mlir::Value vaList) {1412 cir::VAEndOp::create(builder, vaList.getLoc(), vaList);1413}1414 1415// FIXME(cir): This completely abstracts away the ABI with a generic CIR Op. By1416// default this lowers to llvm.va_arg which is incomplete and not ABI-compliant1417// on most targets so cir.va_arg will need some ABI handling in LoweringPrepare1418mlir::Value CIRGenFunction::emitVAArg(VAArgExpr *ve) {1419 assert(!cir::MissingFeatures::msabi());1420 assert(!cir::MissingFeatures::vlas());1421 mlir::Location loc = cgm.getLoc(ve->getExprLoc());1422 mlir::Type type = convertType(ve->getType());1423 mlir::Value vaList = emitVAListRef(ve->getSubExpr()).getPointer();1424 return cir::VAArgOp::create(builder, loc, type, vaList);1425}1426 1427mlir::Value CIRGenFunction::emitBuiltinObjectSize(const Expr *e, unsigned type,1428 cir::IntType resType,1429 mlir::Value emittedE,1430 bool isDynamic) {1431 assert(!cir::MissingFeatures::opCallImplicitObjectSizeArgs());1432 1433 // LLVM can't handle type=3 appropriately, and __builtin_object_size shouldn't1434 // evaluate e for side-effects. In either case, just like original LLVM1435 // lowering, we shouldn't lower to `cir.objsize` but to a constant instead.1436 if (type == 3 || (!emittedE && e->HasSideEffects(getContext())))1437 return builder.getConstInt(getLoc(e->getSourceRange()), resType,1438 (type & 2) ? 0 : -1);1439 1440 mlir::Value ptr = emittedE ? emittedE : emitScalarExpr(e);1441 assert(mlir::isa<cir::PointerType>(ptr.getType()) &&1442 "Non-pointer passed to __builtin_object_size?");1443 1444 assert(!cir::MissingFeatures::countedBySize());1445 1446 // Extract the min/max mode from type. CIR only supports type 01447 // (max, whole object) and type 2 (min, whole object), not type 1 or 31448 // (closest subobject variants).1449 const bool min = ((type & 2) != 0);1450 // For GCC compatibility, __builtin_object_size treats NULL as unknown size.1451 auto op =1452 cir::ObjSizeOp::create(builder, getLoc(e->getSourceRange()), resType, ptr,1453 min, /*nullUnknown=*/true, isDynamic);1454 return op.getResult();1455}1456 1457mlir::Value CIRGenFunction::evaluateOrEmitBuiltinObjectSize(1458 const Expr *e, unsigned type, cir::IntType resType, mlir::Value emittedE,1459 bool isDynamic) {1460 uint64_t objectSize;1461 if (!e->tryEvaluateObjectSize(objectSize, getContext(), type))1462 return emitBuiltinObjectSize(e, type, resType, emittedE, isDynamic);1463 return builder.getConstInt(getLoc(e->getSourceRange()), resType, objectSize);1464}1465