1055 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This contains code to emit x86/x86_64 Builtin calls as CIR or a function10// call to be later resolved.11//12//===----------------------------------------------------------------------===//13 14#include "CIRGenFunction.h"15#include "CIRGenModule.h"16#include "clang/Basic/Builtins.h"17#include "clang/Basic/TargetBuiltins.h"18#include "clang/CIR/MissingFeatures.h"19 20using namespace clang;21using namespace clang::CIRGen;22 23template <typename... Operands>24static mlir::Value emitIntrinsicCallOp(CIRGenBuilderTy &builder,25 mlir::Location loc, const StringRef str,26 const mlir::Type &resTy,27 Operands &&...op) {28 return cir::LLVMIntrinsicCallOp::create(builder, loc,29 builder.getStringAttr(str), resTy,30 std::forward<Operands>(op)...)31 .getResult();32}33 34// OG has unordered comparison as a form of optimization in addition to35// ordered comparison, while CIR doesn't.36//37// This means that we can't encode the comparison code of UGT (unordered38// greater than), at least not at the CIR level.39//40// The boolean shouldInvert compensates for this.41// For example: to get to the comparison code UGT, we pass in42// emitVectorFCmp (OLE, shouldInvert = true) since OLE is the inverse of UGT.43 44// There are several ways to support this otherwise:45// - register extra CmpOpKind for unordered comparison types and build the46// translation code for47// to go from CIR -> LLVM dialect. Notice we get this naturally with48// shouldInvert, benefiting from existing infrastructure, albeit having to49// generate an extra `not` at CIR).50// - Just add extra comparison code to a new VecCmpOpKind instead of51// cluttering CmpOpKind.52// - Add a boolean in VecCmpOp to indicate if it's doing unordered or ordered53// comparison54// - Just emit the intrinsics call instead of calling this helper, see how the55// LLVM lowering handles this.56static mlir::Value emitVectorFCmp(CIRGenBuilderTy &builder,57 llvm::SmallVector<mlir::Value> &ops,58 mlir::Location loc, cir::CmpOpKind pred,59 bool shouldInvert) {60 assert(!cir::MissingFeatures::cgFPOptionsRAII());61 // TODO(cir): Add isSignaling boolean once emitConstrainedFPCall implemented62 assert(!cir::MissingFeatures::emitConstrainedFPCall());63 mlir::Value cmp = builder.createVecCompare(loc, pred, ops[0], ops[1]);64 mlir::Value bitCast = builder.createBitcast(65 shouldInvert ? builder.createNot(cmp) : cmp, ops[0].getType());66 return bitCast;67}68 69static mlir::Value getMaskVecValue(CIRGenBuilderTy &builder, mlir::Location loc,70 mlir::Value mask, unsigned numElems) {71 auto maskTy = cir::VectorType::get(72 builder.getUIntNTy(1), cast<cir::IntType>(mask.getType()).getWidth());73 mlir::Value maskVec = builder.createBitcast(mask, maskTy);74 75 // If we have less than 8 elements, then the starting mask was an i8 and76 // we need to extract down to the right number of elements.77 if (numElems < 8) {78 SmallVector<mlir::Attribute, 4> indices;79 mlir::Type i32Ty = builder.getSInt32Ty();80 for (auto i : llvm::seq<unsigned>(0, numElems))81 indices.push_back(cir::IntAttr::get(i32Ty, i));82 83 maskVec = builder.createVecShuffle(loc, maskVec, maskVec, indices);84 }85 return maskVec;86}87 88static mlir::Value emitX86MaskAddLogic(CIRGenBuilderTy &builder,89 mlir::Location loc,90 const std::string &intrinsicName,91 SmallVectorImpl<mlir::Value> &ops) {92 93 auto intTy = cast<cir::IntType>(ops[0].getType());94 unsigned numElts = intTy.getWidth();95 mlir::Value lhsVec = getMaskVecValue(builder, loc, ops[0], numElts);96 mlir::Value rhsVec = getMaskVecValue(builder, loc, ops[1], numElts);97 mlir::Type vecTy = lhsVec.getType();98 mlir::Value resVec = emitIntrinsicCallOp(builder, loc, intrinsicName, vecTy,99 mlir::ValueRange{lhsVec, rhsVec});100 return builder.createBitcast(resVec, ops[0].getType());101}102 103static mlir::Value emitX86MaskLogic(CIRGenBuilderTy &builder,104 mlir::Location loc,105 cir::BinOpKind binOpKind,106 SmallVectorImpl<mlir::Value> &ops,107 bool invertLHS = false) {108 unsigned numElts = cast<cir::IntType>(ops[0].getType()).getWidth();109 mlir::Value lhs = getMaskVecValue(builder, loc, ops[0], numElts);110 mlir::Value rhs = getMaskVecValue(builder, loc, ops[1], numElts);111 112 if (invertLHS)113 lhs = builder.createNot(lhs);114 return builder.createBitcast(builder.createBinop(loc, lhs, binOpKind, rhs),115 ops[0].getType());116}117 118mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,119 const CallExpr *expr) {120 if (builtinID == Builtin::BI__builtin_cpu_is) {121 cgm.errorNYI(expr->getSourceRange(), "__builtin_cpu_is");122 return {};123 }124 if (builtinID == Builtin::BI__builtin_cpu_supports) {125 cgm.errorNYI(expr->getSourceRange(), "__builtin_cpu_supports");126 return {};127 }128 if (builtinID == Builtin::BI__builtin_cpu_init) {129 cgm.errorNYI(expr->getSourceRange(), "__builtin_cpu_init");130 return {};131 }132 133 // Handle MSVC intrinsics before argument evaluation to prevent double134 // evaluation.135 assert(!cir::MissingFeatures::msvcBuiltins());136 137 // Find out if any arguments are required to be integer constant expressions.138 assert(!cir::MissingFeatures::handleBuiltinICEArguments());139 140 // The operands of the builtin call141 llvm::SmallVector<mlir::Value> ops;142 143 // `ICEArguments` is a bitmap indicating whether the argument at the i-th bit144 // is required to be a constant integer expression.145 unsigned iceArguments = 0;146 ASTContext::GetBuiltinTypeError error;147 getContext().GetBuiltinType(builtinID, error, &iceArguments);148 assert(error == ASTContext::GE_None && "Error while getting builtin type.");149 150 for (auto [idx, arg] : llvm::enumerate(expr->arguments()))151 ops.push_back(emitScalarOrConstFoldImmArg(iceArguments, idx, arg));152 153 CIRGenBuilderTy &builder = getBuilder();154 mlir::Type voidTy = builder.getVoidTy();155 156 switch (builtinID) {157 default:158 return {};159 case X86::BI_mm_clflush:160 return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),161 "x86.sse2.clflush", voidTy, ops[0]);162 case X86::BI_mm_lfence:163 return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),164 "x86.sse2.lfence", voidTy);165 case X86::BI_mm_pause:166 return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),167 "x86.sse2.pause", voidTy);168 case X86::BI_mm_mfence:169 return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),170 "x86.sse2.mfence", voidTy);171 case X86::BI_mm_sfence:172 return emitIntrinsicCallOp(builder, getLoc(expr->getExprLoc()),173 "x86.sse.sfence", voidTy);174 case X86::BI_mm_prefetch:175 case X86::BI__rdtsc:176 case X86::BI__builtin_ia32_rdtscp: {177 cgm.errorNYI(expr->getSourceRange(),178 std::string("unimplemented X86 builtin call: ") +179 getContext().BuiltinInfo.getName(builtinID));180 return {};181 }182 case X86::BI__builtin_ia32_lzcnt_u16:183 case X86::BI__builtin_ia32_lzcnt_u32:184 case X86::BI__builtin_ia32_lzcnt_u64: {185 mlir::Location loc = getLoc(expr->getExprLoc());186 mlir::Value isZeroPoison = builder.getFalse(loc);187 return emitIntrinsicCallOp(builder, loc, "ctlz", ops[0].getType(),188 mlir::ValueRange{ops[0], isZeroPoison});189 }190 case X86::BI__builtin_ia32_tzcnt_u16:191 case X86::BI__builtin_ia32_tzcnt_u32:192 case X86::BI__builtin_ia32_tzcnt_u64: {193 mlir::Location loc = getLoc(expr->getExprLoc());194 mlir::Value isZeroPoison = builder.getFalse(loc);195 return emitIntrinsicCallOp(builder, loc, "cttz", ops[0].getType(),196 mlir::ValueRange{ops[0], isZeroPoison});197 }198 case X86::BI__builtin_ia32_undef128:199 case X86::BI__builtin_ia32_undef256:200 case X86::BI__builtin_ia32_undef512:201 // The x86 definition of "undef" is not the same as the LLVM definition202 // (PR32176). We leave optimizing away an unnecessary zero constant to the203 // IR optimizer and backend.204 // TODO: If we had a "freeze" IR instruction to generate a fixed undef205 // value, we should use that here instead of a zero.206 return builder.getNullValue(convertType(expr->getType()),207 getLoc(expr->getExprLoc()));208 case X86::BI__builtin_ia32_vec_ext_v4hi:209 case X86::BI__builtin_ia32_vec_ext_v16qi:210 case X86::BI__builtin_ia32_vec_ext_v8hi:211 case X86::BI__builtin_ia32_vec_ext_v4si:212 case X86::BI__builtin_ia32_vec_ext_v4sf:213 case X86::BI__builtin_ia32_vec_ext_v2di:214 case X86::BI__builtin_ia32_vec_ext_v32qi:215 case X86::BI__builtin_ia32_vec_ext_v16hi:216 case X86::BI__builtin_ia32_vec_ext_v8si:217 case X86::BI__builtin_ia32_vec_ext_v4di: {218 unsigned numElts = cast<cir::VectorType>(ops[0].getType()).getSize();219 220 uint64_t index =221 ops[1].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue();222 223 index &= numElts - 1;224 225 cir::ConstantOp indexVal =226 builder.getUInt64(index, getLoc(expr->getExprLoc()));227 228 // These builtins exist so we can ensure the index is an ICE and in range.229 // Otherwise we could just do this in the header file.230 return cir::VecExtractOp::create(builder, getLoc(expr->getExprLoc()),231 ops[0], indexVal);232 }233 case X86::BI__builtin_ia32_vec_set_v4hi:234 case X86::BI__builtin_ia32_vec_set_v16qi:235 case X86::BI__builtin_ia32_vec_set_v8hi:236 case X86::BI__builtin_ia32_vec_set_v4si:237 case X86::BI__builtin_ia32_vec_set_v2di:238 case X86::BI__builtin_ia32_vec_set_v32qi:239 case X86::BI__builtin_ia32_vec_set_v16hi:240 case X86::BI__builtin_ia32_vec_set_v8si:241 case X86::BI__builtin_ia32_vec_set_v4di:242 cgm.errorNYI(expr->getSourceRange(),243 std::string("unimplemented X86 builtin call: ") +244 getContext().BuiltinInfo.getName(builtinID));245 return {};246 case X86::BI_mm_setcsr:247 case X86::BI__builtin_ia32_ldmxcsr: {248 mlir::Location loc = getLoc(expr->getExprLoc());249 Address tmp = createMemTemp(expr->getArg(0)->getType(), loc);250 builder.createStore(loc, ops[0], tmp);251 return emitIntrinsicCallOp(builder, loc, "x86.sse.ldmxcsr",252 builder.getVoidTy(), tmp.getPointer());253 }254 case X86::BI_mm_getcsr:255 case X86::BI__builtin_ia32_stmxcsr: {256 mlir::Location loc = getLoc(expr->getExprLoc());257 Address tmp = createMemTemp(expr->getType(), loc);258 emitIntrinsicCallOp(builder, loc, "x86.sse.stmxcsr", builder.getVoidTy(),259 tmp.getPointer());260 return builder.createLoad(loc, tmp);261 }262 case X86::BI__builtin_ia32_xsave:263 case X86::BI__builtin_ia32_xsave64:264 case X86::BI__builtin_ia32_xrstor:265 case X86::BI__builtin_ia32_xrstor64:266 case X86::BI__builtin_ia32_xsaveopt:267 case X86::BI__builtin_ia32_xsaveopt64:268 case X86::BI__builtin_ia32_xrstors:269 case X86::BI__builtin_ia32_xrstors64:270 case X86::BI__builtin_ia32_xsavec:271 case X86::BI__builtin_ia32_xsavec64:272 case X86::BI__builtin_ia32_xsaves:273 case X86::BI__builtin_ia32_xsaves64:274 case X86::BI__builtin_ia32_xsetbv:275 case X86::BI_xsetbv:276 case X86::BI__builtin_ia32_xgetbv:277 case X86::BI_xgetbv:278 case X86::BI__builtin_ia32_storedqudi128_mask:279 case X86::BI__builtin_ia32_storedqusi128_mask:280 case X86::BI__builtin_ia32_storedquhi128_mask:281 case X86::BI__builtin_ia32_storedquqi128_mask:282 case X86::BI__builtin_ia32_storeupd128_mask:283 case X86::BI__builtin_ia32_storeups128_mask:284 case X86::BI__builtin_ia32_storedqudi256_mask:285 case X86::BI__builtin_ia32_storedqusi256_mask:286 case X86::BI__builtin_ia32_storedquhi256_mask:287 case X86::BI__builtin_ia32_storedquqi256_mask:288 case X86::BI__builtin_ia32_storeupd256_mask:289 case X86::BI__builtin_ia32_storeups256_mask:290 case X86::BI__builtin_ia32_storedqudi512_mask:291 case X86::BI__builtin_ia32_storedqusi512_mask:292 case X86::BI__builtin_ia32_storedquhi512_mask:293 case X86::BI__builtin_ia32_storedquqi512_mask:294 case X86::BI__builtin_ia32_storeupd512_mask:295 case X86::BI__builtin_ia32_storeups512_mask:296 case X86::BI__builtin_ia32_storesbf16128_mask:297 case X86::BI__builtin_ia32_storesh128_mask:298 case X86::BI__builtin_ia32_storess128_mask:299 case X86::BI__builtin_ia32_storesd128_mask:300 case X86::BI__builtin_ia32_cvtmask2b128:301 case X86::BI__builtin_ia32_cvtmask2b256:302 case X86::BI__builtin_ia32_cvtmask2b512:303 case X86::BI__builtin_ia32_cvtmask2w128:304 case X86::BI__builtin_ia32_cvtmask2w256:305 case X86::BI__builtin_ia32_cvtmask2w512:306 case X86::BI__builtin_ia32_cvtmask2d128:307 case X86::BI__builtin_ia32_cvtmask2d256:308 case X86::BI__builtin_ia32_cvtmask2d512:309 case X86::BI__builtin_ia32_cvtmask2q128:310 case X86::BI__builtin_ia32_cvtmask2q256:311 case X86::BI__builtin_ia32_cvtmask2q512:312 case X86::BI__builtin_ia32_cvtb2mask128:313 case X86::BI__builtin_ia32_cvtb2mask256:314 case X86::BI__builtin_ia32_cvtb2mask512:315 case X86::BI__builtin_ia32_cvtw2mask128:316 case X86::BI__builtin_ia32_cvtw2mask256:317 case X86::BI__builtin_ia32_cvtw2mask512:318 case X86::BI__builtin_ia32_cvtd2mask128:319 case X86::BI__builtin_ia32_cvtd2mask256:320 case X86::BI__builtin_ia32_cvtd2mask512:321 case X86::BI__builtin_ia32_cvtq2mask128:322 case X86::BI__builtin_ia32_cvtq2mask256:323 case X86::BI__builtin_ia32_cvtq2mask512:324 case X86::BI__builtin_ia32_cvtdq2ps512_mask:325 case X86::BI__builtin_ia32_cvtqq2ps512_mask:326 case X86::BI__builtin_ia32_cvtqq2pd512_mask:327 case X86::BI__builtin_ia32_vcvtw2ph512_mask:328 case X86::BI__builtin_ia32_vcvtdq2ph512_mask:329 case X86::BI__builtin_ia32_vcvtqq2ph512_mask:330 case X86::BI__builtin_ia32_cvtudq2ps512_mask:331 case X86::BI__builtin_ia32_cvtuqq2ps512_mask:332 case X86::BI__builtin_ia32_cvtuqq2pd512_mask:333 case X86::BI__builtin_ia32_vcvtuw2ph512_mask:334 case X86::BI__builtin_ia32_vcvtudq2ph512_mask:335 case X86::BI__builtin_ia32_vcvtuqq2ph512_mask:336 case X86::BI__builtin_ia32_vfmaddsh3_mask:337 case X86::BI__builtin_ia32_vfmaddss3_mask:338 case X86::BI__builtin_ia32_vfmaddsd3_mask:339 case X86::BI__builtin_ia32_vfmaddsh3_maskz:340 case X86::BI__builtin_ia32_vfmaddss3_maskz:341 case X86::BI__builtin_ia32_vfmaddsd3_maskz:342 case X86::BI__builtin_ia32_vfmaddsh3_mask3:343 case X86::BI__builtin_ia32_vfmaddss3_mask3:344 case X86::BI__builtin_ia32_vfmaddsd3_mask3:345 case X86::BI__builtin_ia32_vfmsubsh3_mask3:346 case X86::BI__builtin_ia32_vfmsubss3_mask3:347 case X86::BI__builtin_ia32_vfmsubsd3_mask3:348 case X86::BI__builtin_ia32_vfmaddph512_mask:349 case X86::BI__builtin_ia32_vfmaddph512_maskz:350 case X86::BI__builtin_ia32_vfmaddph512_mask3:351 case X86::BI__builtin_ia32_vfmaddps512_mask:352 case X86::BI__builtin_ia32_vfmaddps512_maskz:353 case X86::BI__builtin_ia32_vfmaddps512_mask3:354 case X86::BI__builtin_ia32_vfmsubps512_mask3:355 case X86::BI__builtin_ia32_vfmaddpd512_mask:356 case X86::BI__builtin_ia32_vfmaddpd512_maskz:357 case X86::BI__builtin_ia32_vfmaddpd512_mask3:358 case X86::BI__builtin_ia32_vfmsubpd512_mask3:359 case X86::BI__builtin_ia32_vfmsubph512_mask3:360 case X86::BI__builtin_ia32_vfmaddsubph512_mask:361 case X86::BI__builtin_ia32_vfmaddsubph512_maskz:362 case X86::BI__builtin_ia32_vfmaddsubph512_mask3:363 case X86::BI__builtin_ia32_vfmsubaddph512_mask3:364 case X86::BI__builtin_ia32_vfmaddsubps512_mask:365 case X86::BI__builtin_ia32_vfmaddsubps512_maskz:366 case X86::BI__builtin_ia32_vfmaddsubps512_mask3:367 case X86::BI__builtin_ia32_vfmsubaddps512_mask3:368 case X86::BI__builtin_ia32_vfmaddsubpd512_mask:369 case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:370 case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:371 case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:372 case X86::BI__builtin_ia32_movdqa32store128_mask:373 case X86::BI__builtin_ia32_movdqa64store128_mask:374 case X86::BI__builtin_ia32_storeaps128_mask:375 case X86::BI__builtin_ia32_storeapd128_mask:376 case X86::BI__builtin_ia32_movdqa32store256_mask:377 case X86::BI__builtin_ia32_movdqa64store256_mask:378 case X86::BI__builtin_ia32_storeaps256_mask:379 case X86::BI__builtin_ia32_storeapd256_mask:380 case X86::BI__builtin_ia32_movdqa32store512_mask:381 case X86::BI__builtin_ia32_movdqa64store512_mask:382 case X86::BI__builtin_ia32_storeaps512_mask:383 case X86::BI__builtin_ia32_storeapd512_mask:384 case X86::BI__builtin_ia32_loadups128_mask:385 case X86::BI__builtin_ia32_loadups256_mask:386 case X86::BI__builtin_ia32_loadups512_mask:387 case X86::BI__builtin_ia32_loadupd128_mask:388 case X86::BI__builtin_ia32_loadupd256_mask:389 case X86::BI__builtin_ia32_loadupd512_mask:390 case X86::BI__builtin_ia32_loaddquqi128_mask:391 case X86::BI__builtin_ia32_loaddquqi256_mask:392 case X86::BI__builtin_ia32_loaddquqi512_mask:393 case X86::BI__builtin_ia32_loaddquhi128_mask:394 case X86::BI__builtin_ia32_loaddquhi256_mask:395 case X86::BI__builtin_ia32_loaddquhi512_mask:396 case X86::BI__builtin_ia32_loaddqusi128_mask:397 case X86::BI__builtin_ia32_loaddqusi256_mask:398 case X86::BI__builtin_ia32_loaddqusi512_mask:399 case X86::BI__builtin_ia32_loaddqudi128_mask:400 case X86::BI__builtin_ia32_loaddqudi256_mask:401 case X86::BI__builtin_ia32_loaddqudi512_mask:402 case X86::BI__builtin_ia32_loadsbf16128_mask:403 case X86::BI__builtin_ia32_loadsh128_mask:404 case X86::BI__builtin_ia32_loadss128_mask:405 case X86::BI__builtin_ia32_loadsd128_mask:406 case X86::BI__builtin_ia32_loadaps128_mask:407 case X86::BI__builtin_ia32_loadaps256_mask:408 case X86::BI__builtin_ia32_loadaps512_mask:409 case X86::BI__builtin_ia32_loadapd128_mask:410 case X86::BI__builtin_ia32_loadapd256_mask:411 case X86::BI__builtin_ia32_loadapd512_mask:412 case X86::BI__builtin_ia32_movdqa32load128_mask:413 case X86::BI__builtin_ia32_movdqa32load256_mask:414 case X86::BI__builtin_ia32_movdqa32load512_mask:415 case X86::BI__builtin_ia32_movdqa64load128_mask:416 case X86::BI__builtin_ia32_movdqa64load256_mask:417 case X86::BI__builtin_ia32_movdqa64load512_mask:418 case X86::BI__builtin_ia32_expandloaddf128_mask:419 case X86::BI__builtin_ia32_expandloaddf256_mask:420 case X86::BI__builtin_ia32_expandloaddf512_mask:421 case X86::BI__builtin_ia32_expandloadsf128_mask:422 case X86::BI__builtin_ia32_expandloadsf256_mask:423 case X86::BI__builtin_ia32_expandloadsf512_mask:424 case X86::BI__builtin_ia32_expandloaddi128_mask:425 case X86::BI__builtin_ia32_expandloaddi256_mask:426 case X86::BI__builtin_ia32_expandloaddi512_mask:427 case X86::BI__builtin_ia32_expandloadsi128_mask:428 case X86::BI__builtin_ia32_expandloadsi256_mask:429 case X86::BI__builtin_ia32_expandloadsi512_mask:430 case X86::BI__builtin_ia32_expandloadhi128_mask:431 case X86::BI__builtin_ia32_expandloadhi256_mask:432 case X86::BI__builtin_ia32_expandloadhi512_mask:433 case X86::BI__builtin_ia32_expandloadqi128_mask:434 case X86::BI__builtin_ia32_expandloadqi256_mask:435 case X86::BI__builtin_ia32_expandloadqi512_mask:436 case X86::BI__builtin_ia32_compressstoredf128_mask:437 case X86::BI__builtin_ia32_compressstoredf256_mask:438 case X86::BI__builtin_ia32_compressstoredf512_mask:439 case X86::BI__builtin_ia32_compressstoresf128_mask:440 case X86::BI__builtin_ia32_compressstoresf256_mask:441 case X86::BI__builtin_ia32_compressstoresf512_mask:442 case X86::BI__builtin_ia32_compressstoredi128_mask:443 case X86::BI__builtin_ia32_compressstoredi256_mask:444 case X86::BI__builtin_ia32_compressstoredi512_mask:445 case X86::BI__builtin_ia32_compressstoresi128_mask:446 case X86::BI__builtin_ia32_compressstoresi256_mask:447 case X86::BI__builtin_ia32_compressstoresi512_mask:448 case X86::BI__builtin_ia32_compressstorehi128_mask:449 case X86::BI__builtin_ia32_compressstorehi256_mask:450 case X86::BI__builtin_ia32_compressstorehi512_mask:451 case X86::BI__builtin_ia32_compressstoreqi128_mask:452 case X86::BI__builtin_ia32_compressstoreqi256_mask:453 case X86::BI__builtin_ia32_compressstoreqi512_mask:454 case X86::BI__builtin_ia32_expanddf128_mask:455 case X86::BI__builtin_ia32_expanddf256_mask:456 case X86::BI__builtin_ia32_expanddf512_mask:457 case X86::BI__builtin_ia32_expandsf128_mask:458 case X86::BI__builtin_ia32_expandsf256_mask:459 case X86::BI__builtin_ia32_expandsf512_mask:460 case X86::BI__builtin_ia32_expanddi128_mask:461 case X86::BI__builtin_ia32_expanddi256_mask:462 case X86::BI__builtin_ia32_expanddi512_mask:463 case X86::BI__builtin_ia32_expandsi128_mask:464 case X86::BI__builtin_ia32_expandsi256_mask:465 case X86::BI__builtin_ia32_expandsi512_mask:466 case X86::BI__builtin_ia32_expandhi128_mask:467 case X86::BI__builtin_ia32_expandhi256_mask:468 case X86::BI__builtin_ia32_expandhi512_mask:469 case X86::BI__builtin_ia32_expandqi128_mask:470 case X86::BI__builtin_ia32_expandqi256_mask:471 case X86::BI__builtin_ia32_expandqi512_mask:472 case X86::BI__builtin_ia32_compressdf128_mask:473 case X86::BI__builtin_ia32_compressdf256_mask:474 case X86::BI__builtin_ia32_compressdf512_mask:475 case X86::BI__builtin_ia32_compresssf128_mask:476 case X86::BI__builtin_ia32_compresssf256_mask:477 case X86::BI__builtin_ia32_compresssf512_mask:478 case X86::BI__builtin_ia32_compressdi128_mask:479 case X86::BI__builtin_ia32_compressdi256_mask:480 case X86::BI__builtin_ia32_compressdi512_mask:481 case X86::BI__builtin_ia32_compresssi128_mask:482 case X86::BI__builtin_ia32_compresssi256_mask:483 case X86::BI__builtin_ia32_compresssi512_mask:484 case X86::BI__builtin_ia32_compresshi128_mask:485 case X86::BI__builtin_ia32_compresshi256_mask:486 case X86::BI__builtin_ia32_compresshi512_mask:487 case X86::BI__builtin_ia32_compressqi128_mask:488 case X86::BI__builtin_ia32_compressqi256_mask:489 case X86::BI__builtin_ia32_compressqi512_mask:490 case X86::BI__builtin_ia32_gather3div2df:491 case X86::BI__builtin_ia32_gather3div2di:492 case X86::BI__builtin_ia32_gather3div4df:493 case X86::BI__builtin_ia32_gather3div4di:494 case X86::BI__builtin_ia32_gather3div4sf:495 case X86::BI__builtin_ia32_gather3div4si:496 case X86::BI__builtin_ia32_gather3div8sf:497 case X86::BI__builtin_ia32_gather3div8si:498 case X86::BI__builtin_ia32_gather3siv2df:499 case X86::BI__builtin_ia32_gather3siv2di:500 case X86::BI__builtin_ia32_gather3siv4df:501 case X86::BI__builtin_ia32_gather3siv4di:502 case X86::BI__builtin_ia32_gather3siv4sf:503 case X86::BI__builtin_ia32_gather3siv4si:504 case X86::BI__builtin_ia32_gather3siv8sf:505 case X86::BI__builtin_ia32_gather3siv8si:506 case X86::BI__builtin_ia32_gathersiv8df:507 case X86::BI__builtin_ia32_gathersiv16sf:508 case X86::BI__builtin_ia32_gatherdiv8df:509 case X86::BI__builtin_ia32_gatherdiv16sf:510 case X86::BI__builtin_ia32_gathersiv8di:511 case X86::BI__builtin_ia32_gathersiv16si:512 case X86::BI__builtin_ia32_gatherdiv8di:513 case X86::BI__builtin_ia32_gatherdiv16si:514 case X86::BI__builtin_ia32_scattersiv8df:515 case X86::BI__builtin_ia32_scattersiv16sf:516 case X86::BI__builtin_ia32_scatterdiv8df:517 case X86::BI__builtin_ia32_scatterdiv16sf:518 case X86::BI__builtin_ia32_scattersiv8di:519 case X86::BI__builtin_ia32_scattersiv16si:520 case X86::BI__builtin_ia32_scatterdiv8di:521 case X86::BI__builtin_ia32_scatterdiv16si:522 case X86::BI__builtin_ia32_scatterdiv2df:523 case X86::BI__builtin_ia32_scatterdiv2di:524 case X86::BI__builtin_ia32_scatterdiv4df:525 case X86::BI__builtin_ia32_scatterdiv4di:526 case X86::BI__builtin_ia32_scatterdiv4sf:527 case X86::BI__builtin_ia32_scatterdiv4si:528 case X86::BI__builtin_ia32_scatterdiv8sf:529 case X86::BI__builtin_ia32_scatterdiv8si:530 case X86::BI__builtin_ia32_scattersiv2df:531 case X86::BI__builtin_ia32_scattersiv2di:532 case X86::BI__builtin_ia32_scattersiv4df:533 case X86::BI__builtin_ia32_scattersiv4di:534 case X86::BI__builtin_ia32_scattersiv4sf:535 case X86::BI__builtin_ia32_scattersiv4si:536 case X86::BI__builtin_ia32_scattersiv8sf:537 case X86::BI__builtin_ia32_scattersiv8si:538 case X86::BI__builtin_ia32_vextractf128_pd256:539 case X86::BI__builtin_ia32_vextractf128_ps256:540 case X86::BI__builtin_ia32_vextractf128_si256:541 case X86::BI__builtin_ia32_extract128i256:542 case X86::BI__builtin_ia32_extractf64x4_mask:543 case X86::BI__builtin_ia32_extractf32x4_mask:544 case X86::BI__builtin_ia32_extracti64x4_mask:545 case X86::BI__builtin_ia32_extracti32x4_mask:546 case X86::BI__builtin_ia32_extractf32x8_mask:547 case X86::BI__builtin_ia32_extracti32x8_mask:548 case X86::BI__builtin_ia32_extractf32x4_256_mask:549 case X86::BI__builtin_ia32_extracti32x4_256_mask:550 case X86::BI__builtin_ia32_extractf64x2_256_mask:551 case X86::BI__builtin_ia32_extracti64x2_256_mask:552 case X86::BI__builtin_ia32_extractf64x2_512_mask:553 case X86::BI__builtin_ia32_extracti64x2_512_mask:554 case X86::BI__builtin_ia32_vinsertf128_pd256:555 case X86::BI__builtin_ia32_vinsertf128_ps256:556 case X86::BI__builtin_ia32_vinsertf128_si256:557 case X86::BI__builtin_ia32_insert128i256:558 case X86::BI__builtin_ia32_insertf64x4:559 case X86::BI__builtin_ia32_insertf32x4:560 case X86::BI__builtin_ia32_inserti64x4:561 case X86::BI__builtin_ia32_inserti32x4:562 case X86::BI__builtin_ia32_insertf32x8:563 case X86::BI__builtin_ia32_inserti32x8:564 case X86::BI__builtin_ia32_insertf32x4_256:565 case X86::BI__builtin_ia32_inserti32x4_256:566 case X86::BI__builtin_ia32_insertf64x2_256:567 case X86::BI__builtin_ia32_inserti64x2_256:568 case X86::BI__builtin_ia32_insertf64x2_512:569 case X86::BI__builtin_ia32_inserti64x2_512:570 case X86::BI__builtin_ia32_pmovqd512_mask:571 case X86::BI__builtin_ia32_pmovwb512_mask:572 case X86::BI__builtin_ia32_pblendw128:573 case X86::BI__builtin_ia32_blendpd:574 case X86::BI__builtin_ia32_blendps:575 case X86::BI__builtin_ia32_blendpd256:576 case X86::BI__builtin_ia32_blendps256:577 case X86::BI__builtin_ia32_pblendw256:578 case X86::BI__builtin_ia32_pblendd128:579 case X86::BI__builtin_ia32_pblendd256:580 case X86::BI__builtin_ia32_pshuflw:581 case X86::BI__builtin_ia32_pshuflw256:582 case X86::BI__builtin_ia32_pshuflw512:583 case X86::BI__builtin_ia32_pshufhw:584 case X86::BI__builtin_ia32_pshufhw256:585 case X86::BI__builtin_ia32_pshufhw512:586 case X86::BI__builtin_ia32_pshufd:587 case X86::BI__builtin_ia32_pshufd256:588 case X86::BI__builtin_ia32_pshufd512:589 case X86::BI__builtin_ia32_vpermilpd:590 case X86::BI__builtin_ia32_vpermilps:591 case X86::BI__builtin_ia32_vpermilpd256:592 case X86::BI__builtin_ia32_vpermilps256:593 case X86::BI__builtin_ia32_vpermilpd512:594 case X86::BI__builtin_ia32_vpermilps512:595 case X86::BI__builtin_ia32_shufpd:596 case X86::BI__builtin_ia32_shufpd256:597 case X86::BI__builtin_ia32_shufpd512:598 case X86::BI__builtin_ia32_shufps:599 case X86::BI__builtin_ia32_shufps256:600 case X86::BI__builtin_ia32_shufps512:601 case X86::BI__builtin_ia32_permdi256:602 case X86::BI__builtin_ia32_permdf256:603 case X86::BI__builtin_ia32_permdi512:604 case X86::BI__builtin_ia32_permdf512:605 case X86::BI__builtin_ia32_palignr128:606 case X86::BI__builtin_ia32_palignr256:607 case X86::BI__builtin_ia32_palignr512:608 case X86::BI__builtin_ia32_alignd128:609 case X86::BI__builtin_ia32_alignd256:610 case X86::BI__builtin_ia32_alignd512:611 case X86::BI__builtin_ia32_alignq128:612 case X86::BI__builtin_ia32_alignq256:613 case X86::BI__builtin_ia32_alignq512:614 case X86::BI__builtin_ia32_shuf_f32x4_256:615 case X86::BI__builtin_ia32_shuf_f64x2_256:616 case X86::BI__builtin_ia32_shuf_i32x4_256:617 case X86::BI__builtin_ia32_shuf_i64x2_256:618 case X86::BI__builtin_ia32_shuf_f32x4:619 case X86::BI__builtin_ia32_shuf_f64x2:620 case X86::BI__builtin_ia32_shuf_i32x4:621 case X86::BI__builtin_ia32_shuf_i64x2:622 case X86::BI__builtin_ia32_vperm2f128_pd256:623 case X86::BI__builtin_ia32_vperm2f128_ps256:624 case X86::BI__builtin_ia32_vperm2f128_si256:625 case X86::BI__builtin_ia32_permti256:626 case X86::BI__builtin_ia32_pslldqi128_byteshift:627 case X86::BI__builtin_ia32_pslldqi256_byteshift:628 case X86::BI__builtin_ia32_pslldqi512_byteshift:629 case X86::BI__builtin_ia32_psrldqi128_byteshift:630 case X86::BI__builtin_ia32_psrldqi256_byteshift:631 case X86::BI__builtin_ia32_psrldqi512_byteshift:632 cgm.errorNYI(expr->getSourceRange(),633 std::string("unimplemented X86 builtin call: ") +634 getContext().BuiltinInfo.getName(builtinID));635 return {};636 case X86::BI__builtin_ia32_kshiftliqi:637 case X86::BI__builtin_ia32_kshiftlihi:638 case X86::BI__builtin_ia32_kshiftlisi:639 case X86::BI__builtin_ia32_kshiftlidi: {640 mlir::Location loc = getLoc(expr->getExprLoc());641 unsigned shiftVal =642 ops[1].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue() &643 0xff;644 unsigned numElems = cast<cir::IntType>(ops[0].getType()).getWidth();645 646 if (shiftVal >= numElems)647 return builder.getNullValue(ops[0].getType(), loc);648 649 mlir::Value in = getMaskVecValue(builder, loc, ops[0], numElems);650 651 SmallVector<mlir::Attribute, 64> indices;652 mlir::Type i32Ty = builder.getSInt32Ty();653 for (auto i : llvm::seq<unsigned>(0, numElems))654 indices.push_back(cir::IntAttr::get(i32Ty, numElems + i - shiftVal));655 656 mlir::Value zero = builder.getNullValue(in.getType(), loc);657 mlir::Value sv = builder.createVecShuffle(loc, zero, in, indices);658 return builder.createBitcast(sv, ops[0].getType());659 }660 case X86::BI__builtin_ia32_kshiftriqi:661 case X86::BI__builtin_ia32_kshiftrihi:662 case X86::BI__builtin_ia32_kshiftrisi:663 case X86::BI__builtin_ia32_kshiftridi: {664 mlir::Location loc = getLoc(expr->getExprLoc());665 unsigned shiftVal =666 ops[1].getDefiningOp<cir::ConstantOp>().getIntValue().getZExtValue() &667 0xff;668 unsigned numElems = cast<cir::IntType>(ops[0].getType()).getWidth();669 670 if (shiftVal >= numElems)671 return builder.getNullValue(ops[0].getType(), loc);672 673 mlir::Value in = getMaskVecValue(builder, loc, ops[0], numElems);674 675 SmallVector<mlir::Attribute, 64> indices;676 mlir::Type i32Ty = builder.getSInt32Ty();677 for (auto i : llvm::seq<unsigned>(0, numElems))678 indices.push_back(cir::IntAttr::get(i32Ty, i + shiftVal));679 680 mlir::Value zero = builder.getNullValue(in.getType(), loc);681 mlir::Value sv = builder.createVecShuffle(loc, in, zero, indices);682 return builder.createBitcast(sv, ops[0].getType());683 }684 case X86::BI__builtin_ia32_vprotbi:685 case X86::BI__builtin_ia32_vprotwi:686 case X86::BI__builtin_ia32_vprotdi:687 case X86::BI__builtin_ia32_vprotqi:688 case X86::BI__builtin_ia32_prold128:689 case X86::BI__builtin_ia32_prold256:690 case X86::BI__builtin_ia32_prold512:691 case X86::BI__builtin_ia32_prolq128:692 case X86::BI__builtin_ia32_prolq256:693 case X86::BI__builtin_ia32_prolq512:694 case X86::BI__builtin_ia32_prord128:695 case X86::BI__builtin_ia32_prord256:696 case X86::BI__builtin_ia32_prord512:697 case X86::BI__builtin_ia32_prorq128:698 case X86::BI__builtin_ia32_prorq256:699 case X86::BI__builtin_ia32_prorq512:700 case X86::BI__builtin_ia32_selectb_128:701 case X86::BI__builtin_ia32_selectb_256:702 case X86::BI__builtin_ia32_selectb_512:703 case X86::BI__builtin_ia32_selectw_128:704 case X86::BI__builtin_ia32_selectw_256:705 case X86::BI__builtin_ia32_selectw_512:706 case X86::BI__builtin_ia32_selectd_128:707 case X86::BI__builtin_ia32_selectd_256:708 case X86::BI__builtin_ia32_selectd_512:709 case X86::BI__builtin_ia32_selectq_128:710 case X86::BI__builtin_ia32_selectq_256:711 case X86::BI__builtin_ia32_selectq_512:712 case X86::BI__builtin_ia32_selectph_128:713 case X86::BI__builtin_ia32_selectph_256:714 case X86::BI__builtin_ia32_selectph_512:715 case X86::BI__builtin_ia32_selectpbf_128:716 case X86::BI__builtin_ia32_selectpbf_256:717 case X86::BI__builtin_ia32_selectpbf_512:718 case X86::BI__builtin_ia32_selectps_128:719 case X86::BI__builtin_ia32_selectps_256:720 case X86::BI__builtin_ia32_selectps_512:721 case X86::BI__builtin_ia32_selectpd_128:722 case X86::BI__builtin_ia32_selectpd_256:723 case X86::BI__builtin_ia32_selectpd_512:724 case X86::BI__builtin_ia32_selectsh_128:725 case X86::BI__builtin_ia32_selectsbf_128:726 case X86::BI__builtin_ia32_selectss_128:727 case X86::BI__builtin_ia32_selectsd_128:728 case X86::BI__builtin_ia32_cmpb128_mask:729 case X86::BI__builtin_ia32_cmpb256_mask:730 case X86::BI__builtin_ia32_cmpb512_mask:731 case X86::BI__builtin_ia32_cmpw128_mask:732 case X86::BI__builtin_ia32_cmpw256_mask:733 case X86::BI__builtin_ia32_cmpw512_mask:734 case X86::BI__builtin_ia32_cmpd128_mask:735 case X86::BI__builtin_ia32_cmpd256_mask:736 case X86::BI__builtin_ia32_cmpd512_mask:737 case X86::BI__builtin_ia32_cmpq128_mask:738 case X86::BI__builtin_ia32_cmpq256_mask:739 case X86::BI__builtin_ia32_cmpq512_mask:740 case X86::BI__builtin_ia32_ucmpb128_mask:741 case X86::BI__builtin_ia32_ucmpb256_mask:742 case X86::BI__builtin_ia32_ucmpb512_mask:743 case X86::BI__builtin_ia32_ucmpw128_mask:744 case X86::BI__builtin_ia32_ucmpw256_mask:745 case X86::BI__builtin_ia32_ucmpw512_mask:746 case X86::BI__builtin_ia32_ucmpd128_mask:747 case X86::BI__builtin_ia32_ucmpd256_mask:748 case X86::BI__builtin_ia32_ucmpd512_mask:749 case X86::BI__builtin_ia32_ucmpq128_mask:750 case X86::BI__builtin_ia32_ucmpq256_mask:751 case X86::BI__builtin_ia32_ucmpq512_mask:752 case X86::BI__builtin_ia32_vpcomb:753 case X86::BI__builtin_ia32_vpcomw:754 case X86::BI__builtin_ia32_vpcomd:755 case X86::BI__builtin_ia32_vpcomq:756 case X86::BI__builtin_ia32_vpcomub:757 case X86::BI__builtin_ia32_vpcomuw:758 case X86::BI__builtin_ia32_vpcomud:759 case X86::BI__builtin_ia32_vpcomuq:760 case X86::BI__builtin_ia32_kortestcqi:761 case X86::BI__builtin_ia32_kortestchi:762 case X86::BI__builtin_ia32_kortestcsi:763 case X86::BI__builtin_ia32_kortestcdi:764 case X86::BI__builtin_ia32_kortestzqi:765 case X86::BI__builtin_ia32_kortestzhi:766 case X86::BI__builtin_ia32_kortestzsi:767 case X86::BI__builtin_ia32_kortestzdi:768 case X86::BI__builtin_ia32_ktestcqi:769 case X86::BI__builtin_ia32_ktestzqi:770 case X86::BI__builtin_ia32_ktestchi:771 case X86::BI__builtin_ia32_ktestzhi:772 case X86::BI__builtin_ia32_ktestcsi:773 case X86::BI__builtin_ia32_ktestzsi:774 case X86::BI__builtin_ia32_ktestcdi:775 case X86::BI__builtin_ia32_ktestzdi:776 cgm.errorNYI(expr->getSourceRange(),777 std::string("unimplemented X86 builtin call: ") +778 getContext().BuiltinInfo.getName(builtinID));779 return {};780 case X86::BI__builtin_ia32_kaddqi:781 return emitX86MaskAddLogic(builder, getLoc(expr->getExprLoc()),782 "x86.avx512.kadd.b", ops);783 case X86::BI__builtin_ia32_kaddhi:784 return emitX86MaskAddLogic(builder, getLoc(expr->getExprLoc()),785 "x86.avx512.kadd.w", ops);786 case X86::BI__builtin_ia32_kaddsi:787 return emitX86MaskAddLogic(builder, getLoc(expr->getExprLoc()),788 "x86.avx512.kadd.d", ops);789 case X86::BI__builtin_ia32_kadddi:790 return emitX86MaskAddLogic(builder, getLoc(expr->getExprLoc()),791 "x86.avx512.kadd.q", ops);792 case X86::BI__builtin_ia32_kandqi:793 case X86::BI__builtin_ia32_kandhi:794 case X86::BI__builtin_ia32_kandsi:795 case X86::BI__builtin_ia32_kanddi:796 return emitX86MaskLogic(builder, getLoc(expr->getExprLoc()),797 cir::BinOpKind::And, ops);798 case X86::BI__builtin_ia32_kandnqi:799 case X86::BI__builtin_ia32_kandnhi:800 case X86::BI__builtin_ia32_kandnsi:801 case X86::BI__builtin_ia32_kandndi:802 return emitX86MaskLogic(builder, getLoc(expr->getExprLoc()),803 cir::BinOpKind::And, ops, true);804 case X86::BI__builtin_ia32_korqi:805 case X86::BI__builtin_ia32_korhi:806 case X86::BI__builtin_ia32_korsi:807 case X86::BI__builtin_ia32_kordi:808 return emitX86MaskLogic(builder, getLoc(expr->getExprLoc()),809 cir::BinOpKind::Or, ops);810 case X86::BI__builtin_ia32_kxnorqi:811 case X86::BI__builtin_ia32_kxnorhi:812 case X86::BI__builtin_ia32_kxnorsi:813 case X86::BI__builtin_ia32_kxnordi:814 return emitX86MaskLogic(builder, getLoc(expr->getExprLoc()),815 cir::BinOpKind::Xor, ops, true);816 case X86::BI__builtin_ia32_kxorqi:817 case X86::BI__builtin_ia32_kxorhi:818 case X86::BI__builtin_ia32_kxorsi:819 case X86::BI__builtin_ia32_kxordi:820 return emitX86MaskLogic(builder, getLoc(expr->getExprLoc()),821 cir::BinOpKind::Xor, ops);822 case X86::BI__builtin_ia32_knotqi:823 case X86::BI__builtin_ia32_knothi:824 case X86::BI__builtin_ia32_knotsi:825 case X86::BI__builtin_ia32_knotdi: {826 cir::IntType intTy = cast<cir::IntType>(ops[0].getType());827 unsigned numElts = intTy.getWidth();828 mlir::Value resVec =829 getMaskVecValue(builder, getLoc(expr->getExprLoc()), ops[0], numElts);830 return builder.createBitcast(builder.createNot(resVec), ops[0].getType());831 }832 case X86::BI__builtin_ia32_kmovb:833 case X86::BI__builtin_ia32_kmovw:834 case X86::BI__builtin_ia32_kmovd:835 case X86::BI__builtin_ia32_kmovq: {836 // Bitcast to vXi1 type and then back to integer. This gets the mask837 // register type into the IR, but might be optimized out depending on838 // what's around it.839 cir::IntType intTy = cast<cir::IntType>(ops[0].getType());840 unsigned numElts = intTy.getWidth();841 mlir::Value resVec =842 getMaskVecValue(builder, getLoc(expr->getExprLoc()), ops[0], numElts);843 return builder.createBitcast(resVec, ops[0].getType());844 }845 case X86::BI__builtin_ia32_kunpckdi:846 case X86::BI__builtin_ia32_kunpcksi:847 case X86::BI__builtin_ia32_kunpckhi:848 case X86::BI__builtin_ia32_sqrtsh_round_mask:849 case X86::BI__builtin_ia32_sqrtsd_round_mask:850 case X86::BI__builtin_ia32_sqrtss_round_mask:851 case X86::BI__builtin_ia32_sqrtph512:852 case X86::BI__builtin_ia32_sqrtps512:853 case X86::BI__builtin_ia32_sqrtpd512:854 case X86::BI__builtin_ia32_pmuludq128:855 case X86::BI__builtin_ia32_pmuludq256:856 case X86::BI__builtin_ia32_pmuludq512:857 case X86::BI__builtin_ia32_pmuldq128:858 case X86::BI__builtin_ia32_pmuldq256:859 case X86::BI__builtin_ia32_pmuldq512:860 case X86::BI__builtin_ia32_pternlogd512_mask:861 case X86::BI__builtin_ia32_pternlogq512_mask:862 case X86::BI__builtin_ia32_pternlogd128_mask:863 case X86::BI__builtin_ia32_pternlogd256_mask:864 case X86::BI__builtin_ia32_pternlogq128_mask:865 case X86::BI__builtin_ia32_pternlogq256_mask:866 case X86::BI__builtin_ia32_pternlogd512_maskz:867 case X86::BI__builtin_ia32_pternlogq512_maskz:868 case X86::BI__builtin_ia32_pternlogd128_maskz:869 case X86::BI__builtin_ia32_pternlogd256_maskz:870 case X86::BI__builtin_ia32_pternlogq128_maskz:871 case X86::BI__builtin_ia32_pternlogq256_maskz:872 case X86::BI__builtin_ia32_vpshldd128:873 case X86::BI__builtin_ia32_vpshldd256:874 case X86::BI__builtin_ia32_vpshldd512:875 case X86::BI__builtin_ia32_vpshldq128:876 case X86::BI__builtin_ia32_vpshldq256:877 case X86::BI__builtin_ia32_vpshldq512:878 case X86::BI__builtin_ia32_vpshldw128:879 case X86::BI__builtin_ia32_vpshldw256:880 case X86::BI__builtin_ia32_vpshldw512:881 case X86::BI__builtin_ia32_vpshrdd128:882 case X86::BI__builtin_ia32_vpshrdd256:883 case X86::BI__builtin_ia32_vpshrdd512:884 case X86::BI__builtin_ia32_vpshrdq128:885 case X86::BI__builtin_ia32_vpshrdq256:886 case X86::BI__builtin_ia32_vpshrdq512:887 case X86::BI__builtin_ia32_vpshrdw128:888 case X86::BI__builtin_ia32_vpshrdw256:889 case X86::BI__builtin_ia32_vpshrdw512:890 case X86::BI__builtin_ia32_reduce_fadd_pd512:891 case X86::BI__builtin_ia32_reduce_fadd_ps512:892 case X86::BI__builtin_ia32_reduce_fadd_ph512:893 case X86::BI__builtin_ia32_reduce_fadd_ph256:894 case X86::BI__builtin_ia32_reduce_fadd_ph128:895 case X86::BI__builtin_ia32_reduce_fmul_pd512:896 case X86::BI__builtin_ia32_reduce_fmul_ps512:897 case X86::BI__builtin_ia32_reduce_fmul_ph512:898 case X86::BI__builtin_ia32_reduce_fmul_ph256:899 case X86::BI__builtin_ia32_reduce_fmul_ph128:900 case X86::BI__builtin_ia32_reduce_fmax_pd512:901 case X86::BI__builtin_ia32_reduce_fmax_ps512:902 case X86::BI__builtin_ia32_reduce_fmax_ph512:903 case X86::BI__builtin_ia32_reduce_fmax_ph256:904 case X86::BI__builtin_ia32_reduce_fmax_ph128:905 case X86::BI__builtin_ia32_reduce_fmin_pd512:906 case X86::BI__builtin_ia32_reduce_fmin_ps512:907 case X86::BI__builtin_ia32_reduce_fmin_ph512:908 case X86::BI__builtin_ia32_reduce_fmin_ph256:909 case X86::BI__builtin_ia32_reduce_fmin_ph128:910 case X86::BI__builtin_ia32_rdrand16_step:911 case X86::BI__builtin_ia32_rdrand32_step:912 case X86::BI__builtin_ia32_rdrand64_step:913 case X86::BI__builtin_ia32_rdseed16_step:914 case X86::BI__builtin_ia32_rdseed32_step:915 case X86::BI__builtin_ia32_rdseed64_step:916 case X86::BI__builtin_ia32_addcarryx_u32:917 case X86::BI__builtin_ia32_addcarryx_u64:918 case X86::BI__builtin_ia32_subborrow_u32:919 case X86::BI__builtin_ia32_subborrow_u64:920 case X86::BI__builtin_ia32_fpclassps128_mask:921 case X86::BI__builtin_ia32_fpclassps256_mask:922 case X86::BI__builtin_ia32_fpclassps512_mask:923 case X86::BI__builtin_ia32_vfpclassbf16128_mask:924 case X86::BI__builtin_ia32_vfpclassbf16256_mask:925 case X86::BI__builtin_ia32_vfpclassbf16512_mask:926 case X86::BI__builtin_ia32_fpclassph128_mask:927 case X86::BI__builtin_ia32_fpclassph256_mask:928 case X86::BI__builtin_ia32_fpclassph512_mask:929 case X86::BI__builtin_ia32_fpclasspd128_mask:930 case X86::BI__builtin_ia32_fpclasspd256_mask:931 case X86::BI__builtin_ia32_fpclasspd512_mask:932 case X86::BI__builtin_ia32_vp2intersect_q_512:933 case X86::BI__builtin_ia32_vp2intersect_q_256:934 case X86::BI__builtin_ia32_vp2intersect_q_128:935 case X86::BI__builtin_ia32_vp2intersect_d_512:936 case X86::BI__builtin_ia32_vp2intersect_d_256:937 case X86::BI__builtin_ia32_vp2intersect_d_128:938 case X86::BI__builtin_ia32_vpmultishiftqb128:939 case X86::BI__builtin_ia32_vpmultishiftqb256:940 case X86::BI__builtin_ia32_vpmultishiftqb512:941 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:942 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:943 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:944 case X86::BI__builtin_ia32_cmpeqps:945 case X86::BI__builtin_ia32_cmpeqpd:946 case X86::BI__builtin_ia32_cmpltps:947 case X86::BI__builtin_ia32_cmpltpd:948 case X86::BI__builtin_ia32_cmpleps:949 case X86::BI__builtin_ia32_cmplepd:950 case X86::BI__builtin_ia32_cmpunordps:951 case X86::BI__builtin_ia32_cmpunordpd:952 case X86::BI__builtin_ia32_cmpneqps:953 case X86::BI__builtin_ia32_cmpneqpd:954 cgm.errorNYI(expr->getSourceRange(),955 std::string("unimplemented X86 builtin call: ") +956 getContext().BuiltinInfo.getName(builtinID));957 return {};958 case X86::BI__builtin_ia32_cmpnltps:959 case X86::BI__builtin_ia32_cmpnltpd:960 return emitVectorFCmp(builder, ops, getLoc(expr->getExprLoc()),961 cir::CmpOpKind::lt, /*shouldInvert=*/true);962 case X86::BI__builtin_ia32_cmpnleps:963 case X86::BI__builtin_ia32_cmpnlepd:964 return emitVectorFCmp(builder, ops, getLoc(expr->getExprLoc()),965 cir::CmpOpKind::le, /*shouldInvert=*/true);966 case X86::BI__builtin_ia32_cmpordps:967 case X86::BI__builtin_ia32_cmpordpd:968 case X86::BI__builtin_ia32_cmpph128_mask:969 case X86::BI__builtin_ia32_cmpph256_mask:970 case X86::BI__builtin_ia32_cmpph512_mask:971 case X86::BI__builtin_ia32_cmpps128_mask:972 case X86::BI__builtin_ia32_cmpps256_mask:973 case X86::BI__builtin_ia32_cmpps512_mask:974 case X86::BI__builtin_ia32_cmppd128_mask:975 case X86::BI__builtin_ia32_cmppd256_mask:976 case X86::BI__builtin_ia32_cmppd512_mask:977 case X86::BI__builtin_ia32_vcmpbf16512_mask:978 case X86::BI__builtin_ia32_vcmpbf16256_mask:979 case X86::BI__builtin_ia32_vcmpbf16128_mask:980 case X86::BI__builtin_ia32_cmpps:981 case X86::BI__builtin_ia32_cmpps256:982 case X86::BI__builtin_ia32_cmppd:983 case X86::BI__builtin_ia32_cmppd256:984 case X86::BI__builtin_ia32_cmpeqss:985 case X86::BI__builtin_ia32_cmpltss:986 case X86::BI__builtin_ia32_cmpless:987 case X86::BI__builtin_ia32_cmpunordss:988 case X86::BI__builtin_ia32_cmpneqss:989 case X86::BI__builtin_ia32_cmpnltss:990 case X86::BI__builtin_ia32_cmpnless:991 case X86::BI__builtin_ia32_cmpordss:992 case X86::BI__builtin_ia32_cmpeqsd:993 case X86::BI__builtin_ia32_cmpltsd:994 case X86::BI__builtin_ia32_cmplesd:995 case X86::BI__builtin_ia32_cmpunordsd:996 case X86::BI__builtin_ia32_cmpneqsd:997 case X86::BI__builtin_ia32_cmpnltsd:998 case X86::BI__builtin_ia32_cmpnlesd:999 case X86::BI__builtin_ia32_cmpordsd:1000 case X86::BI__builtin_ia32_vcvtph2ps_mask:1001 case X86::BI__builtin_ia32_vcvtph2ps256_mask:1002 case X86::BI__builtin_ia32_vcvtph2ps512_mask:1003 case X86::BI__builtin_ia32_cvtneps2bf16_128_mask:1004 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:1005 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask:1006 case X86::BI__cpuid:1007 case X86::BI__cpuidex:1008 case X86::BI__emul:1009 case X86::BI__emulu:1010 case X86::BI__mulh:1011 case X86::BI__umulh:1012 case X86::BI_mul128:1013 case X86::BI_umul128:1014 case X86::BI__faststorefence:1015 case X86::BI__shiftleft128:1016 case X86::BI__shiftright128:1017 case X86::BI_ReadWriteBarrier:1018 case X86::BI_ReadBarrier:1019 case X86::BI_WriteBarrier:1020 case X86::BI_AddressOfReturnAddress:1021 case X86::BI__stosb:1022 case X86::BI__ud2:1023 case X86::BI__int2c:1024 case X86::BI__readfsbyte:1025 case X86::BI__readfsword:1026 case X86::BI__readfsdword:1027 case X86::BI__readfsqword:1028 case X86::BI__readgsbyte:1029 case X86::BI__readgsword:1030 case X86::BI__readgsdword:1031 case X86::BI__readgsqword:1032 case X86::BI__builtin_ia32_encodekey128_u32:1033 case X86::BI__builtin_ia32_encodekey256_u32:1034 case X86::BI__builtin_ia32_aesenc128kl_u8:1035 case X86::BI__builtin_ia32_aesdec128kl_u8:1036 case X86::BI__builtin_ia32_aesenc256kl_u8:1037 case X86::BI__builtin_ia32_aesdec256kl_u8:1038 case X86::BI__builtin_ia32_aesencwide128kl_u8:1039 case X86::BI__builtin_ia32_aesdecwide128kl_u8:1040 case X86::BI__builtin_ia32_aesencwide256kl_u8:1041 case X86::BI__builtin_ia32_aesdecwide256kl_u8:1042 case X86::BI__builtin_ia32_vfcmaddcph512_mask:1043 case X86::BI__builtin_ia32_vfmaddcph512_mask:1044 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask:1045 case X86::BI__builtin_ia32_vfmaddcsh_round_mask:1046 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:1047 case X86::BI__builtin_ia32_vfmaddcsh_round_mask3:1048 case X86::BI__builtin_ia32_prefetchi:1049 cgm.errorNYI(expr->getSourceRange(),1050 std::string("unimplemented X86 builtin call: ") +1051 getContext().BuiltinInfo.getName(builtinID));1052 return {};1053 }1054}1055