brintos / llvm-project-archived public Read only

1//====- LowerToLLVM.cpp - Lowering from CIR to LLVMIR ---------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements lowering of CIR operations to LLVMIR.10//11//===----------------------------------------------------------------------===//12 13#include "LowerToLLVM.h"14 15#include <deque>16#include <optional>17 18#include "mlir/Conversion/LLVMCommon/TypeConverter.h"19#include "mlir/Dialect/DLTI/DLTI.h"20#include "mlir/Dialect/Func/IR/FuncOps.h"21#include "mlir/Dialect/LLVMIR/LLVMDialect.h"22#include "mlir/Dialect/LLVMIR/LLVMTypes.h"23#include "mlir/IR/BuiltinAttributes.h"24#include "mlir/IR/BuiltinDialect.h"25#include "mlir/IR/BuiltinOps.h"26#include "mlir/IR/Types.h"27#include "mlir/Pass/Pass.h"28#include "mlir/Pass/PassManager.h"29#include "mlir/Target/LLVMIR/Dialect/Builtin/BuiltinToLLVMIRTranslation.h"30#include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"31#include "mlir/Target/LLVMIR/Export.h"32#include "mlir/Transforms/DialectConversion.h"33#include "clang/CIR/Dialect/IR/CIRAttrs.h"34#include "clang/CIR/Dialect/IR/CIRDialect.h"35#include "clang/CIR/Dialect/IR/CIRTypes.h"36#include "clang/CIR/Dialect/Passes.h"37#include "clang/CIR/LoweringHelpers.h"38#include "clang/CIR/MissingFeatures.h"39#include "clang/CIR/Passes.h"40#include "llvm/ADT/TypeSwitch.h"41#include "llvm/IR/Module.h"42#include "llvm/Support/ErrorHandling.h"43#include "llvm/Support/TimeProfiler.h"44 45using namespace cir;46using namespace llvm;47 48namespace cir {49namespace direct {50 51//===----------------------------------------------------------------------===//52// Helper Methods53//===----------------------------------------------------------------------===//54 55namespace {56/// If the given type is a vector type, return the vector's element type.57/// Otherwise return the given type unchanged.58mlir::Type elementTypeIfVector(mlir::Type type) {59  return llvm::TypeSwitch<mlir::Type, mlir::Type>(type)60      .Case<cir::VectorType, mlir::VectorType>(61          [](auto p) { return p.getElementType(); })62      .Default([](mlir::Type p) { return p; });63}64} // namespace65 66/// Given a type convertor and a data layout, convert the given type to a type67/// that is suitable for memory operations. For example, this can be used to68/// lower cir.bool accesses to i8.69static mlir::Type convertTypeForMemory(const mlir::TypeConverter &converter,70                                       mlir::DataLayout const &dataLayout,71                                       mlir::Type type) {72  // TODO(cir): Handle other types similarly to clang's codegen73  // convertTypeForMemory74  if (isa<cir::BoolType>(type)) {75    return mlir::IntegerType::get(type.getContext(),76                                  dataLayout.getTypeSizeInBits(type));77  }78 79  return converter.convertType(type);80}81 82static mlir::Value createIntCast(mlir::OpBuilder &bld, mlir::Value src,83                                 mlir::IntegerType dstTy,84                                 bool isSigned = false) {85  mlir::Type srcTy = src.getType();86  assert(mlir::isa<mlir::IntegerType>(srcTy));87 88  unsigned srcWidth = mlir::cast<mlir::IntegerType>(srcTy).getWidth();89  unsigned dstWidth = mlir::cast<mlir::IntegerType>(dstTy).getWidth();90  mlir::Location loc = src.getLoc();91 92  if (dstWidth > srcWidth && isSigned)93    return mlir::LLVM::SExtOp::create(bld, loc, dstTy, src);94  if (dstWidth > srcWidth)95    return mlir::LLVM::ZExtOp::create(bld, loc, dstTy, src);96  if (dstWidth < srcWidth)97    return mlir::LLVM::TruncOp::create(bld, loc, dstTy, src);98  return mlir::LLVM::BitcastOp::create(bld, loc, dstTy, src);99}100 101static mlir::LLVM::Visibility102lowerCIRVisibilityToLLVMVisibility(cir::VisibilityKind visibilityKind) {103  switch (visibilityKind) {104  case cir::VisibilityKind::Default:105    return ::mlir::LLVM::Visibility::Default;106  case cir::VisibilityKind::Hidden:107    return ::mlir::LLVM::Visibility::Hidden;108  case cir::VisibilityKind::Protected:109    return ::mlir::LLVM::Visibility::Protected;110  }111}112 113/// Emits the value from memory as expected by its users. Should be called when114/// the memory represetnation of a CIR type is not equal to its scalar115/// representation.116static mlir::Value emitFromMemory(mlir::ConversionPatternRewriter &rewriter,117                                  mlir::DataLayout const &dataLayout,118                                  cir::LoadOp op, mlir::Value value) {119 120  // TODO(cir): Handle other types similarly to clang's codegen EmitFromMemory121  if (auto boolTy = mlir::dyn_cast<cir::BoolType>(op.getType())) {122    // Create a cast value from specified size in datalayout to i1123    assert(value.getType().isInteger(dataLayout.getTypeSizeInBits(boolTy)));124    return createIntCast(rewriter, value, rewriter.getI1Type());125  }126 127  return value;128}129 130/// Emits a value to memory with the expected scalar type. Should be called when131/// the memory represetnation of a CIR type is not equal to its scalar132/// representation.133static mlir::Value emitToMemory(mlir::ConversionPatternRewriter &rewriter,134                                mlir::DataLayout const &dataLayout,135                                mlir::Type origType, mlir::Value value) {136 137  // TODO(cir): Handle other types similarly to clang's codegen EmitToMemory138  if (auto boolTy = mlir::dyn_cast<cir::BoolType>(origType)) {139    // Create zext of value from i1 to i8140    mlir::IntegerType memType =141        rewriter.getIntegerType(dataLayout.getTypeSizeInBits(boolTy));142    return createIntCast(rewriter, value, memType);143  }144 145  return value;146}147 148mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage) {149  using CIR = cir::GlobalLinkageKind;150  using LLVM = mlir::LLVM::Linkage;151 152  switch (linkage) {153  case CIR::AvailableExternallyLinkage:154    return LLVM::AvailableExternally;155  case CIR::CommonLinkage:156    return LLVM::Common;157  case CIR::ExternalLinkage:158    return LLVM::External;159  case CIR::ExternalWeakLinkage:160    return LLVM::ExternWeak;161  case CIR::InternalLinkage:162    return LLVM::Internal;163  case CIR::LinkOnceAnyLinkage:164    return LLVM::Linkonce;165  case CIR::LinkOnceODRLinkage:166    return LLVM::LinkonceODR;167  case CIR::PrivateLinkage:168    return LLVM::Private;169  case CIR::WeakAnyLinkage:170    return LLVM::Weak;171  case CIR::WeakODRLinkage:172    return LLVM::WeakODR;173  };174  llvm_unreachable("Unknown CIR linkage type");175}176 177mlir::LogicalResult CIRToLLVMCopyOpLowering::matchAndRewrite(178    cir::CopyOp op, OpAdaptor adaptor,179    mlir::ConversionPatternRewriter &rewriter) const {180  mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>());181  const mlir::Value length = mlir::LLVM::ConstantOp::create(182      rewriter, op.getLoc(), rewriter.getI32Type(), op.getLength(layout));183  assert(!cir::MissingFeatures::aggValueSlotVolatile());184  rewriter.replaceOpWithNewOp<mlir::LLVM::MemcpyOp>(185      op, adaptor.getDst(), adaptor.getSrc(), length, op.getIsVolatile());186  return mlir::success();187}188 189mlir::LogicalResult CIRToLLVMCosOpLowering::matchAndRewrite(190    cir::CosOp op, OpAdaptor adaptor,191    mlir::ConversionPatternRewriter &rewriter) const {192  mlir::Type resTy = typeConverter->convertType(op.getType());193  rewriter.replaceOpWithNewOp<mlir::LLVM::CosOp>(op, resTy, adaptor.getSrc());194  return mlir::success();195}196 197mlir::LogicalResult CIRToLLVMExpOpLowering::matchAndRewrite(198    cir::ExpOp op, OpAdaptor adaptor,199    mlir::ConversionPatternRewriter &rewriter) const {200  mlir::Type resTy = typeConverter->convertType(op.getType());201  rewriter.replaceOpWithNewOp<mlir::LLVM::ExpOp>(op, resTy, adaptor.getSrc());202  return mlir::success();203}204 205mlir::LogicalResult CIRToLLVMExp2OpLowering::matchAndRewrite(206    cir::Exp2Op op, OpAdaptor adaptor,207    mlir::ConversionPatternRewriter &rewriter) const {208  mlir::Type resTy = typeConverter->convertType(op.getType());209  rewriter.replaceOpWithNewOp<mlir::LLVM::Exp2Op>(op, resTy, adaptor.getSrc());210  return mlir::success();211}212 213static mlir::Value getLLVMIntCast(mlir::ConversionPatternRewriter &rewriter,214                                  mlir::Value llvmSrc, mlir::Type llvmDstIntTy,215                                  bool isUnsigned, uint64_t cirSrcWidth,216                                  uint64_t cirDstIntWidth) {217  if (cirSrcWidth == cirDstIntWidth)218    return llvmSrc;219 220  auto loc = llvmSrc.getLoc();221  if (cirSrcWidth < cirDstIntWidth) {222    if (isUnsigned)223      return mlir::LLVM::ZExtOp::create(rewriter, loc, llvmDstIntTy, llvmSrc);224    return mlir::LLVM::SExtOp::create(rewriter, loc, llvmDstIntTy, llvmSrc);225  }226 227  // Otherwise truncate228  return mlir::LLVM::TruncOp::create(rewriter, loc, llvmDstIntTy, llvmSrc);229}230 231class CIRAttrToValue {232public:233  CIRAttrToValue(mlir::Operation *parentOp,234                 mlir::ConversionPatternRewriter &rewriter,235                 const mlir::TypeConverter *converter)236      : parentOp(parentOp), rewriter(rewriter), converter(converter) {}237 238  mlir::Value visit(mlir::Attribute attr) {239    return llvm::TypeSwitch<mlir::Attribute, mlir::Value>(attr)240        .Case<cir::IntAttr, cir::FPAttr, cir::ConstComplexAttr,241              cir::ConstArrayAttr, cir::ConstRecordAttr, cir::ConstVectorAttr,242              cir::ConstPtrAttr, cir::GlobalViewAttr, cir::TypeInfoAttr,243              cir::UndefAttr, cir::VTableAttr, cir::ZeroAttr>(244            [&](auto attrT) { return visitCirAttr(attrT); })245        .Default([&](auto attrT) { return mlir::Value(); });246  }247 248  mlir::Value visitCirAttr(cir::IntAttr intAttr);249  mlir::Value visitCirAttr(cir::FPAttr fltAttr);250  mlir::Value visitCirAttr(cir::ConstComplexAttr complexAttr);251  mlir::Value visitCirAttr(cir::ConstPtrAttr ptrAttr);252  mlir::Value visitCirAttr(cir::ConstArrayAttr attr);253  mlir::Value visitCirAttr(cir::ConstRecordAttr attr);254  mlir::Value visitCirAttr(cir::ConstVectorAttr attr);255  mlir::Value visitCirAttr(cir::GlobalViewAttr attr);256  mlir::Value visitCirAttr(cir::TypeInfoAttr attr);257  mlir::Value visitCirAttr(cir::UndefAttr attr);258  mlir::Value visitCirAttr(cir::VTableAttr attr);259  mlir::Value visitCirAttr(cir::ZeroAttr attr);260 261private:262  mlir::Operation *parentOp;263  mlir::ConversionPatternRewriter &rewriter;264  const mlir::TypeConverter *converter;265};266 267/// Switches on the type of attribute and calls the appropriate conversion.268mlir::Value lowerCirAttrAsValue(mlir::Operation *parentOp,269                                const mlir::Attribute attr,270                                mlir::ConversionPatternRewriter &rewriter,271                                const mlir::TypeConverter *converter) {272  CIRAttrToValue valueConverter(parentOp, rewriter, converter);273  mlir::Value value = valueConverter.visit(attr);274  if (!value)275    llvm_unreachable("unhandled attribute type");276  return value;277}278 279void convertSideEffectForCall(mlir::Operation *callOp, bool isNothrow,280                              cir::SideEffect sideEffect,281                              mlir::LLVM::MemoryEffectsAttr &memoryEffect,282                              bool &noUnwind, bool &willReturn) {283  using mlir::LLVM::ModRefInfo;284 285  switch (sideEffect) {286  case cir::SideEffect::All:287    memoryEffect = {};288    noUnwind = isNothrow;289    willReturn = false;290    break;291 292  case cir::SideEffect::Pure:293    memoryEffect = mlir::LLVM::MemoryEffectsAttr::get(294        callOp->getContext(), /*other=*/ModRefInfo::Ref,295        /*argMem=*/ModRefInfo::Ref,296        /*inaccessibleMem=*/ModRefInfo::Ref,297        /*errnoMem=*/ModRefInfo::Ref,298        /*targetMem0=*/ModRefInfo::Ref,299        /*targetMem1=*/ModRefInfo::Ref);300    noUnwind = true;301    willReturn = true;302    break;303 304  case cir::SideEffect::Const:305    memoryEffect = mlir::LLVM::MemoryEffectsAttr::get(306        callOp->getContext(), /*other=*/ModRefInfo::NoModRef,307        /*argMem=*/ModRefInfo::NoModRef,308        /*inaccessibleMem=*/ModRefInfo::NoModRef,309        /*errnoMem=*/ModRefInfo::NoModRef,310        /*targetMem0=*/ModRefInfo::NoModRef,311        /*targetMem1=*/ModRefInfo::NoModRef);312    noUnwind = true;313    willReturn = true;314    break;315  }316}317 318static mlir::LLVM::CallIntrinsicOp319createCallLLVMIntrinsicOp(mlir::ConversionPatternRewriter &rewriter,320                          mlir::Location loc, const llvm::Twine &intrinsicName,321                          mlir::Type resultTy, mlir::ValueRange operands) {322  auto intrinsicNameAttr =323      mlir::StringAttr::get(rewriter.getContext(), intrinsicName);324  return mlir::LLVM::CallIntrinsicOp::create(rewriter, loc, resultTy,325                                             intrinsicNameAttr, operands);326}327 328static mlir::LLVM::CallIntrinsicOp replaceOpWithCallLLVMIntrinsicOp(329    mlir::ConversionPatternRewriter &rewriter, mlir::Operation *op,330    const llvm::Twine &intrinsicName, mlir::Type resultTy,331    mlir::ValueRange operands) {332  mlir::LLVM::CallIntrinsicOp callIntrinOp = createCallLLVMIntrinsicOp(333      rewriter, op->getLoc(), intrinsicName, resultTy, operands);334  rewriter.replaceOp(op, callIntrinOp.getOperation());335  return callIntrinOp;336}337 338mlir::LogicalResult CIRToLLVMLLVMIntrinsicCallOpLowering::matchAndRewrite(339    cir::LLVMIntrinsicCallOp op, OpAdaptor adaptor,340    mlir::ConversionPatternRewriter &rewriter) const {341  mlir::Type llvmResTy =342      getTypeConverter()->convertType(op->getResultTypes()[0]);343  if (!llvmResTy)344    return op.emitError("expected LLVM result type");345  StringRef name = op.getIntrinsicName();346 347  // Some LLVM intrinsics require ElementType attribute to be attached to348  // the argument of pointer type. That prevents us from generating LLVM IR349  // because from LLVM dialect, we have LLVM IR like the below which fails350  // LLVM IR verification.351  // %3 = call i64 @llvm.aarch64.ldxr.p0(ptr %2)352  // The expected LLVM IR should be like353  // %3 = call i64 @llvm.aarch64.ldxr.p0(ptr elementtype(i32) %2)354  // TODO(cir): MLIR LLVM dialect should handle this part as CIR has no way355  // to set LLVM IR attribute.356  assert(!cir::MissingFeatures::intrinsicElementTypeSupport());357  replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm." + name, llvmResTy,358                                   adaptor.getOperands());359  return mlir::success();360}361 362/// IntAttr visitor.363mlir::Value CIRAttrToValue::visitCirAttr(cir::IntAttr intAttr) {364  mlir::Location loc = parentOp->getLoc();365  return mlir::LLVM::ConstantOp::create(366      rewriter, loc, converter->convertType(intAttr.getType()),367      intAttr.getValue());368}369 370/// FPAttr visitor.371mlir::Value CIRAttrToValue::visitCirAttr(cir::FPAttr fltAttr) {372  mlir::Location loc = parentOp->getLoc();373  return mlir::LLVM::ConstantOp::create(374      rewriter, loc, converter->convertType(fltAttr.getType()),375      fltAttr.getValue());376}377 378/// ConstComplexAttr visitor.379mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstComplexAttr complexAttr) {380  auto complexType = mlir::cast<cir::ComplexType>(complexAttr.getType());381  mlir::Type complexElemTy = complexType.getElementType();382  mlir::Type complexElemLLVMTy = converter->convertType(complexElemTy);383 384  mlir::Attribute components[2];385  if (const auto intType = mlir::dyn_cast<cir::IntType>(complexElemTy)) {386    components[0] = rewriter.getIntegerAttr(387        complexElemLLVMTy,388        mlir::cast<cir::IntAttr>(complexAttr.getReal()).getValue());389    components[1] = rewriter.getIntegerAttr(390        complexElemLLVMTy,391        mlir::cast<cir::IntAttr>(complexAttr.getImag()).getValue());392  } else {393    components[0] = rewriter.getFloatAttr(394        complexElemLLVMTy,395        mlir::cast<cir::FPAttr>(complexAttr.getReal()).getValue());396    components[1] = rewriter.getFloatAttr(397        complexElemLLVMTy,398        mlir::cast<cir::FPAttr>(complexAttr.getImag()).getValue());399  }400 401  mlir::Location loc = parentOp->getLoc();402  return mlir::LLVM::ConstantOp::create(403      rewriter, loc, converter->convertType(complexAttr.getType()),404      rewriter.getArrayAttr(components));405}406 407/// ConstPtrAttr visitor.408mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstPtrAttr ptrAttr) {409  mlir::Location loc = parentOp->getLoc();410  if (ptrAttr.isNullValue()) {411    return mlir::LLVM::ZeroOp::create(412        rewriter, loc, converter->convertType(ptrAttr.getType()));413  }414  mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>());415  mlir::Value ptrVal = mlir::LLVM::ConstantOp::create(416      rewriter, loc,417      rewriter.getIntegerType(layout.getTypeSizeInBits(ptrAttr.getType())),418      ptrAttr.getValue().getInt());419  return mlir::LLVM::IntToPtrOp::create(420      rewriter, loc, converter->convertType(ptrAttr.getType()), ptrVal);421}422 423// ConstArrayAttr visitor424mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstArrayAttr attr) {425  mlir::Type llvmTy = converter->convertType(attr.getType());426  mlir::Location loc = parentOp->getLoc();427  mlir::Value result;428 429  if (attr.hasTrailingZeros()) {430    mlir::Type arrayTy = attr.getType();431    result = mlir::LLVM::ZeroOp::create(rewriter, loc,432                                        converter->convertType(arrayTy));433  } else {434    result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);435  }436 437  // Iteratively lower each constant element of the array.438  if (auto arrayAttr = mlir::dyn_cast<mlir::ArrayAttr>(attr.getElts())) {439    for (auto [idx, elt] : llvm::enumerate(arrayAttr)) {440      mlir::DataLayout dataLayout(parentOp->getParentOfType<mlir::ModuleOp>());441      mlir::Value init = visit(elt);442      result =443          mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);444    }445  } else if (auto strAttr = mlir::dyn_cast<mlir::StringAttr>(attr.getElts())) {446    // TODO(cir): this diverges from traditional lowering. Normally the string447    // would be a global constant that is memcopied.448    auto arrayTy = mlir::dyn_cast<cir::ArrayType>(strAttr.getType());449    assert(arrayTy && "String attribute must have an array type");450    mlir::Type eltTy = arrayTy.getElementType();451    for (auto [idx, elt] : llvm::enumerate(strAttr)) {452      auto init = mlir::LLVM::ConstantOp::create(453          rewriter, loc, converter->convertType(eltTy), elt);454      result =455          mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);456    }457  } else {458    llvm_unreachable("unexpected ConstArrayAttr elements");459  }460 461  return result;462}463 464/// ConstRecord visitor.465mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstRecordAttr constRecord) {466  const mlir::Type llvmTy = converter->convertType(constRecord.getType());467  const mlir::Location loc = parentOp->getLoc();468  mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);469 470  // Iteratively lower each constant element of the record.471  for (auto [idx, elt] : llvm::enumerate(constRecord.getMembers())) {472    mlir::Value init = visit(elt);473    result =474        mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);475  }476 477  return result;478}479 480/// ConstVectorAttr visitor.481mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstVectorAttr attr) {482  const mlir::Type llvmTy = converter->convertType(attr.getType());483  const mlir::Location loc = parentOp->getLoc();484 485  SmallVector<mlir::Attribute> mlirValues;486  for (const mlir::Attribute elementAttr : attr.getElts()) {487    mlir::Attribute mlirAttr;488    if (auto intAttr = mlir::dyn_cast<cir::IntAttr>(elementAttr)) {489      mlirAttr = rewriter.getIntegerAttr(490          converter->convertType(intAttr.getType()), intAttr.getValue());491    } else if (auto floatAttr = mlir::dyn_cast<cir::FPAttr>(elementAttr)) {492      mlirAttr = rewriter.getFloatAttr(493          converter->convertType(floatAttr.getType()), floatAttr.getValue());494    } else {495      llvm_unreachable(496          "vector constant with an element that is neither an int nor a float");497    }498    mlirValues.push_back(mlirAttr);499  }500 501  return mlir::LLVM::ConstantOp::create(502      rewriter, loc, llvmTy,503      mlir::DenseElementsAttr::get(mlir::cast<mlir::ShapedType>(llvmTy),504                                   mlirValues));505}506 507// GlobalViewAttr visitor.508mlir::Value CIRAttrToValue::visitCirAttr(cir::GlobalViewAttr globalAttr) {509  auto moduleOp = parentOp->getParentOfType<mlir::ModuleOp>();510  mlir::DataLayout dataLayout(moduleOp);511  mlir::Type sourceType;512  assert(!cir::MissingFeatures::addressSpace());513  llvm::StringRef symName;514  mlir::Operation *sourceSymbol =515      mlir::SymbolTable::lookupSymbolIn(moduleOp, globalAttr.getSymbol());516  if (auto llvmSymbol = dyn_cast<mlir::LLVM::GlobalOp>(sourceSymbol)) {517    sourceType = llvmSymbol.getType();518    symName = llvmSymbol.getSymName();519  } else if (auto cirSymbol = dyn_cast<cir::GlobalOp>(sourceSymbol)) {520    sourceType =521        convertTypeForMemory(*converter, dataLayout, cirSymbol.getSymType());522    symName = cirSymbol.getSymName();523  } else if (auto llvmFun = dyn_cast<mlir::LLVM::LLVMFuncOp>(sourceSymbol)) {524    sourceType = llvmFun.getFunctionType();525    symName = llvmFun.getSymName();526  } else if (auto fun = dyn_cast<cir::FuncOp>(sourceSymbol)) {527    sourceType = converter->convertType(fun.getFunctionType());528    symName = fun.getSymName();529  } else if (auto alias = dyn_cast<mlir::LLVM::AliasOp>(sourceSymbol)) {530    sourceType = alias.getType();531    symName = alias.getSymName();532  } else {533    llvm_unreachable("Unexpected GlobalOp type");534  }535 536  mlir::Location loc = parentOp->getLoc();537  mlir::Value addrOp = mlir::LLVM::AddressOfOp::create(538      rewriter, loc, mlir::LLVM::LLVMPointerType::get(rewriter.getContext()),539      symName);540 541  if (globalAttr.getIndices()) {542    llvm::SmallVector<mlir::LLVM::GEPArg> indices;543 544    if (mlir::isa<mlir::LLVM::LLVMArrayType, mlir::LLVM::LLVMStructType>(545            sourceType))546      indices.push_back(0);547 548    for (mlir::Attribute idx : globalAttr.getIndices()) {549      auto intAttr = mlir::cast<mlir::IntegerAttr>(idx);550      indices.push_back(intAttr.getValue().getSExtValue());551    }552    mlir::Type resTy = addrOp.getType();553    mlir::Type eltTy = converter->convertType(sourceType);554    addrOp =555        mlir::LLVM::GEPOp::create(rewriter, loc, resTy, eltTy, addrOp, indices,556                                  mlir::LLVM::GEPNoWrapFlags::none);557  }558 559  // The incubator has handling here for the attribute having integer type, but560  // the only test case I could find that reaches it is a direct CIR-to-LLVM IR561  // lowering with no clear indication of how the CIR might have been generated.562  // We'll hit the unreachable below if this happens.563  assert(!cir::MissingFeatures::globalViewIntLowering());564 565  if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(globalAttr.getType())) {566    mlir::Type llvmEltTy =567        convertTypeForMemory(*converter, dataLayout, ptrTy.getPointee());568 569    if (llvmEltTy == sourceType)570      return addrOp;571 572    mlir::Type llvmDstTy = converter->convertType(globalAttr.getType());573    return mlir::LLVM::BitcastOp::create(rewriter, parentOp->getLoc(),574                                         llvmDstTy, addrOp);575  }576 577  llvm_unreachable("Expecting pointer or integer type for GlobalViewAttr");578}579 580// TypeInfoAttr visitor.581mlir::Value CIRAttrToValue::visitCirAttr(cir::TypeInfoAttr typeInfoAttr) {582  mlir::Type llvmTy = converter->convertType(typeInfoAttr.getType());583  mlir::Location loc = parentOp->getLoc();584  mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);585 586  for (auto [idx, elt] : llvm::enumerate(typeInfoAttr.getData())) {587    mlir::Value init = visit(elt);588    result =589        mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);590  }591 592  return result;593}594 595/// UndefAttr visitor.596mlir::Value CIRAttrToValue::visitCirAttr(cir::UndefAttr undefAttr) {597  mlir::Location loc = parentOp->getLoc();598  return mlir::LLVM::UndefOp::create(599      rewriter, loc, converter->convertType(undefAttr.getType()));600}601 602// VTableAttr visitor.603mlir::Value CIRAttrToValue::visitCirAttr(cir::VTableAttr vtableArr) {604  mlir::Type llvmTy = converter->convertType(vtableArr.getType());605  mlir::Location loc = parentOp->getLoc();606  mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);607 608  for (auto [idx, elt] : llvm::enumerate(vtableArr.getData())) {609    mlir::Value init = visit(elt);610    result =611        mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);612  }613 614  return result;615}616 617/// ZeroAttr visitor.618mlir::Value CIRAttrToValue::visitCirAttr(cir::ZeroAttr attr) {619  mlir::Location loc = parentOp->getLoc();620  return mlir::LLVM::ZeroOp::create(rewriter, loc,621                                    converter->convertType(attr.getType()));622}623 624// This class handles rewriting initializer attributes for types that do not625// require region initialization.626class GlobalInitAttrRewriter {627public:628  GlobalInitAttrRewriter(mlir::Type type,629                         mlir::ConversionPatternRewriter &rewriter)630      : llvmType(type), rewriter(rewriter) {}631 632  mlir::Attribute visit(mlir::Attribute attr) {633    return llvm::TypeSwitch<mlir::Attribute, mlir::Attribute>(attr)634        .Case<cir::IntAttr, cir::FPAttr, cir::BoolAttr>(635            [&](auto attrT) { return visitCirAttr(attrT); })636        .Default([&](auto attrT) { return mlir::Attribute(); });637  }638 639  mlir::Attribute visitCirAttr(cir::IntAttr attr) {640    return rewriter.getIntegerAttr(llvmType, attr.getValue());641  }642 643  mlir::Attribute visitCirAttr(cir::FPAttr attr) {644    return rewriter.getFloatAttr(llvmType, attr.getValue());645  }646 647  mlir::Attribute visitCirAttr(cir::BoolAttr attr) {648    return rewriter.getBoolAttr(attr.getValue());649  }650 651private:652  mlir::Type llvmType;653  mlir::ConversionPatternRewriter &rewriter;654};655 656// This pass requires the CIR to be in a "flat" state. All blocks in each657// function must belong to the parent region. Once scopes and control flow658// are implemented in CIR, a pass will be run before this one to flatten659// the CIR and get it into the state that this pass requires.660struct ConvertCIRToLLVMPass661    : public mlir::PassWrapper<ConvertCIRToLLVMPass,662                               mlir::OperationPass<mlir::ModuleOp>> {663  void getDependentDialects(mlir::DialectRegistry &registry) const override {664    registry.insert<mlir::BuiltinDialect, mlir::DLTIDialect,665                    mlir::LLVM::LLVMDialect, mlir::func::FuncDialect>();666  }667  void runOnOperation() final;668 669  void processCIRAttrs(mlir::ModuleOp module);670 671  StringRef getDescription() const override {672    return "Convert the prepared CIR dialect module to LLVM dialect";673  }674 675  StringRef getArgument() const override { return "cir-flat-to-llvm"; }676};677 678mlir::LogicalResult CIRToLLVMACosOpLowering::matchAndRewrite(679    cir::ACosOp op, OpAdaptor adaptor,680    mlir::ConversionPatternRewriter &rewriter) const {681  mlir::Type resTy = typeConverter->convertType(op.getType());682  rewriter.replaceOpWithNewOp<mlir::LLVM::ACosOp>(op, resTy,683                                                  adaptor.getOperands()[0]);684  return mlir::success();685}686 687mlir::LogicalResult CIRToLLVMASinOpLowering::matchAndRewrite(688    cir::ASinOp op, OpAdaptor adaptor,689    mlir::ConversionPatternRewriter &rewriter) const {690  mlir::Type resTy = typeConverter->convertType(op.getType());691  rewriter.replaceOpWithNewOp<mlir::LLVM::ASinOp>(op, resTy, adaptor.getSrc());692  return mlir::success();693}694 695mlir::LogicalResult CIRToLLVMIsFPClassOpLowering::matchAndRewrite(696    cir::IsFPClassOp op, OpAdaptor adaptor,697    mlir::ConversionPatternRewriter &rewriter) const {698  mlir::Value src = adaptor.getSrc();699  cir::FPClassTest flags = adaptor.getFlags();700  mlir::IntegerType retTy = rewriter.getI1Type();701 702  rewriter.replaceOpWithNewOp<mlir::LLVM::IsFPClass>(703      op, retTy, src, static_cast<uint32_t>(flags));704  return mlir::success();705}706 707mlir::LogicalResult CIRToLLVMAssumeOpLowering::matchAndRewrite(708    cir::AssumeOp op, OpAdaptor adaptor,709    mlir::ConversionPatternRewriter &rewriter) const {710  auto cond = adaptor.getPredicate();711  rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>(op, cond);712  return mlir::success();713}714 715mlir::LogicalResult CIRToLLVMAssumeAlignedOpLowering::matchAndRewrite(716    cir::AssumeAlignedOp op, OpAdaptor adaptor,717    mlir::ConversionPatternRewriter &rewriter) const {718  SmallVector<mlir::Value, 3> opBundleArgs{adaptor.getPointer()};719 720  auto alignment = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),721                                                  adaptor.getAlignmentAttr());722  opBundleArgs.push_back(alignment);723 724  if (mlir::Value offset = adaptor.getOffset())725    opBundleArgs.push_back(offset);726 727  auto cond = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),728                                             rewriter.getI1Type(), 1);729  mlir::LLVM::AssumeOp::create(rewriter, op.getLoc(), cond, "align",730                               opBundleArgs);731 732  // The llvm.assume operation does not have a result, so we need to replace733  // all uses of this cir.assume_aligned operation with the input ptr itself.734  rewriter.replaceOp(op, adaptor.getPointer());735  return mlir::success();736}737 738mlir::LogicalResult CIRToLLVMAssumeSepStorageOpLowering::matchAndRewrite(739    cir::AssumeSepStorageOp op, OpAdaptor adaptor,740    mlir::ConversionPatternRewriter &rewriter) const {741  auto cond = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),742                                             rewriter.getI1Type(), 1);743  rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>(744      op, cond, mlir::LLVM::AssumeSeparateStorageTag{}, adaptor.getPtr1(),745      adaptor.getPtr2());746  return mlir::success();747}748 749static mlir::LLVM::AtomicOrdering750getLLVMMemOrder(std::optional<cir::MemOrder> memorder) {751  if (!memorder)752    return mlir::LLVM::AtomicOrdering::not_atomic;753  switch (*memorder) {754  case cir::MemOrder::Relaxed:755    return mlir::LLVM::AtomicOrdering::monotonic;756  case cir::MemOrder::Consume:757  case cir::MemOrder::Acquire:758    return mlir::LLVM::AtomicOrdering::acquire;759  case cir::MemOrder::Release:760    return mlir::LLVM::AtomicOrdering::release;761  case cir::MemOrder::AcquireRelease:762    return mlir::LLVM::AtomicOrdering::acq_rel;763  case cir::MemOrder::SequentiallyConsistent:764    return mlir::LLVM::AtomicOrdering::seq_cst;765  }766  llvm_unreachable("unknown memory order");767}768 769mlir::LogicalResult CIRToLLVMAtomicCmpXchgOpLowering::matchAndRewrite(770    cir::AtomicCmpXchgOp op, OpAdaptor adaptor,771    mlir::ConversionPatternRewriter &rewriter) const {772  mlir::Value expected = adaptor.getExpected();773  mlir::Value desired = adaptor.getDesired();774 775  auto cmpxchg = mlir::LLVM::AtomicCmpXchgOp::create(776      rewriter, op.getLoc(), adaptor.getPtr(), expected, desired,777      getLLVMMemOrder(adaptor.getSuccOrder()),778      getLLVMMemOrder(adaptor.getFailOrder()));779  assert(!cir::MissingFeatures::atomicScope());780  cmpxchg.setAlignment(adaptor.getAlignment());781  cmpxchg.setWeak(adaptor.getWeak());782  cmpxchg.setVolatile_(adaptor.getIsVolatile());783 784  // Check result and apply stores accordingly.785  auto old = mlir::LLVM::ExtractValueOp::create(rewriter, op.getLoc(),786                                                cmpxchg.getResult(), 0);787  auto cmp = mlir::LLVM::ExtractValueOp::create(rewriter, op.getLoc(),788                                                cmpxchg.getResult(), 1);789 790  rewriter.replaceOp(op, {old, cmp});791  return mlir::success();792}793 794mlir::LogicalResult CIRToLLVMAtomicXchgOpLowering::matchAndRewrite(795    cir::AtomicXchgOp op, OpAdaptor adaptor,796    mlir::ConversionPatternRewriter &rewriter) const {797  assert(!cir::MissingFeatures::atomicSyncScopeID());798  mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(adaptor.getMemOrder());799  rewriter.replaceOpWithNewOp<mlir::LLVM::AtomicRMWOp>(800      op, mlir::LLVM::AtomicBinOp::xchg, adaptor.getPtr(), adaptor.getVal(),801      llvmOrder);802  return mlir::success();803}804 805mlir::LogicalResult CIRToLLVMAtomicTestAndSetOpLowering::matchAndRewrite(806    cir::AtomicTestAndSetOp op, OpAdaptor adaptor,807    mlir::ConversionPatternRewriter &rewriter) const {808  assert(!cir::MissingFeatures::atomicSyncScopeID());809 810  mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(op.getMemOrder());811 812  auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),813                                            rewriter.getI8Type(), 1);814  auto rmw = mlir::LLVM::AtomicRMWOp::create(815      rewriter, op.getLoc(), mlir::LLVM::AtomicBinOp::xchg, adaptor.getPtr(),816      one, llvmOrder, /*syncscope=*/llvm::StringRef(),817      adaptor.getAlignment().value_or(0), op.getIsVolatile());818 819  auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),820                                             rewriter.getI8Type(), 0);821  auto cmp = mlir::LLVM::ICmpOp::create(822      rewriter, op.getLoc(), mlir::LLVM::ICmpPredicate::ne, rmw, zero);823 824  rewriter.replaceOp(op, cmp);825  return mlir::success();826}827 828mlir::LogicalResult CIRToLLVMAtomicClearOpLowering::matchAndRewrite(829    cir::AtomicClearOp op, OpAdaptor adaptor,830    mlir::ConversionPatternRewriter &rewriter) const {831  assert(!cir::MissingFeatures::atomicSyncScopeID());832 833  mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(op.getMemOrder());834  auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),835                                             rewriter.getI8Type(), 0);836  auto store = mlir::LLVM::StoreOp::create(837      rewriter, op.getLoc(), zero, adaptor.getPtr(),838      adaptor.getAlignment().value_or(0), op.getIsVolatile(),839      /*isNonTemporal=*/false, /*isInvariantGroup=*/false, llvmOrder);840 841  rewriter.replaceOp(op, store);842  return mlir::success();843}844 845static mlir::LLVM::AtomicBinOp846getLLVMAtomicBinOp(cir::AtomicFetchKind k, bool isInt, bool isSignedInt) {847  switch (k) {848  case cir::AtomicFetchKind::Add:849    return isInt ? mlir::LLVM::AtomicBinOp::add : mlir::LLVM::AtomicBinOp::fadd;850  case cir::AtomicFetchKind::Sub:851    return isInt ? mlir::LLVM::AtomicBinOp::sub : mlir::LLVM::AtomicBinOp::fsub;852  case cir::AtomicFetchKind::And:853    return mlir::LLVM::AtomicBinOp::_and;854  case cir::AtomicFetchKind::Xor:855    return mlir::LLVM::AtomicBinOp::_xor;856  case cir::AtomicFetchKind::Or:857    return mlir::LLVM::AtomicBinOp::_or;858  case cir::AtomicFetchKind::Nand:859    return mlir::LLVM::AtomicBinOp::nand;860  case cir::AtomicFetchKind::Max: {861    if (!isInt)862      return mlir::LLVM::AtomicBinOp::fmax;863    return isSignedInt ? mlir::LLVM::AtomicBinOp::max864                       : mlir::LLVM::AtomicBinOp::umax;865  }866  case cir::AtomicFetchKind::Min: {867    if (!isInt)868      return mlir::LLVM::AtomicBinOp::fmin;869    return isSignedInt ? mlir::LLVM::AtomicBinOp::min870                       : mlir::LLVM::AtomicBinOp::umin;871  }872  }873  llvm_unreachable("Unknown atomic fetch opcode");874}875 876static llvm::StringLiteral getLLVMBinop(cir::AtomicFetchKind k, bool isInt) {877  switch (k) {878  case cir::AtomicFetchKind::Add:879    return isInt ? mlir::LLVM::AddOp::getOperationName()880                 : mlir::LLVM::FAddOp::getOperationName();881  case cir::AtomicFetchKind::Sub:882    return isInt ? mlir::LLVM::SubOp::getOperationName()883                 : mlir::LLVM::FSubOp::getOperationName();884  case cir::AtomicFetchKind::And:885    return mlir::LLVM::AndOp::getOperationName();886  case cir::AtomicFetchKind::Xor:887    return mlir::LLVM::XOrOp::getOperationName();888  case cir::AtomicFetchKind::Or:889    return mlir::LLVM::OrOp::getOperationName();890  case cir::AtomicFetchKind::Nand:891    // There's no nand binop in LLVM, this is later fixed with a not.892    return mlir::LLVM::AndOp::getOperationName();893  case cir::AtomicFetchKind::Max:894  case cir::AtomicFetchKind::Min:895    llvm_unreachable("handled in buildMinMaxPostOp");896  }897  llvm_unreachable("Unknown atomic fetch opcode");898}899 900mlir::Value CIRToLLVMAtomicFetchOpLowering::buildPostOp(901    cir::AtomicFetchOp op, OpAdaptor adaptor,902    mlir::ConversionPatternRewriter &rewriter, mlir::Value rmwVal,903    bool isInt) const {904  SmallVector<mlir::Value> atomicOperands = {rmwVal, adaptor.getVal()};905  SmallVector<mlir::Type> atomicResTys = {rmwVal.getType()};906  return rewriter907      .create(op.getLoc(),908              rewriter.getStringAttr(getLLVMBinop(op.getBinop(), isInt)),909              atomicOperands, atomicResTys, {})910      ->getResult(0);911}912 913mlir::Value CIRToLLVMAtomicFetchOpLowering::buildMinMaxPostOp(914    cir::AtomicFetchOp op, OpAdaptor adaptor,915    mlir::ConversionPatternRewriter &rewriter, mlir::Value rmwVal, bool isInt,916    bool isSigned) const {917  mlir::Location loc = op.getLoc();918 919  if (!isInt) {920    if (op.getBinop() == cir::AtomicFetchKind::Max)921      return mlir::LLVM::MaxNumOp::create(rewriter, loc, rmwVal,922                                          adaptor.getVal());923    return mlir::LLVM::MinNumOp::create(rewriter, loc, rmwVal,924                                        adaptor.getVal());925  }926 927  mlir::LLVM::ICmpPredicate pred;928  if (op.getBinop() == cir::AtomicFetchKind::Max) {929    pred = isSigned ? mlir::LLVM::ICmpPredicate::sgt930                    : mlir::LLVM::ICmpPredicate::ugt;931  } else { // Min932    pred = isSigned ? mlir::LLVM::ICmpPredicate::slt933                    : mlir::LLVM::ICmpPredicate::ult;934  }935  mlir::Value cmp = mlir::LLVM::ICmpOp::create(936      rewriter, loc,937      mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(), pred), rmwVal,938      adaptor.getVal());939  return mlir::LLVM::SelectOp::create(rewriter, loc, cmp, rmwVal,940                                      adaptor.getVal());941}942 943mlir::LogicalResult CIRToLLVMAtomicFetchOpLowering::matchAndRewrite(944    cir::AtomicFetchOp op, OpAdaptor adaptor,945    mlir::ConversionPatternRewriter &rewriter) const {946  bool isInt = false;947  bool isSignedInt = false;948  if (auto intTy = mlir::dyn_cast<cir::IntType>(op.getVal().getType())) {949    isInt = true;950    isSignedInt = intTy.isSigned();951  } else if (mlir::isa<cir::SingleType, cir::DoubleType>(952                 op.getVal().getType())) {953    isInt = false;954  } else {955    return op.emitError() << "Unsupported type: " << op.getVal().getType();956  }957 958  mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(op.getMemOrder());959  mlir::LLVM::AtomicBinOp llvmBinOp =960      getLLVMAtomicBinOp(op.getBinop(), isInt, isSignedInt);961  auto rmwVal = mlir::LLVM::AtomicRMWOp::create(rewriter, op.getLoc(),962                                                llvmBinOp, adaptor.getPtr(),963                                                adaptor.getVal(), llvmOrder);964 965  mlir::Value result = rmwVal.getResult();966  if (!op.getFetchFirst()) {967    if (op.getBinop() == cir::AtomicFetchKind::Max ||968        op.getBinop() == cir::AtomicFetchKind::Min)969      result = buildMinMaxPostOp(op, adaptor, rewriter, rmwVal.getRes(), isInt,970                                 isSignedInt);971    else972      result = buildPostOp(op, adaptor, rewriter, rmwVal.getRes(), isInt);973 974    // Compensate lack of nand binop in LLVM IR.975    if (op.getBinop() == cir::AtomicFetchKind::Nand) {976      auto negOne = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),977                                                   result.getType(), -1);978      result = mlir::LLVM::XOrOp::create(rewriter, op.getLoc(), result, negOne);979    }980  }981 982  rewriter.replaceOp(op, result);983  return mlir::success();984}985 986mlir::LogicalResult CIRToLLVMBitClrsbOpLowering::matchAndRewrite(987    cir::BitClrsbOp op, OpAdaptor adaptor,988    mlir::ConversionPatternRewriter &rewriter) const {989  auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),990                                             adaptor.getInput().getType(), 0);991  auto isNeg = mlir::LLVM::ICmpOp::create(992      rewriter, op.getLoc(),993      mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(),994                                         mlir::LLVM::ICmpPredicate::slt),995      adaptor.getInput(), zero);996 997  auto negOne = mlir::LLVM::ConstantOp::create(998      rewriter, op.getLoc(), adaptor.getInput().getType(), -1);999  auto flipped = mlir::LLVM::XOrOp::create(rewriter, op.getLoc(),1000                                           adaptor.getInput(), negOne);1001 1002  auto select = mlir::LLVM::SelectOp::create(rewriter, op.getLoc(), isNeg,1003                                             flipped, adaptor.getInput());1004 1005  auto resTy = getTypeConverter()->convertType(op.getType());1006  auto clz = mlir::LLVM::CountLeadingZerosOp::create(1007      rewriter, op.getLoc(), resTy, select, /*is_zero_poison=*/false);1008 1009  auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 1);1010  auto res = mlir::LLVM::SubOp::create(rewriter, op.getLoc(), clz, one);1011  rewriter.replaceOp(op, res);1012 1013  return mlir::LogicalResult::success();1014}1015 1016mlir::LogicalResult CIRToLLVMBitClzOpLowering::matchAndRewrite(1017    cir::BitClzOp op, OpAdaptor adaptor,1018    mlir::ConversionPatternRewriter &rewriter) const {1019  auto resTy = getTypeConverter()->convertType(op.getType());1020  auto llvmOp = mlir::LLVM::CountLeadingZerosOp::create(1021      rewriter, op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero());1022  rewriter.replaceOp(op, llvmOp);1023  return mlir::LogicalResult::success();1024}1025 1026mlir::LogicalResult CIRToLLVMBitCtzOpLowering::matchAndRewrite(1027    cir::BitCtzOp op, OpAdaptor adaptor,1028    mlir::ConversionPatternRewriter &rewriter) const {1029  auto resTy = getTypeConverter()->convertType(op.getType());1030  auto llvmOp = mlir::LLVM::CountTrailingZerosOp::create(1031      rewriter, op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero());1032  rewriter.replaceOp(op, llvmOp);1033  return mlir::LogicalResult::success();1034}1035 1036mlir::LogicalResult CIRToLLVMBitFfsOpLowering::matchAndRewrite(1037    cir::BitFfsOp op, OpAdaptor adaptor,1038    mlir::ConversionPatternRewriter &rewriter) const {1039  auto resTy = getTypeConverter()->convertType(op.getType());1040  auto ctz = mlir::LLVM::CountTrailingZerosOp::create(rewriter, op.getLoc(),1041                                                      resTy, adaptor.getInput(),1042                                                      /*is_zero_poison=*/true);1043 1044  auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 1);1045  auto ctzAddOne = mlir::LLVM::AddOp::create(rewriter, op.getLoc(), ctz, one);1046 1047  auto zeroInputTy = mlir::LLVM::ConstantOp::create(1048      rewriter, op.getLoc(), adaptor.getInput().getType(), 0);1049  auto isZero = mlir::LLVM::ICmpOp::create(1050      rewriter, op.getLoc(),1051      mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(),1052                                         mlir::LLVM::ICmpPredicate::eq),1053      adaptor.getInput(), zeroInputTy);1054 1055  auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 0);1056  auto res = mlir::LLVM::SelectOp::create(rewriter, op.getLoc(), isZero, zero,1057                                          ctzAddOne);1058  rewriter.replaceOp(op, res);1059 1060  return mlir::LogicalResult::success();1061}1062 1063mlir::LogicalResult CIRToLLVMBitParityOpLowering::matchAndRewrite(1064    cir::BitParityOp op, OpAdaptor adaptor,1065    mlir::ConversionPatternRewriter &rewriter) const {1066  auto resTy = getTypeConverter()->convertType(op.getType());1067  auto popcnt = mlir::LLVM::CtPopOp::create(rewriter, op.getLoc(), resTy,1068                                            adaptor.getInput());1069 1070  auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 1);1071  auto popcntMod2 =1072      mlir::LLVM::AndOp::create(rewriter, op.getLoc(), popcnt, one);1073  rewriter.replaceOp(op, popcntMod2);1074 1075  return mlir::LogicalResult::success();1076}1077 1078mlir::LogicalResult CIRToLLVMBitPopcountOpLowering::matchAndRewrite(1079    cir::BitPopcountOp op, OpAdaptor adaptor,1080    mlir::ConversionPatternRewriter &rewriter) const {1081  auto resTy = getTypeConverter()->convertType(op.getType());1082  auto llvmOp = mlir::LLVM::CtPopOp::create(rewriter, op.getLoc(), resTy,1083                                            adaptor.getInput());1084  rewriter.replaceOp(op, llvmOp);1085  return mlir::LogicalResult::success();1086}1087 1088mlir::LogicalResult CIRToLLVMBitReverseOpLowering::matchAndRewrite(1089    cir::BitReverseOp op, OpAdaptor adaptor,1090    mlir::ConversionPatternRewriter &rewriter) const {1091  rewriter.replaceOpWithNewOp<mlir::LLVM::BitReverseOp>(op, adaptor.getInput());1092  return mlir::success();1093}1094 1095mlir::LogicalResult CIRToLLVMBrCondOpLowering::matchAndRewrite(1096    cir::BrCondOp brOp, OpAdaptor adaptor,1097    mlir::ConversionPatternRewriter &rewriter) const {1098  // When ZExtOp is implemented, we'll need to check if the condition is a1099  // ZExtOp and if so, delete it if it has a single use.1100  assert(!cir::MissingFeatures::zextOp());1101 1102  mlir::Value i1Condition = adaptor.getCond();1103 1104  rewriter.replaceOpWithNewOp<mlir::LLVM::CondBrOp>(1105      brOp, i1Condition, brOp.getDestTrue(), adaptor.getDestOperandsTrue(),1106      brOp.getDestFalse(), adaptor.getDestOperandsFalse());1107 1108  return mlir::success();1109}1110 1111mlir::LogicalResult CIRToLLVMByteSwapOpLowering::matchAndRewrite(1112    cir::ByteSwapOp op, OpAdaptor adaptor,1113    mlir::ConversionPatternRewriter &rewriter) const {1114  rewriter.replaceOpWithNewOp<mlir::LLVM::ByteSwapOp>(op, adaptor.getInput());1115  return mlir::LogicalResult::success();1116}1117 1118mlir::Type CIRToLLVMCastOpLowering::convertTy(mlir::Type ty) const {1119  return getTypeConverter()->convertType(ty);1120}1121 1122mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(1123    cir::CastOp castOp, OpAdaptor adaptor,1124    mlir::ConversionPatternRewriter &rewriter) const {1125  // For arithmetic conversions, LLVM IR uses the same instruction to convert1126  // both individual scalars and entire vectors. This lowering pass handles1127  // both situations.1128 1129  switch (castOp.getKind()) {1130  case cir::CastKind::array_to_ptrdecay: {1131    const auto ptrTy = mlir::cast<cir::PointerType>(castOp.getType());1132    mlir::Value sourceValue = adaptor.getSrc();1133    mlir::Type targetType = convertTy(ptrTy);1134    mlir::Type elementTy = convertTypeForMemory(*getTypeConverter(), dataLayout,1135                                                ptrTy.getPointee());1136    llvm::SmallVector<mlir::LLVM::GEPArg> offset{0};1137    rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(1138        castOp, targetType, elementTy, sourceValue, offset);1139    break;1140  }1141  case cir::CastKind::int_to_bool: {1142    mlir::Value llvmSrcVal = adaptor.getSrc();1143    mlir::Value zeroInt = mlir::LLVM::ConstantOp::create(1144        rewriter, castOp.getLoc(), llvmSrcVal.getType(), 0);1145    rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(1146        castOp, mlir::LLVM::ICmpPredicate::ne, llvmSrcVal, zeroInt);1147    break;1148  }1149  case cir::CastKind::integral: {1150    mlir::Type srcType = castOp.getSrc().getType();1151    mlir::Type dstType = castOp.getType();1152    mlir::Value llvmSrcVal = adaptor.getSrc();1153    mlir::Type llvmDstType = getTypeConverter()->convertType(dstType);1154    cir::IntType srcIntType =1155        mlir::cast<cir::IntType>(elementTypeIfVector(srcType));1156    cir::IntType dstIntType =1157        mlir::cast<cir::IntType>(elementTypeIfVector(dstType));1158    rewriter.replaceOp(castOp, getLLVMIntCast(rewriter, llvmSrcVal, llvmDstType,1159                                              srcIntType.isUnsigned(),1160                                              srcIntType.getWidth(),1161                                              dstIntType.getWidth()));1162    break;1163  }1164  case cir::CastKind::floating: {1165    mlir::Value llvmSrcVal = adaptor.getSrc();1166    mlir::Type llvmDstTy = getTypeConverter()->convertType(castOp.getType());1167 1168    mlir::Type srcTy = elementTypeIfVector(castOp.getSrc().getType());1169    mlir::Type dstTy = elementTypeIfVector(castOp.getType());1170 1171    if (!mlir::isa<cir::FPTypeInterface>(dstTy) ||1172        !mlir::isa<cir::FPTypeInterface>(srcTy))1173      return castOp.emitError() << "NYI cast from " << srcTy << " to " << dstTy;1174 1175    auto getFloatWidth = [](mlir::Type ty) -> unsigned {1176      return mlir::cast<cir::FPTypeInterface>(ty).getWidth();1177    };1178 1179    if (getFloatWidth(srcTy) > getFloatWidth(dstTy))1180      rewriter.replaceOpWithNewOp<mlir::LLVM::FPTruncOp>(castOp, llvmDstTy,1181                                                         llvmSrcVal);1182    else1183      rewriter.replaceOpWithNewOp<mlir::LLVM::FPExtOp>(castOp, llvmDstTy,1184                                                       llvmSrcVal);1185    return mlir::success();1186  }1187  case cir::CastKind::int_to_ptr: {1188    auto dstTy = mlir::cast<cir::PointerType>(castOp.getType());1189    mlir::Value llvmSrcVal = adaptor.getSrc();1190    mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);1191    rewriter.replaceOpWithNewOp<mlir::LLVM::IntToPtrOp>(castOp, llvmDstTy,1192                                                        llvmSrcVal);1193    return mlir::success();1194  }1195  case cir::CastKind::ptr_to_int: {1196    auto dstTy = mlir::cast<cir::IntType>(castOp.getType());1197    mlir::Value llvmSrcVal = adaptor.getSrc();1198    mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);1199    rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(castOp, llvmDstTy,1200                                                        llvmSrcVal);1201    return mlir::success();1202  }1203  case cir::CastKind::float_to_bool: {1204    mlir::Value llvmSrcVal = adaptor.getSrc();1205    auto kind = mlir::LLVM::FCmpPredicate::une;1206 1207    // Check if float is not equal to zero.1208    auto zeroFloat = mlir::LLVM::ConstantOp::create(1209        rewriter, castOp.getLoc(), llvmSrcVal.getType(),1210        mlir::FloatAttr::get(llvmSrcVal.getType(), 0.0));1211 1212    // Extend comparison result to either bool (C++) or int (C).1213    rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>(castOp, kind, llvmSrcVal,1214                                                    zeroFloat);1215 1216    return mlir::success();1217  }1218  case cir::CastKind::bool_to_int: {1219    auto dstTy = mlir::cast<cir::IntType>(castOp.getType());1220    mlir::Value llvmSrcVal = adaptor.getSrc();1221    auto llvmSrcTy = mlir::cast<mlir::IntegerType>(llvmSrcVal.getType());1222    auto llvmDstTy =1223        mlir::cast<mlir::IntegerType>(getTypeConverter()->convertType(dstTy));1224    if (llvmSrcTy.getWidth() == llvmDstTy.getWidth())1225      rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(castOp, llvmDstTy,1226                                                         llvmSrcVal);1227    else1228      rewriter.replaceOpWithNewOp<mlir::LLVM::ZExtOp>(castOp, llvmDstTy,1229                                                      llvmSrcVal);1230    return mlir::success();1231  }1232  case cir::CastKind::bool_to_float: {1233    mlir::Type dstTy = castOp.getType();1234    mlir::Value llvmSrcVal = adaptor.getSrc();1235    mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);1236    rewriter.replaceOpWithNewOp<mlir::LLVM::UIToFPOp>(castOp, llvmDstTy,1237                                                      llvmSrcVal);1238    return mlir::success();1239  }1240  case cir::CastKind::int_to_float: {1241    mlir::Type dstTy = castOp.getType();1242    mlir::Value llvmSrcVal = adaptor.getSrc();1243    mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);1244    if (mlir::cast<cir::IntType>(elementTypeIfVector(castOp.getSrc().getType()))1245            .isSigned())1246      rewriter.replaceOpWithNewOp<mlir::LLVM::SIToFPOp>(castOp, llvmDstTy,1247                                                        llvmSrcVal);1248    else1249      rewriter.replaceOpWithNewOp<mlir::LLVM::UIToFPOp>(castOp, llvmDstTy,1250                                                        llvmSrcVal);1251    return mlir::success();1252  }1253  case cir::CastKind::float_to_int: {1254    mlir::Type dstTy = castOp.getType();1255    mlir::Value llvmSrcVal = adaptor.getSrc();1256    mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);1257    if (mlir::cast<cir::IntType>(elementTypeIfVector(castOp.getType()))1258            .isSigned())1259      rewriter.replaceOpWithNewOp<mlir::LLVM::FPToSIOp>(castOp, llvmDstTy,1260                                                        llvmSrcVal);1261    else1262      rewriter.replaceOpWithNewOp<mlir::LLVM::FPToUIOp>(castOp, llvmDstTy,1263                                                        llvmSrcVal);1264    return mlir::success();1265  }1266  case cir::CastKind::bitcast: {1267    mlir::Type dstTy = castOp.getType();1268    mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);1269 1270    assert(!MissingFeatures::cxxABI());1271    assert(!MissingFeatures::dataMemberType());1272 1273    mlir::Value llvmSrcVal = adaptor.getSrc();1274    rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(castOp, llvmDstTy,1275                                                       llvmSrcVal);1276    return mlir::success();1277  }1278  case cir::CastKind::ptr_to_bool: {1279    mlir::Value llvmSrcVal = adaptor.getSrc();1280    mlir::Value zeroPtr = mlir::LLVM::ZeroOp::create(rewriter, castOp.getLoc(),1281                                                     llvmSrcVal.getType());1282    rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(1283        castOp, mlir::LLVM::ICmpPredicate::ne, llvmSrcVal, zeroPtr);1284    break;1285  }1286  case cir::CastKind::address_space: {1287    mlir::Type dstTy = castOp.getType();1288    mlir::Value llvmSrcVal = adaptor.getSrc();1289    mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);1290    rewriter.replaceOpWithNewOp<mlir::LLVM::AddrSpaceCastOp>(castOp, llvmDstTy,1291                                                             llvmSrcVal);1292    break;1293  }1294  case cir::CastKind::member_ptr_to_bool:1295    assert(!MissingFeatures::cxxABI());1296    assert(!MissingFeatures::methodType());1297    break;1298  default: {1299    return castOp.emitError("Unhandled cast kind: ")1300           << castOp.getKindAttrName();1301  }1302  }1303 1304  return mlir::success();1305}1306 1307mlir::LogicalResult CIRToLLVMPtrStrideOpLowering::matchAndRewrite(1308    cir::PtrStrideOp ptrStrideOp, OpAdaptor adaptor,1309    mlir::ConversionPatternRewriter &rewriter) const {1310 1311  const mlir::TypeConverter *tc = getTypeConverter();1312  const mlir::Type resultTy = tc->convertType(ptrStrideOp.getType());1313 1314  mlir::Type elementTy =1315      convertTypeForMemory(*tc, dataLayout, ptrStrideOp.getElementType());1316  mlir::MLIRContext *ctx = elementTy.getContext();1317 1318  // void and function types doesn't really have a layout to use in GEPs,1319  // make it i8 instead.1320  if (mlir::isa<mlir::LLVM::LLVMVoidType>(elementTy) ||1321      mlir::isa<mlir::LLVM::LLVMFunctionType>(elementTy))1322    elementTy = mlir::IntegerType::get(elementTy.getContext(), 8,1323                                       mlir::IntegerType::Signless);1324  // Zero-extend, sign-extend or trunc the pointer value.1325  mlir::Value index = adaptor.getStride();1326  const unsigned width =1327      mlir::cast<mlir::IntegerType>(index.getType()).getWidth();1328  const std::optional<std::uint64_t> layoutWidth =1329      dataLayout.getTypeIndexBitwidth(adaptor.getBase().getType());1330 1331  mlir::Operation *indexOp = index.getDefiningOp();1332  if (indexOp && layoutWidth && width != *layoutWidth) {1333    // If the index comes from a subtraction, make sure the extension happens1334    // before it. To achieve that, look at unary minus, which already got1335    // lowered to "sub 0, x".1336    const auto sub = dyn_cast<mlir::LLVM::SubOp>(indexOp);1337    auto unary = ptrStrideOp.getStride().getDefiningOp<cir::UnaryOp>();1338    bool rewriteSub =1339        unary && unary.getKind() == cir::UnaryOpKind::Minus && sub;1340    if (rewriteSub)1341      index = indexOp->getOperand(1);1342 1343    // Handle the cast1344    const auto llvmDstType = mlir::IntegerType::get(ctx, *layoutWidth);1345    index = getLLVMIntCast(rewriter, index, llvmDstType,1346                           ptrStrideOp.getStride().getType().isUnsigned(),1347                           width, *layoutWidth);1348 1349    // Rewrite the sub in front of extensions/trunc1350    if (rewriteSub) {1351      index = mlir::LLVM::SubOp::create(1352          rewriter, index.getLoc(), index.getType(),1353          mlir::LLVM::ConstantOp::create(rewriter, index.getLoc(),1354                                         index.getType(), 0),1355          index);1356      rewriter.eraseOp(sub);1357    }1358  }1359 1360  rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(1361      ptrStrideOp, resultTy, elementTy, adaptor.getBase(), index);1362  return mlir::success();1363}1364 1365mlir::LogicalResult CIRToLLVMBaseClassAddrOpLowering::matchAndRewrite(1366    cir::BaseClassAddrOp baseClassOp, OpAdaptor adaptor,1367    mlir::ConversionPatternRewriter &rewriter) const {1368  const mlir::Type resultType =1369      getTypeConverter()->convertType(baseClassOp.getType());1370  mlir::Value derivedAddr = adaptor.getDerivedAddr();1371  llvm::SmallVector<mlir::LLVM::GEPArg, 1> offset = {1372      adaptor.getOffset().getZExtValue()};1373  mlir::Type byteType = mlir::IntegerType::get(resultType.getContext(), 8,1374                                               mlir::IntegerType::Signless);1375  if (adaptor.getOffset().getZExtValue() == 0) {1376    rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(1377        baseClassOp, resultType, adaptor.getDerivedAddr());1378    return mlir::success();1379  }1380 1381  if (baseClassOp.getAssumeNotNull()) {1382    rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(1383        baseClassOp, resultType, byteType, derivedAddr, offset);1384  } else {1385    auto loc = baseClassOp.getLoc();1386    mlir::Value isNull = mlir::LLVM::ICmpOp::create(1387        rewriter, loc, mlir::LLVM::ICmpPredicate::eq, derivedAddr,1388        mlir::LLVM::ZeroOp::create(rewriter, loc, derivedAddr.getType()));1389    mlir::Value adjusted = mlir::LLVM::GEPOp::create(1390        rewriter, loc, resultType, byteType, derivedAddr, offset);1391    rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(baseClassOp, isNull,1392                                                      derivedAddr, adjusted);1393  }1394  return mlir::success();1395}1396 1397mlir::LogicalResult CIRToLLVMDerivedClassAddrOpLowering::matchAndRewrite(1398    cir::DerivedClassAddrOp derivedClassOp, OpAdaptor adaptor,1399    mlir::ConversionPatternRewriter &rewriter) const {1400  const mlir::Type resultType =1401      getTypeConverter()->convertType(derivedClassOp.getType());1402  mlir::Value baseAddr = adaptor.getBaseAddr();1403  // The offset is set in the operation as an unsigned value, but it must be1404  // applied as a negative offset.1405  int64_t offsetVal = -(adaptor.getOffset().getZExtValue());1406  if (offsetVal == 0) {1407    // If the offset is zero, we can just return the base address,1408    rewriter.replaceOp(derivedClassOp, baseAddr);1409    return mlir::success();1410  }1411  llvm::SmallVector<mlir::LLVM::GEPArg, 1> offset = {offsetVal};1412  mlir::Type byteType = mlir::IntegerType::get(resultType.getContext(), 8,1413                                               mlir::IntegerType::Signless);1414  if (derivedClassOp.getAssumeNotNull()) {1415    rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(1416        derivedClassOp, resultType, byteType, baseAddr, offset,1417        mlir::LLVM::GEPNoWrapFlags::inbounds);1418  } else {1419    mlir::Location loc = derivedClassOp.getLoc();1420    mlir::Value isNull = mlir::LLVM::ICmpOp::create(1421        rewriter, loc, mlir::LLVM::ICmpPredicate::eq, baseAddr,1422        mlir::LLVM::ZeroOp::create(rewriter, loc, baseAddr.getType()));1423    mlir::Value adjusted =1424        mlir::LLVM::GEPOp::create(rewriter, loc, resultType, byteType, baseAddr,1425                                  offset, mlir::LLVM::GEPNoWrapFlags::inbounds);1426    rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(derivedClassOp, isNull,1427                                                      baseAddr, adjusted);1428  }1429  return mlir::success();1430}1431 1432mlir::LogicalResult CIRToLLVMATanOpLowering::matchAndRewrite(1433    cir::ATanOp op, OpAdaptor adaptor,1434    mlir::ConversionPatternRewriter &rewriter) const {1435  mlir::Type resTy = typeConverter->convertType(op.getType());1436  rewriter.replaceOpWithNewOp<mlir::LLVM::ATanOp>(op, resTy, adaptor.getSrc());1437  return mlir::success();1438}1439 1440mlir::LogicalResult CIRToLLVMCeilOpLowering::matchAndRewrite(1441    cir::CeilOp op, OpAdaptor adaptor,1442    mlir::ConversionPatternRewriter &rewriter) const {1443  mlir::Type resTy = typeConverter->convertType(op.getType());1444  rewriter.replaceOpWithNewOp<mlir::LLVM::FCeilOp>(op, resTy, adaptor.getSrc());1445  return mlir::success();1446}1447 1448mlir::LogicalResult CIRToLLVMAllocaOpLowering::matchAndRewrite(1449    cir::AllocaOp op, OpAdaptor adaptor,1450    mlir::ConversionPatternRewriter &rewriter) const {1451  mlir::Value size =1452      op.isDynamic()1453          ? adaptor.getDynAllocSize()1454          : mlir::LLVM::ConstantOp::create(1455                rewriter, op.getLoc(),1456                typeConverter->convertType(rewriter.getIndexType()), 1);1457  mlir::Type elementTy =1458      convertTypeForMemory(*getTypeConverter(), dataLayout, op.getAllocaType());1459  mlir::Type resultTy =1460      convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType());1461 1462  assert(!cir::MissingFeatures::addressSpace());1463  assert(!cir::MissingFeatures::opAllocaAnnotations());1464 1465  rewriter.replaceOpWithNewOp<mlir::LLVM::AllocaOp>(1466      op, resultTy, elementTy, size, op.getAlignmentAttr().getInt());1467 1468  return mlir::success();1469}1470 1471mlir::LogicalResult CIRToLLVMReturnOpLowering::matchAndRewrite(1472    cir::ReturnOp op, OpAdaptor adaptor,1473    mlir::ConversionPatternRewriter &rewriter) const {1474  rewriter.replaceOpWithNewOp<mlir::LLVM::ReturnOp>(op, adaptor.getOperands());1475  return mlir::LogicalResult::success();1476}1477 1478mlir::LogicalResult CIRToLLVMRotateOpLowering::matchAndRewrite(1479    cir::RotateOp op, OpAdaptor adaptor,1480    mlir::ConversionPatternRewriter &rewriter) const {1481  // Note that LLVM intrinsic calls to @llvm.fsh{r,l}.i* have the same type as1482  // the operand.1483  mlir::Value input = adaptor.getInput();1484  if (op.isRotateLeft())1485    rewriter.replaceOpWithNewOp<mlir::LLVM::FshlOp>(op, input, input,1486                                                    adaptor.getAmount());1487  else1488    rewriter.replaceOpWithNewOp<mlir::LLVM::FshrOp>(op, input, input,1489                                                    adaptor.getAmount());1490  return mlir::LogicalResult::success();1491}1492 1493static mlir::LogicalResult1494rewriteCallOrInvoke(mlir::Operation *op, mlir::ValueRange callOperands,1495                    mlir::ConversionPatternRewriter &rewriter,1496                    const mlir::TypeConverter *converter,1497                    mlir::FlatSymbolRefAttr calleeAttr) {1498  llvm::SmallVector<mlir::Type, 8> llvmResults;1499  mlir::ValueTypeRange<mlir::ResultRange> cirResults = op->getResultTypes();1500  auto call = cast<cir::CIRCallOpInterface>(op);1501 1502  if (converter->convertTypes(cirResults, llvmResults).failed())1503    return mlir::failure();1504 1505  assert(!cir::MissingFeatures::opCallCallConv());1506 1507  mlir::LLVM::MemoryEffectsAttr memoryEffects;1508  bool noUnwind = false;1509  bool willReturn = false;1510  convertSideEffectForCall(op, call.getNothrow(), call.getSideEffect(),1511                           memoryEffects, noUnwind, willReturn);1512 1513  mlir::LLVM::LLVMFunctionType llvmFnTy;1514 1515  // Temporary to handle the case where we need to prepend an operand if the1516  // callee is an alias.1517  SmallVector<mlir::Value> adjustedCallOperands;1518 1519  if (calleeAttr) { // direct call1520    mlir::Operation *callee =1521        mlir::SymbolTable::lookupNearestSymbolFrom(op, calleeAttr);1522    if (auto fn = mlir::dyn_cast<mlir::FunctionOpInterface>(callee)) {1523      llvmFnTy = converter->convertType<mlir::LLVM::LLVMFunctionType>(1524          fn.getFunctionType());1525      assert(llvmFnTy && "Failed to convert function type");1526    } else if (auto alias = mlir::cast<mlir::LLVM::AliasOp>(callee)) {1527      // If the callee was an alias. In that case,1528      // we need to prepend the address of the alias to the operands. The1529      // way aliases work in the LLVM dialect is a little counter-intuitive.1530      // The AliasOp itself is a pseudo-function that returns the address of1531      // the global value being aliased, but when we generate the call we1532      // need to insert an operation that gets the address of the AliasOp.1533      // This all gets sorted out when the LLVM dialect is lowered to LLVM IR.1534      auto symAttr = mlir::cast<mlir::FlatSymbolRefAttr>(calleeAttr);1535      auto addrOfAlias =1536          mlir::LLVM::AddressOfOp::create(1537              rewriter, op->getLoc(),1538              mlir::LLVM::LLVMPointerType::get(rewriter.getContext()), symAttr)1539              .getResult();1540      adjustedCallOperands.push_back(addrOfAlias);1541 1542      // Now add the regular operands and assign this to the range value.1543      llvm::append_range(adjustedCallOperands, callOperands);1544      callOperands = adjustedCallOperands;1545 1546      // Clear the callee attribute because we're calling an alias.1547      calleeAttr = {};1548      llvmFnTy = mlir::cast<mlir::LLVM::LLVMFunctionType>(alias.getType());1549    } else {1550      // Was this an ifunc?1551      return op->emitError("Unexpected callee type!");1552    }1553  } else { // indirect call1554    assert(!op->getOperands().empty() &&1555           "operands list must no be empty for the indirect call");1556    auto calleeTy = op->getOperands().front().getType();1557    auto calleePtrTy = cast<cir::PointerType>(calleeTy);1558    auto calleeFuncTy = cast<cir::FuncType>(calleePtrTy.getPointee());1559    llvm::append_range(adjustedCallOperands, callOperands);1560    llvmFnTy = cast<mlir::LLVM::LLVMFunctionType>(1561        converter->convertType(calleeFuncTy));1562  }1563 1564  assert(!cir::MissingFeatures::opCallLandingPad());1565  assert(!cir::MissingFeatures::opCallContinueBlock());1566  assert(!cir::MissingFeatures::opCallCallConv());1567 1568  auto newOp = rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(1569      op, llvmFnTy, calleeAttr, callOperands);1570  if (memoryEffects)1571    newOp.setMemoryEffectsAttr(memoryEffects);1572  newOp.setNoUnwind(noUnwind);1573  newOp.setWillReturn(willReturn);1574 1575  return mlir::success();1576}1577 1578mlir::LogicalResult CIRToLLVMCallOpLowering::matchAndRewrite(1579    cir::CallOp op, OpAdaptor adaptor,1580    mlir::ConversionPatternRewriter &rewriter) const {1581  return rewriteCallOrInvoke(op.getOperation(), adaptor.getOperands(), rewriter,1582                             getTypeConverter(), op.getCalleeAttr());1583}1584 1585mlir::LogicalResult CIRToLLVMReturnAddrOpLowering::matchAndRewrite(1586    cir::ReturnAddrOp op, OpAdaptor adaptor,1587    mlir::ConversionPatternRewriter &rewriter) const {1588  auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());1589  replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.returnaddress",1590                                   llvmPtrTy, adaptor.getOperands());1591  return mlir::success();1592}1593 1594mlir::LogicalResult CIRToLLVMFrameAddrOpLowering::matchAndRewrite(1595    cir::FrameAddrOp op, OpAdaptor adaptor,1596    mlir::ConversionPatternRewriter &rewriter) const {1597  auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());1598  replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.frameaddress", llvmPtrTy,1599                                   adaptor.getOperands());1600  return mlir::success();1601}1602 1603mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(1604    cir::LoadOp op, OpAdaptor adaptor,1605    mlir::ConversionPatternRewriter &rewriter) const {1606  const mlir::Type llvmTy =1607      convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType());1608  mlir::LLVM::AtomicOrdering ordering = getLLVMMemOrder(op.getMemOrder());1609  std::optional<size_t> opAlign = op.getAlignment();1610  unsigned alignment =1611      (unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmTy));1612 1613  assert(!cir::MissingFeatures::lowerModeOptLevel());1614 1615  // TODO: nontemporal, syncscope.1616  assert(!cir::MissingFeatures::opLoadStoreNontemporal());1617  mlir::LLVM::LoadOp newLoad = mlir::LLVM::LoadOp::create(1618      rewriter, op->getLoc(), llvmTy, adaptor.getAddr(), alignment,1619      op.getIsVolatile(), /*isNonTemporal=*/false,1620      /*isInvariant=*/false, /*isInvariantGroup=*/false, ordering);1621 1622  // Convert adapted result to its original type if needed.1623  mlir::Value result =1624      emitFromMemory(rewriter, dataLayout, op, newLoad.getResult());1625  rewriter.replaceOp(op, result);1626  assert(!cir::MissingFeatures::opLoadStoreTbaa());1627  return mlir::LogicalResult::success();1628}1629 1630mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite(1631    cir::StoreOp op, OpAdaptor adaptor,1632    mlir::ConversionPatternRewriter &rewriter) const {1633  mlir::LLVM::AtomicOrdering memorder = getLLVMMemOrder(op.getMemOrder());1634  const mlir::Type llvmTy =1635      getTypeConverter()->convertType(op.getValue().getType());1636  std::optional<size_t> opAlign = op.getAlignment();1637  unsigned alignment =1638      (unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmTy));1639 1640  assert(!cir::MissingFeatures::lowerModeOptLevel());1641 1642  // Convert adapted value to its memory type if needed.1643  mlir::Value value = emitToMemory(rewriter, dataLayout,1644                                   op.getValue().getType(), adaptor.getValue());1645  // TODO: nontemporal, syncscope.1646  assert(!cir::MissingFeatures::opLoadStoreNontemporal());1647  assert(!cir::MissingFeatures::opLoadStoreTbaa());1648  mlir::LLVM::StoreOp storeOp = mlir::LLVM::StoreOp::create(1649      rewriter, op->getLoc(), value, adaptor.getAddr(), alignment,1650      op.getIsVolatile(),1651      /*isNonTemporal=*/false, /*isInvariantGroup=*/false, memorder);1652  rewriter.replaceOp(op, storeOp);1653  assert(!cir::MissingFeatures::opLoadStoreTbaa());1654  return mlir::LogicalResult::success();1655}1656 1657bool hasTrailingZeros(cir::ConstArrayAttr attr) {1658  auto array = mlir::dyn_cast<mlir::ArrayAttr>(attr.getElts());1659  return attr.hasTrailingZeros() ||1660         (array && std::count_if(array.begin(), array.end(), [](auto elt) {1661            auto ar = dyn_cast<cir::ConstArrayAttr>(elt);1662            return ar && hasTrailingZeros(ar);1663          }));1664}1665 1666mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(1667    cir::ConstantOp op, OpAdaptor adaptor,1668    mlir::ConversionPatternRewriter &rewriter) const {1669  mlir::Attribute attr = op.getValue();1670 1671  if (mlir::isa<cir::PoisonAttr>(attr)) {1672    rewriter.replaceOpWithNewOp<mlir::LLVM::PoisonOp>(1673        op, getTypeConverter()->convertType(op.getType()));1674    return mlir::success();1675  }1676 1677  if (mlir::isa<mlir::IntegerType>(op.getType())) {1678    // Verified cir.const operations cannot actually be of these types, but the1679    // lowering pass may generate temporary cir.const operations with these1680    // types. This is OK since MLIR allows unverified operations to be alive1681    // during a pass as long as they don't live past the end of the pass.1682    attr = op.getValue();1683  } else if (mlir::isa<cir::BoolType>(op.getType())) {1684    int value = mlir::cast<cir::BoolAttr>(op.getValue()).getValue();1685    attr = rewriter.getIntegerAttr(typeConverter->convertType(op.getType()),1686                                   value);1687  } else if (mlir::isa<cir::IntType>(op.getType())) {1688    // Lower GlobalViewAttr to llvm.mlir.addressof + llvm.mlir.ptrtoint1689    if (auto ga = mlir::dyn_cast<cir::GlobalViewAttr>(op.getValue())) {1690      // See the comment in visitCirAttr for why this isn't implemented.1691      assert(!cir::MissingFeatures::globalViewIntLowering());1692      op.emitError() << "global view with integer type";1693      return mlir::failure();1694    }1695 1696    attr = rewriter.getIntegerAttr(1697        typeConverter->convertType(op.getType()),1698        mlir::cast<cir::IntAttr>(op.getValue()).getValue());1699  } else if (mlir::isa<cir::FPTypeInterface>(op.getType())) {1700    attr = rewriter.getFloatAttr(1701        typeConverter->convertType(op.getType()),1702        mlir::cast<cir::FPAttr>(op.getValue()).getValue());1703  } else if (mlir::isa<cir::PointerType>(op.getType())) {1704    // Optimize with dedicated LLVM op for null pointers.1705    if (mlir::isa<cir::ConstPtrAttr>(op.getValue())) {1706      if (mlir::cast<cir::ConstPtrAttr>(op.getValue()).isNullValue()) {1707        rewriter.replaceOpWithNewOp<mlir::LLVM::ZeroOp>(1708            op, typeConverter->convertType(op.getType()));1709        return mlir::success();1710      }1711    }1712    // Lower GlobalViewAttr to llvm.mlir.addressof1713    if (auto gv = mlir::dyn_cast<cir::GlobalViewAttr>(op.getValue())) {1714      auto newOp = lowerCirAttrAsValue(op, gv, rewriter, getTypeConverter());1715      rewriter.replaceOp(op, newOp);1716      return mlir::success();1717    }1718    attr = op.getValue();1719  } else if (const auto arrTy = mlir::dyn_cast<cir::ArrayType>(op.getType())) {1720    const auto constArr = mlir::dyn_cast<cir::ConstArrayAttr>(op.getValue());1721    if (!constArr && !isa<cir::ZeroAttr, cir::UndefAttr>(op.getValue()))1722      return op.emitError() << "array does not have a constant initializer";1723 1724    std::optional<mlir::Attribute> denseAttr;1725    if (constArr && hasTrailingZeros(constArr)) {1726      const mlir::Value newOp =1727          lowerCirAttrAsValue(op, constArr, rewriter, getTypeConverter());1728      rewriter.replaceOp(op, newOp);1729      return mlir::success();1730    } else if (constArr &&1731               (denseAttr = lowerConstArrayAttr(constArr, typeConverter))) {1732      attr = denseAttr.value();1733    } else {1734      const mlir::Value initVal =1735          lowerCirAttrAsValue(op, op.getValue(), rewriter, typeConverter);1736      rewriter.replaceOp(op, initVal);1737      return mlir::success();1738    }1739  } else if (const auto recordAttr =1740                 mlir::dyn_cast<cir::ConstRecordAttr>(op.getValue())) {1741    auto initVal = lowerCirAttrAsValue(op, recordAttr, rewriter, typeConverter);1742    rewriter.replaceOp(op, initVal);1743    return mlir::success();1744  } else if (const auto vecTy = mlir::dyn_cast<cir::VectorType>(op.getType())) {1745    rewriter.replaceOp(op, lowerCirAttrAsValue(op, op.getValue(), rewriter,1746                                               getTypeConverter()));1747    return mlir::success();1748  } else if (auto recTy = mlir::dyn_cast<cir::RecordType>(op.getType())) {1749    if (mlir::isa<cir::ZeroAttr, cir::UndefAttr>(attr)) {1750      mlir::Value initVal =1751          lowerCirAttrAsValue(op, attr, rewriter, typeConverter);1752      rewriter.replaceOp(op, initVal);1753      return mlir::success();1754    }1755    return op.emitError() << "unsupported lowering for record constant type "1756                          << op.getType();1757  } else if (auto complexTy = mlir::dyn_cast<cir::ComplexType>(op.getType())) {1758    mlir::Type complexElemTy = complexTy.getElementType();1759    mlir::Type complexElemLLVMTy = typeConverter->convertType(complexElemTy);1760 1761    if (auto zeroInitAttr = mlir::dyn_cast<cir::ZeroAttr>(op.getValue())) {1762      mlir::TypedAttr zeroAttr = rewriter.getZeroAttr(complexElemLLVMTy);1763      mlir::ArrayAttr array = rewriter.getArrayAttr({zeroAttr, zeroAttr});1764      rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(1765          op, getTypeConverter()->convertType(op.getType()), array);1766      return mlir::success();1767    }1768 1769    auto complexAttr = mlir::cast<cir::ConstComplexAttr>(op.getValue());1770 1771    mlir::Attribute components[2];1772    if (mlir::isa<cir::IntType>(complexElemTy)) {1773      components[0] = rewriter.getIntegerAttr(1774          complexElemLLVMTy,1775          mlir::cast<cir::IntAttr>(complexAttr.getReal()).getValue());1776      components[1] = rewriter.getIntegerAttr(1777          complexElemLLVMTy,1778          mlir::cast<cir::IntAttr>(complexAttr.getImag()).getValue());1779    } else {1780      components[0] = rewriter.getFloatAttr(1781          complexElemLLVMTy,1782          mlir::cast<cir::FPAttr>(complexAttr.getReal()).getValue());1783      components[1] = rewriter.getFloatAttr(1784          complexElemLLVMTy,1785          mlir::cast<cir::FPAttr>(complexAttr.getImag()).getValue());1786    }1787 1788    attr = rewriter.getArrayAttr(components);1789  } else {1790    return op.emitError() << "unsupported constant type " << op.getType();1791  }1792 1793  rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(1794      op, getTypeConverter()->convertType(op.getType()), attr);1795 1796  return mlir::success();1797}1798 1799static uint64_t getTypeSize(mlir::Type type, mlir::Operation &op) {1800  mlir::DataLayout layout(op.getParentOfType<mlir::ModuleOp>());1801  // For LLVM purposes we treat void as u8.1802  if (isa<cir::VoidType>(type))1803    type = cir::IntType::get(type.getContext(), 8, /*isSigned=*/false);1804  return llvm::divideCeil(layout.getTypeSizeInBits(type), 8);1805}1806 1807mlir::LogicalResult CIRToLLVMPrefetchOpLowering::matchAndRewrite(1808    cir::PrefetchOp op, OpAdaptor adaptor,1809    mlir::ConversionPatternRewriter &rewriter) const {1810  rewriter.replaceOpWithNewOp<mlir::LLVM::Prefetch>(1811      op, adaptor.getAddr(), adaptor.getIsWrite(), adaptor.getLocality(),1812      /*DataCache=*/1);1813  return mlir::success();1814}1815 1816mlir::LogicalResult CIRToLLVMPtrDiffOpLowering::matchAndRewrite(1817    cir::PtrDiffOp op, OpAdaptor adaptor,1818    mlir::ConversionPatternRewriter &rewriter) const {1819  auto dstTy = mlir::cast<cir::IntType>(op.getType());1820  mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);1821 1822  auto lhs = mlir::LLVM::PtrToIntOp::create(rewriter, op.getLoc(), llvmDstTy,1823                                            adaptor.getLhs());1824  auto rhs = mlir::LLVM::PtrToIntOp::create(rewriter, op.getLoc(), llvmDstTy,1825                                            adaptor.getRhs());1826 1827  auto diff =1828      mlir::LLVM::SubOp::create(rewriter, op.getLoc(), llvmDstTy, lhs, rhs);1829 1830  cir::PointerType ptrTy = op.getLhs().getType();1831  assert(!cir::MissingFeatures::llvmLoweringPtrDiffConsidersPointee());1832  uint64_t typeSize = getTypeSize(ptrTy.getPointee(), *op);1833 1834  // Avoid silly division by 1.1835  mlir::Value resultVal = diff.getResult();1836  if (typeSize != 1) {1837    auto typeSizeVal = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),1838                                                      llvmDstTy, typeSize);1839 1840    if (dstTy.isUnsigned()) {1841      auto uDiv =1842          mlir::LLVM::UDivOp::create(rewriter, op.getLoc(), diff, typeSizeVal);1843      uDiv.setIsExact(true);1844      resultVal = uDiv.getResult();1845    } else {1846      auto sDiv =1847          mlir::LLVM::SDivOp::create(rewriter, op.getLoc(), diff, typeSizeVal);1848      sDiv.setIsExact(true);1849      resultVal = sDiv.getResult();1850    }1851  }1852  rewriter.replaceOp(op, resultVal);1853  return mlir::success();1854}1855 1856mlir::LogicalResult CIRToLLVMExpectOpLowering::matchAndRewrite(1857    cir::ExpectOp op, OpAdaptor adaptor,1858    mlir::ConversionPatternRewriter &rewriter) const {1859  // TODO(cir): do not generate LLVM intrinsics under -O01860  assert(!cir::MissingFeatures::optInfoAttr());1861 1862  std::optional<llvm::APFloat> prob = op.getProb();1863  if (prob)1864    rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectWithProbabilityOp>(1865        op, adaptor.getVal(), adaptor.getExpected(), prob.value());1866  else1867    rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectOp>(op, adaptor.getVal(),1868                                                      adaptor.getExpected());1869  return mlir::success();1870}1871 1872mlir::LogicalResult CIRToLLVMFAbsOpLowering::matchAndRewrite(1873    cir::FAbsOp op, OpAdaptor adaptor,1874    mlir::ConversionPatternRewriter &rewriter) const {1875  mlir::Type resTy = typeConverter->convertType(op.getType());1876  rewriter.replaceOpWithNewOp<mlir::LLVM::FAbsOp>(op, resTy,1877                                                  adaptor.getOperands()[0]);1878  return mlir::success();1879}1880 1881/// Convert the `cir.func` attributes to `llvm.func` attributes.1882/// Only retain those attributes that are not constructed by1883/// `LLVMFuncOp::build`. If `filterArgAttrs` is set, also filter out1884/// argument attributes.1885void CIRToLLVMFuncOpLowering::lowerFuncAttributes(1886    cir::FuncOp func, bool filterArgAndResAttrs,1887    SmallVectorImpl<mlir::NamedAttribute> &result) const {1888  assert(!cir::MissingFeatures::opFuncCallingConv());1889  for (mlir::NamedAttribute attr : func->getAttrs()) {1890    assert(!cir::MissingFeatures::opFuncCallingConv());1891    if (attr.getName() == mlir::SymbolTable::getSymbolAttrName() ||1892        attr.getName() == func.getFunctionTypeAttrName() ||1893        attr.getName() == getLinkageAttrNameString() ||1894        attr.getName() == func.getGlobalVisibilityAttrName() ||1895        attr.getName() == func.getDsoLocalAttrName() ||1896        attr.getName() == func.getInlineKindAttrName() ||1897        (filterArgAndResAttrs &&1898         (attr.getName() == func.getArgAttrsAttrName() ||1899          attr.getName() == func.getResAttrsAttrName())))1900      continue;1901 1902    assert(!cir::MissingFeatures::opFuncExtraAttrs());1903    result.push_back(attr);1904  }1905}1906 1907mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewriteAlias(1908    cir::FuncOp op, llvm::StringRef aliasee, mlir::Type ty, OpAdaptor adaptor,1909    mlir::ConversionPatternRewriter &rewriter) const {1910  SmallVector<mlir::NamedAttribute, 4> attributes;1911  lowerFuncAttributes(op, /*filterArgAndResAttrs=*/false, attributes);1912 1913  mlir::Location loc = op.getLoc();1914  auto aliasOp = rewriter.replaceOpWithNewOp<mlir::LLVM::AliasOp>(1915      op, ty, convertLinkage(op.getLinkage()), op.getName(), op.getDsoLocal(),1916      /*threadLocal=*/false, attributes);1917 1918  // Create the alias body1919  mlir::OpBuilder builder(op.getContext());1920  mlir::Block *block = builder.createBlock(&aliasOp.getInitializerRegion());1921  builder.setInsertionPointToStart(block);1922  // The type of AddressOfOp is always a pointer.1923  assert(!cir::MissingFeatures::addressSpace());1924  mlir::Type ptrTy = mlir::LLVM::LLVMPointerType::get(ty.getContext());1925  auto addrOp = mlir::LLVM::AddressOfOp::create(builder, loc, ptrTy, aliasee);1926  mlir::LLVM::ReturnOp::create(builder, loc, addrOp);1927 1928  return mlir::success();1929}1930 1931mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewrite(1932    cir::FuncOp op, OpAdaptor adaptor,1933    mlir::ConversionPatternRewriter &rewriter) const {1934 1935  cir::FuncType fnType = op.getFunctionType();1936  bool isDsoLocal = op.getDsoLocal();1937  mlir::TypeConverter::SignatureConversion signatureConversion(1938      fnType.getNumInputs());1939 1940  for (const auto &argType : llvm::enumerate(fnType.getInputs())) {1941    mlir::Type convertedType = typeConverter->convertType(argType.value());1942    if (!convertedType)1943      return mlir::failure();1944    signatureConversion.addInputs(argType.index(), convertedType);1945  }1946 1947  mlir::Type resultType =1948      getTypeConverter()->convertType(fnType.getReturnType());1949 1950  // Create the LLVM function operation.1951  mlir::Type llvmFnTy = mlir::LLVM::LLVMFunctionType::get(1952      resultType ? resultType : mlir::LLVM::LLVMVoidType::get(getContext()),1953      signatureConversion.getConvertedTypes(),1954      /*isVarArg=*/fnType.isVarArg());1955 1956  // If this is an alias, it needs to be lowered to llvm::AliasOp.1957  if (std::optional<llvm::StringRef> aliasee = op.getAliasee())1958    return matchAndRewriteAlias(op, *aliasee, llvmFnTy, adaptor, rewriter);1959 1960  // LLVMFuncOp expects a single FileLine Location instead of a fused1961  // location.1962  mlir::Location loc = op.getLoc();1963  if (mlir::FusedLoc fusedLoc = mlir::dyn_cast<mlir::FusedLoc>(loc))1964    loc = fusedLoc.getLocations()[0];1965  assert((mlir::isa<mlir::FileLineColLoc>(loc) ||1966          mlir::isa<mlir::UnknownLoc>(loc)) &&1967         "expected single location or unknown location here");1968 1969  mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage());1970  assert(!cir::MissingFeatures::opFuncCallingConv());1971  mlir::LLVM::CConv cconv = mlir::LLVM::CConv::C;1972  SmallVector<mlir::NamedAttribute, 4> attributes;1973  lowerFuncAttributes(op, /*filterArgAndResAttrs=*/false, attributes);1974 1975  mlir::LLVM::LLVMFuncOp fn = mlir::LLVM::LLVMFuncOp::create(1976      rewriter, loc, op.getName(), llvmFnTy, linkage, isDsoLocal, cconv,1977      mlir::SymbolRefAttr(), attributes);1978 1979  assert(!cir::MissingFeatures::opFuncMultipleReturnVals());1980 1981  if (auto inlineKind = op.getInlineKind()) {1982    fn.setNoInline(inlineKind == cir::InlineKind::NoInline);1983    fn.setInlineHint(inlineKind == cir::InlineKind::InlineHint);1984    fn.setAlwaysInline(inlineKind == cir::InlineKind::AlwaysInline);1985  }1986 1987  fn.setVisibility_Attr(mlir::LLVM::VisibilityAttr::get(1988      getContext(), lowerCIRVisibilityToLLVMVisibility(1989                        op.getGlobalVisibilityAttr().getValue())));1990 1991  rewriter.inlineRegionBefore(op.getBody(), fn.getBody(), fn.end());1992  if (failed(rewriter.convertRegionTypes(&fn.getBody(), *typeConverter,1993                                         &signatureConversion)))1994    return mlir::failure();1995 1996  rewriter.eraseOp(op);1997 1998  return mlir::LogicalResult::success();1999}2000 2001mlir::LogicalResult CIRToLLVMGetGlobalOpLowering::matchAndRewrite(2002    cir::GetGlobalOp op, OpAdaptor adaptor,2003    mlir::ConversionPatternRewriter &rewriter) const {2004  // FIXME(cir): Premature DCE to avoid lowering stuff we're not using.2005  // CIRGen should mitigate this and not emit the get_global.2006  if (op->getUses().empty()) {2007    rewriter.eraseOp(op);2008    return mlir::success();2009  }2010 2011  mlir::Type type = getTypeConverter()->convertType(op.getType());2012  mlir::Operation *newop = mlir::LLVM::AddressOfOp::create(2013      rewriter, op.getLoc(), type, op.getName());2014 2015  assert(!cir::MissingFeatures::opGlobalThreadLocal());2016 2017  rewriter.replaceOp(op, newop);2018  return mlir::success();2019}2020 2021/// Replace CIR global with a region initialized LLVM global and update2022/// insertion point to the end of the initializer block.2023void CIRToLLVMGlobalOpLowering::setupRegionInitializedLLVMGlobalOp(2024    cir::GlobalOp op, mlir::ConversionPatternRewriter &rewriter) const {2025  const mlir::Type llvmType =2026      convertTypeForMemory(*getTypeConverter(), dataLayout, op.getSymType());2027 2028  // FIXME: These default values are placeholders until the the equivalent2029  //        attributes are available on cir.global ops. This duplicates code2030  //        in CIRToLLVMGlobalOpLowering::matchAndRewrite() but that will go2031  //        away when the placeholders are no longer needed.2032  const bool isConst = op.getConstant();2033  assert(!cir::MissingFeatures::addressSpace());2034  const unsigned addrSpace = 0;2035  const bool isDsoLocal = op.getDsoLocal();2036  assert(!cir::MissingFeatures::opGlobalThreadLocal());2037  const bool isThreadLocal = false;2038  const uint64_t alignment = op.getAlignment().value_or(0);2039  const mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage());2040  const StringRef symbol = op.getSymName();2041  mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter);2042 2043  SmallVector<mlir::NamedAttribute> attributes;2044  mlir::LLVM::GlobalOp newGlobalOp =2045      rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>(2046          op, llvmType, isConst, linkage, symbol, nullptr, alignment, addrSpace,2047          isDsoLocal, isThreadLocal, comdatAttr, attributes);2048  newGlobalOp.getRegion().emplaceBlock();2049  rewriter.setInsertionPointToEnd(newGlobalOp.getInitializerBlock());2050}2051 2052mlir::LogicalResult2053CIRToLLVMGlobalOpLowering::matchAndRewriteRegionInitializedGlobal(2054    cir::GlobalOp op, mlir::Attribute init,2055    mlir::ConversionPatternRewriter &rewriter) const {2056  // TODO: Generalize this handling when more types are needed here.2057  assert(2058      (isa<cir::ConstArrayAttr, cir::ConstRecordAttr, cir::ConstVectorAttr,2059           cir::ConstPtrAttr, cir::ConstComplexAttr, cir::GlobalViewAttr,2060           cir::TypeInfoAttr, cir::UndefAttr, cir::VTableAttr, cir::ZeroAttr>(2061          init)));2062 2063  // TODO(cir): once LLVM's dialect has proper equivalent attributes this2064  // should be updated. For now, we use a custom op to initialize globals2065  // to the appropriate value.2066  const mlir::Location loc = op.getLoc();2067  setupRegionInitializedLLVMGlobalOp(op, rewriter);2068  CIRAttrToValue valueConverter(op, rewriter, typeConverter);2069  mlir::Value value = valueConverter.visit(init);2070  mlir::LLVM::ReturnOp::create(rewriter, loc, value);2071  return mlir::success();2072}2073 2074mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite(2075    cir::GlobalOp op, OpAdaptor adaptor,2076    mlir::ConversionPatternRewriter &rewriter) const {2077  // If this global requires non-trivial initialization or destruction,2078  // that needs to be moved to runtime handlers during LoweringPrepare.2079  if (!op.getCtorRegion().empty() || !op.getDtorRegion().empty())2080    return op.emitError() << "GlobalOp ctor and dtor regions should be removed "2081                             "in LoweringPrepare";2082 2083  std::optional<mlir::Attribute> init = op.getInitialValue();2084 2085  // Fetch required values to create LLVM op.2086  const mlir::Type cirSymType = op.getSymType();2087 2088  // This is the LLVM dialect type.2089  const mlir::Type llvmType =2090      convertTypeForMemory(*getTypeConverter(), dataLayout, cirSymType);2091  // FIXME: These default values are placeholders until the the equivalent2092  //        attributes are available on cir.global ops.2093  const bool isConst = op.getConstant();2094  assert(!cir::MissingFeatures::addressSpace());2095  const unsigned addrSpace = 0;2096  const bool isDsoLocal = op.getDsoLocal();2097  assert(!cir::MissingFeatures::opGlobalThreadLocal());2098  const bool isThreadLocal = false;2099  const uint64_t alignment = op.getAlignment().value_or(0);2100  const mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage());2101  const StringRef symbol = op.getSymName();2102  SmallVector<mlir::NamedAttribute> attributes;2103 2104  if (init.has_value()) {2105    if (mlir::isa<cir::FPAttr, cir::IntAttr, cir::BoolAttr>(init.value())) {2106      GlobalInitAttrRewriter initRewriter(llvmType, rewriter);2107      init = initRewriter.visit(init.value());2108      // If initRewriter returned a null attribute, init will have a value but2109      // the value will be null. If that happens, initRewriter didn't handle the2110      // attribute type. It probably needs to be added to2111      // GlobalInitAttrRewriter.2112      if (!init.value()) {2113        op.emitError() << "unsupported initializer '" << init.value() << "'";2114        return mlir::failure();2115      }2116    } else if (mlir::isa<cir::ConstArrayAttr, cir::ConstVectorAttr,2117                         cir::ConstRecordAttr, cir::ConstPtrAttr,2118                         cir::ConstComplexAttr, cir::GlobalViewAttr,2119                         cir::TypeInfoAttr, cir::UndefAttr, cir::VTableAttr,2120                         cir::ZeroAttr>(init.value())) {2121      // TODO(cir): once LLVM's dialect has proper equivalent attributes this2122      // should be updated. For now, we use a custom op to initialize globals2123      // to the appropriate value.2124      return matchAndRewriteRegionInitializedGlobal(op, init.value(), rewriter);2125    } else {2126      // We will only get here if new initializer types are added and this2127      // code is not updated to handle them.2128      op.emitError() << "unsupported initializer '" << init.value() << "'";2129      return mlir::failure();2130    }2131  }2132 2133  // Rewrite op.2134  mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter);2135  auto newOp = rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>(2136      op, llvmType, isConst, linkage, symbol, init.value_or(mlir::Attribute()),2137      alignment, addrSpace, isDsoLocal, isThreadLocal, comdatAttr, attributes);2138  newOp.setVisibility_Attr(mlir::LLVM::VisibilityAttr::get(2139      getContext(), lowerCIRVisibilityToLLVMVisibility(2140                        op.getGlobalVisibilityAttr().getValue())));2141 2142  return mlir::success();2143}2144 2145mlir::SymbolRefAttr2146CIRToLLVMGlobalOpLowering::getComdatAttr(cir::GlobalOp &op,2147                                         mlir::OpBuilder &builder) const {2148  if (!op.getComdat())2149    return mlir::SymbolRefAttr{};2150 2151  mlir::ModuleOp module = op->getParentOfType<mlir::ModuleOp>();2152  mlir::OpBuilder::InsertionGuard guard(builder);2153  StringRef comdatName("__llvm_comdat_globals");2154  if (!comdatOp) {2155    builder.setInsertionPointToStart(module.getBody());2156    comdatOp =2157        mlir::LLVM::ComdatOp::create(builder, module.getLoc(), comdatName);2158  }2159 2160  if (auto comdatSelector = comdatOp.lookupSymbol<mlir::LLVM::ComdatSelectorOp>(2161          op.getSymName())) {2162    return mlir::SymbolRefAttr::get(2163        builder.getContext(), comdatName,2164        mlir::FlatSymbolRefAttr::get(comdatSelector.getSymNameAttr()));2165  }2166 2167  builder.setInsertionPointToStart(&comdatOp.getBody().back());2168  auto selectorOp = mlir::LLVM::ComdatSelectorOp::create(2169      builder, comdatOp.getLoc(), op.getSymName(),2170      mlir::LLVM::comdat::Comdat::Any);2171  return mlir::SymbolRefAttr::get(2172      builder.getContext(), comdatName,2173      mlir::FlatSymbolRefAttr::get(selectorOp.getSymNameAttr()));2174}2175 2176mlir::LogicalResult CIRToLLVMSwitchFlatOpLowering::matchAndRewrite(2177    cir::SwitchFlatOp op, OpAdaptor adaptor,2178    mlir::ConversionPatternRewriter &rewriter) const {2179 2180  llvm::SmallVector<mlir::APInt, 8> caseValues;2181  for (mlir::Attribute val : op.getCaseValues()) {2182    auto intAttr = cast<cir::IntAttr>(val);2183    caseValues.push_back(intAttr.getValue());2184  }2185 2186  llvm::SmallVector<mlir::Block *, 8> caseDestinations;2187  llvm::SmallVector<mlir::ValueRange, 8> caseOperands;2188 2189  for (mlir::Block *x : op.getCaseDestinations())2190    caseDestinations.push_back(x);2191 2192  for (mlir::OperandRange x : op.getCaseOperands())2193    caseOperands.push_back(x);2194 2195  // Set switch op to branch to the newly created blocks.2196  rewriter.setInsertionPoint(op);2197  rewriter.replaceOpWithNewOp<mlir::LLVM::SwitchOp>(2198      op, adaptor.getCondition(), op.getDefaultDestination(),2199      op.getDefaultOperands(), caseValues, caseDestinations, caseOperands);2200  return mlir::success();2201}2202 2203mlir::LogicalResult CIRToLLVMUnaryOpLowering::matchAndRewrite(2204    cir::UnaryOp op, OpAdaptor adaptor,2205    mlir::ConversionPatternRewriter &rewriter) const {2206  assert(op.getType() == op.getInput().getType() &&2207         "Unary operation's operand type and result type are different");2208  mlir::Type type = op.getType();2209  mlir::Type elementType = elementTypeIfVector(type);2210  bool isVector = mlir::isa<cir::VectorType>(type);2211  mlir::Type llvmType = getTypeConverter()->convertType(type);2212  mlir::Location loc = op.getLoc();2213 2214  // Integer unary operations: + - ~ ++ --2215  if (mlir::isa<cir::IntType>(elementType)) {2216    mlir::LLVM::IntegerOverflowFlags maybeNSW =2217        op.getNoSignedWrap() ? mlir::LLVM::IntegerOverflowFlags::nsw2218                             : mlir::LLVM::IntegerOverflowFlags::none;2219    switch (op.getKind()) {2220    case cir::UnaryOpKind::Inc: {2221      assert(!isVector && "++ not allowed on vector types");2222      auto one = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 1);2223      rewriter.replaceOpWithNewOp<mlir::LLVM::AddOp>(2224          op, llvmType, adaptor.getInput(), one, maybeNSW);2225      return mlir::success();2226    }2227    case cir::UnaryOpKind::Dec: {2228      assert(!isVector && "-- not allowed on vector types");2229      auto one = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 1);2230      rewriter.replaceOpWithNewOp<mlir::LLVM::SubOp>(op, adaptor.getInput(),2231                                                     one, maybeNSW);2232      return mlir::success();2233    }2234    case cir::UnaryOpKind::Plus:2235      rewriter.replaceOp(op, adaptor.getInput());2236      return mlir::success();2237    case cir::UnaryOpKind::Minus: {2238      mlir::Value zero;2239      if (isVector)2240        zero = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmType);2241      else2242        zero = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 0);2243      rewriter.replaceOpWithNewOp<mlir::LLVM::SubOp>(2244          op, zero, adaptor.getInput(), maybeNSW);2245      return mlir::success();2246    }2247    case cir::UnaryOpKind::Not: {2248      // bit-wise compliment operator, implemented as an XOR with -1.2249      mlir::Value minusOne;2250      if (isVector) {2251        const uint64_t numElements =2252            mlir::dyn_cast<cir::VectorType>(type).getSize();2253        std::vector<int32_t> values(numElements, -1);2254        mlir::DenseIntElementsAttr denseVec = rewriter.getI32VectorAttr(values);2255        minusOne =2256            mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, denseVec);2257      } else {2258        minusOne = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, -1);2259      }2260      rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getInput(),2261                                                     minusOne);2262      return mlir::success();2263    }2264    }2265    llvm_unreachable("Unexpected unary op for int");2266  }2267 2268  // Floating point unary operations: + - ++ --2269  if (mlir::isa<cir::FPTypeInterface>(elementType)) {2270    switch (op.getKind()) {2271    case cir::UnaryOpKind::Inc: {2272      assert(!isVector && "++ not allowed on vector types");2273      mlir::LLVM::ConstantOp one = mlir::LLVM::ConstantOp::create(2274          rewriter, loc, llvmType, rewriter.getFloatAttr(llvmType, 1.0));2275      rewriter.replaceOpWithNewOp<mlir::LLVM::FAddOp>(op, llvmType, one,2276                                                      adaptor.getInput());2277      return mlir::success();2278    }2279    case cir::UnaryOpKind::Dec: {2280      assert(!isVector && "-- not allowed on vector types");2281      mlir::LLVM::ConstantOp minusOne = mlir::LLVM::ConstantOp::create(2282          rewriter, loc, llvmType, rewriter.getFloatAttr(llvmType, -1.0));2283      rewriter.replaceOpWithNewOp<mlir::LLVM::FAddOp>(op, llvmType, minusOne,2284                                                      adaptor.getInput());2285      return mlir::success();2286    }2287    case cir::UnaryOpKind::Plus:2288      rewriter.replaceOp(op, adaptor.getInput());2289      return mlir::success();2290    case cir::UnaryOpKind::Minus:2291      rewriter.replaceOpWithNewOp<mlir::LLVM::FNegOp>(op, llvmType,2292                                                      adaptor.getInput());2293      return mlir::success();2294    case cir::UnaryOpKind::Not:2295      return op.emitError() << "Unary not is invalid for floating-point types";2296    }2297    llvm_unreachable("Unexpected unary op for float");2298  }2299 2300  // Boolean unary operations: ! only. (For all others, the operand has2301  // already been promoted to int.)2302  if (mlir::isa<cir::BoolType>(elementType)) {2303    switch (op.getKind()) {2304    case cir::UnaryOpKind::Inc:2305    case cir::UnaryOpKind::Dec:2306    case cir::UnaryOpKind::Plus:2307    case cir::UnaryOpKind::Minus:2308      // Some of these are allowed in source code, but we shouldn't get here2309      // with a boolean type.2310      return op.emitError() << "Unsupported unary operation on boolean type";2311    case cir::UnaryOpKind::Not: {2312      assert(!isVector && "NYI: op! on vector mask");2313      auto one = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 1);2314      rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getInput(),2315                                                     one);2316      return mlir::success();2317    }2318    }2319    llvm_unreachable("Unexpected unary op for bool");2320  }2321 2322  // Pointer unary operations: + only.  (++ and -- of pointers are implemented2323  // with cir.ptr_stride, not cir.unary.)2324  if (mlir::isa<cir::PointerType>(elementType)) {2325    switch (op.getKind()) {2326    case cir::UnaryOpKind::Plus:2327      rewriter.replaceOp(op, adaptor.getInput());2328      return mlir::success();2329    default:2330      op.emitError() << "Unknown pointer unary operation during CIR lowering";2331      return mlir::failure();2332    }2333  }2334 2335  return op.emitError() << "Unary operation has unsupported type: "2336                        << elementType;2337}2338 2339mlir::LLVM::IntegerOverflowFlags2340CIRToLLVMBinOpLowering::getIntOverflowFlag(cir::BinOp op) const {2341  if (op.getNoUnsignedWrap())2342    return mlir::LLVM::IntegerOverflowFlags::nuw;2343 2344  if (op.getNoSignedWrap())2345    return mlir::LLVM::IntegerOverflowFlags::nsw;2346 2347  return mlir::LLVM::IntegerOverflowFlags::none;2348}2349 2350static bool isIntTypeUnsigned(mlir::Type type) {2351  // TODO: Ideally, we should only need to check cir::IntType here.2352  return mlir::isa<cir::IntType>(type)2353             ? mlir::cast<cir::IntType>(type).isUnsigned()2354             : mlir::cast<mlir::IntegerType>(type).isUnsigned();2355}2356 2357mlir::LogicalResult CIRToLLVMBinOpLowering::matchAndRewrite(2358    cir::BinOp op, OpAdaptor adaptor,2359    mlir::ConversionPatternRewriter &rewriter) const {2360  if (adaptor.getLhs().getType() != adaptor.getRhs().getType())2361    return op.emitError() << "inconsistent operands' types not supported yet";2362 2363  mlir::Type type = op.getRhs().getType();2364  if (!mlir::isa<cir::IntType, cir::BoolType, cir::FPTypeInterface,2365                 mlir::IntegerType, cir::VectorType>(type))2366    return op.emitError() << "operand type not supported yet";2367 2368  const mlir::Type llvmTy = getTypeConverter()->convertType(op.getType());2369  const mlir::Type llvmEltTy = elementTypeIfVector(llvmTy);2370 2371  const mlir::Value rhs = adaptor.getRhs();2372  const mlir::Value lhs = adaptor.getLhs();2373  type = elementTypeIfVector(type);2374 2375  switch (op.getKind()) {2376  case cir::BinOpKind::Add:2377    if (mlir::isa<mlir::IntegerType>(llvmEltTy)) {2378      if (op.getSaturated()) {2379        if (isIntTypeUnsigned(type)) {2380          rewriter.replaceOpWithNewOp<mlir::LLVM::UAddSat>(op, lhs, rhs);2381          break;2382        }2383        rewriter.replaceOpWithNewOp<mlir::LLVM::SAddSat>(op, lhs, rhs);2384        break;2385      }2386      rewriter.replaceOpWithNewOp<mlir::LLVM::AddOp>(op, llvmTy, lhs, rhs,2387                                                     getIntOverflowFlag(op));2388    } else {2389      rewriter.replaceOpWithNewOp<mlir::LLVM::FAddOp>(op, lhs, rhs);2390    }2391    break;2392  case cir::BinOpKind::Sub:2393    if (mlir::isa<mlir::IntegerType>(llvmEltTy)) {2394      if (op.getSaturated()) {2395        if (isIntTypeUnsigned(type)) {2396          rewriter.replaceOpWithNewOp<mlir::LLVM::USubSat>(op, lhs, rhs);2397          break;2398        }2399        rewriter.replaceOpWithNewOp<mlir::LLVM::SSubSat>(op, lhs, rhs);2400        break;2401      }2402      rewriter.replaceOpWithNewOp<mlir::LLVM::SubOp>(op, llvmTy, lhs, rhs,2403                                                     getIntOverflowFlag(op));2404    } else {2405      rewriter.replaceOpWithNewOp<mlir::LLVM::FSubOp>(op, lhs, rhs);2406    }2407    break;2408  case cir::BinOpKind::Mul:2409    if (mlir::isa<mlir::IntegerType>(llvmEltTy))2410      rewriter.replaceOpWithNewOp<mlir::LLVM::MulOp>(op, llvmTy, lhs, rhs,2411                                                     getIntOverflowFlag(op));2412    else2413      rewriter.replaceOpWithNewOp<mlir::LLVM::FMulOp>(op, lhs, rhs);2414    break;2415  case cir::BinOpKind::Div:2416    if (mlir::isa<mlir::IntegerType>(llvmEltTy)) {2417      auto isUnsigned = isIntTypeUnsigned(type);2418      if (isUnsigned)2419        rewriter.replaceOpWithNewOp<mlir::LLVM::UDivOp>(op, lhs, rhs);2420      else2421        rewriter.replaceOpWithNewOp<mlir::LLVM::SDivOp>(op, lhs, rhs);2422    } else {2423      rewriter.replaceOpWithNewOp<mlir::LLVM::FDivOp>(op, lhs, rhs);2424    }2425    break;2426  case cir::BinOpKind::Rem:2427    if (mlir::isa<mlir::IntegerType>(llvmEltTy)) {2428      auto isUnsigned = isIntTypeUnsigned(type);2429      if (isUnsigned)2430        rewriter.replaceOpWithNewOp<mlir::LLVM::URemOp>(op, lhs, rhs);2431      else2432        rewriter.replaceOpWithNewOp<mlir::LLVM::SRemOp>(op, lhs, rhs);2433    } else {2434      rewriter.replaceOpWithNewOp<mlir::LLVM::FRemOp>(op, lhs, rhs);2435    }2436    break;2437  case cir::BinOpKind::And:2438    rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op, lhs, rhs);2439    break;2440  case cir::BinOpKind::Or:2441    rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(op, lhs, rhs);2442    break;2443  case cir::BinOpKind::Xor:2444    rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, lhs, rhs);2445    break;2446  case cir::BinOpKind::Max:2447    if (mlir::isa<mlir::IntegerType>(llvmEltTy)) {2448      auto isUnsigned = isIntTypeUnsigned(type);2449      if (isUnsigned)2450        rewriter.replaceOpWithNewOp<mlir::LLVM::UMaxOp>(op, llvmTy, lhs, rhs);2451      else2452        rewriter.replaceOpWithNewOp<mlir::LLVM::SMaxOp>(op, llvmTy, lhs, rhs);2453    }2454    break;2455  }2456  return mlir::LogicalResult::success();2457}2458 2459/// Convert from a CIR comparison kind to an LLVM IR integral comparison kind.2460static mlir::LLVM::ICmpPredicate2461convertCmpKindToICmpPredicate(cir::CmpOpKind kind, bool isSigned) {2462  using CIR = cir::CmpOpKind;2463  using LLVMICmp = mlir::LLVM::ICmpPredicate;2464  switch (kind) {2465  case CIR::eq:2466    return LLVMICmp::eq;2467  case CIR::ne:2468    return LLVMICmp::ne;2469  case CIR::lt:2470    return (isSigned ? LLVMICmp::slt : LLVMICmp::ult);2471  case CIR::le:2472    return (isSigned ? LLVMICmp::sle : LLVMICmp::ule);2473  case CIR::gt:2474    return (isSigned ? LLVMICmp::sgt : LLVMICmp::ugt);2475  case CIR::ge:2476    return (isSigned ? LLVMICmp::sge : LLVMICmp::uge);2477  }2478  llvm_unreachable("Unknown CmpOpKind");2479}2480 2481/// Convert from a CIR comparison kind to an LLVM IR floating-point comparison2482/// kind.2483static mlir::LLVM::FCmpPredicate2484convertCmpKindToFCmpPredicate(cir::CmpOpKind kind) {2485  using CIR = cir::CmpOpKind;2486  using LLVMFCmp = mlir::LLVM::FCmpPredicate;2487  switch (kind) {2488  case CIR::eq:2489    return LLVMFCmp::oeq;2490  case CIR::ne:2491    return LLVMFCmp::une;2492  case CIR::lt:2493    return LLVMFCmp::olt;2494  case CIR::le:2495    return LLVMFCmp::ole;2496  case CIR::gt:2497    return LLVMFCmp::ogt;2498  case CIR::ge:2499    return LLVMFCmp::oge;2500  }2501  llvm_unreachable("Unknown CmpOpKind");2502}2503 2504mlir::LogicalResult CIRToLLVMCmpOpLowering::matchAndRewrite(2505    cir::CmpOp cmpOp, OpAdaptor adaptor,2506    mlir::ConversionPatternRewriter &rewriter) const {2507  mlir::Type type = cmpOp.getLhs().getType();2508 2509  assert(!cir::MissingFeatures::dataMemberType());2510  assert(!cir::MissingFeatures::methodType());2511 2512  if (mlir::isa<cir::IntType, mlir::IntegerType>(type)) {2513    bool isSigned = mlir::isa<cir::IntType>(type)2514                        ? mlir::cast<cir::IntType>(type).isSigned()2515                        : mlir::cast<mlir::IntegerType>(type).isSigned();2516    mlir::LLVM::ICmpPredicate kind =2517        convertCmpKindToICmpPredicate(cmpOp.getKind(), isSigned);2518    rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(2519        cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());2520    return mlir::success();2521  }2522 2523  if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(type)) {2524    mlir::LLVM::ICmpPredicate kind =2525        convertCmpKindToICmpPredicate(cmpOp.getKind(),2526                                      /* isSigned=*/false);2527    rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(2528        cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());2529    return mlir::success();2530  }2531 2532  if (auto vptrTy = mlir::dyn_cast<cir::VPtrType>(type)) {2533    // !cir.vptr is a special case, but it's just a pointer to LLVM.2534    auto kind = convertCmpKindToICmpPredicate(cmpOp.getKind(),2535                                              /* isSigned=*/false);2536    rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(2537        cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());2538    return mlir::success();2539  }2540 2541  if (mlir::isa<cir::FPTypeInterface>(type)) {2542    mlir::LLVM::FCmpPredicate kind =2543        convertCmpKindToFCmpPredicate(cmpOp.getKind());2544    rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>(2545        cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());2546    return mlir::success();2547  }2548 2549  if (mlir::isa<cir::ComplexType>(type)) {2550    mlir::Value lhs = adaptor.getLhs();2551    mlir::Value rhs = adaptor.getRhs();2552    mlir::Location loc = cmpOp.getLoc();2553 2554    auto complexType = mlir::cast<cir::ComplexType>(cmpOp.getLhs().getType());2555    mlir::Type complexElemTy =2556        getTypeConverter()->convertType(complexType.getElementType());2557 2558    auto lhsReal = mlir::LLVM::ExtractValueOp::create(2559        rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{0}));2560    auto lhsImag = mlir::LLVM::ExtractValueOp::create(2561        rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{1}));2562    auto rhsReal = mlir::LLVM::ExtractValueOp::create(2563        rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{0}));2564    auto rhsImag = mlir::LLVM::ExtractValueOp::create(2565        rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{1}));2566 2567    if (cmpOp.getKind() == cir::CmpOpKind::eq) {2568      if (complexElemTy.isInteger()) {2569        auto realCmp = mlir::LLVM::ICmpOp::create(2570            rewriter, loc, mlir::LLVM::ICmpPredicate::eq, lhsReal, rhsReal);2571        auto imagCmp = mlir::LLVM::ICmpOp::create(2572            rewriter, loc, mlir::LLVM::ICmpPredicate::eq, lhsImag, rhsImag);2573        rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp);2574        return mlir::success();2575      }2576 2577      auto realCmp = mlir::LLVM::FCmpOp::create(2578          rewriter, loc, mlir::LLVM::FCmpPredicate::oeq, lhsReal, rhsReal);2579      auto imagCmp = mlir::LLVM::FCmpOp::create(2580          rewriter, loc, mlir::LLVM::FCmpPredicate::oeq, lhsImag, rhsImag);2581      rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp);2582      return mlir::success();2583    }2584 2585    if (cmpOp.getKind() == cir::CmpOpKind::ne) {2586      if (complexElemTy.isInteger()) {2587        auto realCmp = mlir::LLVM::ICmpOp::create(2588            rewriter, loc, mlir::LLVM::ICmpPredicate::ne, lhsReal, rhsReal);2589        auto imagCmp = mlir::LLVM::ICmpOp::create(2590            rewriter, loc, mlir::LLVM::ICmpPredicate::ne, lhsImag, rhsImag);2591        rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp);2592        return mlir::success();2593      }2594 2595      auto realCmp = mlir::LLVM::FCmpOp::create(2596          rewriter, loc, mlir::LLVM::FCmpPredicate::une, lhsReal, rhsReal);2597      auto imagCmp = mlir::LLVM::FCmpOp::create(2598          rewriter, loc, mlir::LLVM::FCmpPredicate::une, lhsImag, rhsImag);2599      rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp);2600      return mlir::success();2601    }2602  }2603 2604  return cmpOp.emitError() << "unsupported type for CmpOp: " << type;2605}2606 2607mlir::LogicalResult CIRToLLVMBinOpOverflowOpLowering::matchAndRewrite(2608    cir::BinOpOverflowOp op, OpAdaptor adaptor,2609    mlir::ConversionPatternRewriter &rewriter) const {2610  mlir::Location loc = op.getLoc();2611  cir::BinOpOverflowKind arithKind = op.getKind();2612  cir::IntType operandTy = op.getLhs().getType();2613  cir::IntType resultTy = op.getResult().getType();2614 2615  EncompassedTypeInfo encompassedTyInfo =2616      computeEncompassedTypeWidth(operandTy, resultTy);2617  mlir::IntegerType encompassedLLVMTy =2618      rewriter.getIntegerType(encompassedTyInfo.width);2619 2620  mlir::Value lhs = adaptor.getLhs();2621  mlir::Value rhs = adaptor.getRhs();2622  if (operandTy.getWidth() < encompassedTyInfo.width) {2623    if (operandTy.isSigned()) {2624      lhs = mlir::LLVM::SExtOp::create(rewriter, loc, encompassedLLVMTy, lhs);2625      rhs = mlir::LLVM::SExtOp::create(rewriter, loc, encompassedLLVMTy, rhs);2626    } else {2627      lhs = mlir::LLVM::ZExtOp::create(rewriter, loc, encompassedLLVMTy, lhs);2628      rhs = mlir::LLVM::ZExtOp::create(rewriter, loc, encompassedLLVMTy, rhs);2629    }2630  }2631 2632  std::string intrinName = getLLVMIntrinName(arithKind, encompassedTyInfo.sign,2633                                             encompassedTyInfo.width);2634  auto intrinNameAttr = mlir::StringAttr::get(op.getContext(), intrinName);2635 2636  mlir::IntegerType overflowLLVMTy = rewriter.getI1Type();2637  auto intrinRetTy = mlir::LLVM::LLVMStructType::getLiteral(2638      rewriter.getContext(), {encompassedLLVMTy, overflowLLVMTy});2639 2640  auto callLLVMIntrinOp = mlir::LLVM::CallIntrinsicOp::create(2641      rewriter, loc, intrinRetTy, intrinNameAttr, mlir::ValueRange{lhs, rhs});2642  mlir::Value intrinRet = callLLVMIntrinOp.getResult(0);2643 2644  mlir::Value result = mlir::LLVM::ExtractValueOp::create(2645                           rewriter, loc, intrinRet, ArrayRef<int64_t>{0})2646                           .getResult();2647  mlir::Value overflow = mlir::LLVM::ExtractValueOp::create(2648                             rewriter, loc, intrinRet, ArrayRef<int64_t>{1})2649                             .getResult();2650 2651  if (resultTy.getWidth() < encompassedTyInfo.width) {2652    mlir::Type resultLLVMTy = getTypeConverter()->convertType(resultTy);2653    auto truncResult =2654        mlir::LLVM::TruncOp::create(rewriter, loc, resultLLVMTy, result);2655 2656    // Extend the truncated result back to the encompassing type to check for2657    // any overflows during the truncation.2658    mlir::Value truncResultExt;2659    if (resultTy.isSigned())2660      truncResultExt = mlir::LLVM::SExtOp::create(2661          rewriter, loc, encompassedLLVMTy, truncResult);2662    else2663      truncResultExt = mlir::LLVM::ZExtOp::create(2664          rewriter, loc, encompassedLLVMTy, truncResult);2665    auto truncOverflow = mlir::LLVM::ICmpOp::create(2666        rewriter, loc, mlir::LLVM::ICmpPredicate::ne, truncResultExt, result);2667 2668    result = truncResult;2669    overflow = mlir::LLVM::OrOp::create(rewriter, loc, overflow, truncOverflow);2670  }2671 2672  mlir::Type boolLLVMTy =2673      getTypeConverter()->convertType(op.getOverflow().getType());2674  if (boolLLVMTy != rewriter.getI1Type())2675    overflow = mlir::LLVM::ZExtOp::create(rewriter, loc, boolLLVMTy, overflow);2676 2677  rewriter.replaceOp(op, mlir::ValueRange{result, overflow});2678 2679  return mlir::success();2680}2681 2682std::string CIRToLLVMBinOpOverflowOpLowering::getLLVMIntrinName(2683    cir::BinOpOverflowKind opKind, bool isSigned, unsigned width) {2684  // The intrinsic name is `@llvm.{s|u}{opKind}.with.overflow.i{width}`2685 2686  std::string name = "llvm.";2687 2688  if (isSigned)2689    name.push_back('s');2690  else2691    name.push_back('u');2692 2693  switch (opKind) {2694  case cir::BinOpOverflowKind::Add:2695    name.append("add.");2696    break;2697  case cir::BinOpOverflowKind::Sub:2698    name.append("sub.");2699    break;2700  case cir::BinOpOverflowKind::Mul:2701    name.append("mul.");2702    break;2703  }2704 2705  name.append("with.overflow.i");2706  name.append(std::to_string(width));2707 2708  return name;2709}2710 2711CIRToLLVMBinOpOverflowOpLowering::EncompassedTypeInfo2712CIRToLLVMBinOpOverflowOpLowering::computeEncompassedTypeWidth(2713    cir::IntType operandTy, cir::IntType resultTy) {2714  bool sign = operandTy.getIsSigned() || resultTy.getIsSigned();2715  unsigned width =2716      std::max(operandTy.getWidth() + (sign && operandTy.isUnsigned()),2717               resultTy.getWidth() + (sign && resultTy.isUnsigned()));2718  return {sign, width};2719}2720 2721mlir::LogicalResult CIRToLLVMShiftOpLowering::matchAndRewrite(2722    cir::ShiftOp op, OpAdaptor adaptor,2723    mlir::ConversionPatternRewriter &rewriter) const {2724  assert((op.getValue().getType() == op.getType()) &&2725         "inconsistent operands' types NYI");2726 2727  const mlir::Type llvmTy = getTypeConverter()->convertType(op.getType());2728  mlir::Value amt = adaptor.getAmount();2729  mlir::Value val = adaptor.getValue();2730 2731  auto cirAmtTy = mlir::dyn_cast<cir::IntType>(op.getAmount().getType());2732  bool isUnsigned;2733  if (cirAmtTy) {2734    auto cirValTy = mlir::cast<cir::IntType>(op.getValue().getType());2735    isUnsigned = cirValTy.isUnsigned();2736 2737    // Ensure shift amount is the same type as the value. Some undefined2738    // behavior might occur in the casts below as per [C99 6.5.7.3].2739    // Vector type shift amount needs no cast as type consistency is expected to2740    // be already be enforced at CIRGen.2741    if (cirAmtTy)2742      amt = getLLVMIntCast(rewriter, amt, llvmTy, true, cirAmtTy.getWidth(),2743                           cirValTy.getWidth());2744  } else {2745    auto cirValVTy = mlir::cast<cir::VectorType>(op.getValue().getType());2746    isUnsigned =2747        mlir::cast<cir::IntType>(cirValVTy.getElementType()).isUnsigned();2748  }2749 2750  // Lower to the proper LLVM shift operation.2751  if (op.getIsShiftleft()) {2752    rewriter.replaceOpWithNewOp<mlir::LLVM::ShlOp>(op, llvmTy, val, amt);2753    return mlir::success();2754  }2755 2756  if (isUnsigned)2757    rewriter.replaceOpWithNewOp<mlir::LLVM::LShrOp>(op, llvmTy, val, amt);2758  else2759    rewriter.replaceOpWithNewOp<mlir::LLVM::AShrOp>(op, llvmTy, val, amt);2760  return mlir::success();2761}2762 2763mlir::LogicalResult CIRToLLVMSelectOpLowering::matchAndRewrite(2764    cir::SelectOp op, OpAdaptor adaptor,2765    mlir::ConversionPatternRewriter &rewriter) const {2766  auto getConstantBool = [](mlir::Value value) -> cir::BoolAttr {2767    auto definingOp = value.getDefiningOp<cir::ConstantOp>();2768    if (!definingOp)2769      return {};2770 2771    auto constValue = definingOp.getValueAttr<cir::BoolAttr>();2772    if (!constValue)2773      return {};2774 2775    return constValue;2776  };2777 2778  // Two special cases in the LLVMIR codegen of select op:2779  // - select %0, %1, false => and %0, %12780  // - select %0, true, %1 => or %0, %12781  if (mlir::isa<cir::BoolType>(op.getTrueValue().getType())) {2782    cir::BoolAttr trueValue = getConstantBool(op.getTrueValue());2783    cir::BoolAttr falseValue = getConstantBool(op.getFalseValue());2784    if (falseValue && !falseValue.getValue()) {2785      // select %0, %1, false => and %0, %12786      rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op, adaptor.getCondition(),2787                                                     adaptor.getTrueValue());2788      return mlir::success();2789    }2790    if (trueValue && trueValue.getValue()) {2791      // select %0, true, %1 => or %0, %12792      rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(op, adaptor.getCondition(),2793                                                    adaptor.getFalseValue());2794      return mlir::success();2795    }2796  }2797 2798  mlir::Value llvmCondition = adaptor.getCondition();2799  rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(2800      op, llvmCondition, adaptor.getTrueValue(), adaptor.getFalseValue());2801 2802  return mlir::success();2803}2804 2805static void prepareTypeConverter(mlir::LLVMTypeConverter &converter,2806                                 mlir::DataLayout &dataLayout) {2807  converter.addConversion([&](cir::PointerType type) -> mlir::Type {2808    unsigned addrSpace =2809        type.getAddrSpace() ? type.getAddrSpace().getValue().getUInt() : 0;2810    return mlir::LLVM::LLVMPointerType::get(type.getContext(), addrSpace);2811  });2812  converter.addConversion([&](cir::VPtrType type) -> mlir::Type {2813    assert(!cir::MissingFeatures::addressSpace());2814    return mlir::LLVM::LLVMPointerType::get(type.getContext());2815  });2816  converter.addConversion([&](cir::ArrayType type) -> mlir::Type {2817    mlir::Type ty =2818        convertTypeForMemory(converter, dataLayout, type.getElementType());2819    return mlir::LLVM::LLVMArrayType::get(ty, type.getSize());2820  });2821  converter.addConversion([&](cir::VectorType type) -> mlir::Type {2822    const mlir::Type ty = converter.convertType(type.getElementType());2823    return mlir::VectorType::get(type.getSize(), ty);2824  });2825  converter.addConversion([&](cir::BoolType type) -> mlir::Type {2826    return mlir::IntegerType::get(type.getContext(), 1,2827                                  mlir::IntegerType::Signless);2828  });2829  converter.addConversion([&](cir::IntType type) -> mlir::Type {2830    // LLVM doesn't work with signed types, so we drop the CIR signs here.2831    return mlir::IntegerType::get(type.getContext(), type.getWidth());2832  });2833  converter.addConversion([&](cir::SingleType type) -> mlir::Type {2834    return mlir::Float32Type::get(type.getContext());2835  });2836  converter.addConversion([&](cir::DoubleType type) -> mlir::Type {2837    return mlir::Float64Type::get(type.getContext());2838  });2839  converter.addConversion([&](cir::FP80Type type) -> mlir::Type {2840    return mlir::Float80Type::get(type.getContext());2841  });2842  converter.addConversion([&](cir::FP128Type type) -> mlir::Type {2843    return mlir::Float128Type::get(type.getContext());2844  });2845  converter.addConversion([&](cir::LongDoubleType type) -> mlir::Type {2846    return converter.convertType(type.getUnderlying());2847  });2848  converter.addConversion([&](cir::FP16Type type) -> mlir::Type {2849    return mlir::Float16Type::get(type.getContext());2850  });2851  converter.addConversion([&](cir::BF16Type type) -> mlir::Type {2852    return mlir::BFloat16Type::get(type.getContext());2853  });2854  converter.addConversion([&](cir::ComplexType type) -> mlir::Type {2855    // A complex type is lowered to an LLVM struct that contains the real and2856    // imaginary part as data fields.2857    mlir::Type elementTy = converter.convertType(type.getElementType());2858    mlir::Type structFields[2] = {elementTy, elementTy};2859    return mlir::LLVM::LLVMStructType::getLiteral(type.getContext(),2860                                                  structFields);2861  });2862  converter.addConversion([&](cir::FuncType type) -> std::optional<mlir::Type> {2863    auto result = converter.convertType(type.getReturnType());2864    llvm::SmallVector<mlir::Type> arguments;2865    arguments.reserve(type.getNumInputs());2866    if (converter.convertTypes(type.getInputs(), arguments).failed())2867      return std::nullopt;2868    auto varArg = type.isVarArg();2869    return mlir::LLVM::LLVMFunctionType::get(result, arguments, varArg);2870  });2871  converter.addConversion([&](cir::RecordType type) -> mlir::Type {2872    // Convert struct members.2873    llvm::SmallVector<mlir::Type> llvmMembers;2874    switch (type.getKind()) {2875    case cir::RecordType::Class:2876    case cir::RecordType::Struct:2877      for (mlir::Type ty : type.getMembers())2878        llvmMembers.push_back(convertTypeForMemory(converter, dataLayout, ty));2879      break;2880    // Unions are lowered as only the largest member.2881    case cir::RecordType::Union:2882      if (auto largestMember = type.getLargestMember(dataLayout))2883        llvmMembers.push_back(2884            convertTypeForMemory(converter, dataLayout, largestMember));2885      if (type.getPadded()) {2886        auto last = *type.getMembers().rbegin();2887        llvmMembers.push_back(2888            convertTypeForMemory(converter, dataLayout, last));2889      }2890      break;2891    }2892 2893    // Record has a name: lower as an identified record.2894    mlir::LLVM::LLVMStructType llvmStruct;2895    if (type.getName()) {2896      llvmStruct = mlir::LLVM::LLVMStructType::getIdentified(2897          type.getContext(), type.getPrefixedName());2898      if (llvmStruct.setBody(llvmMembers, type.getPacked()).failed())2899        llvm_unreachable("Failed to set body of record");2900    } else { // Record has no name: lower as literal record.2901      llvmStruct = mlir::LLVM::LLVMStructType::getLiteral(2902          type.getContext(), llvmMembers, type.getPacked());2903    }2904 2905    return llvmStruct;2906  });2907  converter.addConversion([&](cir::VoidType type) -> mlir::Type {2908    return mlir::LLVM::LLVMVoidType::get(type.getContext());2909  });2910}2911 2912static void buildCtorDtorList(2913    mlir::ModuleOp module, StringRef globalXtorName, StringRef llvmXtorName,2914    llvm::function_ref<std::pair<StringRef, int>(mlir::Attribute)> createXtor) {2915  llvm::SmallVector<std::pair<StringRef, int>> globalXtors;2916  for (const mlir::NamedAttribute namedAttr : module->getAttrs()) {2917    if (namedAttr.getName() == globalXtorName) {2918      for (auto attr : mlir::cast<mlir::ArrayAttr>(namedAttr.getValue()))2919        globalXtors.emplace_back(createXtor(attr));2920      break;2921    }2922  }2923 2924  if (globalXtors.empty())2925    return;2926 2927  mlir::OpBuilder builder(module.getContext());2928  builder.setInsertionPointToEnd(&module.getBodyRegion().back());2929 2930  // Create a global array llvm.global_ctors with element type of2931  // struct { i32, ptr, ptr }2932  auto ctorPFTy = mlir::LLVM::LLVMPointerType::get(builder.getContext());2933  llvm::SmallVector<mlir::Type> ctorStructFields;2934  ctorStructFields.push_back(builder.getI32Type());2935  ctorStructFields.push_back(ctorPFTy);2936  ctorStructFields.push_back(ctorPFTy);2937 2938  auto ctorStructTy = mlir::LLVM::LLVMStructType::getLiteral(2939      builder.getContext(), ctorStructFields);2940  auto ctorStructArrayTy =2941      mlir::LLVM::LLVMArrayType::get(ctorStructTy, globalXtors.size());2942 2943  mlir::Location loc = module.getLoc();2944  auto newGlobalOp = mlir::LLVM::GlobalOp::create(2945      builder, loc, ctorStructArrayTy, /*constant=*/false,2946      mlir::LLVM::Linkage::Appending, llvmXtorName, mlir::Attribute());2947 2948  builder.createBlock(&newGlobalOp.getRegion());2949  builder.setInsertionPointToEnd(newGlobalOp.getInitializerBlock());2950 2951  mlir::Value result =2952      mlir::LLVM::UndefOp::create(builder, loc, ctorStructArrayTy);2953 2954  for (auto [index, fn] : llvm::enumerate(globalXtors)) {2955    mlir::Value structInit =2956        mlir::LLVM::UndefOp::create(builder, loc, ctorStructTy);2957    mlir::Value initPriority = mlir::LLVM::ConstantOp::create(2958        builder, loc, ctorStructFields[0], fn.second);2959    mlir::Value initFuncAddr = mlir::LLVM::AddressOfOp::create(2960        builder, loc, ctorStructFields[1], fn.first);2961    mlir::Value initAssociate =2962        mlir::LLVM::ZeroOp::create(builder, loc, ctorStructFields[2]);2963    // Literal zero makes the InsertValueOp::create ambiguous.2964    llvm::SmallVector<int64_t> zero{0};2965    structInit = mlir::LLVM::InsertValueOp::create(builder, loc, structInit,2966                                                   initPriority, zero);2967    structInit = mlir::LLVM::InsertValueOp::create(builder, loc, structInit,2968                                                   initFuncAddr, 1);2969    // TODO: handle associated data for initializers.2970    structInit = mlir::LLVM::InsertValueOp::create(builder, loc, structInit,2971                                                   initAssociate, 2);2972    result = mlir::LLVM::InsertValueOp::create(builder, loc, result, structInit,2973                                               index);2974  }2975 2976  mlir::LLVM::ReturnOp::create(builder, loc, result);2977}2978 2979// The applyPartialConversion function traverses blocks in the dominance order,2980// so it does not lower and operations that are not reachachable from the2981// operations passed in as arguments. Since we do need to lower such code in2982// order to avoid verification errors occur, we cannot just pass the module op2983// to applyPartialConversion. We must build a set of unreachable ops and2984// explicitly add them, along with the module, to the vector we pass to2985// applyPartialConversion.2986//2987// For instance, this CIR code:2988//2989//    cir.func @foo(%arg0: !s32i) -> !s32i {2990//      %4 = cir.cast int_to_bool %arg0 : !s32i -> !cir.bool2991//      cir.if %4 {2992//        %5 = cir.const #cir.int<1> : !s32i2993//        cir.return %5 : !s32i2994//      } else {2995//        %5 = cir.const #cir.int<0> : !s32i2996//       cir.return %5 : !s32i2997//      }2998//      cir.return %arg0 : !s32i2999//    }3000//3001// contains an unreachable return operation (the last one). After the flattening3002// pass it will be placed into the unreachable block. The possible error3003// after the lowering pass is: error: 'cir.return' op expects parent op to be3004// one of 'cir.func, cir.scope, cir.if ... The reason that this operation was3005// not lowered and the new parent is llvm.func.3006//3007// In the future we may want to get rid of this function and use a DCE pass or3008// something similar. But for now we need to guarantee the absence of the3009// dialect verification errors.3010static void collectUnreachable(mlir::Operation *parent,3011                               llvm::SmallVector<mlir::Operation *> &ops) {3012 3013  llvm::SmallVector<mlir::Block *> unreachableBlocks;3014  parent->walk([&](mlir::Block *blk) { // check3015    if (blk->hasNoPredecessors() && !blk->isEntryBlock())3016      unreachableBlocks.push_back(blk);3017  });3018 3019  std::set<mlir::Block *> visited;3020  for (mlir::Block *root : unreachableBlocks) {3021    // We create a work list for each unreachable block.3022    // Thus we traverse operations in some order.3023    std::deque<mlir::Block *> workList;3024    workList.push_back(root);3025 3026    while (!workList.empty()) {3027      mlir::Block *blk = workList.back();3028      workList.pop_back();3029      if (visited.count(blk))3030        continue;3031      visited.emplace(blk);3032 3033      for (mlir::Operation &op : *blk)3034        ops.push_back(&op);3035 3036      for (mlir::Block *succ : blk->getSuccessors())3037        workList.push_back(succ);3038    }3039  }3040}3041 3042mlir::LogicalResult CIRToLLVMObjSizeOpLowering::matchAndRewrite(3043    cir::ObjSizeOp op, OpAdaptor adaptor,3044    mlir::ConversionPatternRewriter &rewriter) const {3045  mlir::Type llvmResTy = getTypeConverter()->convertType(op.getType());3046  mlir::Location loc = op->getLoc();3047 3048  mlir::IntegerType i1Ty = rewriter.getI1Type();3049 3050  auto i1Val = [&rewriter, &loc, &i1Ty](bool val) {3051    return mlir::LLVM::ConstantOp::create(rewriter, loc, i1Ty, val);3052  };3053 3054  replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.objectsize", llvmResTy,3055                                   {3056                                       adaptor.getPtr(),3057                                       i1Val(op.getMin()),3058                                       i1Val(op.getNullunknown()),3059                                       i1Val(op.getDynamic()),3060                                   });3061 3062  return mlir::LogicalResult::success();3063}3064 3065void ConvertCIRToLLVMPass::processCIRAttrs(mlir::ModuleOp module) {3066  // Lower the module attributes to LLVM equivalents.3067  if (mlir::Attribute tripleAttr =3068          module->getAttr(cir::CIRDialect::getTripleAttrName()))3069    module->setAttr(mlir::LLVM::LLVMDialect::getTargetTripleAttrName(),3070                    tripleAttr);3071 3072  if (mlir::Attribute asmAttr =3073          module->getAttr(cir::CIRDialect::getModuleLevelAsmAttrName()))3074    module->setAttr(mlir::LLVM::LLVMDialect::getModuleLevelAsmAttrName(),3075                    asmAttr);3076}3077 3078void ConvertCIRToLLVMPass::runOnOperation() {3079  llvm::TimeTraceScope scope("Convert CIR to LLVM Pass");3080 3081  mlir::ModuleOp module = getOperation();3082  mlir::DataLayout dl(module);3083  mlir::LLVMTypeConverter converter(&getContext());3084  prepareTypeConverter(converter, dl);3085 3086  mlir::RewritePatternSet patterns(&getContext());3087 3088  patterns.add<3089#define GET_LLVM_LOWERING_PATTERNS_LIST3090#include "clang/CIR/Dialect/IR/CIRLowering.inc"3091#undef GET_LLVM_LOWERING_PATTERNS_LIST3092      >(converter, patterns.getContext(), dl);3093 3094  processCIRAttrs(module);3095 3096  mlir::ConversionTarget target(getContext());3097  target.addLegalOp<mlir::ModuleOp>();3098  target.addLegalDialect<mlir::LLVM::LLVMDialect>();3099  target.addIllegalDialect<mlir::BuiltinDialect, cir::CIRDialect,3100                           mlir::func::FuncDialect>();3101 3102  llvm::SmallVector<mlir::Operation *> ops;3103  ops.push_back(module);3104  collectUnreachable(module, ops);3105 3106  if (failed(applyPartialConversion(ops, target, std::move(patterns))))3107    signalPassFailure();3108 3109  // Emit the llvm.global_ctors array.3110  buildCtorDtorList(module, cir::CIRDialect::getGlobalCtorsAttrName(),3111                    "llvm.global_ctors", [](mlir::Attribute attr) {3112                      auto ctorAttr = mlir::cast<cir::GlobalCtorAttr>(attr);3113                      return std::make_pair(ctorAttr.getName(),3114                                            ctorAttr.getPriority());3115                    });3116  // Emit the llvm.global_dtors array.3117  buildCtorDtorList(module, cir::CIRDialect::getGlobalDtorsAttrName(),3118                    "llvm.global_dtors", [](mlir::Attribute attr) {3119                      auto dtorAttr = mlir::cast<cir::GlobalDtorAttr>(attr);3120                      return std::make_pair(dtorAttr.getName(),3121                                            dtorAttr.getPriority());3122                    });3123}3124 3125mlir::LogicalResult CIRToLLVMBrOpLowering::matchAndRewrite(3126    cir::BrOp op, OpAdaptor adaptor,3127    mlir::ConversionPatternRewriter &rewriter) const {3128  rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(op, adaptor.getOperands(),3129                                                op.getDest());3130  return mlir::LogicalResult::success();3131}3132 3133mlir::LogicalResult CIRToLLVMGetMemberOpLowering::matchAndRewrite(3134    cir::GetMemberOp op, OpAdaptor adaptor,3135    mlir::ConversionPatternRewriter &rewriter) const {3136  mlir::Type llResTy = getTypeConverter()->convertType(op.getType());3137  const auto recordTy =3138      mlir::cast<cir::RecordType>(op.getAddrTy().getPointee());3139  assert(recordTy && "expected record type");3140 3141  switch (recordTy.getKind()) {3142  case cir::RecordType::Class:3143  case cir::RecordType::Struct: {3144    // Since the base address is a pointer to an aggregate, the first offset3145    // is always zero. The second offset tell us which member it will access.3146    llvm::SmallVector<mlir::LLVM::GEPArg, 2> offset{0, op.getIndex()};3147    const mlir::Type elementTy = getTypeConverter()->convertType(recordTy);3148    rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, llResTy, elementTy,3149                                                   adaptor.getAddr(), offset);3150    return mlir::success();3151  }3152  case cir::RecordType::Union:3153    // Union members share the address space, so we just need a bitcast to3154    // conform to type-checking.3155    rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(op, llResTy,3156                                                       adaptor.getAddr());3157    return mlir::success();3158  }3159}3160 3161mlir::LogicalResult CIRToLLVMUnreachableOpLowering::matchAndRewrite(3162    cir::UnreachableOp op, OpAdaptor adaptor,3163    mlir::ConversionPatternRewriter &rewriter) const {3164  rewriter.replaceOpWithNewOp<mlir::LLVM::UnreachableOp>(op);3165  return mlir::success();3166}3167 3168void createLLVMFuncOpIfNotExist(mlir::ConversionPatternRewriter &rewriter,3169                                mlir::Operation *srcOp, llvm::StringRef fnName,3170                                mlir::Type fnTy) {3171  auto modOp = srcOp->getParentOfType<mlir::ModuleOp>();3172  auto enclosingFnOp = srcOp->getParentOfType<mlir::LLVM::LLVMFuncOp>();3173  mlir::Operation *sourceSymbol =3174      mlir::SymbolTable::lookupSymbolIn(modOp, fnName);3175  if (!sourceSymbol) {3176    mlir::OpBuilder::InsertionGuard guard(rewriter);3177    rewriter.setInsertionPoint(enclosingFnOp);3178    mlir::LLVM::LLVMFuncOp::create(rewriter, srcOp->getLoc(), fnName, fnTy);3179  }3180}3181 3182mlir::LogicalResult CIRToLLVMThrowOpLowering::matchAndRewrite(3183    cir::ThrowOp op, OpAdaptor adaptor,3184    mlir::ConversionPatternRewriter &rewriter) const {3185  mlir::Location loc = op.getLoc();3186  auto voidTy = mlir::LLVM::LLVMVoidType::get(getContext());3187 3188  if (op.rethrows()) {3189    auto funcTy = mlir::LLVM::LLVMFunctionType::get(voidTy, {});3190 3191    // Get or create `declare void @__cxa_rethrow()`3192    const llvm::StringRef functionName = "__cxa_rethrow";3193    createLLVMFuncOpIfNotExist(rewriter, op, functionName, funcTy);3194 3195    auto cxaRethrow = mlir::LLVM::CallOp::create(3196        rewriter, loc, mlir::TypeRange{}, functionName);3197 3198    rewriter.replaceOp(op, cxaRethrow);3199    return mlir::success();3200  }3201 3202  auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());3203  auto fnTy = mlir::LLVM::LLVMFunctionType::get(3204      voidTy, {llvmPtrTy, llvmPtrTy, llvmPtrTy});3205 3206  // Get or create `declare void @__cxa_throw(ptr, ptr, ptr)`3207  const llvm::StringRef fnName = "__cxa_throw";3208  createLLVMFuncOpIfNotExist(rewriter, op, fnName, fnTy);3209 3210  mlir::Value typeInfo = mlir::LLVM::AddressOfOp::create(3211      rewriter, loc, mlir::LLVM::LLVMPointerType::get(rewriter.getContext()),3212      adaptor.getTypeInfoAttr());3213 3214  mlir::Value dtor;3215  if (op.getDtor()) {3216    dtor = mlir::LLVM::AddressOfOp::create(rewriter, loc, llvmPtrTy,3217                                           adaptor.getDtorAttr());3218  } else {3219    dtor = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmPtrTy);3220  }3221 3222  auto cxaThrowCall = mlir::LLVM::CallOp::create(3223      rewriter, loc, mlir::TypeRange{}, fnName,3224      mlir::ValueRange{adaptor.getExceptionPtr(), typeInfo, dtor});3225 3226  rewriter.replaceOp(op, cxaThrowCall);3227  return mlir::success();3228}3229 3230mlir::LogicalResult CIRToLLVMAllocExceptionOpLowering::matchAndRewrite(3231    cir::AllocExceptionOp op, OpAdaptor adaptor,3232    mlir::ConversionPatternRewriter &rewriter) const {3233  // Get or create `declare ptr @__cxa_allocate_exception(i64)`3234  StringRef fnName = "__cxa_allocate_exception";3235  auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());3236  auto int64Ty = mlir::IntegerType::get(rewriter.getContext(), 64);3237  auto fnTy = mlir::LLVM::LLVMFunctionType::get(llvmPtrTy, {int64Ty});3238 3239  createLLVMFuncOpIfNotExist(rewriter, op, fnName, fnTy);3240  auto exceptionSize = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),3241                                                      adaptor.getSizeAttr());3242 3243  auto allocaExceptionCall = mlir::LLVM::CallOp::create(3244      rewriter, op.getLoc(), mlir::TypeRange{llvmPtrTy}, fnName,3245      mlir::ValueRange{exceptionSize});3246 3247  rewriter.replaceOp(op, allocaExceptionCall);3248  return mlir::success();3249}3250 3251static mlir::LLVM::LLVMStructType3252getLLVMLandingPadStructTy(mlir::ConversionPatternRewriter &rewriter) {3253  // Create the landing pad type: struct { ptr, i32 }3254  mlir::MLIRContext *ctx = rewriter.getContext();3255  auto llvmPtr = mlir::LLVM::LLVMPointerType::get(ctx);3256  llvm::SmallVector<mlir::Type> structFields = {llvmPtr, rewriter.getI32Type()};3257  return mlir::LLVM::LLVMStructType::getLiteral(ctx, structFields);3258}3259 3260mlir::LogicalResult CIRToLLVMEhInflightOpLowering::matchAndRewrite(3261    cir::EhInflightOp op, OpAdaptor adaptor,3262    mlir::ConversionPatternRewriter &rewriter) const {3263  auto llvmFn = op->getParentOfType<mlir::LLVM::LLVMFuncOp>();3264  assert(llvmFn && "expected LLVM function parent");3265  mlir::Block *entryBlock = &llvmFn.getRegion().front();3266  assert(entryBlock->isEntryBlock());3267 3268  mlir::ArrayAttr catchListAttr = op.getCatchTypeListAttr();3269  mlir::SmallVector<mlir::Value> catchSymAddrs;3270 3271  auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());3272  mlir::Location loc = op.getLoc();3273 3274  // %landingpad = landingpad { ptr, i32 }3275  // Note that since llvm.landingpad has to be the first operation on the3276  // block, any needed value for its operands has to be added somewhere else.3277  if (catchListAttr) {3278    //   catch ptr @_ZTIi3279    //   catch ptr @_ZTIPKc3280    for (mlir::Attribute catchAttr : catchListAttr) {3281      auto symAttr = cast<mlir::FlatSymbolRefAttr>(catchAttr);3282      // Generate `llvm.mlir.addressof` for each symbol, and place those3283      // operations in the LLVM function entry basic block.3284      mlir::OpBuilder::InsertionGuard guard(rewriter);3285      rewriter.setInsertionPointToStart(entryBlock);3286      mlir::Value addrOp = mlir::LLVM::AddressOfOp::create(3287          rewriter, loc, llvmPtrTy, symAttr.getValue());3288      catchSymAddrs.push_back(addrOp);3289    }3290  } else if (!op.getCleanup()) {3291    // We need to emit catch-all only if cleanup is not set, because when we3292    // have catch-all handler, there is no case when we set would unwind past3293    // the handler3294    mlir::OpBuilder::InsertionGuard guard(rewriter);3295    rewriter.setInsertionPointToStart(entryBlock);3296    mlir::Value nullOp = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmPtrTy);3297    catchSymAddrs.push_back(nullOp);3298  }3299 3300  // %slot = extractvalue { ptr, i32 } %x, 03301  // %selector = extractvalue { ptr, i32 } %x, 13302  mlir::LLVM::LLVMStructType llvmLandingPadStructTy =3303      getLLVMLandingPadStructTy(rewriter);3304  auto landingPadOp = mlir::LLVM::LandingpadOp::create(3305      rewriter, loc, llvmLandingPadStructTy, catchSymAddrs);3306 3307  if (op.getCleanup())3308    landingPadOp.setCleanup(true);3309 3310  mlir::Value slot =3311      mlir::LLVM::ExtractValueOp::create(rewriter, loc, landingPadOp, 0);3312  mlir::Value selector =3313      mlir::LLVM::ExtractValueOp::create(rewriter, loc, landingPadOp, 1);3314  rewriter.replaceOp(op, mlir::ValueRange{slot, selector});3315 3316  // Landing pads are required to be in LLVM functions with personality3317  // attribute.3318  // TODO(cir): for now hardcode personality creation in order to start3319  // adding exception tests, once we annotate CIR with such information,3320  // change it to be in FuncOp lowering instead.3321  mlir::OpBuilder::InsertionGuard guard(rewriter);3322  // Insert personality decl before the current function.3323  rewriter.setInsertionPoint(llvmFn);3324  auto personalityFnTy =3325      mlir::LLVM::LLVMFunctionType::get(rewriter.getI32Type(), {},3326                                        /*isVarArg=*/true);3327 3328  const StringRef fnName = "__gxx_personality_v0";3329  createLLVMFuncOpIfNotExist(rewriter, op, fnName, personalityFnTy);3330  llvmFn.setPersonality(fnName);3331 3332  return mlir::success();3333}3334 3335mlir::LogicalResult CIRToLLVMTrapOpLowering::matchAndRewrite(3336    cir::TrapOp op, OpAdaptor adaptor,3337    mlir::ConversionPatternRewriter &rewriter) const {3338  mlir::Location loc = op->getLoc();3339  rewriter.eraseOp(op);3340 3341  mlir::LLVM::Trap::create(rewriter, loc);3342 3343  // Note that the call to llvm.trap is not a terminator in LLVM dialect.3344  // So we must emit an additional llvm.unreachable to terminate the current3345  // block.3346  mlir::LLVM::UnreachableOp::create(rewriter, loc);3347 3348  return mlir::success();3349}3350 3351static mlir::Value3352getValueForVTableSymbol(mlir::Operation *op,3353                        mlir::ConversionPatternRewriter &rewriter,3354                        const mlir::TypeConverter *converter,3355                        mlir::FlatSymbolRefAttr nameAttr, mlir::Type &eltType) {3356  auto module = op->getParentOfType<mlir::ModuleOp>();3357  mlir::Operation *symbol = mlir::SymbolTable::lookupSymbolIn(module, nameAttr);3358  if (auto llvmSymbol = mlir::dyn_cast<mlir::LLVM::GlobalOp>(symbol)) {3359    eltType = llvmSymbol.getType();3360  } else if (auto cirSymbol = mlir::dyn_cast<cir::GlobalOp>(symbol)) {3361    eltType = converter->convertType(cirSymbol.getSymType());3362  } else {3363    op->emitError() << "unexpected symbol type for " << symbol;3364    return {};3365  }3366 3367  return mlir::LLVM::AddressOfOp::create(3368      rewriter, op->getLoc(),3369      mlir::LLVM::LLVMPointerType::get(op->getContext()), nameAttr.getValue());3370}3371 3372mlir::LogicalResult CIRToLLVMVTableAddrPointOpLowering::matchAndRewrite(3373    cir::VTableAddrPointOp op, OpAdaptor adaptor,3374    mlir::ConversionPatternRewriter &rewriter) const {3375  const mlir::TypeConverter *converter = getTypeConverter();3376  mlir::Type targetType = converter->convertType(op.getType());3377  llvm::SmallVector<mlir::LLVM::GEPArg> offsets;3378  mlir::Type eltType;3379  mlir::Value symAddr = getValueForVTableSymbol(op, rewriter, converter,3380                                                op.getNameAttr(), eltType);3381  if (!symAddr)3382    return op.emitError() << "Unable to get value for vtable symbol";3383 3384  offsets = llvm::SmallVector<mlir::LLVM::GEPArg>{3385      0, op.getAddressPointAttr().getIndex(),3386      op.getAddressPointAttr().getOffset()};3387 3388  assert(eltType && "Shouldn't ever be missing an eltType here");3389  mlir::LLVM::GEPNoWrapFlags inboundsNuw =3390      mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;3391  rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, targetType, eltType,3392                                                 symAddr, offsets, inboundsNuw);3393  return mlir::success();3394}3395 3396mlir::LogicalResult CIRToLLVMVTableGetVPtrOpLowering::matchAndRewrite(3397    cir::VTableGetVPtrOp op, OpAdaptor adaptor,3398    mlir::ConversionPatternRewriter &rewriter) const {3399  // cir.vtable.get_vptr is equivalent to a bitcast from the source object3400  // pointer to the vptr type. Since the LLVM dialect uses opaque pointers3401  // we can just replace uses of this operation with the original pointer.3402  mlir::Value srcVal = adaptor.getSrc();3403  rewriter.replaceOp(op, srcVal);3404  return mlir::success();3405}3406 3407mlir::LogicalResult CIRToLLVMVTableGetVirtualFnAddrOpLowering::matchAndRewrite(3408    cir::VTableGetVirtualFnAddrOp op, OpAdaptor adaptor,3409    mlir::ConversionPatternRewriter &rewriter) const {3410  mlir::Type targetType = getTypeConverter()->convertType(op.getType());3411  auto eltType = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());3412  llvm::SmallVector<mlir::LLVM::GEPArg> offsets =3413      llvm::SmallVector<mlir::LLVM::GEPArg>{op.getIndex()};3414  rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(3415      op, targetType, eltType, adaptor.getVptr(), offsets,3416      mlir::LLVM::GEPNoWrapFlags::inbounds);3417  return mlir::success();3418}3419 3420mlir::LogicalResult CIRToLLVMVTTAddrPointOpLowering::matchAndRewrite(3421    cir::VTTAddrPointOp op, OpAdaptor adaptor,3422    mlir::ConversionPatternRewriter &rewriter) const {3423  const mlir::Type resultType = getTypeConverter()->convertType(op.getType());3424  llvm::SmallVector<mlir::LLVM::GEPArg> offsets;3425  mlir::Type eltType;3426  mlir::Value llvmAddr = adaptor.getSymAddr();3427 3428  if (op.getSymAddr()) {3429    if (op.getOffset() == 0) {3430      rewriter.replaceOp(op, {llvmAddr});3431      return mlir::success();3432    }3433 3434    offsets.push_back(adaptor.getOffset());3435    eltType = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());3436  } else {3437    llvmAddr = getValueForVTableSymbol(op, rewriter, getTypeConverter(),3438                                       op.getNameAttr(), eltType);3439    assert(eltType && "Shouldn't ever be missing an eltType here");3440    offsets.push_back(0);3441    offsets.push_back(adaptor.getOffset());3442  }3443  rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(3444      op, resultType, eltType, llvmAddr, offsets,3445      mlir::LLVM::GEPNoWrapFlags::inbounds);3446  return mlir::success();3447}3448 3449mlir::LogicalResult CIRToLLVMStackSaveOpLowering::matchAndRewrite(3450    cir::StackSaveOp op, OpAdaptor adaptor,3451    mlir::ConversionPatternRewriter &rewriter) const {3452  const mlir::Type ptrTy = getTypeConverter()->convertType(op.getType());3453  rewriter.replaceOpWithNewOp<mlir::LLVM::StackSaveOp>(op, ptrTy);3454  return mlir::success();3455}3456 3457mlir::LogicalResult CIRToLLVMStackRestoreOpLowering::matchAndRewrite(3458    cir::StackRestoreOp op, OpAdaptor adaptor,3459    mlir::ConversionPatternRewriter &rewriter) const {3460  rewriter.replaceOpWithNewOp<mlir::LLVM::StackRestoreOp>(op, adaptor.getPtr());3461  return mlir::success();3462}3463 3464mlir::LogicalResult CIRToLLVMVecCreateOpLowering::matchAndRewrite(3465    cir::VecCreateOp op, OpAdaptor adaptor,3466    mlir::ConversionPatternRewriter &rewriter) const {3467  // Start with an 'undef' value for the vector.  Then 'insertelement' for3468  // each of the vector elements.3469  const auto vecTy = mlir::cast<cir::VectorType>(op.getType());3470  const mlir::Type llvmTy = typeConverter->convertType(vecTy);3471  const mlir::Location loc = op.getLoc();3472  mlir::Value result = mlir::LLVM::PoisonOp::create(rewriter, loc, llvmTy);3473  assert(vecTy.getSize() == op.getElements().size() &&3474         "cir.vec.create op count doesn't match vector type elements count");3475 3476  for (uint64_t i = 0; i < vecTy.getSize(); ++i) {3477    const mlir::Value indexValue =3478        mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), i);3479    result = mlir::LLVM::InsertElementOp::create(3480        rewriter, loc, result, adaptor.getElements()[i], indexValue);3481  }3482 3483  rewriter.replaceOp(op, result);3484  return mlir::success();3485}3486 3487mlir::LogicalResult CIRToLLVMVecExtractOpLowering::matchAndRewrite(3488    cir::VecExtractOp op, OpAdaptor adaptor,3489    mlir::ConversionPatternRewriter &rewriter) const {3490  rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractElementOp>(3491      op, adaptor.getVec(), adaptor.getIndex());3492  return mlir::success();3493}3494 3495mlir::LogicalResult CIRToLLVMVecInsertOpLowering::matchAndRewrite(3496    cir::VecInsertOp op, OpAdaptor adaptor,3497    mlir::ConversionPatternRewriter &rewriter) const {3498  rewriter.replaceOpWithNewOp<mlir::LLVM::InsertElementOp>(3499      op, adaptor.getVec(), adaptor.getValue(), adaptor.getIndex());3500  return mlir::success();3501}3502 3503mlir::LogicalResult CIRToLLVMVecCmpOpLowering::matchAndRewrite(3504    cir::VecCmpOp op, OpAdaptor adaptor,3505    mlir::ConversionPatternRewriter &rewriter) const {3506  mlir::Type elementType = elementTypeIfVector(op.getLhs().getType());3507  mlir::Value bitResult;3508  if (auto intType = mlir::dyn_cast<cir::IntType>(elementType)) {3509    bitResult = mlir::LLVM::ICmpOp::create(3510        rewriter, op.getLoc(),3511        convertCmpKindToICmpPredicate(op.getKind(), intType.isSigned()),3512        adaptor.getLhs(), adaptor.getRhs());3513  } else if (mlir::isa<cir::FPTypeInterface>(elementType)) {3514    bitResult = mlir::LLVM::FCmpOp::create(3515        rewriter, op.getLoc(), convertCmpKindToFCmpPredicate(op.getKind()),3516        adaptor.getLhs(), adaptor.getRhs());3517  } else {3518    return op.emitError() << "unsupported type for VecCmpOp: " << elementType;3519  }3520 3521  // LLVM IR vector comparison returns a vector of i1. This one-bit vector3522  // must be sign-extended to the correct result type.3523  rewriter.replaceOpWithNewOp<mlir::LLVM::SExtOp>(3524      op, typeConverter->convertType(op.getType()), bitResult);3525  return mlir::success();3526}3527 3528mlir::LogicalResult CIRToLLVMVecSplatOpLowering::matchAndRewrite(3529    cir::VecSplatOp op, OpAdaptor adaptor,3530    mlir::ConversionPatternRewriter &rewriter) const {3531  // Vector splat can be implemented with an `insertelement` and a3532  // `shufflevector`, which is better than an `insertelement` for each3533  // element in the vector. Start with an undef vector. Insert the value into3534  // the first element. Then use a `shufflevector` with a mask of all 0 to3535  // fill out the entire vector with that value.3536  cir::VectorType vecTy = op.getType();3537  mlir::Type llvmTy = typeConverter->convertType(vecTy);3538  mlir::Location loc = op.getLoc();3539  mlir::Value poison = mlir::LLVM::PoisonOp::create(rewriter, loc, llvmTy);3540 3541  mlir::Value elementValue = adaptor.getValue();3542  if (elementValue.getDefiningOp<mlir::LLVM::PoisonOp>()) {3543    // If the splat value is poison, then we can just use poison value3544    // for the entire vector.3545    rewriter.replaceOp(op, poison);3546    return mlir::success();3547  }3548 3549  if (auto constValue = elementValue.getDefiningOp<mlir::LLVM::ConstantOp>()) {3550    if (auto intAttr = dyn_cast<mlir::IntegerAttr>(constValue.getValue())) {3551      mlir::DenseIntElementsAttr denseVec = mlir::DenseIntElementsAttr::get(3552          mlir::cast<mlir::ShapedType>(llvmTy), intAttr.getValue());3553      rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(3554          op, denseVec.getType(), denseVec);3555      return mlir::success();3556    }3557 3558    if (auto fpAttr = dyn_cast<mlir::FloatAttr>(constValue.getValue())) {3559      mlir::DenseFPElementsAttr denseVec = mlir::DenseFPElementsAttr::get(3560          mlir::cast<mlir::ShapedType>(llvmTy), fpAttr.getValue());3561      rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(3562          op, denseVec.getType(), denseVec);3563      return mlir::success();3564    }3565  }3566 3567  mlir::Value indexValue =3568      mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), 0);3569  mlir::Value oneElement = mlir::LLVM::InsertElementOp::create(3570      rewriter, loc, poison, elementValue, indexValue);3571  SmallVector<int32_t> zeroValues(vecTy.getSize(), 0);3572  rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(op, oneElement,3573                                                           poison, zeroValues);3574  return mlir::success();3575}3576 3577mlir::LogicalResult CIRToLLVMVecShuffleOpLowering::matchAndRewrite(3578    cir::VecShuffleOp op, OpAdaptor adaptor,3579    mlir::ConversionPatternRewriter &rewriter) const {3580  // LLVM::ShuffleVectorOp takes an ArrayRef of int for the list of indices.3581  // Convert the ClangIR ArrayAttr of IntAttr constants into a3582  // SmallVector<int>.3583  SmallVector<int, 8> indices;3584  std::transform(3585      op.getIndices().begin(), op.getIndices().end(),3586      std::back_inserter(indices), [](mlir::Attribute intAttr) {3587        return mlir::cast<cir::IntAttr>(intAttr).getValue().getSExtValue();3588      });3589  rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(3590      op, adaptor.getVec1(), adaptor.getVec2(), indices);3591  return mlir::success();3592}3593 3594mlir::LogicalResult CIRToLLVMVecShuffleDynamicOpLowering::matchAndRewrite(3595    cir::VecShuffleDynamicOp op, OpAdaptor adaptor,3596    mlir::ConversionPatternRewriter &rewriter) const {3597  // LLVM IR does not have an operation that corresponds to this form of3598  // the built-in.3599  //     __builtin_shufflevector(V, I)3600  // is implemented as this pseudocode, where the for loop is unrolled3601  // and N is the number of elements:3602  //3603  // result = undef3604  // maskbits = NextPowerOf2(N - 1)3605  // masked = I & maskbits3606  // for (i in 0 <= i < N)3607  //    result[i] = V[masked[i]]3608  mlir::Location loc = op.getLoc();3609  mlir::Value input = adaptor.getVec();3610  mlir::Type llvmIndexVecType =3611      getTypeConverter()->convertType(op.getIndices().getType());3612  mlir::Type llvmIndexType = getTypeConverter()->convertType(3613      elementTypeIfVector(op.getIndices().getType()));3614  uint64_t numElements =3615      mlir::cast<cir::VectorType>(op.getVec().getType()).getSize();3616 3617  uint64_t maskBits = llvm::NextPowerOf2(numElements - 1) - 1;3618  mlir::Value maskValue = mlir::LLVM::ConstantOp::create(3619      rewriter, loc, llvmIndexType,3620      rewriter.getIntegerAttr(llvmIndexType, maskBits));3621  mlir::Value maskVector =3622      mlir::LLVM::UndefOp::create(rewriter, loc, llvmIndexVecType);3623 3624  for (uint64_t i = 0; i < numElements; ++i) {3625    mlir::Value idxValue =3626        mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), i);3627    maskVector = mlir::LLVM::InsertElementOp::create(rewriter, loc, maskVector,3628                                                     maskValue, idxValue);3629  }3630 3631  mlir::Value maskedIndices = mlir::LLVM::AndOp::create(3632      rewriter, loc, llvmIndexVecType, adaptor.getIndices(), maskVector);3633  mlir::Value result = mlir::LLVM::UndefOp::create(3634      rewriter, loc, getTypeConverter()->convertType(op.getVec().getType()));3635  for (uint64_t i = 0; i < numElements; ++i) {3636    mlir::Value iValue =3637        mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), i);3638    mlir::Value indexValue = mlir::LLVM::ExtractElementOp::create(3639        rewriter, loc, maskedIndices, iValue);3640    mlir::Value valueAtIndex =3641        mlir::LLVM::ExtractElementOp::create(rewriter, loc, input, indexValue);3642    result = mlir::LLVM::InsertElementOp::create(rewriter, loc, result,3643                                                 valueAtIndex, iValue);3644  }3645  rewriter.replaceOp(op, result);3646  return mlir::success();3647}3648 3649mlir::LogicalResult CIRToLLVMVecTernaryOpLowering::matchAndRewrite(3650    cir::VecTernaryOp op, OpAdaptor adaptor,3651    mlir::ConversionPatternRewriter &rewriter) const {3652  // Convert `cond` into a vector of i1, then use that in a `select` op.3653  mlir::Value bitVec = mlir::LLVM::ICmpOp::create(3654      rewriter, op.getLoc(), mlir::LLVM::ICmpPredicate::ne, adaptor.getCond(),3655      mlir::LLVM::ZeroOp::create(3656          rewriter, op.getCond().getLoc(),3657          typeConverter->convertType(op.getCond().getType())));3658  rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(3659      op, bitVec, adaptor.getLhs(), adaptor.getRhs());3660  return mlir::success();3661}3662 3663mlir::LogicalResult CIRToLLVMComplexAddOpLowering::matchAndRewrite(3664    cir::ComplexAddOp op, OpAdaptor adaptor,3665    mlir::ConversionPatternRewriter &rewriter) const {3666  mlir::Value lhs = adaptor.getLhs();3667  mlir::Value rhs = adaptor.getRhs();3668  mlir::Location loc = op.getLoc();3669 3670  auto complexType = mlir::cast<cir::ComplexType>(op.getLhs().getType());3671  mlir::Type complexElemTy =3672      getTypeConverter()->convertType(complexType.getElementType());3673  auto lhsReal = mlir::LLVM::ExtractValueOp::create(3674      rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{0}));3675  auto lhsImag = mlir::LLVM::ExtractValueOp::create(3676      rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{1}));3677  auto rhsReal = mlir::LLVM::ExtractValueOp::create(3678      rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{0}));3679  auto rhsImag = mlir::LLVM::ExtractValueOp::create(3680      rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{1}));3681 3682  mlir::Value newReal;3683  mlir::Value newImag;3684  if (complexElemTy.isInteger()) {3685    newReal = mlir::LLVM::AddOp::create(rewriter, loc, complexElemTy, lhsReal,3686                                        rhsReal);3687    newImag = mlir::LLVM::AddOp::create(rewriter, loc, complexElemTy, lhsImag,3688                                        rhsImag);3689  } else {3690    assert(!cir::MissingFeatures::fastMathFlags());3691    assert(!cir::MissingFeatures::fpConstraints());3692    newReal = mlir::LLVM::FAddOp::create(rewriter, loc, complexElemTy, lhsReal,3693                                         rhsReal);3694    newImag = mlir::LLVM::FAddOp::create(rewriter, loc, complexElemTy, lhsImag,3695                                         rhsImag);3696  }3697 3698  mlir::Type complexLLVMTy =3699      getTypeConverter()->convertType(op.getResult().getType());3700  auto initialComplex =3701      mlir::LLVM::PoisonOp::create(rewriter, op->getLoc(), complexLLVMTy);3702 3703  auto realComplex = mlir::LLVM::InsertValueOp::create(3704      rewriter, op->getLoc(), initialComplex, newReal, ArrayRef(int64_t{0}));3705 3706  rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(3707      op, realComplex, newImag, ArrayRef(int64_t{1}));3708 3709  return mlir::success();3710}3711 3712mlir::LogicalResult CIRToLLVMComplexCreateOpLowering::matchAndRewrite(3713    cir::ComplexCreateOp op, OpAdaptor adaptor,3714    mlir::ConversionPatternRewriter &rewriter) const {3715  mlir::Type complexLLVMTy =3716      getTypeConverter()->convertType(op.getResult().getType());3717  auto initialComplex =3718      mlir::LLVM::UndefOp::create(rewriter, op->getLoc(), complexLLVMTy);3719 3720  auto realComplex = mlir::LLVM::InsertValueOp::create(3721      rewriter, op->getLoc(), initialComplex, adaptor.getReal(),3722      ArrayRef(int64_t{0}));3723 3724  auto complex = mlir::LLVM::InsertValueOp::create(3725      rewriter, op->getLoc(), realComplex, adaptor.getImag(),3726      ArrayRef(int64_t{1}));3727 3728  rewriter.replaceOp(op, complex);3729  return mlir::success();3730}3731 3732mlir::LogicalResult CIRToLLVMComplexRealOpLowering::matchAndRewrite(3733    cir::ComplexRealOp op, OpAdaptor adaptor,3734    mlir::ConversionPatternRewriter &rewriter) const {3735  mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());3736  mlir::Value operand = adaptor.getOperand();3737  if (mlir::isa<cir::ComplexType>(op.getOperand().getType())) {3738    operand = mlir::LLVM::ExtractValueOp::create(3739        rewriter, op.getLoc(), resultLLVMTy, operand,3740        llvm::ArrayRef<std::int64_t>{0});3741  }3742  rewriter.replaceOp(op, operand);3743  return mlir::success();3744}3745 3746mlir::LogicalResult CIRToLLVMComplexSubOpLowering::matchAndRewrite(3747    cir::ComplexSubOp op, OpAdaptor adaptor,3748    mlir::ConversionPatternRewriter &rewriter) const {3749  mlir::Value lhs = adaptor.getLhs();3750  mlir::Value rhs = adaptor.getRhs();3751  mlir::Location loc = op.getLoc();3752 3753  auto complexType = mlir::cast<cir::ComplexType>(op.getLhs().getType());3754  mlir::Type complexElemTy =3755      getTypeConverter()->convertType(complexType.getElementType());3756  auto lhsReal = mlir::LLVM::ExtractValueOp::create(3757      rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{0}));3758  auto lhsImag = mlir::LLVM::ExtractValueOp::create(3759      rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{1}));3760  auto rhsReal = mlir::LLVM::ExtractValueOp::create(3761      rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{0}));3762  auto rhsImag = mlir::LLVM::ExtractValueOp::create(3763      rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{1}));3764 3765  mlir::Value newReal;3766  mlir::Value newImag;3767  if (complexElemTy.isInteger()) {3768    newReal = mlir::LLVM::SubOp::create(rewriter, loc, complexElemTy, lhsReal,3769                                        rhsReal);3770    newImag = mlir::LLVM::SubOp::create(rewriter, loc, complexElemTy, lhsImag,3771                                        rhsImag);3772  } else {3773    assert(!cir::MissingFeatures::fastMathFlags());3774    assert(!cir::MissingFeatures::fpConstraints());3775    newReal = mlir::LLVM::FSubOp::create(rewriter, loc, complexElemTy, lhsReal,3776                                         rhsReal);3777    newImag = mlir::LLVM::FSubOp::create(rewriter, loc, complexElemTy, lhsImag,3778                                         rhsImag);3779  }3780 3781  mlir::Type complexLLVMTy =3782      getTypeConverter()->convertType(op.getResult().getType());3783  auto initialComplex =3784      mlir::LLVM::PoisonOp::create(rewriter, op->getLoc(), complexLLVMTy);3785 3786  auto realComplex = mlir::LLVM::InsertValueOp::create(3787      rewriter, op->getLoc(), initialComplex, newReal, ArrayRef(int64_t{0}));3788 3789  rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(3790      op, realComplex, newImag, ArrayRef(int64_t{1}));3791 3792  return mlir::success();3793}3794 3795mlir::LogicalResult CIRToLLVMComplexImagOpLowering::matchAndRewrite(3796    cir::ComplexImagOp op, OpAdaptor adaptor,3797    mlir::ConversionPatternRewriter &rewriter) const {3798  mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());3799  mlir::Value operand = adaptor.getOperand();3800  mlir::Location loc = op.getLoc();3801 3802  if (mlir::isa<cir::ComplexType>(op.getOperand().getType())) {3803    operand = mlir::LLVM::ExtractValueOp::create(3804        rewriter, loc, resultLLVMTy, operand, llvm::ArrayRef<std::int64_t>{1});3805  } else {3806    mlir::TypedAttr zeroAttr = rewriter.getZeroAttr(resultLLVMTy);3807    operand =3808        mlir::LLVM::ConstantOp::create(rewriter, loc, resultLLVMTy, zeroAttr);3809  }3810 3811  rewriter.replaceOp(op, operand);3812  return mlir::success();3813}3814 3815mlir::IntegerType computeBitfieldIntType(mlir::Type storageType,3816                                         mlir::MLIRContext *context,3817                                         unsigned &storageSize) {3818  return TypeSwitch<mlir::Type, mlir::IntegerType>(storageType)3819      .Case<cir::ArrayType>([&](cir::ArrayType atTy) {3820        storageSize = atTy.getSize() * 8;3821        return mlir::IntegerType::get(context, storageSize);3822      })3823      .Case<cir::IntType>([&](cir::IntType intTy) {3824        storageSize = intTy.getWidth();3825        return mlir::IntegerType::get(context, storageSize);3826      })3827      .Default([](mlir::Type) -> mlir::IntegerType {3828        llvm_unreachable(3829            "Either ArrayType or IntType expected for bitfields storage");3830      });3831}3832 3833mlir::LogicalResult CIRToLLVMSetBitfieldOpLowering::matchAndRewrite(3834    cir::SetBitfieldOp op, OpAdaptor adaptor,3835    mlir::ConversionPatternRewriter &rewriter) const {3836  mlir::OpBuilder::InsertionGuard guard(rewriter);3837  rewriter.setInsertionPoint(op);3838 3839  cir::BitfieldInfoAttr info = op.getBitfieldInfo();3840  uint64_t size = info.getSize();3841  uint64_t offset = info.getOffset();3842  mlir::Type storageType = info.getStorageType();3843  mlir::MLIRContext *context = storageType.getContext();3844 3845  unsigned storageSize = 0;3846 3847  mlir::IntegerType intType =3848      computeBitfieldIntType(storageType, context, storageSize);3849 3850  mlir::Value srcVal = createIntCast(rewriter, adaptor.getSrc(), intType);3851  unsigned srcWidth = storageSize;3852  mlir::Value resultVal = srcVal;3853 3854  if (storageSize != size) {3855    assert(storageSize > size && "Invalid bitfield size.");3856 3857    mlir::Value val = mlir::LLVM::LoadOp::create(3858        rewriter, op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(),3859        op.getIsVolatile());3860 3861    srcVal =3862        createAnd(rewriter, srcVal, llvm::APInt::getLowBitsSet(srcWidth, size));3863    resultVal = srcVal;3864    srcVal = createShL(rewriter, srcVal, offset);3865 3866    // Mask out the original value.3867    val = createAnd(rewriter, val,3868                    ~llvm::APInt::getBitsSet(srcWidth, offset, offset + size));3869 3870    // Or together the unchanged values and the source value.3871    srcVal = mlir::LLVM::OrOp::create(rewriter, op.getLoc(), val, srcVal);3872  }3873 3874  mlir::LLVM::StoreOp::create(rewriter, op.getLoc(), srcVal, adaptor.getAddr(),3875                              op.getAlignment(), op.getIsVolatile());3876 3877  mlir::Type resultTy = getTypeConverter()->convertType(op.getType());3878 3879  if (info.getIsSigned()) {3880    assert(size <= storageSize);3881    unsigned highBits = storageSize - size;3882 3883    if (highBits) {3884      resultVal = createShL(rewriter, resultVal, highBits);3885      resultVal = createAShR(rewriter, resultVal, highBits);3886    }3887  }3888 3889  resultVal = createIntCast(rewriter, resultVal,3890                            mlir::cast<mlir::IntegerType>(resultTy),3891                            info.getIsSigned());3892 3893  rewriter.replaceOp(op, resultVal);3894  return mlir::success();3895}3896 3897mlir::LogicalResult CIRToLLVMComplexImagPtrOpLowering::matchAndRewrite(3898    cir::ComplexImagPtrOp op, OpAdaptor adaptor,3899    mlir::ConversionPatternRewriter &rewriter) const {3900  cir::PointerType operandTy = op.getOperand().getType();3901  mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());3902  mlir::Type elementLLVMTy =3903      getTypeConverter()->convertType(operandTy.getPointee());3904 3905  mlir::LLVM::GEPArg gepIndices[2] = {{0}, {1}};3906  mlir::LLVM::GEPNoWrapFlags inboundsNuw =3907      mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;3908  rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(3909      op, resultLLVMTy, elementLLVMTy, adaptor.getOperand(), gepIndices,3910      inboundsNuw);3911  return mlir::success();3912}3913 3914mlir::LogicalResult CIRToLLVMComplexRealPtrOpLowering::matchAndRewrite(3915    cir::ComplexRealPtrOp op, OpAdaptor adaptor,3916    mlir::ConversionPatternRewriter &rewriter) const {3917  cir::PointerType operandTy = op.getOperand().getType();3918  mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());3919  mlir::Type elementLLVMTy =3920      getTypeConverter()->convertType(operandTy.getPointee());3921 3922  mlir::LLVM::GEPArg gepIndices[2] = {0, 0};3923  mlir::LLVM::GEPNoWrapFlags inboundsNuw =3924      mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;3925  rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(3926      op, resultLLVMTy, elementLLVMTy, adaptor.getOperand(), gepIndices,3927      inboundsNuw);3928  return mlir::success();3929}3930 3931mlir::LogicalResult CIRToLLVMGetBitfieldOpLowering::matchAndRewrite(3932    cir::GetBitfieldOp op, OpAdaptor adaptor,3933    mlir::ConversionPatternRewriter &rewriter) const {3934 3935  mlir::OpBuilder::InsertionGuard guard(rewriter);3936  rewriter.setInsertionPoint(op);3937 3938  cir::BitfieldInfoAttr info = op.getBitfieldInfo();3939  uint64_t size = info.getSize();3940  uint64_t offset = info.getOffset();3941  mlir::Type storageType = info.getStorageType();3942  mlir::MLIRContext *context = storageType.getContext();3943  unsigned storageSize = 0;3944 3945  mlir::IntegerType intType =3946      computeBitfieldIntType(storageType, context, storageSize);3947 3948  mlir::Value val = mlir::LLVM::LoadOp::create(3949      rewriter, op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(),3950      op.getIsVolatile());3951  val = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), intType, val);3952 3953  if (info.getIsSigned()) {3954    assert(static_cast<unsigned>(offset + size) <= storageSize);3955    unsigned highBits = storageSize - offset - size;3956    val = createShL(rewriter, val, highBits);3957    val = createAShR(rewriter, val, offset + highBits);3958  } else {3959    val = createLShR(rewriter, val, offset);3960 3961    if (static_cast<unsigned>(offset) + size < storageSize)3962      val = createAnd(rewriter, val,3963                      llvm::APInt::getLowBitsSet(storageSize, size));3964  }3965 3966  mlir::Type resTy = getTypeConverter()->convertType(op.getType());3967  mlir::Value newOp = createIntCast(3968      rewriter, val, mlir::cast<mlir::IntegerType>(resTy), info.getIsSigned());3969  rewriter.replaceOp(op, newOp);3970  return mlir::success();3971}3972 3973mlir::LogicalResult CIRToLLVMInlineAsmOpLowering::matchAndRewrite(3974    cir::InlineAsmOp op, OpAdaptor adaptor,3975    mlir::ConversionPatternRewriter &rewriter) const {3976  mlir::Type llResTy;3977  if (op.getNumResults())3978    llResTy = getTypeConverter()->convertType(op.getType(0));3979 3980  cir::AsmFlavor dialect = op.getAsmFlavor();3981  mlir::LLVM::AsmDialect llDialect = dialect == cir::AsmFlavor::x86_att3982                                         ? mlir::LLVM::AsmDialect::AD_ATT3983                                         : mlir::LLVM::AsmDialect::AD_Intel;3984 3985  SmallVector<mlir::Attribute> opAttrs;3986  StringRef llvmAttrName = mlir::LLVM::InlineAsmOp::getElementTypeAttrName();3987 3988  // this is for the lowering to LLVM from LLVM dialect. Otherwise, if we3989  // don't have the result (i.e. void type as a result of operation), the3990  // element type attribute will be attached to the whole instruction, but not3991  // to the operand3992  if (!op.getNumResults())3993    opAttrs.push_back(mlir::Attribute());3994 3995  SmallVector<mlir::Value> llvmOperands;3996  SmallVector<mlir::Value> cirOperands;3997  for (auto const &[llvmOp, cirOp] :3998       zip(adaptor.getAsmOperands(), op.getAsmOperands())) {3999    append_range(llvmOperands, llvmOp);4000    append_range(cirOperands, cirOp);4001  }4002 4003  // so far we infer the llvm dialect element type attr from4004  // CIR operand type.4005  for (auto const &[cirOpAttr, cirOp] :4006       zip(op.getOperandAttrs(), cirOperands)) {4007    if (!cirOpAttr) {4008      opAttrs.push_back(mlir::Attribute());4009      continue;4010    }4011 4012    llvm::SmallVector<mlir::NamedAttribute, 1> attrs;4013    cir::PointerType typ = mlir::cast<cir::PointerType>(cirOp.getType());4014    mlir::TypeAttr typAttr = mlir::TypeAttr::get(convertTypeForMemory(4015        *getTypeConverter(), dataLayout, typ.getPointee()));4016 4017    attrs.push_back(rewriter.getNamedAttr(llvmAttrName, typAttr));4018    mlir::DictionaryAttr newDict = rewriter.getDictionaryAttr(attrs);4019    opAttrs.push_back(newDict);4020  }4021 4022  rewriter.replaceOpWithNewOp<mlir::LLVM::InlineAsmOp>(4023      op, llResTy, llvmOperands, op.getAsmStringAttr(), op.getConstraintsAttr(),4024      op.getSideEffectsAttr(),4025      /*is_align_stack*/ mlir::UnitAttr(),4026      /*tail_call_kind*/4027      mlir::LLVM::TailCallKindAttr::get(4028          getContext(), mlir::LLVM::tailcallkind::TailCallKind::None),4029      mlir::LLVM::AsmDialectAttr::get(getContext(), llDialect),4030      rewriter.getArrayAttr(opAttrs));4031 4032  return mlir::success();4033}4034 4035mlir::LogicalResult CIRToLLVMVAStartOpLowering::matchAndRewrite(4036    cir::VAStartOp op, OpAdaptor adaptor,4037    mlir::ConversionPatternRewriter &rewriter) const {4038  auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());4039  auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,4040                                              adaptor.getArgList());4041  rewriter.replaceOpWithNewOp<mlir::LLVM::VaStartOp>(op, vaList);4042  return mlir::success();4043}4044 4045mlir::LogicalResult CIRToLLVMVAEndOpLowering::matchAndRewrite(4046    cir::VAEndOp op, OpAdaptor adaptor,4047    mlir::ConversionPatternRewriter &rewriter) const {4048  auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());4049  auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,4050                                              adaptor.getArgList());4051  rewriter.replaceOpWithNewOp<mlir::LLVM::VaEndOp>(op, vaList);4052  return mlir::success();4053}4054 4055mlir::LogicalResult CIRToLLVMVAArgOpLowering::matchAndRewrite(4056    cir::VAArgOp op, OpAdaptor adaptor,4057    mlir::ConversionPatternRewriter &rewriter) const {4058  assert(!cir::MissingFeatures::vaArgABILowering());4059  auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());4060  auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,4061                                              adaptor.getArgList());4062 4063  mlir::Type llvmType =4064      getTypeConverter()->convertType(op->getResultTypes().front());4065  if (!llvmType)4066    return mlir::failure();4067 4068  rewriter.replaceOpWithNewOp<mlir::LLVM::VaArgOp>(op, llvmType, vaList);4069  return mlir::success();4070}4071 4072mlir::LogicalResult CIRToLLVMBlockAddressOpLowering::matchAndRewrite(4073    cir::BlockAddressOp op, OpAdaptor adaptor,4074    mlir::ConversionPatternRewriter &rewriter) const {4075  return mlir::failure();4076}4077 4078mlir::LogicalResult CIRToLLVMAwaitOpLowering::matchAndRewrite(4079    cir::AwaitOp op, OpAdaptor adaptor,4080    mlir::ConversionPatternRewriter &rewriter) const {4081  return mlir::failure();4082}4083 4084std::unique_ptr<mlir::Pass> createConvertCIRToLLVMPass() {4085  return std::make_unique<ConvertCIRToLLVMPass>();4086}4087 4088void populateCIRToLLVMPasses(mlir::OpPassManager &pm) {4089  mlir::populateCIRPreLoweringPasses(pm);4090  pm.addPass(createConvertCIRToLLVMPass());4091}4092 4093std::unique_ptr<llvm::Module>4094lowerDirectlyFromCIRToLLVMIR(mlir::ModuleOp mlirModule, LLVMContext &llvmCtx) {4095  llvm::TimeTraceScope scope("lower from CIR to LLVM directly");4096 4097  mlir::MLIRContext *mlirCtx = mlirModule.getContext();4098 4099  mlir::PassManager pm(mlirCtx);4100  populateCIRToLLVMPasses(pm);4101 4102  (void)mlir::applyPassManagerCLOptions(pm);4103 4104  if (mlir::failed(pm.run(mlirModule))) {4105    // FIXME: Handle any errors where they occurs and return a nullptr here.4106    report_fatal_error(4107        "The pass manager failed to lower CIR to LLVMIR dialect!");4108  }4109 4110  mlir::registerBuiltinDialectTranslation(*mlirCtx);4111  mlir::registerLLVMDialectTranslation(*mlirCtx);4112  mlir::registerCIRDialectTranslation(*mlirCtx);4113 4114  llvm::TimeTraceScope translateScope("translateModuleToLLVMIR");4115 4116  StringRef moduleName = mlirModule.getName().value_or("CIRToLLVMModule");4117  std::unique_ptr<llvm::Module> llvmModule =4118      mlir::translateModuleToLLVMIR(mlirModule, llvmCtx, moduleName);4119 4120  if (!llvmModule) {4121    // FIXME: Handle any errors where they occurs and return a nullptr here.4122    report_fatal_error("Lowering from LLVMIR dialect to llvm IR failed!");4123  }4124 4125  return llvmModule;4126}4127} // namespace direct4128} // namespace cir4129