1145 lines · cpp
1//===- LoweringPrepare.cpp - pareparation work for LLVM lowering ----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "LoweringPrepareCXXABI.h"10#include "PassDetail.h"11#include "mlir/IR/Attributes.h"12#include "clang/AST/ASTContext.h"13#include "clang/Basic/Module.h"14#include "clang/Basic/TargetInfo.h"15#include "clang/CIR/Dialect/Builder/CIRBaseBuilder.h"16#include "clang/CIR/Dialect/IR/CIRAttrs.h"17#include "clang/CIR/Dialect/IR/CIRDialect.h"18#include "clang/CIR/Dialect/IR/CIROpsEnums.h"19#include "clang/CIR/Dialect/Passes.h"20#include "clang/CIR/MissingFeatures.h"21#include "llvm/Support/Path.h"22 23#include <memory>24 25using namespace mlir;26using namespace cir;27 28namespace mlir {29#define GEN_PASS_DEF_LOWERINGPREPARE30#include "clang/CIR/Dialect/Passes.h.inc"31} // namespace mlir32 33static SmallString<128> getTransformedFileName(mlir::ModuleOp mlirModule) {34 SmallString<128> fileName;35 36 if (mlirModule.getSymName())37 fileName = llvm::sys::path::filename(mlirModule.getSymName()->str());38 39 if (fileName.empty())40 fileName = "<null>";41 42 for (size_t i = 0; i < fileName.size(); ++i) {43 // Replace everything that's not [a-zA-Z0-9._] with a _. This set happens44 // to be the set of C preprocessing numbers.45 if (!clang::isPreprocessingNumberBody(fileName[i]))46 fileName[i] = '_';47 }48 49 return fileName;50}51 52/// Return the FuncOp called by `callOp`.53static cir::FuncOp getCalledFunction(cir::CallOp callOp) {54 mlir::SymbolRefAttr sym = llvm::dyn_cast_if_present<mlir::SymbolRefAttr>(55 callOp.getCallableForCallee());56 if (!sym)57 return nullptr;58 return dyn_cast_or_null<cir::FuncOp>(59 mlir::SymbolTable::lookupNearestSymbolFrom(callOp, sym));60}61 62namespace {63struct LoweringPreparePass64 : public impl::LoweringPrepareBase<LoweringPreparePass> {65 LoweringPreparePass() = default;66 void runOnOperation() override;67 68 void runOnOp(mlir::Operation *op);69 void lowerCastOp(cir::CastOp op);70 void lowerComplexDivOp(cir::ComplexDivOp op);71 void lowerComplexMulOp(cir::ComplexMulOp op);72 void lowerUnaryOp(cir::UnaryOp op);73 void lowerGlobalOp(cir::GlobalOp op);74 void lowerDynamicCastOp(cir::DynamicCastOp op);75 void lowerArrayDtor(cir::ArrayDtor op);76 void lowerArrayCtor(cir::ArrayCtor op);77 78 /// Build the function that initializes the specified global79 cir::FuncOp buildCXXGlobalVarDeclInitFunc(cir::GlobalOp op);80 81 /// Handle the dtor region by registering destructor with __cxa_atexit82 cir::FuncOp getOrCreateDtorFunc(CIRBaseBuilderTy &builder, cir::GlobalOp op,83 mlir::Region &dtorRegion,84 cir::CallOp &dtorCall);85 86 /// Build a module init function that calls all the dynamic initializers.87 void buildCXXGlobalInitFunc();88 89 /// Materialize global ctor/dtor list90 void buildGlobalCtorDtorList();91 92 cir::FuncOp buildRuntimeFunction(93 mlir::OpBuilder &builder, llvm::StringRef name, mlir::Location loc,94 cir::FuncType type,95 cir::GlobalLinkageKind linkage = cir::GlobalLinkageKind::ExternalLinkage);96 97 cir::GlobalOp buildRuntimeVariable(98 mlir::OpBuilder &builder, llvm::StringRef name, mlir::Location loc,99 mlir::Type type,100 cir::GlobalLinkageKind linkage = cir::GlobalLinkageKind::ExternalLinkage,101 cir::VisibilityKind visibility = cir::VisibilityKind::Default);102 103 ///104 /// AST related105 /// -----------106 107 clang::ASTContext *astCtx;108 109 // Helper for lowering C++ ABI specific operations.110 std::shared_ptr<cir::LoweringPrepareCXXABI> cxxABI;111 112 /// Tracks current module.113 mlir::ModuleOp mlirModule;114 115 /// Tracks existing dynamic initializers.116 llvm::StringMap<uint32_t> dynamicInitializerNames;117 llvm::SmallVector<cir::FuncOp> dynamicInitializers;118 119 /// List of ctors and their priorities to be called before main()120 llvm::SmallVector<std::pair<std::string, uint32_t>, 4> globalCtorList;121 /// List of dtors and their priorities to be called when unloading module.122 llvm::SmallVector<std::pair<std::string, uint32_t>, 4> globalDtorList;123 124 void setASTContext(clang::ASTContext *c) {125 astCtx = c;126 switch (c->getCXXABIKind()) {127 case clang::TargetCXXABI::GenericItanium:128 // We'll need X86-specific support for handling vaargs lowering, but for129 // now the Itanium ABI will work.130 assert(!cir::MissingFeatures::loweringPrepareX86CXXABI());131 cxxABI.reset(cir::LoweringPrepareCXXABI::createItaniumABI());132 break;133 case clang::TargetCXXABI::GenericAArch64:134 case clang::TargetCXXABI::AppleARM64:135 assert(!cir::MissingFeatures::loweringPrepareAArch64XXABI());136 cxxABI.reset(cir::LoweringPrepareCXXABI::createItaniumABI());137 break;138 default:139 llvm_unreachable("NYI");140 }141 }142};143 144} // namespace145 146cir::GlobalOp LoweringPreparePass::buildRuntimeVariable(147 mlir::OpBuilder &builder, llvm::StringRef name, mlir::Location loc,148 mlir::Type type, cir::GlobalLinkageKind linkage,149 cir::VisibilityKind visibility) {150 cir::GlobalOp g = dyn_cast_or_null<cir::GlobalOp>(151 mlir::SymbolTable::lookupNearestSymbolFrom(152 mlirModule, mlir::StringAttr::get(mlirModule->getContext(), name)));153 if (!g) {154 g = cir::GlobalOp::create(builder, loc, name, type);155 g.setLinkageAttr(156 cir::GlobalLinkageKindAttr::get(builder.getContext(), linkage));157 mlir::SymbolTable::setSymbolVisibility(158 g, mlir::SymbolTable::Visibility::Private);159 g.setGlobalVisibilityAttr(160 cir::VisibilityAttr::get(builder.getContext(), visibility));161 }162 return g;163}164 165cir::FuncOp LoweringPreparePass::buildRuntimeFunction(166 mlir::OpBuilder &builder, llvm::StringRef name, mlir::Location loc,167 cir::FuncType type, cir::GlobalLinkageKind linkage) {168 cir::FuncOp f = dyn_cast_or_null<FuncOp>(SymbolTable::lookupNearestSymbolFrom(169 mlirModule, StringAttr::get(mlirModule->getContext(), name)));170 if (!f) {171 f = cir::FuncOp::create(builder, loc, name, type);172 f.setLinkageAttr(173 cir::GlobalLinkageKindAttr::get(builder.getContext(), linkage));174 mlir::SymbolTable::setSymbolVisibility(175 f, mlir::SymbolTable::Visibility::Private);176 177 assert(!cir::MissingFeatures::opFuncExtraAttrs());178 }179 return f;180}181 182static mlir::Value lowerScalarToComplexCast(mlir::MLIRContext &ctx,183 cir::CastOp op) {184 cir::CIRBaseBuilderTy builder(ctx);185 builder.setInsertionPoint(op);186 187 mlir::Value src = op.getSrc();188 mlir::Value imag = builder.getNullValue(src.getType(), op.getLoc());189 return builder.createComplexCreate(op.getLoc(), src, imag);190}191 192static mlir::Value lowerComplexToScalarCast(mlir::MLIRContext &ctx,193 cir::CastOp op,194 cir::CastKind elemToBoolKind) {195 cir::CIRBaseBuilderTy builder(ctx);196 builder.setInsertionPoint(op);197 198 mlir::Value src = op.getSrc();199 if (!mlir::isa<cir::BoolType>(op.getType()))200 return builder.createComplexReal(op.getLoc(), src);201 202 // Complex cast to bool: (bool)(a+bi) => (bool)a || (bool)b203 mlir::Value srcReal = builder.createComplexReal(op.getLoc(), src);204 mlir::Value srcImag = builder.createComplexImag(op.getLoc(), src);205 206 cir::BoolType boolTy = builder.getBoolTy();207 mlir::Value srcRealToBool =208 builder.createCast(op.getLoc(), elemToBoolKind, srcReal, boolTy);209 mlir::Value srcImagToBool =210 builder.createCast(op.getLoc(), elemToBoolKind, srcImag, boolTy);211 return builder.createLogicalOr(op.getLoc(), srcRealToBool, srcImagToBool);212}213 214static mlir::Value lowerComplexToComplexCast(mlir::MLIRContext &ctx,215 cir::CastOp op,216 cir::CastKind scalarCastKind) {217 CIRBaseBuilderTy builder(ctx);218 builder.setInsertionPoint(op);219 220 mlir::Value src = op.getSrc();221 auto dstComplexElemTy =222 mlir::cast<cir::ComplexType>(op.getType()).getElementType();223 224 mlir::Value srcReal = builder.createComplexReal(op.getLoc(), src);225 mlir::Value srcImag = builder.createComplexImag(op.getLoc(), src);226 227 mlir::Value dstReal = builder.createCast(op.getLoc(), scalarCastKind, srcReal,228 dstComplexElemTy);229 mlir::Value dstImag = builder.createCast(op.getLoc(), scalarCastKind, srcImag,230 dstComplexElemTy);231 return builder.createComplexCreate(op.getLoc(), dstReal, dstImag);232}233 234void LoweringPreparePass::lowerCastOp(cir::CastOp op) {235 mlir::MLIRContext &ctx = getContext();236 mlir::Value loweredValue = [&]() -> mlir::Value {237 switch (op.getKind()) {238 case cir::CastKind::float_to_complex:239 case cir::CastKind::int_to_complex:240 return lowerScalarToComplexCast(ctx, op);241 case cir::CastKind::float_complex_to_real:242 case cir::CastKind::int_complex_to_real:243 return lowerComplexToScalarCast(ctx, op, op.getKind());244 case cir::CastKind::float_complex_to_bool:245 return lowerComplexToScalarCast(ctx, op, cir::CastKind::float_to_bool);246 case cir::CastKind::int_complex_to_bool:247 return lowerComplexToScalarCast(ctx, op, cir::CastKind::int_to_bool);248 case cir::CastKind::float_complex:249 return lowerComplexToComplexCast(ctx, op, cir::CastKind::floating);250 case cir::CastKind::float_complex_to_int_complex:251 return lowerComplexToComplexCast(ctx, op, cir::CastKind::float_to_int);252 case cir::CastKind::int_complex:253 return lowerComplexToComplexCast(ctx, op, cir::CastKind::integral);254 case cir::CastKind::int_complex_to_float_complex:255 return lowerComplexToComplexCast(ctx, op, cir::CastKind::int_to_float);256 default:257 return nullptr;258 }259 }();260 261 if (loweredValue) {262 op.replaceAllUsesWith(loweredValue);263 op.erase();264 }265}266 267static mlir::Value buildComplexBinOpLibCall(268 LoweringPreparePass &pass, CIRBaseBuilderTy &builder,269 llvm::StringRef (*libFuncNameGetter)(llvm::APFloat::Semantics),270 mlir::Location loc, cir::ComplexType ty, mlir::Value lhsReal,271 mlir::Value lhsImag, mlir::Value rhsReal, mlir::Value rhsImag) {272 cir::FPTypeInterface elementTy =273 mlir::cast<cir::FPTypeInterface>(ty.getElementType());274 275 llvm::StringRef libFuncName = libFuncNameGetter(276 llvm::APFloat::SemanticsToEnum(elementTy.getFloatSemantics()));277 llvm::SmallVector<mlir::Type, 4> libFuncInputTypes(4, elementTy);278 279 cir::FuncType libFuncTy = cir::FuncType::get(libFuncInputTypes, ty);280 281 // Insert a declaration for the runtime function to be used in Complex282 // multiplication and division when needed283 cir::FuncOp libFunc;284 {285 mlir::OpBuilder::InsertionGuard ipGuard{builder};286 builder.setInsertionPointToStart(pass.mlirModule.getBody());287 libFunc = pass.buildRuntimeFunction(builder, libFuncName, loc, libFuncTy);288 }289 290 cir::CallOp call =291 builder.createCallOp(loc, libFunc, {lhsReal, lhsImag, rhsReal, rhsImag});292 return call.getResult();293}294 295static llvm::StringRef296getComplexDivLibCallName(llvm::APFloat::Semantics semantics) {297 switch (semantics) {298 case llvm::APFloat::S_IEEEhalf:299 return "__divhc3";300 case llvm::APFloat::S_IEEEsingle:301 return "__divsc3";302 case llvm::APFloat::S_IEEEdouble:303 return "__divdc3";304 case llvm::APFloat::S_PPCDoubleDouble:305 return "__divtc3";306 case llvm::APFloat::S_x87DoubleExtended:307 return "__divxc3";308 case llvm::APFloat::S_IEEEquad:309 return "__divtc3";310 default:311 llvm_unreachable("unsupported floating point type");312 }313}314 315static mlir::Value316buildAlgebraicComplexDiv(CIRBaseBuilderTy &builder, mlir::Location loc,317 mlir::Value lhsReal, mlir::Value lhsImag,318 mlir::Value rhsReal, mlir::Value rhsImag) {319 // (a+bi) / (c+di) = ((ac+bd)/(cc+dd)) + ((bc-ad)/(cc+dd))i320 mlir::Value &a = lhsReal;321 mlir::Value &b = lhsImag;322 mlir::Value &c = rhsReal;323 mlir::Value &d = rhsImag;324 325 mlir::Value ac = builder.createBinop(loc, a, cir::BinOpKind::Mul, c); // a*c326 mlir::Value bd = builder.createBinop(loc, b, cir::BinOpKind::Mul, d); // b*d327 mlir::Value cc = builder.createBinop(loc, c, cir::BinOpKind::Mul, c); // c*c328 mlir::Value dd = builder.createBinop(loc, d, cir::BinOpKind::Mul, d); // d*d329 mlir::Value acbd =330 builder.createBinop(loc, ac, cir::BinOpKind::Add, bd); // ac+bd331 mlir::Value ccdd =332 builder.createBinop(loc, cc, cir::BinOpKind::Add, dd); // cc+dd333 mlir::Value resultReal =334 builder.createBinop(loc, acbd, cir::BinOpKind::Div, ccdd);335 336 mlir::Value bc = builder.createBinop(loc, b, cir::BinOpKind::Mul, c); // b*c337 mlir::Value ad = builder.createBinop(loc, a, cir::BinOpKind::Mul, d); // a*d338 mlir::Value bcad =339 builder.createBinop(loc, bc, cir::BinOpKind::Sub, ad); // bc-ad340 mlir::Value resultImag =341 builder.createBinop(loc, bcad, cir::BinOpKind::Div, ccdd);342 return builder.createComplexCreate(loc, resultReal, resultImag);343}344 345static mlir::Value346buildRangeReductionComplexDiv(CIRBaseBuilderTy &builder, mlir::Location loc,347 mlir::Value lhsReal, mlir::Value lhsImag,348 mlir::Value rhsReal, mlir::Value rhsImag) {349 // Implements Smith's algorithm for complex division.350 // SMITH, R. L. Algorithm 116: Complex division. Commun. ACM 5, 8 (1962).351 352 // Let:353 // - lhs := a+bi354 // - rhs := c+di355 // - result := lhs / rhs = e+fi356 //357 // The algorithm pseudocode looks like follows:358 // if fabs(c) >= fabs(d):359 // r := d / c360 // tmp := c + r*d361 // e = (a + b*r) / tmp362 // f = (b - a*r) / tmp363 // else:364 // r := c / d365 // tmp := d + r*c366 // e = (a*r + b) / tmp367 // f = (b*r - a) / tmp368 369 mlir::Value &a = lhsReal;370 mlir::Value &b = lhsImag;371 mlir::Value &c = rhsReal;372 mlir::Value &d = rhsImag;373 374 auto trueBranchBuilder = [&](mlir::OpBuilder &, mlir::Location) {375 mlir::Value r = builder.createBinop(loc, d, cir::BinOpKind::Div,376 c); // r := d / c377 mlir::Value rd = builder.createBinop(loc, r, cir::BinOpKind::Mul, d); // r*d378 mlir::Value tmp = builder.createBinop(loc, c, cir::BinOpKind::Add,379 rd); // tmp := c + r*d380 381 mlir::Value br = builder.createBinop(loc, b, cir::BinOpKind::Mul, r); // b*r382 mlir::Value abr =383 builder.createBinop(loc, a, cir::BinOpKind::Add, br); // a + b*r384 mlir::Value e = builder.createBinop(loc, abr, cir::BinOpKind::Div, tmp);385 386 mlir::Value ar = builder.createBinop(loc, a, cir::BinOpKind::Mul, r); // a*r387 mlir::Value bar =388 builder.createBinop(loc, b, cir::BinOpKind::Sub, ar); // b - a*r389 mlir::Value f = builder.createBinop(loc, bar, cir::BinOpKind::Div, tmp);390 391 mlir::Value result = builder.createComplexCreate(loc, e, f);392 builder.createYield(loc, result);393 };394 395 auto falseBranchBuilder = [&](mlir::OpBuilder &, mlir::Location) {396 mlir::Value r = builder.createBinop(loc, c, cir::BinOpKind::Div,397 d); // r := c / d398 mlir::Value rc = builder.createBinop(loc, r, cir::BinOpKind::Mul, c); // r*c399 mlir::Value tmp = builder.createBinop(loc, d, cir::BinOpKind::Add,400 rc); // tmp := d + r*c401 402 mlir::Value ar = builder.createBinop(loc, a, cir::BinOpKind::Mul, r); // a*r403 mlir::Value arb =404 builder.createBinop(loc, ar, cir::BinOpKind::Add, b); // a*r + b405 mlir::Value e = builder.createBinop(loc, arb, cir::BinOpKind::Div, tmp);406 407 mlir::Value br = builder.createBinop(loc, b, cir::BinOpKind::Mul, r); // b*r408 mlir::Value bra =409 builder.createBinop(loc, br, cir::BinOpKind::Sub, a); // b*r - a410 mlir::Value f = builder.createBinop(loc, bra, cir::BinOpKind::Div, tmp);411 412 mlir::Value result = builder.createComplexCreate(loc, e, f);413 builder.createYield(loc, result);414 };415 416 auto cFabs = cir::FAbsOp::create(builder, loc, c);417 auto dFabs = cir::FAbsOp::create(builder, loc, d);418 cir::CmpOp cmpResult =419 builder.createCompare(loc, cir::CmpOpKind::ge, cFabs, dFabs);420 auto ternary = cir::TernaryOp::create(builder, loc, cmpResult,421 trueBranchBuilder, falseBranchBuilder);422 423 return ternary.getResult();424}425 426static mlir::Type higherPrecisionElementTypeForComplexArithmetic(427 mlir::MLIRContext &context, clang::ASTContext &cc,428 CIRBaseBuilderTy &builder, mlir::Type elementType) {429 430 auto getHigherPrecisionFPType = [&context](mlir::Type type) -> mlir::Type {431 if (mlir::isa<cir::FP16Type>(type))432 return cir::SingleType::get(&context);433 434 if (mlir::isa<cir::SingleType>(type) || mlir::isa<cir::BF16Type>(type))435 return cir::DoubleType::get(&context);436 437 if (mlir::isa<cir::DoubleType>(type))438 return cir::LongDoubleType::get(&context, type);439 440 return type;441 };442 443 auto getFloatTypeSemantics =444 [&cc](mlir::Type type) -> const llvm::fltSemantics & {445 const clang::TargetInfo &info = cc.getTargetInfo();446 if (mlir::isa<cir::FP16Type>(type))447 return info.getHalfFormat();448 449 if (mlir::isa<cir::BF16Type>(type))450 return info.getBFloat16Format();451 452 if (mlir::isa<cir::SingleType>(type))453 return info.getFloatFormat();454 455 if (mlir::isa<cir::DoubleType>(type))456 return info.getDoubleFormat();457 458 if (mlir::isa<cir::LongDoubleType>(type)) {459 if (cc.getLangOpts().OpenMP && cc.getLangOpts().OpenMPIsTargetDevice)460 llvm_unreachable("NYI Float type semantics with OpenMP");461 return info.getLongDoubleFormat();462 }463 464 if (mlir::isa<cir::FP128Type>(type)) {465 if (cc.getLangOpts().OpenMP && cc.getLangOpts().OpenMPIsTargetDevice)466 llvm_unreachable("NYI Float type semantics with OpenMP");467 return info.getFloat128Format();468 }469 470 llvm_unreachable("Unsupported float type semantics");471 };472 473 const mlir::Type higherElementType = getHigherPrecisionFPType(elementType);474 const llvm::fltSemantics &elementTypeSemantics =475 getFloatTypeSemantics(elementType);476 const llvm::fltSemantics &higherElementTypeSemantics =477 getFloatTypeSemantics(higherElementType);478 479 // Check that the promoted type can handle the intermediate values without480 // overflowing. This can be interpreted as:481 // (SmallerType.LargestFiniteVal * SmallerType.LargestFiniteVal) * 2 <=482 // LargerType.LargestFiniteVal.483 // In terms of exponent it gives this formula:484 // (SmallerType.LargestFiniteVal * SmallerType.LargestFiniteVal485 // doubles the exponent of SmallerType.LargestFiniteVal)486 if (llvm::APFloat::semanticsMaxExponent(elementTypeSemantics) * 2 + 1 <=487 llvm::APFloat::semanticsMaxExponent(higherElementTypeSemantics)) {488 return higherElementType;489 }490 491 // The intermediate values can't be represented in the promoted type492 // without overflowing.493 return {};494}495 496static mlir::Value497lowerComplexDiv(LoweringPreparePass &pass, CIRBaseBuilderTy &builder,498 mlir::Location loc, cir::ComplexDivOp op, mlir::Value lhsReal,499 mlir::Value lhsImag, mlir::Value rhsReal, mlir::Value rhsImag,500 mlir::MLIRContext &mlirCx, clang::ASTContext &cc) {501 cir::ComplexType complexTy = op.getType();502 if (mlir::isa<cir::FPTypeInterface>(complexTy.getElementType())) {503 cir::ComplexRangeKind range = op.getRange();504 if (range == cir::ComplexRangeKind::Improved)505 return buildRangeReductionComplexDiv(builder, loc, lhsReal, lhsImag,506 rhsReal, rhsImag);507 508 if (range == cir::ComplexRangeKind::Full)509 return buildComplexBinOpLibCall(pass, builder, &getComplexDivLibCallName,510 loc, complexTy, lhsReal, lhsImag, rhsReal,511 rhsImag);512 513 if (range == cir::ComplexRangeKind::Promoted) {514 mlir::Type originalElementType = complexTy.getElementType();515 mlir::Type higherPrecisionElementType =516 higherPrecisionElementTypeForComplexArithmetic(mlirCx, cc, builder,517 originalElementType);518 519 if (!higherPrecisionElementType)520 return buildRangeReductionComplexDiv(builder, loc, lhsReal, lhsImag,521 rhsReal, rhsImag);522 523 cir::CastKind floatingCastKind = cir::CastKind::floating;524 lhsReal = builder.createCast(floatingCastKind, lhsReal,525 higherPrecisionElementType);526 lhsImag = builder.createCast(floatingCastKind, lhsImag,527 higherPrecisionElementType);528 rhsReal = builder.createCast(floatingCastKind, rhsReal,529 higherPrecisionElementType);530 rhsImag = builder.createCast(floatingCastKind, rhsImag,531 higherPrecisionElementType);532 533 mlir::Value algebraicResult = buildAlgebraicComplexDiv(534 builder, loc, lhsReal, lhsImag, rhsReal, rhsImag);535 536 mlir::Value resultReal = builder.createComplexReal(loc, algebraicResult);537 mlir::Value resultImag = builder.createComplexImag(loc, algebraicResult);538 539 mlir::Value finalReal =540 builder.createCast(floatingCastKind, resultReal, originalElementType);541 mlir::Value finalImag =542 builder.createCast(floatingCastKind, resultImag, originalElementType);543 return builder.createComplexCreate(loc, finalReal, finalImag);544 }545 }546 547 return buildAlgebraicComplexDiv(builder, loc, lhsReal, lhsImag, rhsReal,548 rhsImag);549}550 551void LoweringPreparePass::lowerComplexDivOp(cir::ComplexDivOp op) {552 cir::CIRBaseBuilderTy builder(getContext());553 builder.setInsertionPointAfter(op);554 mlir::Location loc = op.getLoc();555 mlir::TypedValue<cir::ComplexType> lhs = op.getLhs();556 mlir::TypedValue<cir::ComplexType> rhs = op.getRhs();557 mlir::Value lhsReal = builder.createComplexReal(loc, lhs);558 mlir::Value lhsImag = builder.createComplexImag(loc, lhs);559 mlir::Value rhsReal = builder.createComplexReal(loc, rhs);560 mlir::Value rhsImag = builder.createComplexImag(loc, rhs);561 562 mlir::Value loweredResult =563 lowerComplexDiv(*this, builder, loc, op, lhsReal, lhsImag, rhsReal,564 rhsImag, getContext(), *astCtx);565 op.replaceAllUsesWith(loweredResult);566 op.erase();567}568 569static llvm::StringRef570getComplexMulLibCallName(llvm::APFloat::Semantics semantics) {571 switch (semantics) {572 case llvm::APFloat::S_IEEEhalf:573 return "__mulhc3";574 case llvm::APFloat::S_IEEEsingle:575 return "__mulsc3";576 case llvm::APFloat::S_IEEEdouble:577 return "__muldc3";578 case llvm::APFloat::S_PPCDoubleDouble:579 return "__multc3";580 case llvm::APFloat::S_x87DoubleExtended:581 return "__mulxc3";582 case llvm::APFloat::S_IEEEquad:583 return "__multc3";584 default:585 llvm_unreachable("unsupported floating point type");586 }587}588 589static mlir::Value lowerComplexMul(LoweringPreparePass &pass,590 CIRBaseBuilderTy &builder,591 mlir::Location loc, cir::ComplexMulOp op,592 mlir::Value lhsReal, mlir::Value lhsImag,593 mlir::Value rhsReal, mlir::Value rhsImag) {594 // (a+bi) * (c+di) = (ac-bd) + (ad+bc)i595 mlir::Value resultRealLhs =596 builder.createBinop(loc, lhsReal, cir::BinOpKind::Mul, rhsReal);597 mlir::Value resultRealRhs =598 builder.createBinop(loc, lhsImag, cir::BinOpKind::Mul, rhsImag);599 mlir::Value resultImagLhs =600 builder.createBinop(loc, lhsReal, cir::BinOpKind::Mul, rhsImag);601 mlir::Value resultImagRhs =602 builder.createBinop(loc, lhsImag, cir::BinOpKind::Mul, rhsReal);603 mlir::Value resultReal = builder.createBinop(604 loc, resultRealLhs, cir::BinOpKind::Sub, resultRealRhs);605 mlir::Value resultImag = builder.createBinop(606 loc, resultImagLhs, cir::BinOpKind::Add, resultImagRhs);607 mlir::Value algebraicResult =608 builder.createComplexCreate(loc, resultReal, resultImag);609 610 cir::ComplexType complexTy = op.getType();611 cir::ComplexRangeKind rangeKind = op.getRange();612 if (mlir::isa<cir::IntType>(complexTy.getElementType()) ||613 rangeKind == cir::ComplexRangeKind::Basic ||614 rangeKind == cir::ComplexRangeKind::Improved ||615 rangeKind == cir::ComplexRangeKind::Promoted)616 return algebraicResult;617 618 assert(!cir::MissingFeatures::fastMathFlags());619 620 // Check whether the real part and the imaginary part of the result are both621 // NaN. If so, emit a library call to compute the multiplication instead.622 // We check a value against NaN by comparing the value against itself.623 mlir::Value resultRealIsNaN = builder.createIsNaN(loc, resultReal);624 mlir::Value resultImagIsNaN = builder.createIsNaN(loc, resultImag);625 mlir::Value resultRealAndImagAreNaN =626 builder.createLogicalAnd(loc, resultRealIsNaN, resultImagIsNaN);627 628 return cir::TernaryOp::create(629 builder, loc, resultRealAndImagAreNaN,630 [&](mlir::OpBuilder &, mlir::Location) {631 mlir::Value libCallResult = buildComplexBinOpLibCall(632 pass, builder, &getComplexMulLibCallName, loc, complexTy,633 lhsReal, lhsImag, rhsReal, rhsImag);634 builder.createYield(loc, libCallResult);635 },636 [&](mlir::OpBuilder &, mlir::Location) {637 builder.createYield(loc, algebraicResult);638 })639 .getResult();640}641 642void LoweringPreparePass::lowerComplexMulOp(cir::ComplexMulOp op) {643 cir::CIRBaseBuilderTy builder(getContext());644 builder.setInsertionPointAfter(op);645 mlir::Location loc = op.getLoc();646 mlir::TypedValue<cir::ComplexType> lhs = op.getLhs();647 mlir::TypedValue<cir::ComplexType> rhs = op.getRhs();648 mlir::Value lhsReal = builder.createComplexReal(loc, lhs);649 mlir::Value lhsImag = builder.createComplexImag(loc, lhs);650 mlir::Value rhsReal = builder.createComplexReal(loc, rhs);651 mlir::Value rhsImag = builder.createComplexImag(loc, rhs);652 mlir::Value loweredResult = lowerComplexMul(*this, builder, loc, op, lhsReal,653 lhsImag, rhsReal, rhsImag);654 op.replaceAllUsesWith(loweredResult);655 op.erase();656}657 658void LoweringPreparePass::lowerUnaryOp(cir::UnaryOp op) {659 mlir::Type ty = op.getType();660 if (!mlir::isa<cir::ComplexType>(ty))661 return;662 663 mlir::Location loc = op.getLoc();664 cir::UnaryOpKind opKind = op.getKind();665 666 CIRBaseBuilderTy builder(getContext());667 builder.setInsertionPointAfter(op);668 669 mlir::Value operand = op.getInput();670 mlir::Value operandReal = builder.createComplexReal(loc, operand);671 mlir::Value operandImag = builder.createComplexImag(loc, operand);672 673 mlir::Value resultReal;674 mlir::Value resultImag;675 676 switch (opKind) {677 case cir::UnaryOpKind::Inc:678 case cir::UnaryOpKind::Dec:679 resultReal = builder.createUnaryOp(loc, opKind, operandReal);680 resultImag = operandImag;681 break;682 683 case cir::UnaryOpKind::Plus:684 case cir::UnaryOpKind::Minus:685 resultReal = builder.createUnaryOp(loc, opKind, operandReal);686 resultImag = builder.createUnaryOp(loc, opKind, operandImag);687 break;688 689 case cir::UnaryOpKind::Not:690 resultReal = operandReal;691 resultImag =692 builder.createUnaryOp(loc, cir::UnaryOpKind::Minus, operandImag);693 break;694 }695 696 mlir::Value result = builder.createComplexCreate(loc, resultReal, resultImag);697 op.replaceAllUsesWith(result);698 op.erase();699}700 701cir::FuncOp LoweringPreparePass::getOrCreateDtorFunc(CIRBaseBuilderTy &builder,702 cir::GlobalOp op,703 mlir::Region &dtorRegion,704 cir::CallOp &dtorCall) {705 mlir::OpBuilder::InsertionGuard guard(builder);706 assert(!cir::MissingFeatures::astVarDeclInterface());707 assert(!cir::MissingFeatures::opGlobalThreadLocal());708 709 cir::VoidType voidTy = builder.getVoidTy();710 auto voidPtrTy = cir::PointerType::get(voidTy);711 712 // Look for operations in dtorBlock713 mlir::Block &dtorBlock = dtorRegion.front();714 715 // The first operation should be a get_global to retrieve the address716 // of the global variable we're destroying.717 auto opIt = dtorBlock.getOperations().begin();718 cir::GetGlobalOp ggop = mlir::cast<cir::GetGlobalOp>(*opIt);719 720 // The simple case is just a call to a destructor, like this:721 //722 // %0 = cir.get_global %globalS : !cir.ptr<!rec_S>723 // cir.call %_ZN1SD1Ev(%0) : (!cir.ptr<!rec_S>) -> ()724 // (implicit cir.yield)725 //726 // That is, if the second operation is a call that takes the get_global result727 // as its only operand, and the only other operation is a yield, then we can728 // just return the called function.729 if (dtorBlock.getOperations().size() == 3) {730 auto callOp = mlir::dyn_cast<cir::CallOp>(&*(++opIt));731 auto yieldOp = mlir::dyn_cast<cir::YieldOp>(&*(++opIt));732 if (yieldOp && callOp && callOp.getNumOperands() == 1 &&733 callOp.getArgOperand(0) == ggop) {734 dtorCall = callOp;735 return getCalledFunction(callOp);736 }737 }738 739 // Otherwise, we need to create a helper function to replace the dtor region.740 // This name is kind of arbitrary, but it matches the name that classic741 // codegen uses, based on the expected case that gets us here.742 builder.setInsertionPointAfter(op);743 SmallString<256> fnName("__cxx_global_array_dtor");744 uint32_t cnt = dynamicInitializerNames[fnName]++;745 if (cnt)746 fnName += "." + std::to_string(cnt);747 748 // Create the helper function.749 auto fnType = cir::FuncType::get({voidPtrTy}, voidTy);750 cir::FuncOp dtorFunc =751 buildRuntimeFunction(builder, fnName, op.getLoc(), fnType,752 cir::GlobalLinkageKind::InternalLinkage);753 mlir::Block *entryBB = dtorFunc.addEntryBlock();754 755 // Move everything from the dtor region into the helper function.756 entryBB->getOperations().splice(entryBB->begin(), dtorBlock.getOperations(),757 dtorBlock.begin(), dtorBlock.end());758 759 // Before erasing this, clone it back into the dtor region760 cir::GetGlobalOp dtorGGop =761 mlir::cast<cir::GetGlobalOp>(entryBB->getOperations().front());762 builder.setInsertionPointToStart(&dtorBlock);763 builder.clone(*dtorGGop.getOperation());764 765 // Replace all uses of the help function's get_global with the function766 // argument.767 mlir::Value dtorArg = entryBB->getArgument(0);768 dtorGGop.replaceAllUsesWith(dtorArg);769 dtorGGop.erase();770 771 // Replace the yield in the final block with a return772 mlir::Block &finalBlock = dtorFunc.getBody().back();773 auto yieldOp = cast<cir::YieldOp>(finalBlock.getTerminator());774 builder.setInsertionPoint(yieldOp);775 cir::ReturnOp::create(builder, yieldOp->getLoc());776 yieldOp->erase();777 778 // Create a call to the helper function, passing the original get_global op779 // as the argument.780 cir::GetGlobalOp origGGop =781 mlir::cast<cir::GetGlobalOp>(dtorBlock.getOperations().front());782 builder.setInsertionPointAfter(origGGop);783 mlir::Value ggopResult = origGGop.getResult();784 dtorCall = builder.createCallOp(op.getLoc(), dtorFunc, ggopResult);785 786 // Add a yield after the call.787 auto finalYield = cir::YieldOp::create(builder, op.getLoc());788 789 // Erase everything after the yield.790 dtorBlock.getOperations().erase(std::next(mlir::Block::iterator(finalYield)),791 dtorBlock.end());792 dtorRegion.getBlocks().erase(std::next(dtorRegion.begin()), dtorRegion.end());793 794 return dtorFunc;795}796 797cir::FuncOp798LoweringPreparePass::buildCXXGlobalVarDeclInitFunc(cir::GlobalOp op) {799 // TODO(cir): Store this in the GlobalOp.800 // This should come from the MangleContext, but for now I'm hardcoding it.801 SmallString<256> fnName("__cxx_global_var_init");802 // Get a unique name803 uint32_t cnt = dynamicInitializerNames[fnName]++;804 if (cnt)805 fnName += "." + std::to_string(cnt);806 807 // Create a variable initialization function.808 CIRBaseBuilderTy builder(getContext());809 builder.setInsertionPointAfter(op);810 cir::VoidType voidTy = builder.getVoidTy();811 auto fnType = cir::FuncType::get({}, voidTy);812 FuncOp f = buildRuntimeFunction(builder, fnName, op.getLoc(), fnType,813 cir::GlobalLinkageKind::InternalLinkage);814 815 // Move over the initialzation code of the ctor region.816 mlir::Block *entryBB = f.addEntryBlock();817 if (!op.getCtorRegion().empty()) {818 mlir::Block &block = op.getCtorRegion().front();819 entryBB->getOperations().splice(entryBB->begin(), block.getOperations(),820 block.begin(), std::prev(block.end()));821 }822 823 // Register the destructor call with __cxa_atexit824 mlir::Region &dtorRegion = op.getDtorRegion();825 if (!dtorRegion.empty()) {826 assert(!cir::MissingFeatures::astVarDeclInterface());827 assert(!cir::MissingFeatures::opGlobalThreadLocal());828 829 // Create a variable that binds the atexit to this shared object.830 builder.setInsertionPointToStart(&mlirModule.getBodyRegion().front());831 cir::GlobalOp handle = buildRuntimeVariable(832 builder, "__dso_handle", op.getLoc(), builder.getI8Type(),833 cir::GlobalLinkageKind::ExternalLinkage, cir::VisibilityKind::Hidden);834 835 // If this is a simple call to a destructor, get the called function.836 // Otherwise, create a helper function for the entire dtor region,837 // replacing the current dtor region body with a call to the helper838 // function.839 cir::CallOp dtorCall;840 cir::FuncOp dtorFunc =841 getOrCreateDtorFunc(builder, op, dtorRegion, dtorCall);842 843 // Create a runtime helper function:844 // extern "C" int __cxa_atexit(void (*f)(void *), void *p, void *d);845 auto voidPtrTy = cir::PointerType::get(voidTy);846 auto voidFnTy = cir::FuncType::get({voidPtrTy}, voidTy);847 auto voidFnPtrTy = cir::PointerType::get(voidFnTy);848 auto handlePtrTy = cir::PointerType::get(handle.getSymType());849 auto fnAtExitType =850 cir::FuncType::get({voidFnPtrTy, voidPtrTy, handlePtrTy}, voidTy);851 const char *nameAtExit = "__cxa_atexit";852 cir::FuncOp fnAtExit =853 buildRuntimeFunction(builder, nameAtExit, op.getLoc(), fnAtExitType);854 855 // Replace the dtor (or helper) call with a call to856 // __cxa_atexit(&dtor, &var, &__dso_handle)857 builder.setInsertionPointAfter(dtorCall);858 mlir::Value args[3];859 auto dtorPtrTy = cir::PointerType::get(dtorFunc.getFunctionType());860 // dtorPtrTy861 args[0] = cir::GetGlobalOp::create(builder, dtorCall.getLoc(), dtorPtrTy,862 dtorFunc.getSymName());863 args[0] = cir::CastOp::create(builder, dtorCall.getLoc(), voidFnPtrTy,864 cir::CastKind::bitcast, args[0]);865 args[1] =866 cir::CastOp::create(builder, dtorCall.getLoc(), voidPtrTy,867 cir::CastKind::bitcast, dtorCall.getArgOperand(0));868 args[2] = cir::GetGlobalOp::create(builder, handle.getLoc(), handlePtrTy,869 handle.getSymName());870 builder.createCallOp(dtorCall.getLoc(), fnAtExit, args);871 dtorCall->erase();872 mlir::Block &dtorBlock = dtorRegion.front();873 entryBB->getOperations().splice(entryBB->end(), dtorBlock.getOperations(),874 dtorBlock.begin(),875 std::prev(dtorBlock.end()));876 }877 878 // Replace cir.yield with cir.return879 builder.setInsertionPointToEnd(entryBB);880 mlir::Operation *yieldOp = nullptr;881 if (!op.getCtorRegion().empty()) {882 mlir::Block &block = op.getCtorRegion().front();883 yieldOp = &block.getOperations().back();884 } else {885 assert(!dtorRegion.empty());886 mlir::Block &block = dtorRegion.front();887 yieldOp = &block.getOperations().back();888 }889 890 assert(isa<cir::YieldOp>(*yieldOp));891 cir::ReturnOp::create(builder, yieldOp->getLoc());892 return f;893}894 895void LoweringPreparePass::lowerGlobalOp(GlobalOp op) {896 mlir::Region &ctorRegion = op.getCtorRegion();897 mlir::Region &dtorRegion = op.getDtorRegion();898 899 if (!ctorRegion.empty() || !dtorRegion.empty()) {900 // Build a variable initialization function and move the initialzation code901 // in the ctor region over.902 cir::FuncOp f = buildCXXGlobalVarDeclInitFunc(op);903 904 // Clear the ctor and dtor region905 ctorRegion.getBlocks().clear();906 dtorRegion.getBlocks().clear();907 908 assert(!cir::MissingFeatures::astVarDeclInterface());909 dynamicInitializers.push_back(f);910 }911 912 assert(!cir::MissingFeatures::opGlobalAnnotations());913}914 915template <typename AttributeTy>916static llvm::SmallVector<mlir::Attribute>917prepareCtorDtorAttrList(mlir::MLIRContext *context,918 llvm::ArrayRef<std::pair<std::string, uint32_t>> list) {919 llvm::SmallVector<mlir::Attribute> attrs;920 for (const auto &[name, priority] : list)921 attrs.push_back(AttributeTy::get(context, name, priority));922 return attrs;923}924 925void LoweringPreparePass::buildGlobalCtorDtorList() {926 if (!globalCtorList.empty()) {927 llvm::SmallVector<mlir::Attribute> globalCtors =928 prepareCtorDtorAttrList<cir::GlobalCtorAttr>(&getContext(),929 globalCtorList);930 931 mlirModule->setAttr(cir::CIRDialect::getGlobalCtorsAttrName(),932 mlir::ArrayAttr::get(&getContext(), globalCtors));933 }934 935 if (!globalDtorList.empty()) {936 llvm::SmallVector<mlir::Attribute> globalDtors =937 prepareCtorDtorAttrList<cir::GlobalDtorAttr>(&getContext(),938 globalDtorList);939 mlirModule->setAttr(cir::CIRDialect::getGlobalDtorsAttrName(),940 mlir::ArrayAttr::get(&getContext(), globalDtors));941 }942}943 944void LoweringPreparePass::buildCXXGlobalInitFunc() {945 if (dynamicInitializers.empty())946 return;947 948 // TODO: handle globals with a user-specified initialzation priority.949 // TODO: handle default priority more nicely.950 assert(!cir::MissingFeatures::opGlobalCtorPriority());951 952 SmallString<256> fnName;953 // Include the filename in the symbol name. Including "sub_" matches gcc954 // and makes sure these symbols appear lexicographically behind the symbols955 // with priority (TBD). Module implementation units behave the same956 // way as a non-modular TU with imports.957 // TODO: check CXX20ModuleInits958 if (astCtx->getCurrentNamedModule() &&959 !astCtx->getCurrentNamedModule()->isModuleImplementation()) {960 llvm::raw_svector_ostream out(fnName);961 std::unique_ptr<clang::MangleContext> mangleCtx(962 astCtx->createMangleContext());963 cast<clang::ItaniumMangleContext>(*mangleCtx)964 .mangleModuleInitializer(astCtx->getCurrentNamedModule(), out);965 } else {966 fnName += "_GLOBAL__sub_I_";967 fnName += getTransformedFileName(mlirModule);968 }969 970 CIRBaseBuilderTy builder(getContext());971 builder.setInsertionPointToEnd(&mlirModule.getBodyRegion().back());972 auto fnType = cir::FuncType::get({}, builder.getVoidTy());973 cir::FuncOp f =974 buildRuntimeFunction(builder, fnName, mlirModule.getLoc(), fnType,975 cir::GlobalLinkageKind::ExternalLinkage);976 builder.setInsertionPointToStart(f.addEntryBlock());977 for (cir::FuncOp &f : dynamicInitializers)978 builder.createCallOp(f.getLoc(), f, {});979 // Add the global init function (not the individual ctor functions) to the980 // global ctor list.981 globalCtorList.emplace_back(fnName,982 cir::GlobalCtorAttr::getDefaultPriority());983 984 cir::ReturnOp::create(builder, f.getLoc());985}986 987void LoweringPreparePass::lowerDynamicCastOp(DynamicCastOp op) {988 CIRBaseBuilderTy builder(getContext());989 builder.setInsertionPointAfter(op);990 991 assert(astCtx && "AST context is not available during lowering prepare");992 auto loweredValue = cxxABI->lowerDynamicCast(builder, *astCtx, op);993 994 op.replaceAllUsesWith(loweredValue);995 op.erase();996}997 998static void lowerArrayDtorCtorIntoLoop(cir::CIRBaseBuilderTy &builder,999 clang::ASTContext *astCtx,1000 mlir::Operation *op, mlir::Type eltTy,1001 mlir::Value arrayAddr, uint64_t arrayLen,1002 bool isCtor) {1003 // Generate loop to call into ctor/dtor for every element.1004 mlir::Location loc = op->getLoc();1005 1006 // TODO: instead of getting the size from the AST context, create alias for1007 // PtrDiffTy and unify with CIRGen stuff.1008 const unsigned sizeTypeSize =1009 astCtx->getTypeSize(astCtx->getSignedSizeType());1010 uint64_t endOffset = isCtor ? arrayLen : arrayLen - 1;1011 mlir::Value endOffsetVal =1012 builder.getUnsignedInt(loc, endOffset, sizeTypeSize);1013 1014 auto begin = cir::CastOp::create(builder, loc, eltTy,1015 cir::CastKind::array_to_ptrdecay, arrayAddr);1016 mlir::Value end =1017 cir::PtrStrideOp::create(builder, loc, eltTy, begin, endOffsetVal);1018 mlir::Value start = isCtor ? begin : end;1019 mlir::Value stop = isCtor ? end : begin;1020 1021 mlir::Value tmpAddr = builder.createAlloca(1022 loc, /*addr type*/ builder.getPointerTo(eltTy),1023 /*var type*/ eltTy, "__array_idx", builder.getAlignmentAttr(1));1024 builder.createStore(loc, start, tmpAddr);1025 1026 cir::DoWhileOp loop = builder.createDoWhile(1027 loc,1028 /*condBuilder=*/1029 [&](mlir::OpBuilder &b, mlir::Location loc) {1030 auto currentElement = cir::LoadOp::create(b, loc, eltTy, tmpAddr);1031 mlir::Type boolTy = cir::BoolType::get(b.getContext());1032 auto cmp = cir::CmpOp::create(builder, loc, boolTy, cir::CmpOpKind::ne,1033 currentElement, stop);1034 builder.createCondition(cmp);1035 },1036 /*bodyBuilder=*/1037 [&](mlir::OpBuilder &b, mlir::Location loc) {1038 auto currentElement = cir::LoadOp::create(b, loc, eltTy, tmpAddr);1039 1040 cir::CallOp ctorCall;1041 op->walk([&](cir::CallOp c) { ctorCall = c; });1042 assert(ctorCall && "expected ctor call");1043 1044 // Array elements get constructed in order but destructed in reverse.1045 mlir::Value stride;1046 if (isCtor)1047 stride = builder.getUnsignedInt(loc, 1, sizeTypeSize);1048 else1049 stride = builder.getSignedInt(loc, -1, sizeTypeSize);1050 1051 ctorCall->moveBefore(stride.getDefiningOp());1052 ctorCall->setOperand(0, currentElement);1053 auto nextElement = cir::PtrStrideOp::create(builder, loc, eltTy,1054 currentElement, stride);1055 1056 // Store the element pointer to the temporary variable1057 builder.createStore(loc, nextElement, tmpAddr);1058 builder.createYield(loc);1059 });1060 1061 op->replaceAllUsesWith(loop);1062 op->erase();1063}1064 1065void LoweringPreparePass::lowerArrayDtor(cir::ArrayDtor op) {1066 CIRBaseBuilderTy builder(getContext());1067 builder.setInsertionPointAfter(op.getOperation());1068 1069 mlir::Type eltTy = op->getRegion(0).getArgument(0).getType();1070 assert(!cir::MissingFeatures::vlas());1071 auto arrayLen =1072 mlir::cast<cir::ArrayType>(op.getAddr().getType().getPointee()).getSize();1073 lowerArrayDtorCtorIntoLoop(builder, astCtx, op, eltTy, op.getAddr(), arrayLen,1074 false);1075}1076 1077void LoweringPreparePass::lowerArrayCtor(cir::ArrayCtor op) {1078 cir::CIRBaseBuilderTy builder(getContext());1079 builder.setInsertionPointAfter(op.getOperation());1080 1081 mlir::Type eltTy = op->getRegion(0).getArgument(0).getType();1082 assert(!cir::MissingFeatures::vlas());1083 auto arrayLen =1084 mlir::cast<cir::ArrayType>(op.getAddr().getType().getPointee()).getSize();1085 lowerArrayDtorCtorIntoLoop(builder, astCtx, op, eltTy, op.getAddr(), arrayLen,1086 true);1087}1088 1089void LoweringPreparePass::runOnOp(mlir::Operation *op) {1090 if (auto arrayCtor = dyn_cast<cir::ArrayCtor>(op)) {1091 lowerArrayCtor(arrayCtor);1092 } else if (auto arrayDtor = dyn_cast<cir::ArrayDtor>(op)) {1093 lowerArrayDtor(arrayDtor);1094 } else if (auto cast = mlir::dyn_cast<cir::CastOp>(op)) {1095 lowerCastOp(cast);1096 } else if (auto complexDiv = mlir::dyn_cast<cir::ComplexDivOp>(op)) {1097 lowerComplexDivOp(complexDiv);1098 } else if (auto complexMul = mlir::dyn_cast<cir::ComplexMulOp>(op)) {1099 lowerComplexMulOp(complexMul);1100 } else if (auto glob = mlir::dyn_cast<cir::GlobalOp>(op)) {1101 lowerGlobalOp(glob);1102 } else if (auto dynamicCast = mlir::dyn_cast<cir::DynamicCastOp>(op)) {1103 lowerDynamicCastOp(dynamicCast);1104 } else if (auto unary = mlir::dyn_cast<cir::UnaryOp>(op)) {1105 lowerUnaryOp(unary);1106 } else if (auto fnOp = dyn_cast<cir::FuncOp>(op)) {1107 if (auto globalCtor = fnOp.getGlobalCtorPriority())1108 globalCtorList.emplace_back(fnOp.getName(), globalCtor.value());1109 else if (auto globalDtor = fnOp.getGlobalDtorPriority())1110 globalDtorList.emplace_back(fnOp.getName(), globalDtor.value());1111 }1112}1113 1114void LoweringPreparePass::runOnOperation() {1115 mlir::Operation *op = getOperation();1116 if (isa<::mlir::ModuleOp>(op))1117 mlirModule = cast<::mlir::ModuleOp>(op);1118 1119 llvm::SmallVector<mlir::Operation *> opsToTransform;1120 1121 op->walk([&](mlir::Operation *op) {1122 if (mlir::isa<cir::ArrayCtor, cir::ArrayDtor, cir::CastOp,1123 cir::ComplexMulOp, cir::ComplexDivOp, cir::DynamicCastOp,1124 cir::FuncOp, cir::GlobalOp, cir::UnaryOp>(op))1125 opsToTransform.push_back(op);1126 });1127 1128 for (mlir::Operation *o : opsToTransform)1129 runOnOp(o);1130 1131 buildCXXGlobalInitFunc();1132 buildGlobalCtorDtorList();1133}1134 1135std::unique_ptr<Pass> mlir::createLoweringPreparePass() {1136 return std::make_unique<LoweringPreparePass>();1137}1138 1139std::unique_ptr<Pass>1140mlir::createLoweringPreparePass(clang::ASTContext *astCtx) {1141 auto pass = std::make_unique<LoweringPreparePass>();1142 pass->setASTContext(astCtx);1143 return std::move(pass);1144}1145