928 lines · cpp
1//===- BufferizeHLFIR.cpp - Bufferize HLFIR ------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8// This file defines a pass that bufferize hlfir.expr. It translates operations9// producing or consuming hlfir.expr into operations operating on memory.10// An hlfir.expr is translated to a tuple<variable address, cleanupflag>11// where cleanupflag is set to true if storage for the expression was allocated12// on the heap.13//===----------------------------------------------------------------------===//14 15#include "flang/Optimizer/Builder/Character.h"16#include "flang/Optimizer/Builder/FIRBuilder.h"17#include "flang/Optimizer/Builder/HLFIRTools.h"18#include "flang/Optimizer/Builder/MutableBox.h"19#include "flang/Optimizer/Builder/Runtime/Allocatable.h"20#include "flang/Optimizer/Builder/Runtime/Derived.h"21#include "flang/Optimizer/Builder/Todo.h"22#include "flang/Optimizer/Dialect/FIRDialect.h"23#include "flang/Optimizer/Dialect/FIROps.h"24#include "flang/Optimizer/Dialect/FIRType.h"25#include "flang/Optimizer/Dialect/Support/FIRContext.h"26#include "flang/Optimizer/HLFIR/HLFIRDialect.h"27#include "flang/Optimizer/HLFIR/HLFIROps.h"28#include "flang/Optimizer/HLFIR/Passes.h"29#include "flang/Optimizer/OpenMP/Passes.h"30#include "mlir/Dialect/OpenMP/OpenMPDialect.h"31#include "mlir/IR/Dominance.h"32#include "mlir/IR/PatternMatch.h"33#include "mlir/Pass/Pass.h"34#include "mlir/Pass/PassManager.h"35#include "mlir/Transforms/DialectConversion.h"36#include "llvm/ADT/TypeSwitch.h"37 38namespace hlfir {39#define GEN_PASS_DEF_BUFFERIZEHLFIR40#include "flang/Optimizer/HLFIR/Passes.h.inc"41} // namespace hlfir42 43namespace {44 45/// Helper to create tuple from a bufferized expr storage and clean up46/// instruction flag. The storage is an HLFIR variable so that it can47/// be manipulated as a variable later (all shape and length information48/// cam be retrieved from it).49static mlir::Value packageBufferizedExpr(mlir::Location loc,50 fir::FirOpBuilder &builder,51 hlfir::Entity storage,52 mlir::Value mustFree) {53 auto tupleType = mlir::TupleType::get(54 builder.getContext(),55 mlir::TypeRange{storage.getType(), mustFree.getType()});56 auto undef = fir::UndefOp::create(builder, loc, tupleType);57 auto insert = fir::InsertValueOp::create(58 builder, loc, tupleType, undef, mustFree,59 builder.getArrayAttr(60 {builder.getIntegerAttr(builder.getIndexType(), 1)}));61 return fir::InsertValueOp::create(62 builder, loc, tupleType, insert, storage,63 builder.getArrayAttr(64 {builder.getIntegerAttr(builder.getIndexType(), 0)}));65}66 67/// Helper to create tuple from a bufferized expr storage and constant68/// boolean clean-up flag.69static mlir::Value packageBufferizedExpr(mlir::Location loc,70 fir::FirOpBuilder &builder,71 hlfir::Entity storage, bool mustFree) {72 mlir::Value mustFreeValue = builder.createBool(loc, mustFree);73 return packageBufferizedExpr(loc, builder, storage, mustFreeValue);74}75 76/// Helper to extract the storage from a tuple created by packageBufferizedExpr.77/// It assumes no tuples are used as HLFIR operation operands, which is78/// currently enforced by the verifiers that only accept HLFIR value or79/// variable types which do not include tuples.80static hlfir::Entity getBufferizedExprStorage(mlir::Value bufferizedExpr) {81 auto tupleType = mlir::dyn_cast<mlir::TupleType>(bufferizedExpr.getType());82 if (!tupleType)83 return hlfir::Entity{bufferizedExpr};84 assert(tupleType.size() == 2 && "unexpected tuple type");85 if (auto insert = bufferizedExpr.getDefiningOp<fir::InsertValueOp>())86 if (insert.getVal().getType() == tupleType.getType(0))87 return hlfir::Entity{insert.getVal()};88 TODO(bufferizedExpr.getLoc(), "general extract storage case");89}90 91/// Helper to extract the clean-up flag from a tuple created by92/// packageBufferizedExpr.93static mlir::Value getBufferizedExprMustFreeFlag(mlir::Value bufferizedExpr) {94 auto tupleType = mlir::dyn_cast<mlir::TupleType>(bufferizedExpr.getType());95 if (!tupleType)96 return bufferizedExpr;97 assert(tupleType.size() == 2 && "unexpected tuple type");98 if (auto insert = bufferizedExpr.getDefiningOp<fir::InsertValueOp>())99 if (auto insert0 = insert.getAdt().getDefiningOp<fir::InsertValueOp>())100 if (insert0.getVal().getType() == tupleType.getType(1))101 return insert0.getVal();102 TODO(bufferizedExpr.getLoc(), "general extract storage case");103}104 105static std::pair<hlfir::Entity, mlir::Value>106createArrayTemp(mlir::Location loc, fir::FirOpBuilder &builder,107 mlir::Type exprType, mlir::Value shape,108 llvm::ArrayRef<mlir::Value> extents,109 llvm::ArrayRef<mlir::Value> lenParams,110 std::optional<hlfir::Entity> polymorphicMold) {111 auto sequenceType = mlir::cast<fir::SequenceType>(112 hlfir::getFortranElementOrSequenceType(exprType));113 114 auto genTempDeclareOp =115 [](fir::FirOpBuilder &builder, mlir::Location loc, mlir::Value memref,116 llvm::StringRef name, mlir::Value shape,117 llvm::ArrayRef<mlir::Value> typeParams,118 fir::FortranVariableFlagsAttr attrs) -> mlir::Value {119 auto declareOp =120 hlfir::DeclareOp::create(builder, loc, memref, name, shape, typeParams,121 /*dummy_scope=*/nullptr, /*storage=*/nullptr,122 /*storage_offset=*/0, attrs);123 return declareOp.getBase();124 };125 126 auto [base, isHeapAlloc] = builder.createArrayTemp(127 loc, sequenceType, shape, extents, lenParams, genTempDeclareOp,128 polymorphicMold ? polymorphicMold->getFirBase() : nullptr);129 hlfir::Entity temp = hlfir::Entity{base};130 assert(!temp.isAllocatable() && "temp must have been allocated");131 return {temp, builder.createBool(loc, isHeapAlloc)};132}133 134/// Copy \p source into a new temporary and package the temporary into a135/// <temp,cleanup> tuple. The temporary may be heap or stack allocated.136static mlir::Value copyInTempAndPackage(mlir::Location loc,137 fir::FirOpBuilder &builder,138 hlfir::Entity source) {139 auto [temp, mustFree] = hlfir::createTempFromMold(loc, builder, source);140 assert(!temp.isAllocatable() && "expect temp to already be allocated");141 hlfir::AssignOp::create(builder, loc, source, temp, /*realloc=*/false,142 /*keep_lhs_length_if_realloc=*/false,143 /*temporary_lhs=*/true);144 return packageBufferizedExpr(loc, builder, temp, mustFree);145}146 147struct AsExprOpConversion : public mlir::OpConversionPattern<hlfir::AsExprOp> {148 using mlir::OpConversionPattern<hlfir::AsExprOp>::OpConversionPattern;149 explicit AsExprOpConversion(mlir::MLIRContext *ctx)150 : mlir::OpConversionPattern<hlfir::AsExprOp>{ctx} {}151 llvm::LogicalResult152 matchAndRewrite(hlfir::AsExprOp asExpr, OpAdaptor adaptor,153 mlir::ConversionPatternRewriter &rewriter) const override {154 mlir::Location loc = asExpr->getLoc();155 auto module = asExpr->getParentOfType<mlir::ModuleOp>();156 fir::FirOpBuilder builder(rewriter, module);157 if (asExpr.isMove()) {158 // Move variable storage for the hlfir.expr buffer.159 mlir::Value bufferizedExpr = packageBufferizedExpr(160 loc, builder, hlfir::Entity{adaptor.getVar()}, adaptor.getMustFree());161 rewriter.replaceOp(asExpr, bufferizedExpr);162 return mlir::success();163 }164 // Otherwise, create a copy in a new buffer.165 hlfir::Entity source = hlfir::Entity{adaptor.getVar()};166 mlir::Value bufferizedExpr = copyInTempAndPackage(loc, builder, source);167 rewriter.replaceOp(asExpr, bufferizedExpr);168 return mlir::success();169 }170};171 172struct ShapeOfOpConversion173 : public mlir::OpConversionPattern<hlfir::ShapeOfOp> {174 using mlir::OpConversionPattern<hlfir::ShapeOfOp>::OpConversionPattern;175 176 llvm::LogicalResult177 matchAndRewrite(hlfir::ShapeOfOp shapeOf, OpAdaptor adaptor,178 mlir::ConversionPatternRewriter &rewriter) const override {179 mlir::Location loc = shapeOf.getLoc();180 mlir::ModuleOp mod = shapeOf->getParentOfType<mlir::ModuleOp>();181 fir::FirOpBuilder builder(rewriter, mod);182 183 mlir::Value shape;184 hlfir::Entity bufferizedExpr{getBufferizedExprStorage(adaptor.getExpr())};185 if (bufferizedExpr.isVariable()) {186 shape = hlfir::genShape(loc, builder, bufferizedExpr);187 } else {188 // everything else failed so try to create a shape from static type info189 hlfir::ExprType exprTy =190 mlir::dyn_cast_or_null<hlfir::ExprType>(adaptor.getExpr().getType());191 if (exprTy)192 shape = hlfir::genExprShape(builder, loc, exprTy);193 }194 // expected to never happen195 if (!shape)196 return emitError(loc,197 "Unresolvable hlfir.shape_of where extents are unknown");198 199 rewriter.replaceOp(shapeOf, shape);200 return mlir::success();201 }202};203 204struct ApplyOpConversion : public mlir::OpConversionPattern<hlfir::ApplyOp> {205 using mlir::OpConversionPattern<hlfir::ApplyOp>::OpConversionPattern;206 explicit ApplyOpConversion(mlir::MLIRContext *ctx)207 : mlir::OpConversionPattern<hlfir::ApplyOp>{ctx} {}208 llvm::LogicalResult209 matchAndRewrite(hlfir::ApplyOp apply, OpAdaptor adaptor,210 mlir::ConversionPatternRewriter &rewriter) const override {211 mlir::Location loc = apply->getLoc();212 hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr());213 mlir::Type resultType = hlfir::getVariableElementType(bufferizedExpr);214 mlir::Value result = hlfir::DesignateOp::create(215 rewriter, loc, resultType, bufferizedExpr, adaptor.getIndices(),216 adaptor.getTypeparams());217 if (fir::isa_trivial(apply.getType())) {218 result = fir::LoadOp::create(rewriter, loc, result);219 } else {220 fir::FirOpBuilder builder(rewriter, apply.getOperation());221 result =222 packageBufferizedExpr(loc, builder, hlfir::Entity{result}, false);223 }224 rewriter.replaceOp(apply, result);225 return mlir::success();226 }227};228 229struct AssignOpConversion : public mlir::OpConversionPattern<hlfir::AssignOp> {230 using mlir::OpConversionPattern<hlfir::AssignOp>::OpConversionPattern;231 explicit AssignOpConversion(mlir::MLIRContext *ctx)232 : mlir::OpConversionPattern<hlfir::AssignOp>{ctx} {}233 llvm::LogicalResult234 matchAndRewrite(hlfir::AssignOp assign, OpAdaptor adaptor,235 mlir::ConversionPatternRewriter &rewriter) const override {236 llvm::SmallVector<mlir::Value> newOperands;237 for (mlir::Value operand : adaptor.getOperands())238 newOperands.push_back(getBufferizedExprStorage(operand));239 rewriter.startOpModification(assign);240 assign->setOperands(newOperands);241 rewriter.finalizeOpModification(assign);242 return mlir::success();243 }244};245 246struct ConcatOpConversion : public mlir::OpConversionPattern<hlfir::ConcatOp> {247 using mlir::OpConversionPattern<hlfir::ConcatOp>::OpConversionPattern;248 explicit ConcatOpConversion(mlir::MLIRContext *ctx)249 : mlir::OpConversionPattern<hlfir::ConcatOp>{ctx} {}250 llvm::LogicalResult251 matchAndRewrite(hlfir::ConcatOp concat, OpAdaptor adaptor,252 mlir::ConversionPatternRewriter &rewriter) const override {253 mlir::Location loc = concat->getLoc();254 fir::FirOpBuilder builder(rewriter, concat.getOperation());255 assert(adaptor.getStrings().size() >= 2 &&256 "must have at least two strings operands");257 if (adaptor.getStrings().size() > 2)258 TODO(loc, "codegen of optimized chained concatenation of more than two "259 "strings");260 hlfir::Entity lhs = getBufferizedExprStorage(adaptor.getStrings()[0]);261 hlfir::Entity rhs = getBufferizedExprStorage(adaptor.getStrings()[1]);262 auto [lhsExv, c1] = hlfir::translateToExtendedValue(loc, builder, lhs);263 auto [rhsExv, c2] = hlfir::translateToExtendedValue(loc, builder, rhs);264 assert(!c1 && !c2 && "expected variables");265 fir::ExtendedValue res =266 fir::factory::CharacterExprHelper{builder, loc}.createConcatenate(267 *lhsExv.getCharBox(), *rhsExv.getCharBox());268 // Ensure the memory type is the same as the result type.269 mlir::Type addrType = fir::ReferenceType::get(270 hlfir::getFortranElementType(concat.getResult().getType()));271 mlir::Value cast = builder.createConvert(loc, addrType, fir::getBase(res));272 res = fir::substBase(res, cast);273 hlfir::Entity hlfirTempRes =274 hlfir::Entity{hlfir::genDeclare(loc, builder, res, "tmp",275 fir::FortranVariableFlagsAttr{})276 .getBase()};277 mlir::Value bufferizedExpr =278 packageBufferizedExpr(loc, builder, hlfirTempRes, false);279 rewriter.replaceOp(concat, bufferizedExpr);280 return mlir::success();281 }282};283 284struct SetLengthOpConversion285 : public mlir::OpConversionPattern<hlfir::SetLengthOp> {286 using mlir::OpConversionPattern<hlfir::SetLengthOp>::OpConversionPattern;287 explicit SetLengthOpConversion(mlir::MLIRContext *ctx)288 : mlir::OpConversionPattern<hlfir::SetLengthOp>{ctx} {}289 llvm::LogicalResult290 matchAndRewrite(hlfir::SetLengthOp setLength, OpAdaptor adaptor,291 mlir::ConversionPatternRewriter &rewriter) const override {292 mlir::Location loc = setLength->getLoc();293 fir::FirOpBuilder builder(rewriter, setLength.getOperation());294 // Create a temp with the new length.295 hlfir::Entity string = getBufferizedExprStorage(adaptor.getString());296 auto charType = hlfir::getFortranElementType(setLength.getType());297 llvm::StringRef tmpName{".tmp"};298 llvm::SmallVector<mlir::Value, 1> lenParams{adaptor.getLength()};299 auto alloca = builder.createTemporary(loc, charType, tmpName,300 /*shape=*/{}, lenParams);301 auto declareOp = hlfir::DeclareOp::create(302 builder, loc, alloca, tmpName, /*shape=*/mlir::Value{}, lenParams);303 hlfir::Entity temp{declareOp.getBase()};304 // Assign string value to the created temp.305 hlfir::AssignOp::create(builder, loc, string, temp,306 /*realloc=*/false,307 /*keep_lhs_length_if_realloc=*/false,308 /*temporary_lhs=*/true);309 mlir::Value bufferizedExpr =310 packageBufferizedExpr(loc, builder, temp, false);311 rewriter.replaceOp(setLength, bufferizedExpr);312 return mlir::success();313 }314};315 316struct GetLengthOpConversion317 : public mlir::OpConversionPattern<hlfir::GetLengthOp> {318 using mlir::OpConversionPattern<hlfir::GetLengthOp>::OpConversionPattern;319 explicit GetLengthOpConversion(mlir::MLIRContext *ctx)320 : mlir::OpConversionPattern<hlfir::GetLengthOp>{ctx} {}321 llvm::LogicalResult322 matchAndRewrite(hlfir::GetLengthOp getLength, OpAdaptor adaptor,323 mlir::ConversionPatternRewriter &rewriter) const override {324 mlir::Location loc = getLength->getLoc();325 fir::FirOpBuilder builder(rewriter, getLength.getOperation());326 hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr());327 mlir::Value length = hlfir::genCharLength(loc, builder, bufferizedExpr);328 if (!length)329 return rewriter.notifyMatchFailure(330 getLength, "could not deduce length from GetLengthOp operand");331 length = builder.createConvert(loc, builder.getIndexType(), length);332 rewriter.replaceOp(getLength, length);333 return mlir::success();334 }335};336 337/// The current hlfir.associate lowering does not handle multiple uses of a338/// non-trivial expression value because it generates the cleanup for the339/// expression bufferization at hlfir.end_associate. If there was more than one340/// hlfir.end_associate, it would be cleaned up multiple times, perhaps before341/// one of the other uses.342/// Note that we have to be careful about expressions used by a single343/// hlfir.end_associate that may be executed more times than the producer344/// of the expression value. This may also cause multiple clean-ups345/// for the same memory (e.g. cause double-free errors). For example,346/// hlfir.end_associate inside hlfir.elemental may cause such issues347/// for expressions produced outside of hlfir.elemental.348static bool allOtherUsesAreSafeForAssociate(mlir::Value value,349 mlir::Operation *currentUse,350 mlir::Operation *endAssociate) {351 // If value producer is from a different region than352 // hlfir.associate/end_associate, then conservatively assume353 // that the hlfir.end_associate may execute more times than354 // the value producer.355 // TODO: this may be improved for operations that cannot356 // result in multiple executions (e.g. ifOp).357 if (value.getParentRegion() != currentUse->getParentRegion() ||358 (endAssociate &&359 value.getParentRegion() != endAssociate->getParentRegion()))360 return false;361 362 for (mlir::Operation *useOp : value.getUsers()) {363 // Ignore DestroyOp's that do not imply finalization.364 // If finalization is implied, then we must delegate365 // the finalization to the correspoding EndAssociateOp,366 // but we currently do not; so we disable the buffer367 // reuse in this case.368 if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(useOp)) {369 if (destroy.mustFinalizeExpr())370 return false;371 else372 continue;373 }374 375 if (useOp != currentUse) {376 // hlfir.shape_of and hlfir.get_length will not disrupt cleanup so it is377 // safe for hlfir.associate. These operations might read from the box and378 // so they need to come before the hflir.end_associate (which may379 // deallocate).380 if (mlir::isa<hlfir::ShapeOfOp>(useOp) ||381 mlir::isa<hlfir::GetLengthOp>(useOp)) {382 if (!endAssociate)383 continue;384 // If useOp dominates the endAssociate, then it is definitely safe.385 if (useOp->getBlock() != endAssociate->getBlock())386 if (mlir::DominanceInfo{}.dominates(useOp, endAssociate))387 continue;388 if (useOp->isBeforeInBlock(endAssociate))389 continue;390 }391 return false;392 }393 }394 return true;395}396 397static void eraseAllUsesInDestroys(mlir::Value value,398 mlir::ConversionPatternRewriter &rewriter) {399 for (mlir::Operation *useOp : value.getUsers())400 if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(useOp)) {401 assert(!destroy.mustFinalizeExpr() &&402 "deleting DestroyOp with finalize attribute");403 rewriter.eraseOp(destroy);404 }405}406 407struct AssociateOpConversion408 : public mlir::OpConversionPattern<hlfir::AssociateOp> {409 using mlir::OpConversionPattern<hlfir::AssociateOp>::OpConversionPattern;410 explicit AssociateOpConversion(mlir::MLIRContext *ctx)411 : mlir::OpConversionPattern<hlfir::AssociateOp>{ctx} {}412 llvm::LogicalResult413 matchAndRewrite(hlfir::AssociateOp associate, OpAdaptor adaptor,414 mlir::ConversionPatternRewriter &rewriter) const override {415 mlir::Location loc = associate->getLoc();416 fir::FirOpBuilder builder(rewriter, associate.getOperation());417 mlir::Value bufferizedExpr = getBufferizedExprStorage(adaptor.getSource());418 const bool isTrivialValue = fir::isa_trivial(bufferizedExpr.getType());419 420 auto getEndAssociate =421 [](hlfir::AssociateOp associate) -> mlir::Operation * {422 for (mlir::Operation *useOp : associate->getUsers())423 if (mlir::isa<hlfir::EndAssociateOp>(useOp))424 return useOp;425 // happens in some hand coded mlir in tests426 return nullptr;427 };428 429 auto replaceWith = [&](mlir::Value hlfirVar, mlir::Value firVar,430 mlir::Value flag) {431 // 0-dim variables may need special handling:432 // %0 = hlfir.as_expr %x move %true :433 // (!fir.box<!fir.heap<!fir.type<_T{y:i32}>>>, i1) ->434 // !hlfir.expr<!fir.type<_T{y:i32}>>435 // %1:3 = hlfir.associate %0 {adapt.valuebyref} :436 // (!hlfir.expr<!fir.type<_T{y:i32}>>) ->437 // (!fir.ref<!fir.type<_T{y:i32}>>,438 // !fir.ref<!fir.type<_T{y:i32}>>,439 // i1)440 //441 // !fir.box<!fir.heap<!fir.type<_T{y:i32}>>> value must be442 // propagated as the box address !fir.ref<!fir.type<_T{y:i32}>>.443 auto adjustVar = [&](mlir::Value sourceVar, mlir::Type assocType) {444 if ((mlir::isa<fir::BaseBoxType>(sourceVar.getType()) &&445 !mlir::isa<fir::BaseBoxType>(assocType)) ||446 ((mlir::isa<fir::BoxCharType>(sourceVar.getType()) &&447 !mlir::isa<fir::BoxCharType>(assocType)))) {448 sourceVar =449 fir::BoxAddrOp::create(builder, loc, assocType, sourceVar);450 } else {451 sourceVar = builder.createConvert(loc, assocType, sourceVar);452 }453 return sourceVar;454 };455 456 mlir::Type associateHlfirVarType = associate.getResultTypes()[0];457 hlfirVar = adjustVar(hlfirVar, associateHlfirVarType);458 mlir::Type associateFirVarType = associate.getResultTypes()[1];459 firVar = adjustVar(firVar, associateFirVarType);460 // FIXME: note that the AssociateOp that is being erased461 // here will continue to be a user of the original Source462 // operand (e.g. a result of hlfir.elemental), because463 // the erasure is not immediate in the rewriter.464 // In case there are multiple uses of the Source operand,465 // the allOtherUsesAreSafeForAssociate() below will always466 // see them, so there is no way to reuse the buffer.467 // I think we have to run this analysis before doing468 // the conversions, so that we can analyze HLFIR in its469 // original form and decide which of the AssociateOp470 // users of hlfir.expr can reuse the buffer (if it can).471 rewriter.replaceOp(associate, {hlfirVar, firVar, flag});472 };473 474 // If this is the last use of the expression value and this is an hlfir.expr475 // that was bufferized, re-use the storage.476 // Otherwise, create a temp and assign the storage to it.477 //478 // WARNING: it is important to use the original Source operand479 // of the AssociateOp to look for the users, because its replacement480 // has zero materialized users at this point.481 // So allOtherUsesAreSafeForAssociate() may incorrectly return482 // true here.483 if (!isTrivialValue && allOtherUsesAreSafeForAssociate(484 associate.getSource(), associate.getOperation(),485 getEndAssociate(associate))) {486 // Re-use hlfir.expr buffer if this is the only use of the hlfir.expr487 // outside of the hlfir.destroy. Take on the cleaning-up responsibility488 // for the related hlfir.end_associate, and erase the hlfir.destroy (if489 // any).490 mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getSource());491 mlir::Value firBase = hlfir::Entity{bufferizedExpr}.getFirBase();492 replaceWith(bufferizedExpr, firBase, mustFree);493 eraseAllUsesInDestroys(associate.getSource(), rewriter);494 // Make sure to erase the hlfir.destroy if there is an indirection through495 // a hlfir.no_reassoc operation.496 if (auto noReassoc = mlir::dyn_cast_or_null<hlfir::NoReassocOp>(497 associate.getSource().getDefiningOp()))498 eraseAllUsesInDestroys(noReassoc.getVal(), rewriter);499 return mlir::success();500 }501 if (isTrivialValue) {502 llvm::SmallVector<mlir::NamedAttribute, 1> attrs;503 if (associate->hasAttr(fir::getAdaptToByRefAttrName())) {504 attrs.push_back(fir::getAdaptToByRefAttr(builder));505 }506 llvm::StringRef name = "";507 if (associate.getUniqName())508 name = *associate.getUniqName();509 auto temp =510 builder.createTemporary(loc, bufferizedExpr.getType(), name, attrs);511 fir::StoreOp::create(builder, loc, bufferizedExpr, temp);512 mlir::Value mustFree = builder.createBool(loc, false);513 replaceWith(temp, temp, mustFree);514 return mlir::success();515 }516 // non-trivial value with more than one use. We will have to make a copy and517 // use that518 hlfir::Entity source = hlfir::Entity{bufferizedExpr};519 mlir::Value bufferTuple = copyInTempAndPackage(loc, builder, source);520 bufferizedExpr = getBufferizedExprStorage(bufferTuple);521 replaceWith(bufferizedExpr, hlfir::Entity{bufferizedExpr}.getFirBase(),522 getBufferizedExprMustFreeFlag(bufferTuple));523 return mlir::success();524 }525};526 527static void genBufferDestruction(mlir::Location loc, fir::FirOpBuilder &builder,528 mlir::Value var, mlir::Value mustFree,529 bool mustFinalize) {530 auto genFreeOrFinalize = [&](bool doFree, bool deallocComponents,531 bool doFinalize) {532 if (!doFree && !deallocComponents && !doFinalize)533 return;534 535 mlir::Value addr = var;536 537 // fir::FreeMemOp operand type must be a fir::HeapType.538 mlir::Type heapType = fir::HeapType::get(539 hlfir::getFortranElementOrSequenceType(var.getType()));540 if (mlir::isa<fir::BaseBoxType, fir::BoxCharType>(var.getType())) {541 if (mustFinalize && !mlir::isa<fir::BaseBoxType>(var.getType()))542 fir::emitFatalError(loc, "non-finalizable variable");543 544 addr = fir::BoxAddrOp::create(builder, loc, heapType, var);545 } else {546 if (!mlir::isa<fir::HeapType>(var.getType()))547 addr = fir::ConvertOp::create(builder, loc, heapType, var);548 549 if (mustFinalize || deallocComponents) {550 // Embox the raw pointer using proper shape and type params551 // (note that the shape might be visible via the array finalization552 // routines).553 if (!hlfir::isFortranEntity(var))554 TODO(loc, "need a Fortran entity to create a box");555 556 hlfir::Entity entity{var};557 llvm::SmallVector<mlir::Value> lenParams;558 hlfir::genLengthParameters(loc, builder, entity, lenParams);559 mlir::Value shape;560 if (entity.isArray())561 shape = hlfir::genShape(loc, builder, entity);562 mlir::Type boxType = fir::BoxType::get(heapType);563 var = builder.createBox(loc, boxType, addr, shape, /*slice=*/nullptr,564 lenParams, /*tdesc=*/nullptr);565 }566 }567 568 if (mustFinalize)569 fir::runtime::genDerivedTypeFinalize(builder, loc, var);570 571 // If there are allocatable components, they need to be deallocated572 // (regardless of the mustFree and mustFinalize settings).573 if (deallocComponents)574 fir::runtime::genDerivedTypeDestroyWithoutFinalization(builder, loc, var);575 576 if (doFree)577 fir::FreeMemOp::create(builder, loc, addr);578 };579 bool deallocComponents = hlfir::mayHaveAllocatableComponent(var.getType());580 581 auto genFree = [&]() {582 genFreeOrFinalize(/*doFree=*/true, /*deallocComponents=*/false,583 /*doFinalize=*/false);584 };585 if (auto cstMustFree = fir::getIntIfConstant(mustFree)) {586 genFreeOrFinalize(*cstMustFree != 0 ? true : false, deallocComponents,587 mustFinalize);588 return;589 }590 591 // If mustFree is dynamic, first, deallocate any allocatable592 // components and finalize.593 genFreeOrFinalize(/*doFree=*/false, deallocComponents,594 /*doFinalize=*/mustFinalize);595 // Conditionally free the memory.596 builder.genIfThen(loc, mustFree).genThen(genFree).end();597}598 599struct EndAssociateOpConversion600 : public mlir::OpConversionPattern<hlfir::EndAssociateOp> {601 using mlir::OpConversionPattern<hlfir::EndAssociateOp>::OpConversionPattern;602 explicit EndAssociateOpConversion(mlir::MLIRContext *ctx)603 : mlir::OpConversionPattern<hlfir::EndAssociateOp>{ctx} {}604 llvm::LogicalResult605 matchAndRewrite(hlfir::EndAssociateOp endAssociate, OpAdaptor adaptor,606 mlir::ConversionPatternRewriter &rewriter) const override {607 mlir::Location loc = endAssociate->getLoc();608 fir::FirOpBuilder builder(rewriter, endAssociate.getOperation());609 genBufferDestruction(loc, builder, adaptor.getVar(), adaptor.getMustFree(),610 /*mustFinalize=*/false);611 rewriter.eraseOp(endAssociate);612 return mlir::success();613 }614};615 616struct DestroyOpConversion617 : public mlir::OpConversionPattern<hlfir::DestroyOp> {618 using mlir::OpConversionPattern<hlfir::DestroyOp>::OpConversionPattern;619 explicit DestroyOpConversion(mlir::MLIRContext *ctx)620 : mlir::OpConversionPattern<hlfir::DestroyOp>{ctx} {}621 llvm::LogicalResult622 matchAndRewrite(hlfir::DestroyOp destroy, OpAdaptor adaptor,623 mlir::ConversionPatternRewriter &rewriter) const override {624 // If expr was bufferized on the heap, now is time to deallocate the buffer.625 mlir::Location loc = destroy->getLoc();626 hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr());627 if (!fir::isa_trivial(bufferizedExpr.getType())) {628 fir::FirOpBuilder builder(rewriter, destroy.getOperation());629 mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getExpr());630 // Passing FIR base might be enough for cases when631 // component deallocation and finalization are not required.632 // If extra BoxAddr operations become a performance problem,633 // we may pass both bases and let genBufferDestruction decide634 // which one to use.635 mlir::Value base = bufferizedExpr.getBase();636 genBufferDestruction(loc, builder, base, mustFree,637 destroy.mustFinalizeExpr());638 }639 640 rewriter.eraseOp(destroy);641 return mlir::success();642 }643};644 645struct NoReassocOpConversion646 : public mlir::OpConversionPattern<hlfir::NoReassocOp> {647 using mlir::OpConversionPattern<hlfir::NoReassocOp>::OpConversionPattern;648 explicit NoReassocOpConversion(mlir::MLIRContext *ctx)649 : mlir::OpConversionPattern<hlfir::NoReassocOp>{ctx} {}650 llvm::LogicalResult651 matchAndRewrite(hlfir::NoReassocOp noreassoc, OpAdaptor adaptor,652 mlir::ConversionPatternRewriter &rewriter) const override {653 mlir::Location loc = noreassoc->getLoc();654 fir::FirOpBuilder builder(rewriter, noreassoc.getOperation());655 mlir::Value bufferizedExpr = getBufferizedExprStorage(adaptor.getVal());656 mlir::Value result =657 hlfir::NoReassocOp::create(builder, loc, bufferizedExpr);658 659 if (!fir::isa_trivial(bufferizedExpr.getType())) {660 // NoReassocOp should not be needed on the mustFree path.661 mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getVal());662 result =663 packageBufferizedExpr(loc, builder, hlfir::Entity{result}, mustFree);664 }665 rewriter.replaceOp(noreassoc, result);666 return mlir::success();667 }668};669 670/// Was \p value created in the mlir block where \p builder is currently set ?671static bool wasCreatedInCurrentBlock(mlir::Value value,672 fir::FirOpBuilder &builder) {673 if (mlir::Operation *op = value.getDefiningOp())674 return op->getBlock() == builder.getBlock();675 return false;676}677 678/// This Listener allows setting both the builder and the rewriter as679/// listeners. This is required when a pattern uses a firBuilder helper that680/// may create illegal operations that will need to be translated and requires681/// notifying the rewriter.682struct HLFIRListener : public mlir::OpBuilder::Listener {683 HLFIRListener(fir::FirOpBuilder &builder,684 mlir::ConversionPatternRewriter &rewriter)685 : builder{builder}, rewriter{rewriter} {}686 void notifyOperationInserted(mlir::Operation *op,687 mlir::OpBuilder::InsertPoint previous) override {688 builder.notifyOperationInserted(op, previous);689 rewriter.getListener()->notifyOperationInserted(op, previous);690 }691 virtual void notifyBlockInserted(mlir::Block *block, mlir::Region *previous,692 mlir::Region::iterator previousIt) override {693 builder.notifyBlockInserted(block, previous, previousIt);694 rewriter.getListener()->notifyBlockInserted(block, previous, previousIt);695 }696 fir::FirOpBuilder &builder;697 mlir::ConversionPatternRewriter &rewriter;698};699 700struct ElementalOpConversion701 : public mlir::OpConversionPattern<hlfir::ElementalOp> {702 using mlir::OpConversionPattern<hlfir::ElementalOp>::OpConversionPattern;703 explicit ElementalOpConversion(mlir::MLIRContext *ctx,704 bool optimizeEmptyElementals = false)705 : mlir::OpConversionPattern<hlfir::ElementalOp>{ctx},706 optimizeEmptyElementals(optimizeEmptyElementals) {707 // This pattern recursively converts nested ElementalOp's708 // by cloning and then converting them, so we have to allow709 // for recursive pattern application. The recursion is bounded710 // by the nesting level of ElementalOp's.711 setHasBoundedRewriteRecursion();712 }713 llvm::LogicalResult714 matchAndRewrite(hlfir::ElementalOp elemental, OpAdaptor adaptor,715 mlir::ConversionPatternRewriter &rewriter) const override {716 mlir::Location loc = elemental->getLoc();717 fir::FirOpBuilder builder(rewriter, elemental.getOperation());718 // The body of the elemental op may contain operation that will require719 // to be translated. Notify the rewriter about the cloned operations.720 HLFIRListener listener{builder, rewriter};721 builder.setListener(&listener);722 723 mlir::Value shape = adaptor.getShape();724 std::optional<hlfir::Entity> mold;725 if (adaptor.getMold())726 mold = getBufferizedExprStorage(adaptor.getMold());727 auto extents = hlfir::getIndexExtents(loc, builder, shape);728 llvm::SmallVector<mlir::Value> typeParams(adaptor.getTypeparams().begin(),729 adaptor.getTypeparams().end());730 auto [temp, cleanup] = createArrayTemp(loc, builder, elemental.getType(),731 shape, extents, typeParams, mold);732 733 if (optimizeEmptyElementals)734 extents = fir::factory::updateRuntimeExtentsForEmptyArrays(builder, loc,735 extents);736 737 // Generate a loop nest looping around the fir.elemental shape and clone738 // fir.elemental region inside the inner loop.739 hlfir::LoopNest loopNest =740 hlfir::genLoopNest(loc, builder, extents, !elemental.isOrdered(),741 flangomp::shouldUseWorkshareLowering(elemental));742 auto insPt = builder.saveInsertionPoint();743 builder.setInsertionPointToStart(loopNest.body);744 auto yield = hlfir::inlineElementalOp(loc, builder, elemental,745 loopNest.oneBasedIndices);746 hlfir::Entity elementValue(yield.getElementValue());747 // Skip final AsExpr if any. It would create an element temporary,748 // which is no needed since the element will be assigned right away in749 // the array temporary. An hlfir.as_expr may have been added if the750 // elemental is a "view" over a variable (e.g parentheses or transpose).751 if (auto asExpr = elementValue.getDefiningOp<hlfir::AsExprOp>()) {752 if (asExpr->hasOneUse() && !asExpr.isMove()) {753 // Check that the asExpr is the final operation before the yield,754 // otherwise, clean-ups could impact the memory being re-used.755 if (asExpr->getNextNode() == yield.getOperation()) {756 elementValue = hlfir::Entity{asExpr.getVar()};757 rewriter.eraseOp(asExpr);758 }759 }760 }761 rewriter.eraseOp(yield);762 // Assign the element value to the temp element for this iteration.763 auto tempElement =764 hlfir::getElementAt(loc, builder, temp, loopNest.oneBasedIndices);765 // If the elemental result is a temporary of a derived type,766 // we can avoid the deep copy implied by the AssignOp and just767 // do the shallow copy with load/store. This helps avoiding the overhead768 // of deallocating allocatable components of the temporary (if any)769 // on each iteration of the elemental operation.770 auto asExpr = elementValue.getDefiningOp<hlfir::AsExprOp>();771 auto elemType = hlfir::getFortranElementType(elementValue.getType());772 if (asExpr && asExpr.isMove() && mlir::isa<fir::RecordType>(elemType) &&773 hlfir::mayHaveAllocatableComponent(elemType) &&774 wasCreatedInCurrentBlock(elementValue, builder)) {775 auto load = fir::LoadOp::create(builder, loc, asExpr.getVar());776 fir::StoreOp::create(builder, loc, load, tempElement);777 } else {778 hlfir::AssignOp::create(builder, loc, elementValue, tempElement,779 /*realloc=*/false,780 /*keep_lhs_length_if_realloc=*/false,781 /*temporary_lhs=*/true);782 783 // hlfir.yield_element implicitly marks the end-of-life its operand if784 // it is an expression created in the hlfir.elemental (since it is its785 // last use and an hlfir.destroy could not be created afterwards)786 // Now that this node has been removed and the expression has been used in787 // the assign, insert an hlfir.destroy to mark the expression end-of-life.788 // If the expression creation allocated a buffer on the heap inside the789 // loop, this will ensure the buffer properly deallocated.790 if (mlir::isa<hlfir::ExprType>(elementValue.getType()) &&791 wasCreatedInCurrentBlock(elementValue, builder))792 hlfir::DestroyOp::create(builder, loc, elementValue);793 }794 builder.restoreInsertionPoint(insPt);795 796 mlir::Value bufferizedExpr =797 packageBufferizedExpr(loc, builder, temp, cleanup);798 // Explicitly delete the body of the elemental to get rid799 // of any users of hlfir.expr values inside the body as early800 // as possible.801 rewriter.startOpModification(elemental);802 rewriter.eraseBlock(elemental.getBody());803 rewriter.finalizeOpModification(elemental);804 rewriter.replaceOp(elemental, bufferizedExpr);805 return mlir::success();806 }807 808private:809 bool optimizeEmptyElementals = false;810};811struct CharExtremumOpConversion812 : public mlir::OpConversionPattern<hlfir::CharExtremumOp> {813 using mlir::OpConversionPattern<hlfir::CharExtremumOp>::OpConversionPattern;814 explicit CharExtremumOpConversion(mlir::MLIRContext *ctx)815 : mlir::OpConversionPattern<hlfir::CharExtremumOp>{ctx} {}816 llvm::LogicalResult817 matchAndRewrite(hlfir::CharExtremumOp char_extremum, OpAdaptor adaptor,818 mlir::ConversionPatternRewriter &rewriter) const override {819 mlir::Location loc = char_extremum->getLoc();820 auto predicate = char_extremum.getPredicate();821 bool predIsMin =822 predicate == hlfir::CharExtremumPredicate::min ? true : false;823 fir::FirOpBuilder builder(rewriter, char_extremum.getOperation());824 assert(adaptor.getStrings().size() >= 2 &&825 "must have at least two strings operands");826 auto numOperands = adaptor.getStrings().size();827 828 std::vector<hlfir::Entity> chars;829 std::vector<830 std::pair<fir::ExtendedValue, std::optional<hlfir::CleanupFunction>>>831 pairs;832 llvm::SmallVector<fir::CharBoxValue> opCBVs;833 for (size_t i = 0; i < numOperands; ++i) {834 chars.emplace_back(getBufferizedExprStorage(adaptor.getStrings()[i]));835 pairs.emplace_back(836 hlfir::translateToExtendedValue(loc, builder, chars[i]));837 assert(!pairs[i].second && "expected variables");838 opCBVs.emplace_back(*pairs[i].first.getCharBox());839 }840 841 fir::ExtendedValue res =842 fir::factory::CharacterExprHelper{builder, loc}.createCharExtremum(843 predIsMin, opCBVs);844 mlir::Type addrType = fir::ReferenceType::get(845 hlfir::getFortranElementType(char_extremum.getResult().getType()));846 mlir::Value cast = builder.createConvert(loc, addrType, fir::getBase(res));847 res = fir::substBase(res, cast);848 hlfir::Entity hlfirTempRes =849 hlfir::Entity{hlfir::genDeclare(loc, builder, res, ".tmp.char_extremum",850 fir::FortranVariableFlagsAttr{})851 .getBase()};852 mlir::Value bufferizedExpr =853 packageBufferizedExpr(loc, builder, hlfirTempRes, false);854 rewriter.replaceOp(char_extremum, bufferizedExpr);855 return mlir::success();856 }857};858 859struct EvaluateInMemoryOpConversion860 : public mlir::OpConversionPattern<hlfir::EvaluateInMemoryOp> {861 using mlir::OpConversionPattern<862 hlfir::EvaluateInMemoryOp>::OpConversionPattern;863 explicit EvaluateInMemoryOpConversion(mlir::MLIRContext *ctx)864 : mlir::OpConversionPattern<hlfir::EvaluateInMemoryOp>{ctx} {}865 llvm::LogicalResult866 matchAndRewrite(hlfir::EvaluateInMemoryOp evalInMemOp, OpAdaptor adaptor,867 mlir::ConversionPatternRewriter &rewriter) const override {868 mlir::Location loc = evalInMemOp->getLoc();869 fir::FirOpBuilder builder(rewriter, evalInMemOp.getOperation());870 auto [temp, isHeapAlloc] = hlfir::computeEvaluateOpInNewTemp(871 loc, builder, evalInMemOp, adaptor.getShape(), adaptor.getTypeparams());872 mlir::Value bufferizedExpr =873 packageBufferizedExpr(loc, builder, temp, isHeapAlloc);874 rewriter.replaceOp(evalInMemOp, bufferizedExpr);875 return mlir::success();876 }877};878 879class BufferizeHLFIR : public hlfir::impl::BufferizeHLFIRBase<BufferizeHLFIR> {880public:881 using BufferizeHLFIRBase<BufferizeHLFIR>::BufferizeHLFIRBase;882 883 void runOnOperation() override {884 // TODO: make this a pass operating on FuncOp. The issue is that885 // FirOpBuilder helpers may generate new FuncOp because of runtime/llvm886 // intrinsics calls creation. This may create race conflict if the pass is887 // scheduled on FuncOp. A solution could be to provide an optional mutex888 // when building a FirOpBuilder and locking around FuncOp and GlobalOp889 // creation, but this needs a bit more thinking, so at this point the pass890 // is scheduled on the moduleOp.891 auto module = this->getOperation();892 auto *context = &getContext();893 mlir::RewritePatternSet patterns(context);894 patterns.insert<ApplyOpConversion, AsExprOpConversion, AssignOpConversion,895 AssociateOpConversion, CharExtremumOpConversion,896 ConcatOpConversion, DestroyOpConversion,897 EndAssociateOpConversion, EvaluateInMemoryOpConversion,898 NoReassocOpConversion, SetLengthOpConversion,899 ShapeOfOpConversion, GetLengthOpConversion>(context);900 patterns.insert<ElementalOpConversion>(context, optimizeEmptyElementals);901 mlir::ConversionTarget target(*context);902 // Note that YieldElementOp is not marked as an illegal operation.903 // It must be erased by its parent converter and there is no explicit904 // conversion pattern to YieldElementOp itself. If any YieldElementOp905 // survives this pass, the verifier will detect it because it has to be906 // a child of ElementalOp and ElementalOp's are explicitly illegal.907 target.addIllegalOp<hlfir::ApplyOp, hlfir::AssociateOp, hlfir::ElementalOp,908 hlfir::EndAssociateOp, hlfir::SetLengthOp>();909 910 target.markUnknownOpDynamicallyLegal([](mlir::Operation *op) {911 return llvm::all_of(op->getResultTypes(),912 [](mlir::Type ty) {913 return !mlir::isa<hlfir::ExprType>(ty);914 }) &&915 llvm::all_of(op->getOperandTypes(), [](mlir::Type ty) {916 return !mlir::isa<hlfir::ExprType>(ty);917 });918 });919 if (mlir::failed(920 mlir::applyFullConversion(module, target, std::move(patterns)))) {921 mlir::emitError(mlir::UnknownLoc::get(context),922 "failure in HLFIR bufferization pass");923 signalPassFailure();924 }925 }926};927} // namespace928