394 lines · cpp
1//===-- LowerRepackArrays.cpp ---------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8/// \file9/// This pass expands fir.pack_array and fir.unpack_array operations10/// into sequences of other FIR operations and Fortran runtime calls.11/// This pass is using structured control flow FIR operations such12/// as fir.if, so its placement in the pipeline should guarantee13/// further lowering of these operations.14///15/// A fir.pack_array operation is converted into a sequence of checks16/// identifying whether an array needs to be copied into a contiguous17/// temporary. When the checks pass, a new memory allocation is done18/// for the temporary array (in either stack or heap memory).19/// If `fir.pack_array` does not have no_copy attribute, then20/// the original array is shallow-copied into the temporary.21///22/// A fir.unpack_array operations is converted into a check23/// of whether the original and the temporary arrays are different24/// memory. When the check passes, the temporary array might be25/// shallow-copied into the original array, and then the temporary26/// array is deallocated (if it was allocated in stack memory,27/// then there is no explicit deallocation).28//===----------------------------------------------------------------------===//29 30#include "flang/Optimizer/CodeGen/CodeGen.h"31 32#include "flang/Optimizer/Builder/Character.h"33#include "flang/Optimizer/Builder/FIRBuilder.h"34#include "flang/Optimizer/Builder/MutableBox.h"35#include "flang/Optimizer/Builder/Runtime/Allocatable.h"36#include "flang/Optimizer/Builder/Runtime/Transformational.h"37#include "flang/Optimizer/Builder/Todo.h"38#include "flang/Optimizer/Dialect/FIRDialect.h"39#include "flang/Optimizer/Dialect/FIROps.h"40#include "flang/Optimizer/Dialect/FIRType.h"41#include "flang/Optimizer/OpenACC/Support/RegisterOpenACCExtensions.h"42#include "flang/Optimizer/OpenMP/Support/RegisterOpenMPExtensions.h"43#include "mlir/Pass/Pass.h"44#include "mlir/Transforms/GreedyPatternRewriteDriver.h"45 46namespace fir {47#define GEN_PASS_DEF_LOWERREPACKARRAYSPASS48#include "flang/Optimizer/CodeGen/CGPasses.h.inc"49} // namespace fir50 51#define DEBUG_TYPE "lower-repack-arrays"52 53namespace {54class PackArrayConversion : public mlir::OpRewritePattern<fir::PackArrayOp> {55public:56 using OpRewritePattern::OpRewritePattern;57 58 mlir::LogicalResult59 matchAndRewrite(fir::PackArrayOp op,60 mlir::PatternRewriter &rewriter) const override;61 62private:63 static constexpr llvm::StringRef bufferName = ".repacked";64 65 // Return value of fir::BaseBoxType that represents a temporary66 // array created for the original box with given lbounds/extents and67 // type parameters. The new box has the same shape as the original68 // array. If useStack is true, then the temporary will be allocated69 // in stack memory (when possible).70 static mlir::Value allocateTempBuffer(fir::FirOpBuilder &builder,71 mlir::Location loc, bool useStack,72 mlir::Value origBox,73 llvm::ArrayRef<mlir::Value> lbounds,74 llvm::ArrayRef<mlir::Value> extents,75 llvm::ArrayRef<mlir::Value> typeParams);76 77 // Generate value of fir::BaseBoxType that represents the result78 // of the given fir.pack_array operation. The original box79 // is assumed to be present (though, it may represent an empty array).80 static mlir::FailureOr<mlir::Value> genRepackedBox(fir::FirOpBuilder &builder,81 mlir::Location loc,82 fir::PackArrayOp packOp);83};84 85class UnpackArrayConversion86 : public mlir::OpRewritePattern<fir::UnpackArrayOp> {87public:88 using OpRewritePattern::OpRewritePattern;89 90 mlir::LogicalResult91 matchAndRewrite(fir::UnpackArrayOp op,92 mlir::PatternRewriter &rewriter) const override;93};94} // anonymous namespace95 96// Return true iff for the given original boxed array we can97// allocate temporary memory in stack memory.98// This function is used to synchronize allocation/deallocation99// implied by fir.pack_array and fir.unpack_array, because100// the presence of the stack attribute does not automatically101// mean that the allocation is actually done in stack memory.102// For example, we always do the heap allocation for polymorphic103// types using Fortran runtime. Currently, we allocate all104// repack temporaries of derived types as polymorphic,105// so that we can preserve the dynamic type of the original.106// Adding the polymorpic mold to fir.alloca and then using107// Fortran runtime to compute the allocation size could probably108// resolve this limitation.109static bool canAllocateTempOnStack(mlir::Value box) {110 return !fir::isPolymorphicType(box.getType());111}112 113/// Return true if array repacking is safe either statically114/// (there are no 'is_safe' attributes) or dynamically115/// (neither of the 'is_safe' attributes claims 'isDynamicallySafe() == false').116/// \p op is either fir.pack_array or fir.unpack_array.117template <typename OP>118static bool repackIsSafe(OP op) {119 bool isSafe = true;120 if (auto isSafeAttrs = op.getIsSafe()) {121 // We currently support only the attributes for which122 // isDynamicallySafe() returns false.123 for (auto attr : *isSafeAttrs) {124 auto iface = mlir::cast<fir::SafeTempArrayCopyAttrInterface>(attr);125 if (iface.isDynamicallySafe())126 TODO(op.getLoc(), "dynamically safe array repacking");127 else128 isSafe = false;129 }130 }131 return isSafe;132}133 134mlir::LogicalResult135PackArrayConversion::matchAndRewrite(fir::PackArrayOp op,136 mlir::PatternRewriter &rewriter) const {137 mlir::Value box = op.getArray();138 // If repacking is not safe, then just use the original box.139 if (!repackIsSafe(op)) {140 rewriter.replaceOp(op, box);141 return mlir::success();142 }143 144 mlir::Location loc = op.getLoc();145 fir::FirOpBuilder builder(rewriter, op.getOperation());146 if (op.getMaxSize() || op.getMaxElementSize() || op.getMinStride())147 TODO(loc, "fir.pack_array with constraints");148 if (op.getHeuristics() != fir::PackArrayHeuristics::None)149 TODO(loc, "fir.pack_array with heuristics");150 151 auto boxType = mlir::cast<fir::BaseBoxType>(box.getType());152 153 // For now we have to always check if the box is present.154 auto isPresent =155 fir::IsPresentOp::create(builder, loc, builder.getI1Type(), box);156 157 fir::IfOp ifOp = fir::IfOp::create(builder, loc, boxType, isPresent,158 /*withElseRegion=*/true);159 builder.setInsertionPointToStart(&ifOp.getThenRegion().front());160 // The box is present.161 auto newBox = genRepackedBox(builder, loc, op);162 if (mlir::failed(newBox))163 return newBox;164 fir::ResultOp::create(builder, loc, *newBox);165 166 // The box is not present. Return original box.167 builder.setInsertionPointToStart(&ifOp.getElseRegion().front());168 fir::ResultOp::create(builder, loc, box);169 170 rewriter.replaceOp(op, ifOp.getResult(0));171 return mlir::success();172}173 174mlir::Value PackArrayConversion::allocateTempBuffer(175 fir::FirOpBuilder &builder, mlir::Location loc, bool useStack,176 mlir::Value origBox, llvm::ArrayRef<mlir::Value> lbounds,177 llvm::ArrayRef<mlir::Value> extents,178 llvm::ArrayRef<mlir::Value> typeParams) {179 auto tempType = mlir::cast<fir::SequenceType>(180 fir::extractSequenceType(origBox.getType()));181 assert(tempType.getDimension() == extents.size() &&182 "number of extents does not match the rank");183 184 mlir::Value shape = builder.genShape(loc, extents);185 auto [base, isHeapAllocation] = builder.createArrayTemp(186 loc, tempType, shape, extents, typeParams,187 fir::FirOpBuilder::genTempDeclareOp,188 fir::isPolymorphicType(origBox.getType()) ? origBox : nullptr, useStack,189 bufferName);190 // Make sure canAllocateTempOnStack() can recognize when191 // the temporary is actually allocated on the stack192 // by createArrayTemp(). Otherwise, we may miss dynamic193 // deallocation when lowering fir.unpack_array.194 if (useStack && canAllocateTempOnStack(origBox))195 assert(!isHeapAllocation && "temp must have been allocated on the stack");196 197 mlir::Type ptrType = base.getType();198 if (auto tempBoxType = mlir::dyn_cast<fir::BaseBoxType>(ptrType)) {199 // We need to reset the CFI_attribute_allocatable before200 // returning the temporary box to avoid any mishandling201 // of the temporary box in Fortran runtime.202 base = fir::BoxAddrOp::create(builder, loc, fir::boxMemRefType(tempBoxType),203 base);204 ptrType = base.getType();205 }206 207 // Create the temporary using dynamic type of the original,208 // if it is polymorphic, or it has a derived type with SEQUENCE209 // or BIND attribute (such dummy arguments may have their dynamic210 // type not exactly matching their static type).211 // Note that for the latter case, the allocation can still be done212 // without the mold, because the dynamic and static types213 // must be storage compatible.214 bool useDynamicType = fir::isBoxedRecordType(origBox.getType()) ||215 fir::isPolymorphicType(origBox.getType());216 mlir::Type tempBoxType =217 fir::wrapInClassOrBoxType(fir::unwrapRefType(ptrType),218 /*isPolymorphic=*/useDynamicType);219 // Use the shape with proper lower bounds for the final box.220 shape = builder.genShape(loc, lbounds, extents);221 mlir::Value newBox =222 builder.createBox(loc, tempBoxType, base, shape, /*slice=*/nullptr,223 typeParams, useDynamicType ? origBox : nullptr);224 // The new box might be !fir.class, while the original might be225 // !fir.box - we have to add a conversion.226 return builder.createConvert(loc, origBox.getType(), newBox);227}228 229mlir::FailureOr<mlir::Value>230PackArrayConversion::genRepackedBox(fir::FirOpBuilder &builder,231 mlir::Location loc, fir::PackArrayOp op) {232 mlir::OpBuilder::InsertionGuard guard(builder);233 mlir::Value box = op.getArray();234 235 llvm::SmallVector<mlir::Value> typeParams(op.getTypeparams().begin(),236 op.getTypeparams().end());237 auto boxType = mlir::cast<fir::BaseBoxType>(box.getType());238 mlir::Type indexType = builder.getIndexType();239 240 // If type parameters are not specified by fir.pack_array,241 // figure out how many of them we need to read from the box.242 unsigned numTypeParams = 0;243 if (typeParams.size() == 0) {244 if (auto recordType =245 mlir::dyn_cast<fir::RecordType>(boxType.unwrapInnerType()))246 if (recordType.getNumLenParams() != 0)247 TODO(loc,248 "allocating temporary for a parameterized derived type array");249 250 if (auto charType =251 mlir::dyn_cast<fir::CharacterType>(boxType.unwrapInnerType())) {252 if (charType.hasDynamicLen()) {253 // Read one length parameter from the box.254 numTypeParams = 1;255 } else {256 // Place the constant length into typeParams.257 mlir::Value length =258 builder.createIntegerConstant(loc, indexType, charType.getLen());259 typeParams.push_back(length);260 }261 }262 }263 264 // Create a temporay iff the original is not contigous and is not empty.265 auto isNotContiguous =266 builder.genNot(loc, fir::IsContiguousBoxOp::create(builder, loc, box,267 op.getInnermost()));268 auto dataAddr =269 fir::BoxAddrOp::create(builder, loc, fir::boxMemRefType(boxType), box);270 auto isNotEmpty =271 fir::IsPresentOp::create(builder, loc, builder.getI1Type(), dataAddr);272 auto doPack =273 mlir::arith::AndIOp::create(builder, loc, isNotContiguous, isNotEmpty);274 275 fir::IfOp ifOp =276 fir::IfOp::create(builder, loc, boxType, doPack, /*withElseRegion=*/true);277 // Assume that the repacking is unlikely.278 ifOp.setUnlikelyIfWeights();279 280 // Return original box.281 builder.setInsertionPointToStart(&ifOp.getElseRegion().front());282 fir::ResultOp::create(builder, loc, box);283 284 // Create a new box.285 builder.setInsertionPointToStart(&ifOp.getThenRegion().front());286 287 // Get lower bounds and extents from the box.288 llvm::SmallVector<mlir::Value, Fortran::common::maxRank> lbounds, extents;289 fir::factory::genDimInfoFromBox(builder, loc, box, &lbounds, &extents,290 /*strides=*/nullptr);291 // Get the type parameters from the box, if needed.292 if (numTypeParams != 0) {293 if (auto charType =294 mlir::dyn_cast<fir::CharacterType>(boxType.unwrapInnerType()))295 if (charType.hasDynamicLen()) {296 fir::factory::CharacterExprHelper charHelper(builder, loc);297 mlir::Value len = charHelper.readLengthFromBox(box, charType);298 typeParams.push_back(builder.createConvert(loc, indexType, len));299 }300 301 if (numTypeParams != typeParams.size())302 return emitError(loc) << "failed to compute the type parameters for "303 << op.getOperation() << '\n';304 }305 306 mlir::Value tempBox = allocateTempBuffer(builder, loc, op.getStack(), box,307 lbounds, extents, typeParams);308 if (!op.getNoCopy())309 fir::runtime::genShallowCopy(builder, loc, tempBox, box,310 /*resultIsAllocated=*/true);311 fir::ResultOp::create(builder, loc, tempBox);312 313 return ifOp.getResult(0);314}315 316mlir::LogicalResult317UnpackArrayConversion::matchAndRewrite(fir::UnpackArrayOp op,318 mlir::PatternRewriter &rewriter) const {319 // If repacking is not safe, then just remove the operation.320 if (!repackIsSafe(op)) {321 rewriter.eraseOp(op);322 return mlir::success();323 }324 325 mlir::Location loc = op.getLoc();326 fir::FirOpBuilder builder(rewriter, op.getOperation());327 mlir::Type predicateType = builder.getI1Type();328 mlir::Value tempBox = op.getTemp();329 mlir::Value originalBox = op.getOriginal();330 331 // For now we have to always check if the box is present.332 auto isPresent =333 fir::IsPresentOp::create(builder, loc, predicateType, originalBox);334 335 builder.genIfThen(loc, isPresent).genThen([&]() {336 mlir::Type addrType =337 fir::HeapType::get(fir::extractSequenceType(tempBox.getType()));338 mlir::Value tempAddr =339 fir::BoxAddrOp::create(builder, loc, addrType, tempBox);340 mlir::Value originalAddr =341 fir::BoxAddrOp::create(builder, loc, addrType, originalBox);342 343 auto isNotSame = builder.genPtrCompare(loc, mlir::arith::CmpIPredicate::ne,344 tempAddr, originalAddr);345 builder.genIfThen(loc, isNotSame)346 .genThen([&]() {347 // Copy from temporary to the original.348 if (!op.getNoCopy())349 fir::runtime::genShallowCopy(builder, loc, originalBox, tempBox,350 /*resultIsAllocated=*/true);351 352 // Deallocate, if it was allocated in heap.353 // Note that the stack attribute does not always mean354 // that the allocation was actually done in stack memory.355 // There are currently cases where we delegate the allocation356 // to the runtime that uses heap memory, even when the stack357 // attribute is set on fir.pack_array.358 if (!op.getStack() || !canAllocateTempOnStack(originalBox))359 fir::FreeMemOp::create(builder, loc, tempAddr);360 })361 .getIfOp()362 .setUnlikelyIfWeights();363 });364 rewriter.eraseOp(op);365 return mlir::success();366}367 368namespace {369class LowerRepackArraysPass370 : public fir::impl::LowerRepackArraysPassBase<LowerRepackArraysPass> {371public:372 using LowerRepackArraysPassBase<373 LowerRepackArraysPass>::LowerRepackArraysPassBase;374 375 void runOnOperation() override final {376 auto *context = &getContext();377 mlir::ModuleOp module = getOperation();378 mlir::RewritePatternSet patterns(context);379 patterns.insert<PackArrayConversion>(context);380 patterns.insert<UnpackArrayConversion>(context);381 mlir::GreedyRewriteConfig config;382 config.setRegionSimplificationLevel(383 mlir::GreedySimplifyRegionLevel::Disabled);384 (void)applyPatternsGreedily(module, std::move(patterns), config);385 }386 387 void getDependentDialects(mlir::DialectRegistry ®istry) const override {388 fir::acc::registerTransformationalAttrsDependentDialects(registry);389 fir::omp::registerTransformationalAttrsDependentDialects(registry);390 }391};392 393} // anonymous namespace394