1395 lines · cpp
1//===- SimplifyIntrinsics.cpp -- replace intrinsics with simpler form -----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9//===----------------------------------------------------------------------===//10/// \file11/// This pass looks for suitable calls to runtime library for intrinsics that12/// can be simplified/specialized and replaces with a specialized function.13///14/// For example, SUM(arr) can be specialized as a simple function with one loop,15/// compared to the three arguments (plus file & line info) that the runtime16/// call has - when the argument is a 1D-array (multiple loops may be needed17// for higher dimension arrays, of course)18///19/// The general idea is that besides making the call simpler, it can also be20/// inlined by other passes that run after this pass, which further improves21/// performance, particularly when the work done in the function is trivial22/// and small in size.23//===----------------------------------------------------------------------===//24 25#include "flang/Optimizer/Builder/BoxValue.h"26#include "flang/Optimizer/Builder/CUFCommon.h"27#include "flang/Optimizer/Builder/FIRBuilder.h"28#include "flang/Optimizer/Builder/LowLevelIntrinsics.h"29#include "flang/Optimizer/Builder/Todo.h"30#include "flang/Optimizer/Dialect/FIROps.h"31#include "flang/Optimizer/Dialect/FIRType.h"32#include "flang/Optimizer/Dialect/Support/FIRContext.h"33#include "flang/Optimizer/HLFIR/HLFIRDialect.h"34#include "flang/Optimizer/Transforms/Passes.h"35#include "flang/Optimizer/Transforms/Utils.h"36#include "flang/Runtime/entry-names.h"37#include "flang/Support/Fortran.h"38#include "mlir/Dialect/LLVMIR/LLVMDialect.h"39#include "mlir/IR/Matchers.h"40#include "mlir/IR/Operation.h"41#include "mlir/Pass/Pass.h"42#include "mlir/Transforms/DialectConversion.h"43#include "mlir/Transforms/GreedyPatternRewriteDriver.h"44#include "mlir/Transforms/RegionUtils.h"45#include "llvm/Support/Debug.h"46#include "llvm/Support/raw_ostream.h"47#include <llvm/Support/ErrorHandling.h>48#include <mlir/Dialect/Arith/IR/Arith.h>49#include <mlir/IR/BuiltinTypes.h>50#include <mlir/IR/Location.h>51#include <mlir/IR/MLIRContext.h>52#include <mlir/IR/Value.h>53#include <mlir/Support/LLVM.h>54#include <optional>55 56namespace fir {57#define GEN_PASS_DEF_SIMPLIFYINTRINSICS58#include "flang/Optimizer/Transforms/Passes.h.inc"59} // namespace fir60 61#define DEBUG_TYPE "flang-simplify-intrinsics"62 63namespace {64 65class SimplifyIntrinsicsPass66 : public fir::impl::SimplifyIntrinsicsBase<SimplifyIntrinsicsPass> {67 using FunctionTypeGeneratorTy =68 llvm::function_ref<mlir::FunctionType(fir::FirOpBuilder &)>;69 using FunctionBodyGeneratorTy =70 llvm::function_ref<void(fir::FirOpBuilder &, mlir::func::FuncOp &)>;71 using GenReductionBodyTy = llvm::function_ref<void(72 fir::FirOpBuilder &builder, mlir::func::FuncOp &funcOp, unsigned rank,73 mlir::Type elementType)>;74 75public:76 using fir::impl::SimplifyIntrinsicsBase<77 SimplifyIntrinsicsPass>::SimplifyIntrinsicsBase;78 79 /// Generate a new function implementing a simplified version80 /// of a Fortran runtime function defined by \p basename name.81 /// \p typeGenerator is a callback that generates the new function's type.82 /// \p bodyGenerator is a callback that generates the new function's body.83 /// The new function is created in the \p builder's Module.84 mlir::func::FuncOp getOrCreateFunction(fir::FirOpBuilder &builder,85 const mlir::StringRef &basename,86 FunctionTypeGeneratorTy typeGenerator,87 FunctionBodyGeneratorTy bodyGenerator);88 void runOnOperation() override;89 void getDependentDialects(mlir::DialectRegistry ®istry) const override;90 91private:92 /// Helper functions to replace a reduction type of call with its93 /// simplified form. The actual function is generated using a callback94 /// function.95 /// \p call is the call to be replaced96 /// \p kindMap is used to create FIROpBuilder97 /// \p genBodyFunc is the callback that builds the replacement function98 void simplifyIntOrFloatReduction(fir::CallOp call,99 const fir::KindMapping &kindMap,100 GenReductionBodyTy genBodyFunc);101 void simplifyLogicalDim0Reduction(fir::CallOp call,102 const fir::KindMapping &kindMap,103 GenReductionBodyTy genBodyFunc);104 void simplifyLogicalDim1Reduction(fir::CallOp call,105 const fir::KindMapping &kindMap,106 GenReductionBodyTy genBodyFunc);107 void simplifyMinMaxlocReduction(fir::CallOp call,108 const fir::KindMapping &kindMap, bool isMax);109 void simplifyReductionBody(fir::CallOp call, const fir::KindMapping &kindMap,110 GenReductionBodyTy genBodyFunc,111 fir::FirOpBuilder &builder,112 const mlir::StringRef &basename,113 mlir::Type elementType);114};115 116} // namespace117 118/// Create FirOpBuilder with the provided \p op insertion point119/// and \p kindMap additionally inheriting FastMathFlags from \p op.120static fir::FirOpBuilder121getSimplificationBuilder(mlir::Operation *op, const fir::KindMapping &kindMap) {122 fir::FirOpBuilder builder{op, kindMap};123 auto fmi = mlir::dyn_cast<mlir::arith::ArithFastMathInterface>(*op);124 if (!fmi)125 return builder;126 127 // Regardless of what default FastMathFlags are used by FirOpBuilder,128 // override them with FastMathFlags attached to the operation.129 builder.setFastMathFlags(fmi.getFastMathFlagsAttr().getValue());130 return builder;131}132 133/// Generate function type for the simplified version of RTNAME(Sum) and134/// similar functions with a fir.box<none> type returning \p elementType.135static mlir::FunctionType genNoneBoxType(fir::FirOpBuilder &builder,136 const mlir::Type &elementType) {137 mlir::Type boxType = fir::BoxType::get(builder.getNoneType());138 return mlir::FunctionType::get(builder.getContext(), {boxType},139 {elementType});140}141 142template <typename Op>143Op expectOp(mlir::Value val) {144 if (Op op = mlir::dyn_cast_or_null<Op>(val.getDefiningOp()))145 return op;146 LLVM_DEBUG(llvm::dbgs() << "Didn't find expected " << Op::getOperationName()147 << '\n');148 return nullptr;149}150 151template <typename Op>152static mlir::Value findDefSingle(fir::ConvertOp op) {153 if (auto defOp = expectOp<Op>(op->getOperand(0))) {154 return defOp.getResult();155 }156 return {};157}158 159template <typename... Ops>160static mlir::Value findDef(fir::ConvertOp op) {161 mlir::Value defOp;162 // Loop over the operation types given to see if any match, exiting once163 // a match is found. Cast to void is needed to avoid compiler complaining164 // that the result of expression is unused165 (void)((defOp = findDefSingle<Ops>(op), (defOp)) || ...);166 return defOp;167}168 169static bool isOperandAbsent(mlir::Value val) {170 if (auto op = expectOp<fir::ConvertOp>(val)) {171 assert(op->getOperands().size() != 0);172 return mlir::isa_and_nonnull<fir::AbsentOp>(173 op->getOperand(0).getDefiningOp());174 }175 return false;176}177 178static bool isTrueOrNotConstant(mlir::Value val) {179 if (auto op = expectOp<mlir::arith::ConstantOp>(val)) {180 return !mlir::matchPattern(val, mlir::m_Zero());181 }182 return true;183}184 185static bool isZero(mlir::Value val) {186 if (auto op = expectOp<fir::ConvertOp>(val)) {187 assert(op->getOperands().size() != 0);188 if (mlir::Operation *defOp = op->getOperand(0).getDefiningOp())189 return mlir::matchPattern(defOp, mlir::m_Zero());190 }191 return false;192}193 194static mlir::Value findBoxDef(mlir::Value val) {195 if (auto op = expectOp<fir::ConvertOp>(val)) {196 assert(op->getOperands().size() != 0);197 return findDef<fir::EmboxOp, fir::ReboxOp>(op);198 }199 return {};200}201 202static mlir::Value findMaskDef(mlir::Value val) {203 if (auto op = expectOp<fir::ConvertOp>(val)) {204 assert(op->getOperands().size() != 0);205 return findDef<fir::EmboxOp, fir::ReboxOp, fir::AbsentOp>(op);206 }207 return {};208}209 210static unsigned getDimCount(mlir::Value val) {211 // In order to find the dimensions count, we look for EmboxOp/ReboxOp212 // and take the count from its *result* type. Note that in case213 // of sliced emboxing the operand and the result of EmboxOp/ReboxOp214 // have different types.215 // Actually, we can take the box type from the operand of216 // the first ConvertOp that has non-opaque box type that we meet217 // going through the ConvertOp chain.218 if (mlir::Value emboxVal = findBoxDef(val))219 if (auto boxTy = mlir::dyn_cast<fir::BoxType>(emboxVal.getType()))220 if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(boxTy.getEleTy()))221 return seqTy.getDimension();222 return 0;223}224 225/// Given the call operation's box argument \p val, discover226/// the element type of the underlying array object.227/// \returns the element type or std::nullopt if the type cannot228/// be reliably found.229/// We expect that the argument is a result of fir.convert230/// with the destination type of !fir.box<none>.231static std::optional<mlir::Type> getArgElementType(mlir::Value val) {232 mlir::Operation *defOp;233 do {234 defOp = val.getDefiningOp();235 // Analyze only sequences of convert operations.236 if (!mlir::isa<fir::ConvertOp>(defOp))237 return std::nullopt;238 val = defOp->getOperand(0);239 // The convert operation is expected to convert from one240 // box type to another box type.241 auto boxType = mlir::cast<fir::BoxType>(val.getType());242 auto elementType = fir::unwrapSeqOrBoxedSeqType(boxType);243 if (!mlir::isa<mlir::NoneType>(elementType))244 return elementType;245 } while (true);246}247 248using BodyOpGeneratorTy = llvm::function_ref<mlir::Value(249 fir::FirOpBuilder &, mlir::Location, const mlir::Type &, mlir::Value,250 mlir::Value)>;251using ContinueLoopGenTy = llvm::function_ref<llvm::SmallVector<mlir::Value>(252 fir::FirOpBuilder &, mlir::Location, mlir::Value)>;253 254/// Generate the reduction loop into \p funcOp.255///256/// \p initVal is a function, called to get the initial value for257/// the reduction value258/// \p genBody is called to fill in the actual reduciton operation259/// for example add for SUM, MAX for MAXVAL, etc.260/// \p rank is the rank of the input argument.261/// \p elementType is the type of the elements in the input array,262/// which may be different to the return type.263/// \p loopCond is called to generate the condition to continue or264/// not for IterWhile loops265/// \p unorderedOrInitalLoopCond contains either a boolean or bool266/// mlir constant, and controls the inital value for while loops267/// or if DoLoop is ordered/unordered.268 269template <typename OP, typename T, int resultIndex>270static void271genReductionLoop(fir::FirOpBuilder &builder, mlir::func::FuncOp &funcOp,272 fir::InitValGeneratorTy initVal, ContinueLoopGenTy loopCond,273 T unorderedOrInitialLoopCond, BodyOpGeneratorTy genBody,274 unsigned rank, mlir::Type elementType, mlir::Location loc) {275 276 mlir::IndexType idxTy = builder.getIndexType();277 278 mlir::Block::BlockArgListType args = funcOp.front().getArguments();279 mlir::Value arg = args[0];280 281 mlir::Value zeroIdx = builder.createIntegerConstant(loc, idxTy, 0);282 283 fir::SequenceType::Shape flatShape(rank,284 fir::SequenceType::getUnknownExtent());285 mlir::Type arrTy = fir::SequenceType::get(flatShape, elementType);286 mlir::Type boxArrTy = fir::BoxType::get(arrTy);287 mlir::Value array = fir::ConvertOp::create(builder, loc, boxArrTy, arg);288 mlir::Type resultType = funcOp.getResultTypes()[0];289 mlir::Value init = initVal(builder, loc, resultType);290 291 llvm::SmallVector<mlir::Value, Fortran::common::maxRank> bounds;292 293 assert(rank > 0 && "rank cannot be zero");294 mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);295 296 // Compute all the upper bounds before the loop nest.297 // It is not strictly necessary for performance, since the loop nest298 // does not have any store operations and any LICM optimization299 // should be able to optimize the redundancy.300 for (unsigned i = 0; i < rank; ++i) {301 mlir::Value dimIdx = builder.createIntegerConstant(loc, idxTy, i);302 auto dims = fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, array,303 dimIdx);304 mlir::Value len = dims.getResult(1);305 // We use C indexing here, so len-1 as loopcount306 mlir::Value loopCount = mlir::arith::SubIOp::create(builder, loc, len, one);307 bounds.push_back(loopCount);308 }309 // Create a loop nest consisting of OP operations.310 // Collect the loops' induction variables into indices array,311 // which will be used in the innermost loop to load the input312 // array's element.313 // The loops are generated such that the innermost loop processes314 // the 0 dimension.315 llvm::SmallVector<mlir::Value, Fortran::common::maxRank> indices;316 for (unsigned i = rank; 0 < i; --i) {317 mlir::Value step = one;318 mlir::Value loopCount = bounds[i - 1];319 auto loop = OP::create(builder, loc, zeroIdx, loopCount, step,320 unorderedOrInitialLoopCond,321 /*finalCountValue=*/false, init);322 init = loop.getRegionIterArgs()[resultIndex];323 indices.push_back(loop.getInductionVar());324 // Set insertion point to the loop body so that the next loop325 // is inserted inside the current one.326 builder.setInsertionPointToStart(loop.getBody());327 }328 329 // Reverse the indices such that they are ordered as:330 // <dim-0-idx, dim-1-idx, ...>331 std::reverse(indices.begin(), indices.end());332 // We are in the innermost loop: generate the reduction body.333 mlir::Type eleRefTy = builder.getRefType(elementType);334 mlir::Value addr =335 fir::CoordinateOp::create(builder, loc, eleRefTy, array, indices);336 mlir::Value elem = fir::LoadOp::create(builder, loc, addr);337 mlir::Value reductionVal = genBody(builder, loc, elementType, elem, init);338 // Generate vector with condition to continue while loop at [0] and result339 // from current loop at [1] for IterWhileOp loops, just result at [0] for340 // DoLoopOp loops.341 llvm::SmallVector<mlir::Value> results = loopCond(builder, loc, reductionVal);342 343 // Unwind the loop nest and insert ResultOp on each level344 // to return the updated value of the reduction to the enclosing345 // loops.346 for (unsigned i = 0; i < rank; ++i) {347 auto result = fir::ResultOp::create(builder, loc, results);348 // Proceed to the outer loop.349 auto loop = mlir::cast<OP>(result->getParentOp());350 results = loop.getResults();351 // Set insertion point after the loop operation that we have352 // just processed.353 builder.setInsertionPointAfter(loop.getOperation());354 }355 // End of loop nest. The insertion point is after the outermost loop.356 // Return the reduction value from the function.357 mlir::func::ReturnOp::create(builder, loc, results[resultIndex]);358}359 360static llvm::SmallVector<mlir::Value> nopLoopCond(fir::FirOpBuilder &builder,361 mlir::Location loc,362 mlir::Value reductionVal) {363 return {reductionVal};364}365 366/// Generate function body of the simplified version of RTNAME(Sum)367/// with signature provided by \p funcOp. The caller is responsible368/// for saving/restoring the original insertion point of \p builder.369/// \p funcOp is expected to be empty on entry to this function.370/// \p rank specifies the rank of the input argument.371static void genRuntimeSumBody(fir::FirOpBuilder &builder,372 mlir::func::FuncOp &funcOp, unsigned rank,373 mlir::Type elementType) {374 // function RTNAME(Sum)<T>x<rank>_simplified(arr)375 // T, dimension(:) :: arr376 // T sum = 0377 // integer iter378 // do iter = 0, extent(arr)379 // sum = sum + arr[iter]380 // end do381 // RTNAME(Sum)<T>x<rank>_simplified = sum382 // end function RTNAME(Sum)<T>x<rank>_simplified383 auto zero = [](fir::FirOpBuilder builder, mlir::Location loc,384 mlir::Type elementType) {385 if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {386 const llvm::fltSemantics &sem = ty.getFloatSemantics();387 return builder.createRealConstant(loc, elementType,388 llvm::APFloat::getZero(sem));389 }390 return builder.createIntegerConstant(loc, elementType, 0);391 };392 393 auto genBodyOp = [](fir::FirOpBuilder builder, mlir::Location loc,394 mlir::Type elementType, mlir::Value elem1,395 mlir::Value elem2) -> mlir::Value {396 if (mlir::isa<mlir::FloatType>(elementType))397 return mlir::arith::AddFOp::create(builder, loc, elem1, elem2);398 if (mlir::isa<mlir::IntegerType>(elementType))399 return mlir::arith::AddIOp::create(builder, loc, elem1, elem2);400 401 llvm_unreachable("unsupported type");402 return {};403 };404 405 mlir::Location loc = mlir::UnknownLoc::get(builder.getContext());406 builder.setInsertionPointToEnd(funcOp.addEntryBlock());407 408 genReductionLoop<fir::DoLoopOp, bool, 0>(builder, funcOp, zero, nopLoopCond,409 false, genBodyOp, rank, elementType,410 loc);411}412 413static void genRuntimeMaxvalBody(fir::FirOpBuilder &builder,414 mlir::func::FuncOp &funcOp, unsigned rank,415 mlir::Type elementType) {416 auto init = [](fir::FirOpBuilder builder, mlir::Location loc,417 mlir::Type elementType) {418 if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {419 const llvm::fltSemantics &sem = ty.getFloatSemantics();420 return builder.createRealConstant(421 loc, elementType, llvm::APFloat::getLargest(sem, /*Negative=*/true));422 }423 unsigned bits = elementType.getIntOrFloatBitWidth();424 int64_t minInt = llvm::APInt::getSignedMinValue(bits).getSExtValue();425 return builder.createIntegerConstant(loc, elementType, minInt);426 };427 428 auto genBodyOp = [](fir::FirOpBuilder builder, mlir::Location loc,429 mlir::Type elementType, mlir::Value elem1,430 mlir::Value elem2) -> mlir::Value {431 if (mlir::isa<mlir::FloatType>(elementType)) {432 // arith.maxf later converted to llvm.intr.maxnum does not work433 // correctly for NaNs and -0.0 (see maxnum/minnum pattern matching434 // in LLVM's InstCombine pass). Moreover, llvm.intr.maxnum435 // for F128 operands is lowered into fmaxl call by LLVM.436 // This libm function may not work properly for F128 arguments437 // on targets where long double is not F128. It is an LLVM issue,438 // but we just use normal select here to resolve all the cases.439 auto compare = mlir::arith::CmpFOp::create(440 builder, loc, mlir::arith::CmpFPredicate::OGT, elem1, elem2);441 return mlir::arith::SelectOp::create(builder, loc, compare, elem1, elem2);442 }443 if (mlir::isa<mlir::IntegerType>(elementType))444 return mlir::arith::MaxSIOp::create(builder, loc, elem1, elem2);445 446 llvm_unreachable("unsupported type");447 return {};448 };449 450 mlir::Location loc = mlir::UnknownLoc::get(builder.getContext());451 builder.setInsertionPointToEnd(funcOp.addEntryBlock());452 453 genReductionLoop<fir::DoLoopOp, bool, 0>(builder, funcOp, init, nopLoopCond,454 false, genBodyOp, rank, elementType,455 loc);456}457 458static void genRuntimeCountBody(fir::FirOpBuilder &builder,459 mlir::func::FuncOp &funcOp, unsigned rank,460 mlir::Type elementType) {461 auto zero = [](fir::FirOpBuilder builder, mlir::Location loc,462 mlir::Type elementType) {463 unsigned bits = elementType.getIntOrFloatBitWidth();464 int64_t zeroInt = llvm::APInt::getZero(bits).getSExtValue();465 return builder.createIntegerConstant(loc, elementType, zeroInt);466 };467 468 auto genBodyOp = [](fir::FirOpBuilder builder, mlir::Location loc,469 mlir::Type elementType, mlir::Value elem1,470 mlir::Value elem2) -> mlir::Value {471 auto zero32 = builder.createIntegerConstant(loc, elementType, 0);472 auto zero64 = builder.createIntegerConstant(loc, builder.getI64Type(), 0);473 auto one64 = builder.createIntegerConstant(loc, builder.getI64Type(), 1);474 475 auto compare = mlir::arith::CmpIOp::create(476 builder, loc, mlir::arith::CmpIPredicate::eq, elem1, zero32);477 auto select =478 mlir::arith::SelectOp::create(builder, loc, compare, zero64, one64);479 return mlir::arith::AddIOp::create(builder, loc, select, elem2);480 };481 482 // Count always gets I32 for elementType as it converts logical input to483 // logical<4> before passing to the function.484 mlir::Location loc = mlir::UnknownLoc::get(builder.getContext());485 builder.setInsertionPointToEnd(funcOp.addEntryBlock());486 487 genReductionLoop<fir::DoLoopOp, bool, 0>(builder, funcOp, zero, nopLoopCond,488 false, genBodyOp, rank, elementType,489 loc);490}491 492static void genRuntimeAnyBody(fir::FirOpBuilder &builder,493 mlir::func::FuncOp &funcOp, unsigned rank,494 mlir::Type elementType) {495 auto zero = [](fir::FirOpBuilder builder, mlir::Location loc,496 mlir::Type elementType) {497 return builder.createIntegerConstant(loc, elementType, 0);498 };499 500 auto genBodyOp = [](fir::FirOpBuilder builder, mlir::Location loc,501 mlir::Type elementType, mlir::Value elem1,502 mlir::Value elem2) -> mlir::Value {503 auto zero = builder.createIntegerConstant(loc, elementType, 0);504 return mlir::arith::CmpIOp::create(505 builder, loc, mlir::arith::CmpIPredicate::ne, elem1, zero);506 };507 508 auto continueCond = [](fir::FirOpBuilder builder, mlir::Location loc,509 mlir::Value reductionVal) {510 auto one1 = builder.createIntegerConstant(loc, builder.getI1Type(), 1);511 auto eor = mlir::arith::XOrIOp::create(builder, loc, reductionVal, one1);512 llvm::SmallVector<mlir::Value> results = {eor, reductionVal};513 return results;514 };515 516 mlir::Location loc = mlir::UnknownLoc::get(builder.getContext());517 builder.setInsertionPointToEnd(funcOp.addEntryBlock());518 mlir::Value ok = builder.createBool(loc, true);519 520 genReductionLoop<fir::IterWhileOp, mlir::Value, 1>(521 builder, funcOp, zero, continueCond, ok, genBodyOp, rank, elementType,522 loc);523}524 525static void genRuntimeAllBody(fir::FirOpBuilder &builder,526 mlir::func::FuncOp &funcOp, unsigned rank,527 mlir::Type elementType) {528 auto one = [](fir::FirOpBuilder builder, mlir::Location loc,529 mlir::Type elementType) {530 return builder.createIntegerConstant(loc, elementType, 1);531 };532 533 auto genBodyOp = [](fir::FirOpBuilder builder, mlir::Location loc,534 mlir::Type elementType, mlir::Value elem1,535 mlir::Value elem2) -> mlir::Value {536 auto zero = builder.createIntegerConstant(loc, elementType, 0);537 return mlir::arith::CmpIOp::create(538 builder, loc, mlir::arith::CmpIPredicate::ne, elem1, zero);539 };540 541 auto continueCond = [](fir::FirOpBuilder builder, mlir::Location loc,542 mlir::Value reductionVal) {543 llvm::SmallVector<mlir::Value> results = {reductionVal, reductionVal};544 return results;545 };546 547 mlir::Location loc = mlir::UnknownLoc::get(builder.getContext());548 builder.setInsertionPointToEnd(funcOp.addEntryBlock());549 mlir::Value ok = builder.createBool(loc, true);550 551 genReductionLoop<fir::IterWhileOp, mlir::Value, 1>(552 builder, funcOp, one, continueCond, ok, genBodyOp, rank, elementType,553 loc);554}555 556static mlir::FunctionType genRuntimeMinlocType(fir::FirOpBuilder &builder,557 unsigned int rank) {558 mlir::Type boxType = fir::BoxType::get(builder.getNoneType());559 mlir::Type boxRefType = builder.getRefType(boxType);560 561 return mlir::FunctionType::get(builder.getContext(),562 {boxRefType, boxType, boxType}, {});563}564 565// Produces a loop nest for a Minloc intrinsic.566void fir::genMinMaxlocReductionLoop(567 fir::FirOpBuilder &builder, mlir::Value array,568 fir::InitValGeneratorTy initVal, fir::MinlocBodyOpGeneratorTy genBody,569 fir::AddrGeneratorTy getAddrFn, unsigned rank, mlir::Type elementType,570 mlir::Location loc, mlir::Type maskElemType, mlir::Value resultArr,571 bool maskMayBeLogicalScalar) {572 mlir::IndexType idxTy = builder.getIndexType();573 574 mlir::Value zeroIdx = builder.createIntegerConstant(loc, idxTy, 0);575 576 fir::SequenceType::Shape flatShape(rank,577 fir::SequenceType::getUnknownExtent());578 mlir::Type arrTy = fir::SequenceType::get(flatShape, elementType);579 mlir::Type boxArrTy = fir::BoxType::get(arrTy);580 array = fir::ConvertOp::create(builder, loc, boxArrTy, array);581 582 mlir::Type resultElemType = hlfir::getFortranElementType(resultArr.getType());583 mlir::Value flagSet = builder.createIntegerConstant(loc, resultElemType, 1);584 mlir::Value zero = builder.createIntegerConstant(loc, resultElemType, 0);585 mlir::Value flagRef = builder.createTemporary(loc, resultElemType);586 fir::StoreOp::create(builder, loc, zero, flagRef);587 588 mlir::Value init = initVal(builder, loc, elementType);589 llvm::SmallVector<mlir::Value, Fortran::common::maxRank> bounds;590 591 assert(rank > 0 && "rank cannot be zero");592 mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);593 594 // Compute all the upper bounds before the loop nest.595 // It is not strictly necessary for performance, since the loop nest596 // does not have any store operations and any LICM optimization597 // should be able to optimize the redundancy.598 for (unsigned i = 0; i < rank; ++i) {599 mlir::Value dimIdx = builder.createIntegerConstant(loc, idxTy, i);600 auto dims = fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, array,601 dimIdx);602 mlir::Value len = dims.getResult(1);603 // We use C indexing here, so len-1 as loopcount604 mlir::Value loopCount = mlir::arith::SubIOp::create(builder, loc, len, one);605 bounds.push_back(loopCount);606 }607 // Create a loop nest consisting of OP operations.608 // Collect the loops' induction variables into indices array,609 // which will be used in the innermost loop to load the input610 // array's element.611 // The loops are generated such that the innermost loop processes612 // the 0 dimension.613 llvm::SmallVector<mlir::Value, Fortran::common::maxRank> indices;614 for (unsigned i = rank; 0 < i; --i) {615 mlir::Value step = one;616 mlir::Value loopCount = bounds[i - 1];617 auto loop =618 fir::DoLoopOp::create(builder, loc, zeroIdx, loopCount, step, false,619 /*finalCountValue=*/false, init);620 init = loop.getRegionIterArgs()[0];621 indices.push_back(loop.getInductionVar());622 // Set insertion point to the loop body so that the next loop623 // is inserted inside the current one.624 builder.setInsertionPointToStart(loop.getBody());625 }626 627 // Reverse the indices such that they are ordered as:628 // <dim-0-idx, dim-1-idx, ...>629 std::reverse(indices.begin(), indices.end());630 mlir::Value reductionVal =631 genBody(builder, loc, elementType, array, flagRef, init, indices);632 633 // Unwind the loop nest and insert ResultOp on each level634 // to return the updated value of the reduction to the enclosing635 // loops.636 for (unsigned i = 0; i < rank; ++i) {637 auto result = fir::ResultOp::create(builder, loc, reductionVal);638 // Proceed to the outer loop.639 auto loop = mlir::cast<fir::DoLoopOp>(result->getParentOp());640 reductionVal = loop.getResult(0);641 // Set insertion point after the loop operation that we have642 // just processed.643 builder.setInsertionPointAfter(loop.getOperation());644 }645 // End of loop nest. The insertion point is after the outermost loop.646 if (maskMayBeLogicalScalar) {647 if (fir::IfOp ifOp =648 mlir::dyn_cast<fir::IfOp>(builder.getBlock()->getParentOp())) {649 fir::ResultOp::create(builder, loc, reductionVal);650 builder.setInsertionPointAfter(ifOp);651 // Redefine flagSet to escape scope of ifOp652 flagSet = builder.createIntegerConstant(loc, resultElemType, 1);653 reductionVal = ifOp.getResult(0);654 }655 }656}657 658static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder,659 mlir::func::FuncOp &funcOp, bool isMax,660 unsigned rank, int maskRank,661 mlir::Type elementType,662 mlir::Type maskElemType,663 mlir::Type resultElemTy, bool isDim) {664 auto init = [isMax](fir::FirOpBuilder builder, mlir::Location loc,665 mlir::Type elementType) {666 if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {667 const llvm::fltSemantics &sem = ty.getFloatSemantics();668 llvm::APFloat limit = llvm::APFloat::getInf(sem, /*Negative=*/isMax);669 return builder.createRealConstant(loc, elementType, limit);670 }671 unsigned bits = elementType.getIntOrFloatBitWidth();672 int64_t initValue = (isMax ? llvm::APInt::getSignedMinValue(bits)673 : llvm::APInt::getSignedMaxValue(bits))674 .getSExtValue();675 return builder.createIntegerConstant(loc, elementType, initValue);676 };677 678 mlir::Location loc = mlir::UnknownLoc::get(builder.getContext());679 builder.setInsertionPointToEnd(funcOp.addEntryBlock());680 681 mlir::Value mask = funcOp.front().getArgument(2);682 683 // Set up result array in case of early exit / 0 length array684 mlir::IndexType idxTy = builder.getIndexType();685 mlir::Type resultTy = fir::SequenceType::get(rank, resultElemTy);686 mlir::Type resultHeapTy = fir::HeapType::get(resultTy);687 mlir::Type resultBoxTy = fir::BoxType::get(resultHeapTy);688 689 mlir::Value returnValue = builder.createIntegerConstant(loc, resultElemTy, 0);690 mlir::Value resultArrSize = builder.createIntegerConstant(loc, idxTy, rank);691 692 mlir::Value resultArrInit = fir::AllocMemOp::create(builder, loc, resultTy);693 mlir::Value resultArrShape =694 fir::ShapeOp::create(builder, loc, resultArrSize);695 mlir::Value resultArr = fir::EmboxOp::create(builder, loc, resultBoxTy,696 resultArrInit, resultArrShape);697 698 mlir::Type resultRefTy = builder.getRefType(resultElemTy);699 700 if (maskRank > 0) {701 fir::SequenceType::Shape flatShape(rank,702 fir::SequenceType::getUnknownExtent());703 mlir::Type maskTy = fir::SequenceType::get(flatShape, maskElemType);704 mlir::Type boxMaskTy = fir::BoxType::get(maskTy);705 mask = fir::ConvertOp::create(builder, loc, boxMaskTy, mask);706 }707 708 for (unsigned int i = 0; i < rank; ++i) {709 mlir::Value index = builder.createIntegerConstant(loc, idxTy, i);710 mlir::Value resultElemAddr =711 fir::CoordinateOp::create(builder, loc, resultRefTy, resultArr, index);712 fir::StoreOp::create(builder, loc, returnValue, resultElemAddr);713 }714 715 auto genBodyOp =716 [&rank, &resultArr, isMax, &mask, &maskElemType, &maskRank](717 fir::FirOpBuilder builder, mlir::Location loc, mlir::Type elementType,718 mlir::Value array, mlir::Value flagRef, mlir::Value reduction,719 const llvm::SmallVectorImpl<mlir::Value> &indices) -> mlir::Value {720 // We are in the innermost loop: generate the reduction body.721 if (maskRank > 0) {722 mlir::Type logicalRef = builder.getRefType(maskElemType);723 mlir::Value maskAddr =724 fir::CoordinateOp::create(builder, loc, logicalRef, mask, indices);725 mlir::Value maskElem = fir::LoadOp::create(builder, loc, maskAddr);726 727 // fir::IfOp requires argument to be I1 - won't accept logical or any728 // other Integer.729 mlir::Type ifCompatType = builder.getI1Type();730 mlir::Value ifCompatElem =731 fir::ConvertOp::create(builder, loc, ifCompatType, maskElem);732 733 fir::IfOp ifOp =734 fir::IfOp::create(builder, loc, elementType, ifCompatElem,735 /*withElseRegion=*/true);736 builder.setInsertionPointToStart(&ifOp.getThenRegion().front());737 }738 739 // Set flag that mask was true at some point740 mlir::Value flagSet = builder.createIntegerConstant(741 loc, mlir::cast<fir::ReferenceType>(flagRef.getType()).getEleTy(), 1);742 mlir::Value isFirst = fir::LoadOp::create(builder, loc, flagRef);743 mlir::Type eleRefTy = builder.getRefType(elementType);744 mlir::Value addr =745 fir::CoordinateOp::create(builder, loc, eleRefTy, array, indices);746 mlir::Value elem = fir::LoadOp::create(builder, loc, addr);747 748 mlir::Value cmp;749 if (mlir::isa<mlir::FloatType>(elementType)) {750 // For FP reductions we want the first smallest value to be used, that751 // is not NaN. A OGL/OLT condition will usually work for this unless all752 // the values are Nan or Inf. This follows the same logic as753 // NumericCompare for Minloc/Maxlox in extrema.cpp.754 cmp = mlir::arith::CmpFOp::create(builder, loc,755 isMax ? mlir::arith::CmpFPredicate::OGT756 : mlir::arith::CmpFPredicate::OLT,757 elem, reduction);758 759 mlir::Value cmpNan = mlir::arith::CmpFOp::create(760 builder, loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);761 mlir::Value cmpNan2 = mlir::arith::CmpFOp::create(762 builder, loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);763 cmpNan = mlir::arith::AndIOp::create(builder, loc, cmpNan, cmpNan2);764 cmp = mlir::arith::OrIOp::create(builder, loc, cmp, cmpNan);765 } else if (mlir::isa<mlir::IntegerType>(elementType)) {766 cmp = mlir::arith::CmpIOp::create(builder, loc,767 isMax ? mlir::arith::CmpIPredicate::sgt768 : mlir::arith::CmpIPredicate::slt,769 elem, reduction);770 } else {771 llvm_unreachable("unsupported type");772 }773 774 // The condition used for the loop is isFirst || <the condition above>.775 isFirst = fir::ConvertOp::create(builder, loc, cmp.getType(), isFirst);776 isFirst = mlir::arith::XOrIOp::create(777 builder, loc, isFirst,778 builder.createIntegerConstant(loc, cmp.getType(), 1));779 cmp = mlir::arith::OrIOp::create(builder, loc, cmp, isFirst);780 fir::IfOp ifOp = fir::IfOp::create(builder, loc, elementType, cmp,781 /*withElseRegion*/ true);782 783 builder.setInsertionPointToStart(&ifOp.getThenRegion().front());784 fir::StoreOp::create(builder, loc, flagSet, flagRef);785 mlir::Type resultElemTy = hlfir::getFortranElementType(resultArr.getType());786 mlir::Type returnRefTy = builder.getRefType(resultElemTy);787 mlir::IndexType idxTy = builder.getIndexType();788 789 mlir::Value one = builder.createIntegerConstant(loc, resultElemTy, 1);790 791 for (unsigned int i = 0; i < rank; ++i) {792 mlir::Value index = builder.createIntegerConstant(loc, idxTy, i);793 mlir::Value resultElemAddr = fir::CoordinateOp::create(794 builder, loc, returnRefTy, resultArr, index);795 mlir::Value convert =796 fir::ConvertOp::create(builder, loc, resultElemTy, indices[i]);797 mlir::Value fortranIndex =798 mlir::arith::AddIOp::create(builder, loc, convert, one);799 fir::StoreOp::create(builder, loc, fortranIndex, resultElemAddr);800 }801 fir::ResultOp::create(builder, loc, elem);802 builder.setInsertionPointToStart(&ifOp.getElseRegion().front());803 fir::ResultOp::create(builder, loc, reduction);804 builder.setInsertionPointAfter(ifOp);805 mlir::Value reductionVal = ifOp.getResult(0);806 807 // Close the mask if needed808 if (maskRank > 0) {809 fir::IfOp ifOp =810 mlir::dyn_cast<fir::IfOp>(builder.getBlock()->getParentOp());811 fir::ResultOp::create(builder, loc, reductionVal);812 builder.setInsertionPointToStart(&ifOp.getElseRegion().front());813 fir::ResultOp::create(builder, loc, reduction);814 reductionVal = ifOp.getResult(0);815 builder.setInsertionPointAfter(ifOp);816 }817 818 return reductionVal;819 };820 821 // if mask is a logical scalar, we can check its value before the main loop822 // and either ignore the fact it is there or exit early.823 if (maskRank == 0) {824 mlir::Type i1Type = builder.getI1Type();825 mlir::Type logical = maskElemType;826 mlir::Type logicalRefTy = builder.getRefType(logical);827 mlir::Value condAddr =828 fir::BoxAddrOp::create(builder, loc, logicalRefTy, mask);829 mlir::Value cond = fir::LoadOp::create(builder, loc, condAddr);830 mlir::Value condI1 = fir::ConvertOp::create(builder, loc, i1Type, cond);831 832 fir::IfOp ifOp = fir::IfOp::create(builder, loc, elementType, condI1,833 /*withElseRegion=*/true);834 835 builder.setInsertionPointToStart(&ifOp.getElseRegion().front());836 mlir::Value basicValue;837 if (mlir::isa<mlir::IntegerType>(elementType)) {838 basicValue = builder.createIntegerConstant(loc, elementType, 0);839 } else {840 basicValue = builder.createRealConstant(loc, elementType, 0);841 }842 fir::ResultOp::create(builder, loc, basicValue);843 844 builder.setInsertionPointToStart(&ifOp.getThenRegion().front());845 }846 auto getAddrFn = [](fir::FirOpBuilder builder, mlir::Location loc,847 const mlir::Type &resultElemType, mlir::Value resultArr,848 mlir::Value index) {849 mlir::Type resultRefTy = builder.getRefType(resultElemType);850 return fir::CoordinateOp::create(builder, loc, resultRefTy, resultArr,851 index);852 };853 854 genMinMaxlocReductionLoop(builder, funcOp.front().getArgument(1), init,855 genBodyOp, getAddrFn, rank, elementType, loc,856 maskElemType, resultArr, maskRank == 0);857 858 // Store newly created output array to the reference passed in859 if (isDim) {860 mlir::Type resultBoxTy =861 fir::BoxType::get(fir::HeapType::get(resultElemTy));862 mlir::Value outputArr =863 fir::ConvertOp::create(builder, loc, builder.getRefType(resultBoxTy),864 funcOp.front().getArgument(0));865 mlir::Value resultArrScalar = fir::ConvertOp::create(866 builder, loc, fir::HeapType::get(resultElemTy), resultArrInit);867 mlir::Value resultBox =868 fir::EmboxOp::create(builder, loc, resultBoxTy, resultArrScalar);869 fir::StoreOp::create(builder, loc, resultBox, outputArr);870 } else {871 fir::SequenceType::Shape resultShape(1, rank);872 mlir::Type outputArrTy = fir::SequenceType::get(resultShape, resultElemTy);873 mlir::Type outputHeapTy = fir::HeapType::get(outputArrTy);874 mlir::Type outputBoxTy = fir::BoxType::get(outputHeapTy);875 mlir::Type outputRefTy = builder.getRefType(outputBoxTy);876 mlir::Value outputArr = fir::ConvertOp::create(877 builder, loc, outputRefTy, funcOp.front().getArgument(0));878 fir::StoreOp::create(builder, loc, resultArr, outputArr);879 }880 881 mlir::func::ReturnOp::create(builder, loc);882}883 884/// Generate function type for the simplified version of RTNAME(DotProduct)885/// operating on the given \p elementType.886static mlir::FunctionType genRuntimeDotType(fir::FirOpBuilder &builder,887 const mlir::Type &elementType) {888 mlir::Type boxType = fir::BoxType::get(builder.getNoneType());889 return mlir::FunctionType::get(builder.getContext(), {boxType, boxType},890 {elementType});891}892 893/// Generate function body of the simplified version of RTNAME(DotProduct)894/// with signature provided by \p funcOp. The caller is responsible895/// for saving/restoring the original insertion point of \p builder.896/// \p funcOp is expected to be empty on entry to this function.897/// \p arg1ElementTy and \p arg2ElementTy specify elements types898/// of the underlying array objects - they are used to generate proper899/// element accesses.900static void genRuntimeDotBody(fir::FirOpBuilder &builder,901 mlir::func::FuncOp &funcOp,902 mlir::Type arg1ElementTy,903 mlir::Type arg2ElementTy) {904 // function RTNAME(DotProduct)<T>_simplified(arr1, arr2)905 // T, dimension(:) :: arr1, arr2906 // T product = 0907 // integer iter908 // do iter = 0, extent(arr1)909 // product = product + arr1[iter] * arr2[iter]910 // end do911 // RTNAME(ADotProduct)<T>_simplified = product912 // end function RTNAME(DotProduct)<T>_simplified913 auto loc = mlir::UnknownLoc::get(builder.getContext());914 mlir::Type resultElementType = funcOp.getResultTypes()[0];915 builder.setInsertionPointToEnd(funcOp.addEntryBlock());916 917 mlir::IndexType idxTy = builder.getIndexType();918 919 mlir::Value zero =920 mlir::isa<mlir::FloatType>(resultElementType)921 ? builder.createRealConstant(loc, resultElementType, 0.0)922 : builder.createIntegerConstant(loc, resultElementType, 0);923 924 mlir::Block::BlockArgListType args = funcOp.front().getArguments();925 mlir::Value arg1 = args[0];926 mlir::Value arg2 = args[1];927 928 mlir::Value zeroIdx = builder.createIntegerConstant(loc, idxTy, 0);929 930 fir::SequenceType::Shape flatShape = {fir::SequenceType::getUnknownExtent()};931 mlir::Type arrTy1 = fir::SequenceType::get(flatShape, arg1ElementTy);932 mlir::Type boxArrTy1 = fir::BoxType::get(arrTy1);933 mlir::Value array1 = fir::ConvertOp::create(builder, loc, boxArrTy1, arg1);934 mlir::Type arrTy2 = fir::SequenceType::get(flatShape, arg2ElementTy);935 mlir::Type boxArrTy2 = fir::BoxType::get(arrTy2);936 mlir::Value array2 = fir::ConvertOp::create(builder, loc, boxArrTy2, arg2);937 // This version takes the loop trip count from the first argument.938 // If the first argument's box has unknown (at compilation time)939 // extent, then it may be better to take the extent from the second940 // argument - so that after inlining the loop may be better optimized, e.g.941 // fully unrolled. This requires generating two versions of the simplified942 // function and some analysis at the call site to choose which version943 // is more profitable to call.944 // Note that we can assume that both arguments have the same extent.945 auto dims = fir::BoxDimsOp::create(builder, loc, idxTy, idxTy, idxTy, array1,946 zeroIdx);947 mlir::Value len = dims.getResult(1);948 mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);949 mlir::Value step = one;950 951 // We use C indexing here, so len-1 as loopcount952 mlir::Value loopCount = mlir::arith::SubIOp::create(builder, loc, len, one);953 auto loop = fir::DoLoopOp::create(builder, loc, zeroIdx, loopCount, step,954 /*unordered=*/false,955 /*finalCountValue=*/false, zero);956 mlir::Value sumVal = loop.getRegionIterArgs()[0];957 958 // Begin loop code959 mlir::OpBuilder::InsertPoint loopEndPt = builder.saveInsertionPoint();960 builder.setInsertionPointToStart(loop.getBody());961 962 mlir::Type eleRef1Ty = builder.getRefType(arg1ElementTy);963 mlir::Value index = loop.getInductionVar();964 mlir::Value addr1 =965 fir::CoordinateOp::create(builder, loc, eleRef1Ty, array1, index);966 mlir::Value elem1 = fir::LoadOp::create(builder, loc, addr1);967 // Convert to the result type.968 elem1 = fir::ConvertOp::create(builder, loc, resultElementType, elem1);969 970 mlir::Type eleRef2Ty = builder.getRefType(arg2ElementTy);971 mlir::Value addr2 =972 fir::CoordinateOp::create(builder, loc, eleRef2Ty, array2, index);973 mlir::Value elem2 = fir::LoadOp::create(builder, loc, addr2);974 // Convert to the result type.975 elem2 = fir::ConvertOp::create(builder, loc, resultElementType, elem2);976 977 if (mlir::isa<mlir::FloatType>(resultElementType))978 sumVal = mlir::arith::AddFOp::create(979 builder, loc, mlir::arith::MulFOp::create(builder, loc, elem1, elem2),980 sumVal);981 else if (mlir::isa<mlir::IntegerType>(resultElementType))982 sumVal = mlir::arith::AddIOp::create(983 builder, loc, mlir::arith::MulIOp::create(builder, loc, elem1, elem2),984 sumVal);985 else986 llvm_unreachable("unsupported type");987 988 fir::ResultOp::create(builder, loc, sumVal);989 // End of loop.990 builder.restoreInsertionPoint(loopEndPt);991 992 mlir::Value resultVal = loop.getResult(0);993 mlir::func::ReturnOp::create(builder, loc, resultVal);994}995 996mlir::func::FuncOp SimplifyIntrinsicsPass::getOrCreateFunction(997 fir::FirOpBuilder &builder, const mlir::StringRef &baseName,998 FunctionTypeGeneratorTy typeGenerator,999 FunctionBodyGeneratorTy bodyGenerator) {1000 // WARNING: if the function generated here changes its signature1001 // or behavior (the body code), we should probably embed some1002 // versioning information into its name, otherwise libraries1003 // statically linked with older versions of Flang may stop1004 // working with object files created with newer Flang.1005 // We can also avoid this by using internal linkage, but1006 // this may increase the size of final executable/shared library.1007 std::string replacementName = mlir::Twine{baseName, "_simplified"}.str();1008 // If we already have a function, just return it.1009 mlir::func::FuncOp newFunc = builder.getNamedFunction(replacementName);1010 mlir::FunctionType fType = typeGenerator(builder);1011 if (newFunc) {1012 assert(newFunc.getFunctionType() == fType &&1013 "type mismatch for simplified function");1014 return newFunc;1015 }1016 1017 // Need to build the function!1018 auto loc = mlir::UnknownLoc::get(builder.getContext());1019 newFunc = builder.createFunction(loc, replacementName, fType);1020 auto inlineLinkage = mlir::LLVM::linkage::Linkage::LinkonceODR;1021 auto linkage =1022 mlir::LLVM::LinkageAttr::get(builder.getContext(), inlineLinkage);1023 newFunc->setAttr("llvm.linkage", linkage);1024 1025 // Save the position of the original call.1026 mlir::OpBuilder::InsertPoint insertPt = builder.saveInsertionPoint();1027 1028 bodyGenerator(builder, newFunc);1029 1030 // Now back to where we were adding code earlier...1031 builder.restoreInsertionPoint(insertPt);1032 1033 return newFunc;1034}1035 1036void SimplifyIntrinsicsPass::simplifyIntOrFloatReduction(1037 fir::CallOp call, const fir::KindMapping &kindMap,1038 GenReductionBodyTy genBodyFunc) {1039 // args[1] and args[2] are source filename and line number, ignored.1040 mlir::Operation::operand_range args = call.getArgs();1041 1042 const mlir::Value &dim = args[3];1043 const mlir::Value &mask = args[4];1044 // dim is zero when it is absent, which is an implementation1045 // detail in the runtime library.1046 1047 bool dimAndMaskAbsent = isZero(dim) && isOperandAbsent(mask);1048 unsigned rank = getDimCount(args[0]);1049 1050 // Rank is set to 0 for assumed shape arrays, don't simplify1051 // in these cases1052 if (!(dimAndMaskAbsent && rank > 0))1053 return;1054 1055 mlir::Type resultType = call.getResult(0).getType();1056 1057 if (!mlir::isa<mlir::FloatType>(resultType) &&1058 !mlir::isa<mlir::IntegerType>(resultType))1059 return;1060 1061 auto argType = getArgElementType(args[0]);1062 if (!argType)1063 return;1064 assert(*argType == resultType &&1065 "Argument/result types mismatch in reduction");1066 1067 mlir::SymbolRefAttr callee = call.getCalleeAttr();1068 1069 fir::FirOpBuilder builder{getSimplificationBuilder(call, kindMap)};1070 std::string fmfString{builder.getFastMathFlagsString()};1071 std::string funcName =1072 (mlir::Twine{callee.getLeafReference().getValue(), "x"} +1073 mlir::Twine{rank} +1074 // We must mangle the generated function name with FastMathFlags1075 // value.1076 (fmfString.empty() ? mlir::Twine{} : mlir::Twine{"_", fmfString}))1077 .str();1078 1079 simplifyReductionBody(call, kindMap, genBodyFunc, builder, funcName,1080 resultType);1081}1082 1083void SimplifyIntrinsicsPass::simplifyLogicalDim0Reduction(1084 fir::CallOp call, const fir::KindMapping &kindMap,1085 GenReductionBodyTy genBodyFunc) {1086 1087 mlir::Operation::operand_range args = call.getArgs();1088 const mlir::Value &dim = args[3];1089 unsigned rank = getDimCount(args[0]);1090 1091 // getDimCount returns a rank of 0 for assumed shape arrays, don't simplify in1092 // these cases.1093 if (!(isZero(dim) && rank > 0))1094 return;1095 1096 mlir::Value inputBox = findBoxDef(args[0]);1097 1098 mlir::Type elementType = hlfir::getFortranElementType(inputBox.getType());1099 mlir::SymbolRefAttr callee = call.getCalleeAttr();1100 1101 fir::FirOpBuilder builder{getSimplificationBuilder(call, kindMap)};1102 1103 // Treating logicals as integers makes things a lot easier1104 fir::LogicalType logicalType = {1105 mlir::dyn_cast<fir::LogicalType>(elementType)};1106 fir::KindTy kind = logicalType.getFKind();1107 mlir::Type intElementType = builder.getIntegerType(kind * 8);1108 1109 // Mangle kind into function name as it is not done by default1110 std::string funcName =1111 (mlir::Twine{callee.getLeafReference().getValue(), "Logical"} +1112 mlir::Twine{kind} + "x" + mlir::Twine{rank})1113 .str();1114 1115 simplifyReductionBody(call, kindMap, genBodyFunc, builder, funcName,1116 intElementType);1117}1118 1119void SimplifyIntrinsicsPass::simplifyLogicalDim1Reduction(1120 fir::CallOp call, const fir::KindMapping &kindMap,1121 GenReductionBodyTy genBodyFunc) {1122 1123 mlir::Operation::operand_range args = call.getArgs();1124 mlir::SymbolRefAttr callee = call.getCalleeAttr();1125 mlir::StringRef funcNameBase = callee.getLeafReference().getValue();1126 unsigned rank = getDimCount(args[0]);1127 1128 // getDimCount returns a rank of 0 for assumed shape arrays, don't simplify in1129 // these cases. We check for Dim at the end as some logical functions (Any,1130 // All) set dim to 1 instead of 0 when the argument is not present.1131 if (funcNameBase.ends_with("Dim") || !(rank > 0))1132 return;1133 1134 mlir::Value inputBox = findBoxDef(args[0]);1135 mlir::Type elementType = hlfir::getFortranElementType(inputBox.getType());1136 1137 fir::FirOpBuilder builder{getSimplificationBuilder(call, kindMap)};1138 1139 // Treating logicals as integers makes things a lot easier1140 fir::LogicalType logicalType = {1141 mlir::dyn_cast<fir::LogicalType>(elementType)};1142 fir::KindTy kind = logicalType.getFKind();1143 mlir::Type intElementType = builder.getIntegerType(kind * 8);1144 1145 // Mangle kind into function name as it is not done by default1146 std::string funcName =1147 (mlir::Twine{callee.getLeafReference().getValue(), "Logical"} +1148 mlir::Twine{kind} + "x" + mlir::Twine{rank})1149 .str();1150 1151 simplifyReductionBody(call, kindMap, genBodyFunc, builder, funcName,1152 intElementType);1153}1154 1155void SimplifyIntrinsicsPass::simplifyMinMaxlocReduction(1156 fir::CallOp call, const fir::KindMapping &kindMap, bool isMax) {1157 1158 mlir::Operation::operand_range args = call.getArgs();1159 1160 mlir::SymbolRefAttr callee = call.getCalleeAttr();1161 mlir::StringRef funcNameBase = callee.getLeafReference().getValue();1162 bool isDim = funcNameBase.ends_with("Dim");1163 mlir::Value back = args[isDim ? 7 : 6];1164 if (isTrueOrNotConstant(back))1165 return;1166 1167 mlir::Value mask = args[isDim ? 6 : 5];1168 mlir::Value maskDef = findMaskDef(mask);1169 1170 // maskDef is set to NULL when the defining op is not one we accept.1171 // This tends to be because it is a selectOp, in which case let the1172 // runtime deal with it.1173 if (maskDef == NULL)1174 return;1175 1176 unsigned rank = getDimCount(args[1]);1177 if ((isDim && rank != 1) || !(rank > 0))1178 return;1179 1180 fir::FirOpBuilder builder{getSimplificationBuilder(call, kindMap)};1181 mlir::Location loc = call.getLoc();1182 auto inputBox = findBoxDef(args[1]);1183 mlir::Type inputType = hlfir::getFortranElementType(inputBox.getType());1184 1185 if (mlir::isa<fir::CharacterType>(inputType))1186 return;1187 1188 int maskRank;1189 fir::KindTy kind = 0;1190 mlir::Type logicalElemType = builder.getI1Type();1191 if (isOperandAbsent(mask)) {1192 maskRank = -1;1193 } else {1194 maskRank = getDimCount(mask);1195 mlir::Type maskElemTy = hlfir::getFortranElementType(maskDef.getType());1196 fir::LogicalType logicalFirType = {1197 mlir::dyn_cast<fir::LogicalType>(maskElemTy)};1198 kind = logicalFirType.getFKind();1199 // Convert fir::LogicalType to mlir::Type1200 logicalElemType = logicalFirType;1201 }1202 1203 mlir::Operation *outputDef = args[0].getDefiningOp();1204 mlir::Value outputAlloc = outputDef->getOperand(0);1205 mlir::Type outType = hlfir::getFortranElementType(outputAlloc.getType());1206 1207 std::string fmfString{builder.getFastMathFlagsString()};1208 std::string funcName =1209 (mlir::Twine{callee.getLeafReference().getValue(), "x"} +1210 mlir::Twine{rank} +1211 (maskRank >= 01212 ? "_Logical" + mlir::Twine{kind} + "x" + mlir::Twine{maskRank}1213 : "") +1214 "_")1215 .str();1216 1217 llvm::raw_string_ostream nameOS(funcName);1218 outType.print(nameOS);1219 if (isDim)1220 nameOS << '_' << inputType;1221 nameOS << '_' << fmfString;1222 1223 auto typeGenerator = [rank](fir::FirOpBuilder &builder) {1224 return genRuntimeMinlocType(builder, rank);1225 };1226 auto bodyGenerator = [rank, maskRank, inputType, logicalElemType, outType,1227 isMax, isDim](fir::FirOpBuilder &builder,1228 mlir::func::FuncOp &funcOp) {1229 genRuntimeMinMaxlocBody(builder, funcOp, isMax, rank, maskRank, inputType,1230 logicalElemType, outType, isDim);1231 };1232 1233 mlir::func::FuncOp newFunc =1234 getOrCreateFunction(builder, funcName, typeGenerator, bodyGenerator);1235 fir::CallOp::create(builder, loc, newFunc,1236 mlir::ValueRange{args[0], args[1], mask});1237 call->dropAllReferences();1238 call->erase();1239}1240 1241void SimplifyIntrinsicsPass::simplifyReductionBody(1242 fir::CallOp call, const fir::KindMapping &kindMap,1243 GenReductionBodyTy genBodyFunc, fir::FirOpBuilder &builder,1244 const mlir::StringRef &funcName, mlir::Type elementType) {1245 1246 mlir::Operation::operand_range args = call.getArgs();1247 1248 mlir::Type resultType = call.getResult(0).getType();1249 unsigned rank = getDimCount(args[0]);1250 1251 mlir::Location loc = call.getLoc();1252 1253 auto typeGenerator = [&resultType](fir::FirOpBuilder &builder) {1254 return genNoneBoxType(builder, resultType);1255 };1256 auto bodyGenerator = [&rank, &genBodyFunc,1257 &elementType](fir::FirOpBuilder &builder,1258 mlir::func::FuncOp &funcOp) {1259 genBodyFunc(builder, funcOp, rank, elementType);1260 };1261 // Mangle the function name with the rank value as "x<rank>".1262 mlir::func::FuncOp newFunc =1263 getOrCreateFunction(builder, funcName, typeGenerator, bodyGenerator);1264 auto newCall =1265 fir::CallOp::create(builder, loc, newFunc, mlir::ValueRange{args[0]});1266 call->replaceAllUsesWith(newCall.getResults());1267 call->dropAllReferences();1268 call->erase();1269}1270 1271void SimplifyIntrinsicsPass::runOnOperation() {1272 LLVM_DEBUG(llvm::dbgs() << "=== Begin " DEBUG_TYPE " ===\n");1273 mlir::ModuleOp module = getOperation();1274 fir::KindMapping kindMap = fir::getKindMapping(module);1275 module.walk([&](mlir::Operation *op) {1276 if (auto call = mlir::dyn_cast<fir::CallOp>(op)) {1277 if (cuf::isCUDADeviceContext(op))1278 return;1279 if (mlir::SymbolRefAttr callee = call.getCalleeAttr()) {1280 mlir::StringRef funcName = callee.getLeafReference().getValue();1281 // Replace call to runtime function for SUM when it has single1282 // argument (no dim or mask argument) for 1D arrays with either1283 // Integer4 or Real8 types. Other forms are ignored.1284 // The new function is added to the module.1285 //1286 // Prototype for runtime call (from sum.cpp):1287 // RTNAME(Sum<T>)(const Descriptor &x, const char *source, int line,1288 // int dim, const Descriptor *mask)1289 //1290 if (funcName.starts_with(RTNAME_STRING(Sum))) {1291 simplifyIntOrFloatReduction(call, kindMap, genRuntimeSumBody);1292 return;1293 }1294 if (funcName.starts_with(RTNAME_STRING(DotProduct))) {1295 LLVM_DEBUG(llvm::dbgs() << "Handling " << funcName << "\n");1296 LLVM_DEBUG(llvm::dbgs() << "Call operation:\n"; op->dump();1297 llvm::dbgs() << "\n");1298 mlir::Operation::operand_range args = call.getArgs();1299 const mlir::Value &v1 = args[0];1300 const mlir::Value &v2 = args[1];1301 mlir::Location loc = call.getLoc();1302 fir::FirOpBuilder builder{getSimplificationBuilder(op, kindMap)};1303 // Stringize the builder's FastMathFlags flags for mangling1304 // the generated function name.1305 std::string fmfString{builder.getFastMathFlagsString()};1306 1307 mlir::Type type = call.getResult(0).getType();1308 if (!mlir::isa<mlir::FloatType>(type) &&1309 !mlir::isa<mlir::IntegerType>(type))1310 return;1311 1312 // Try to find the element types of the boxed arguments.1313 auto arg1Type = getArgElementType(v1);1314 auto arg2Type = getArgElementType(v2);1315 1316 if (!arg1Type || !arg2Type)1317 return;1318 1319 // Support only floating point and integer arguments1320 // now (e.g. logical is skipped here).1321 if (!mlir::isa<mlir::FloatType, mlir::IntegerType>(*arg1Type))1322 return;1323 if (!mlir::isa<mlir::FloatType, mlir::IntegerType>(*arg2Type))1324 return;1325 1326 auto typeGenerator = [&type](fir::FirOpBuilder &builder) {1327 return genRuntimeDotType(builder, type);1328 };1329 auto bodyGenerator = [&arg1Type,1330 &arg2Type](fir::FirOpBuilder &builder,1331 mlir::func::FuncOp &funcOp) {1332 genRuntimeDotBody(builder, funcOp, *arg1Type, *arg2Type);1333 };1334 1335 // Suffix the function name with the element types1336 // of the arguments.1337 std::string typedFuncName(funcName);1338 llvm::raw_string_ostream nameOS(typedFuncName);1339 // We must mangle the generated function name with FastMathFlags1340 // value.1341 if (!fmfString.empty())1342 nameOS << '_' << fmfString;1343 nameOS << '_';1344 arg1Type->print(nameOS);1345 nameOS << '_';1346 arg2Type->print(nameOS);1347 1348 mlir::func::FuncOp newFunc = getOrCreateFunction(1349 builder, typedFuncName, typeGenerator, bodyGenerator);1350 auto newCall = fir::CallOp::create(builder, loc, newFunc,1351 mlir::ValueRange{v1, v2});1352 call->replaceAllUsesWith(newCall.getResults());1353 call->dropAllReferences();1354 call->erase();1355 1356 LLVM_DEBUG(llvm::dbgs() << "Replaced with:\n"; newCall.dump();1357 llvm::dbgs() << "\n");1358 return;1359 }1360 if (funcName.starts_with(RTNAME_STRING(Maxval))) {1361 simplifyIntOrFloatReduction(call, kindMap, genRuntimeMaxvalBody);1362 return;1363 }1364 if (funcName.starts_with(RTNAME_STRING(Count))) {1365 simplifyLogicalDim0Reduction(call, kindMap, genRuntimeCountBody);1366 return;1367 }1368 if (funcName.starts_with(RTNAME_STRING(Any))) {1369 simplifyLogicalDim1Reduction(call, kindMap, genRuntimeAnyBody);1370 return;1371 }1372 if (funcName.ends_with(RTNAME_STRING(All))) {1373 simplifyLogicalDim1Reduction(call, kindMap, genRuntimeAllBody);1374 return;1375 }1376 if (funcName.starts_with(RTNAME_STRING(Minloc))) {1377 simplifyMinMaxlocReduction(call, kindMap, false);1378 return;1379 }1380 if (funcName.starts_with(RTNAME_STRING(Maxloc))) {1381 simplifyMinMaxlocReduction(call, kindMap, true);1382 return;1383 }1384 }1385 }1386 });1387 LLVM_DEBUG(llvm::dbgs() << "=== End " DEBUG_TYPE " ===\n");1388}1389 1390void SimplifyIntrinsicsPass::getDependentDialects(1391 mlir::DialectRegistry ®istry) const {1392 // LLVM::LinkageAttr creation requires that LLVM dialect is loaded.1393 registry.insert<mlir::LLVM::LLVMDialect>();1394}1395