3201 lines · cpp
1//===- SimplifyHLFIRIntrinsics.cpp - Simplify HLFIR Intrinsics ------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8// Normally transformational intrinsics are lowered to calls to runtime9// functions. However, some cases of the intrinsics are faster when inlined10// into the calling function.11//===----------------------------------------------------------------------===//12 13#include "flang/Optimizer/Builder/Character.h"14#include "flang/Optimizer/Builder/Complex.h"15#include "flang/Optimizer/Builder/FIRBuilder.h"16#include "flang/Optimizer/Builder/HLFIRTools.h"17#include "flang/Optimizer/Builder/IntrinsicCall.h"18#include "flang/Optimizer/Dialect/FIRDialect.h"19#include "flang/Optimizer/HLFIR/HLFIRDialect.h"20#include "flang/Optimizer/HLFIR/HLFIROps.h"21#include "flang/Optimizer/HLFIR/Passes.h"22#include "mlir/Dialect/Arith/IR/Arith.h"23#include "mlir/IR/Location.h"24#include "mlir/Pass/Pass.h"25#include "mlir/Transforms/GreedyPatternRewriteDriver.h"26 27namespace hlfir {28#define GEN_PASS_DEF_SIMPLIFYHLFIRINTRINSICS29#include "flang/Optimizer/HLFIR/Passes.h.inc"30} // namespace hlfir31 32#define DEBUG_TYPE "simplify-hlfir-intrinsics"33 34static llvm::cl::opt<bool> forceMatmulAsElemental(35 "flang-inline-matmul-as-elemental",36 llvm::cl::desc("Expand hlfir.matmul as elemental operation"),37 llvm::cl::init(false));38 39namespace {40 41// Helper class to generate operations related to computing42// product of values.43class ProductFactory {44public:45 ProductFactory(mlir::Location loc, fir::FirOpBuilder &builder)46 : loc(loc), builder(builder) {}47 48 // Generate an update of the inner product value:49 // acc += v1 * v2, OR50 // acc += CONJ(v1) * v2, OR51 // acc ||= v1 && v252 //53 // CONJ parameter specifies whether the first complex product argument54 // needs to be conjugated.55 template <bool CONJ = false>56 mlir::Value genAccumulateProduct(mlir::Value acc, mlir::Value v1,57 mlir::Value v2) {58 mlir::Type resultType = acc.getType();59 acc = castToProductType(acc, resultType);60 v1 = castToProductType(v1, resultType);61 v2 = castToProductType(v2, resultType);62 mlir::Value result;63 if (mlir::isa<mlir::FloatType>(resultType)) {64 result = mlir::arith::AddFOp::create(65 builder, loc, acc, mlir::arith::MulFOp::create(builder, loc, v1, v2));66 } else if (mlir::isa<mlir::ComplexType>(resultType)) {67 if constexpr (CONJ)68 result = fir::IntrinsicLibrary{builder, loc}.genConjg(resultType, v1);69 else70 result = v1;71 72 result = fir::AddcOp::create(73 builder, loc, acc, fir::MulcOp::create(builder, loc, result, v2));74 } else if (mlir::isa<mlir::IntegerType>(resultType)) {75 result = mlir::arith::AddIOp::create(76 builder, loc, acc, mlir::arith::MulIOp::create(builder, loc, v1, v2));77 } else if (mlir::isa<fir::LogicalType>(resultType)) {78 result = mlir::arith::OrIOp::create(79 builder, loc, acc, mlir::arith::AndIOp::create(builder, loc, v1, v2));80 } else {81 llvm_unreachable("unsupported type");82 }83 84 return builder.createConvert(loc, resultType, result);85 }86 87private:88 mlir::Location loc;89 fir::FirOpBuilder &builder;90 91 mlir::Value castToProductType(mlir::Value value, mlir::Type type) {92 if (mlir::isa<fir::LogicalType>(type))93 return builder.createConvert(loc, builder.getIntegerType(1), value);94 95 // TODO: the multiplications/additions by/of zero resulting from96 // complex * real are optimized by LLVM under -fno-signed-zeros97 // -fno-honor-nans.98 // We can make them disappear by default if we:99 // * either expand the complex multiplication into real100 // operations, OR101 // * set nnan nsz fast-math flags to the complex operations.102 if (fir::isa_complex(type) && !fir::isa_complex(value.getType())) {103 mlir::Value zeroCmplx = fir::factory::createZeroValue(builder, loc, type);104 fir::factory::Complex helper(builder, loc);105 mlir::Type partType = helper.getComplexPartType(type);106 return helper.insertComplexPart(zeroCmplx,107 castToProductType(value, partType),108 /*isImagPart=*/false);109 }110 return builder.createConvert(loc, type, value);111 }112};113 114class TransposeAsElementalConversion115 : public mlir::OpRewritePattern<hlfir::TransposeOp> {116public:117 using mlir::OpRewritePattern<hlfir::TransposeOp>::OpRewritePattern;118 119 llvm::LogicalResult120 matchAndRewrite(hlfir::TransposeOp transpose,121 mlir::PatternRewriter &rewriter) const override {122 hlfir::ExprType expr = transpose.getType();123 // TODO: hlfir.elemental supports polymorphic data types now,124 // so this can be supported.125 if (expr.isPolymorphic())126 return rewriter.notifyMatchFailure(transpose,127 "TRANSPOSE of polymorphic type");128 129 mlir::Location loc = transpose.getLoc();130 fir::FirOpBuilder builder{rewriter, transpose.getOperation()};131 mlir::Type elementType = expr.getElementType();132 hlfir::Entity array = hlfir::Entity{transpose.getArray()};133 mlir::Value resultShape = genResultShape(loc, builder, array);134 llvm::SmallVector<mlir::Value, 1> typeParams;135 hlfir::genLengthParameters(loc, builder, array, typeParams);136 137 auto genKernel = [&array](mlir::Location loc, fir::FirOpBuilder &builder,138 mlir::ValueRange inputIndices) -> hlfir::Entity {139 assert(inputIndices.size() == 2 && "checked in TransposeOp::validate");140 const std::initializer_list<mlir::Value> initList = {inputIndices[1],141 inputIndices[0]};142 mlir::ValueRange transposedIndices(initList);143 hlfir::Entity element =144 hlfir::getElementAt(loc, builder, array, transposedIndices);145 hlfir::Entity val = hlfir::loadTrivialScalar(loc, builder, element);146 return val;147 };148 hlfir::ElementalOp elementalOp = hlfir::genElementalOp(149 loc, builder, elementType, resultShape, typeParams, genKernel,150 /*isUnordered=*/true, /*polymorphicMold=*/nullptr,151 transpose.getResult().getType());152 153 // it wouldn't be safe to replace block arguments with a different154 // hlfir.expr type. Types can differ due to differing amounts of shape155 // information156 assert(elementalOp.getResult().getType() ==157 transpose.getResult().getType());158 159 rewriter.replaceOp(transpose, elementalOp);160 return mlir::success();161 }162 163private:164 static mlir::Value genResultShape(mlir::Location loc,165 fir::FirOpBuilder &builder,166 hlfir::Entity array) {167 llvm::SmallVector<mlir::Value, 2> inExtents =168 hlfir::genExtentsVector(loc, builder, array);169 170 // transpose indices171 assert(inExtents.size() == 2 && "checked in TransposeOp::validate");172 return fir::ShapeOp::create(builder, loc,173 mlir::ValueRange{inExtents[1], inExtents[0]});174 }175};176 177/// Base class for converting reduction-like operations into178/// a reduction loop[-nest] optionally wrapped into hlfir.elemental.179/// It is used to handle operations produced for ALL, ANY, COUNT,180/// MAXLOC, MAXVAL, MINLOC, MINVAL, SUM intrinsics.181///182/// All of these operations take an input array, and optional183/// dim, mask arguments. ALL, ANY, COUNT do not have mask argument.184class ReductionAsElementalConverter {185public:186 ReductionAsElementalConverter(mlir::Operation *op,187 mlir::PatternRewriter &rewriter)188 : op{op}, rewriter{rewriter}, loc{op->getLoc()}, builder{rewriter, op} {189 assert(op->getNumResults() == 1);190 }191 virtual ~ReductionAsElementalConverter() {}192 193 /// Do the actual conversion or return mlir::failure(),194 /// if conversion is not possible.195 mlir::LogicalResult convert();196 197private:198 // Return fir.shape specifying the shape of the result199 // of a reduction with DIM=dimVal. The second return value200 // is the extent of the DIM dimension.201 std::tuple<mlir::Value, mlir::Value>202 genResultShapeForPartialReduction(hlfir::Entity array, int64_t dimVal);203 204 /// \p mask is a scalar or array logical mask.205 /// If \p isPresentPred is not nullptr, it is a dynamic predicate value206 /// identifying whether the mask's variable is present.207 /// \p indices is a range of one-based indices to access \p mask208 /// when it is an array.209 ///210 /// The method returns the scalar mask value to guard the access211 /// to a single element of the input array.212 mlir::Value genMaskValue(mlir::Value mask, mlir::Value isPresentPred,213 mlir::ValueRange indices);214 215protected:216 /// Return the input array.217 virtual mlir::Value getSource() const = 0;218 219 /// Return DIM or nullptr, if it is not present.220 virtual mlir::Value getDim() const = 0;221 222 /// Return MASK or nullptr, if it is not present.223 virtual mlir::Value getMask() const { return nullptr; }224 225 /// Return FastMathFlags attached to the operation226 /// or arith::FastMathFlags::none, if the operation227 /// does not support FastMathFlags (e.g. ALL, ANY, COUNT).228 virtual mlir::arith::FastMathFlags getFastMath() const {229 return mlir::arith::FastMathFlags::none;230 }231 232 /// Generates initial values for the reduction values used233 /// by the reduction loop. In general, there is a single234 /// loop-carried reduction value (e.g. for SUM), but, for example,235 /// MAXLOC/MINLOC implementation uses multiple reductions.236 /// \p oneBasedIndices contains any array indices predefined237 /// before the reduction loop, i.e. it is empty for total238 /// reductions, and contains the one-based indices of the wrapping239 /// hlfir.elemental.240 /// \p extents are the pre-computed extents of the input array.241 /// For total reductions, \p extents holds extents of all dimensions.242 /// For partial reductions, \p extents holds a single extent243 /// of the DIM dimension.244 virtual llvm::SmallVector<mlir::Value>245 genReductionInitValues(mlir::ValueRange oneBasedIndices,246 const llvm::SmallVectorImpl<mlir::Value> &extents) = 0;247 248 /// Perform reduction(s) update given a single input array's element249 /// identified by \p array and \p oneBasedIndices coordinates.250 /// \p currentValue specifies the current value(s) of the reduction(s)251 /// inside the reduction loop body.252 virtual llvm::SmallVector<mlir::Value>253 reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> ¤tValue,254 hlfir::Entity array, mlir::ValueRange oneBasedIndices) = 0;255 256 /// Given reduction value(s) in \p reductionResults produced257 /// by the reduction loop, apply any required updates and return258 /// new reduction value(s) to be used after the reduction loop259 /// (e.g. as the result yield of the wrapping hlfir.elemental).260 /// NOTE: if the reduction loop is wrapped in hlfir.elemental,261 /// the insertion point of any generated code is inside hlfir.elemental.262 virtual hlfir::Entity263 genFinalResult(const llvm::SmallVectorImpl<mlir::Value> &reductionResults) {264 assert(reductionResults.size() == 1 &&265 "default implementation of genFinalResult expect a single reduction "266 "value");267 return hlfir::Entity{reductionResults[0]};268 }269 270 /// Return mlir::success(), if the operation can be converted.271 /// The default implementation always returns mlir::success().272 /// The derived type may override the default implementation273 /// with its own definition.274 virtual mlir::LogicalResult isConvertible() const { return mlir::success(); }275 276 // Default implementation of isTotalReduction() just checks277 // if the result of the operation is a scalar.278 // True result indicates that the reduction has to be done279 // across all elements, false result indicates that280 // the result is an array expression produced by an hlfir.elemental281 // operation with a single reduction loop across the DIM dimension.282 //283 // MAXLOC/MINLOC must override this.284 virtual bool isTotalReduction() const { return getResultRank() == 0; }285 286 // Return true, if the reduction loop[-nest] may be unordered.287 // In general, FP reductions may only be unordered when288 // FastMathFlags::reassoc transformations are allowed.289 //290 // Some dervied types may need to override this.291 virtual bool isUnordered() const {292 mlir::Type elemType = getSourceElementType();293 if (mlir::isa<mlir::IntegerType, fir::LogicalType, fir::CharacterType>(294 elemType))295 return true;296 return static_cast<bool>(getFastMath() &297 mlir::arith::FastMathFlags::reassoc);298 }299 300 /// Return 0, if DIM is not present or its values does not matter301 /// (for example, a reduction of 1D array does not care about302 /// the DIM value, assuming that it is a valid program).303 /// Return mlir::failure(), if DIM is a constant known304 /// to be invalid for the given array.305 /// Otherwise, return DIM constant value.306 mlir::FailureOr<int64_t> getConstDim() const {307 int64_t dimVal = 0;308 if (!isTotalReduction()) {309 // In case of partial reduction we should ignore the operations310 // with invalid DIM values. They may appear in dead code311 // after constant propagation.312 auto constDim = fir::getIntIfConstant(getDim());313 if (!constDim)314 return rewriter.notifyMatchFailure(op, "Nonconstant DIM");315 dimVal = *constDim;316 317 if ((dimVal <= 0 || dimVal > getSourceRank()))318 return rewriter.notifyMatchFailure(op,319 "Invalid DIM for partial reduction");320 }321 return dimVal;322 }323 324 /// Return hlfir::Entity of the result.325 hlfir::Entity getResultEntity() const {326 return hlfir::Entity{op->getResult(0)};327 }328 329 /// Return type of the result (e.g. !hlfir.expr<?xi32>).330 mlir::Type getResultType() const { return getResultEntity().getType(); }331 332 /// Return the element type of the result (e.g. i32).333 mlir::Type getResultElementType() const {334 return hlfir::getFortranElementType(getResultType());335 }336 337 /// Return rank of the result.338 unsigned getResultRank() const { return getResultEntity().getRank(); }339 340 /// Return the element type of the source.341 mlir::Type getSourceElementType() const {342 return hlfir::getFortranElementType(getSource().getType());343 }344 345 /// Return rank of the input array.346 unsigned getSourceRank() const {347 return hlfir::Entity{getSource()}.getRank();348 }349 350 /// The reduction operation.351 mlir::Operation *op;352 353 mlir::PatternRewriter &rewriter;354 mlir::Location loc;355 fir::FirOpBuilder builder;356};357 358/// Generate initialization value for MIN or MAX reduction359/// of the given \p type.360template <bool IS_MAX>361static mlir::Value genMinMaxInitValue(mlir::Location loc,362 fir::FirOpBuilder &builder,363 mlir::Type type) {364 if (auto ty = mlir::dyn_cast<mlir::FloatType>(type)) {365 const llvm::fltSemantics &sem = ty.getFloatSemantics();366 // We must not use +/-INF here. If the reduction input is empty,367 // the result of reduction must be +/-LARGEST.368 llvm::APFloat limit = llvm::APFloat::getLargest(sem, /*Negative=*/IS_MAX);369 return builder.createRealConstant(loc, type, limit);370 }371 unsigned bits = type.getIntOrFloatBitWidth();372 int64_t limitInt = IS_MAX373 ? llvm::APInt::getSignedMinValue(bits).getSExtValue()374 : llvm::APInt::getSignedMaxValue(bits).getSExtValue();375 return builder.createIntegerConstant(loc, type, limitInt);376}377 378/// Generate a comparison of an array element value \p elem379/// and the current reduction value \p reduction for MIN/MAX reduction.380template <bool IS_MAX>381static mlir::Value382genMinMaxComparison(mlir::Location loc, fir::FirOpBuilder &builder,383 mlir::Value elem, mlir::Value reduction) {384 if (mlir::isa<mlir::FloatType>(reduction.getType())) {385 // For FP reductions we want the first smallest value to be used, that386 // is not NaN. A OGL/OLT condition will usually work for this unless all387 // the values are Nan or Inf. This follows the same logic as388 // NumericCompare for Minloc/Maxloc in extrema.cpp.389 mlir::Value cmp =390 mlir::arith::CmpFOp::create(builder, loc,391 IS_MAX ? mlir::arith::CmpFPredicate::OGT392 : mlir::arith::CmpFPredicate::OLT,393 elem, reduction);394 mlir::Value cmpNan = mlir::arith::CmpFOp::create(395 builder, loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);396 mlir::Value cmpNan2 = mlir::arith::CmpFOp::create(397 builder, loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);398 cmpNan = mlir::arith::AndIOp::create(builder, loc, cmpNan, cmpNan2);399 return mlir::arith::OrIOp::create(builder, loc, cmp, cmpNan);400 } else if (mlir::isa<mlir::IntegerType>(reduction.getType())) {401 return mlir::arith::CmpIOp::create(builder, loc,402 IS_MAX ? mlir::arith::CmpIPredicate::sgt403 : mlir::arith::CmpIPredicate::slt,404 elem, reduction);405 }406 llvm_unreachable("unsupported type");407}408 409// Generate a predicate value indicating that an array with the given410// extents is not empty.411static mlir::Value412genIsNotEmptyArrayExtents(mlir::Location loc, fir::FirOpBuilder &builder,413 const llvm::SmallVectorImpl<mlir::Value> &extents) {414 mlir::Value isNotEmpty = builder.createBool(loc, true);415 for (auto extent : extents) {416 mlir::Value zero =417 fir::factory::createZeroValue(builder, loc, extent.getType());418 mlir::Value cmp = mlir::arith::CmpIOp::create(419 builder, loc, mlir::arith::CmpIPredicate::ne, extent, zero);420 isNotEmpty = mlir::arith::AndIOp::create(builder, loc, isNotEmpty, cmp);421 }422 return isNotEmpty;423}424 425// Helper method for MIN/MAX LOC/VAL reductions.426// It returns a vector of indices such that they address427// the first element of an array (in case of total reduction)428// or its section (in case of partial reduction).429//430// If case of total reduction oneBasedIndices must be empty,431// otherwise, they contain the one based indices of the wrapping432// hlfir.elemental.433// Basically, the method adds the necessary number of constant-one434// indices into oneBasedIndices.435static llvm::SmallVector<mlir::Value> genFirstElementIndicesForReduction(436 mlir::Location loc, fir::FirOpBuilder &builder, bool isTotalReduction,437 mlir::FailureOr<int64_t> dim, unsigned rank,438 mlir::ValueRange oneBasedIndices) {439 llvm::SmallVector<mlir::Value> indices{oneBasedIndices};440 mlir::Value one =441 builder.createIntegerConstant(loc, builder.getIndexType(), 1);442 if (isTotalReduction) {443 assert(oneBasedIndices.size() == 0 &&444 "wrong number of indices for total reduction");445 // Set indices to all-ones.446 indices.append(rank, one);447 } else {448 assert(oneBasedIndices.size() == rank - 1 &&449 "there must be RANK-1 indices for partial reduction");450 assert(mlir::succeeded(dim) && "partial reduction with invalid DIM");451 // Insert constant-one index at DIM dimension.452 indices.insert(indices.begin() + *dim - 1, one);453 }454 return indices;455}456 457/// Implementation of ReductionAsElementalConverter interface458/// for MAXLOC/MINLOC.459template <typename T>460class MinMaxlocAsElementalConverter : public ReductionAsElementalConverter {461 static_assert(std::is_same_v<T, hlfir::MaxlocOp> ||462 std::is_same_v<T, hlfir::MinlocOp>);463 static constexpr unsigned maxRank = Fortran::common::maxRank;464 // We have the following reduction values in the reduction loop:465 // * N integer coordinates, where N is:466 // - RANK(ARRAY) for total reductions.467 // - 1 for partial reductions.468 // * 1 reduction value holding the current MIN/MAX.469 // * 1 boolean indicating whether it is the first time470 // the mask is true.471 //472 // If useIsFirst() returns false, then the boolean loop-carried473 // value is not used.474 static constexpr unsigned maxNumReductions = Fortran::common::maxRank + 2;475 static constexpr bool isMax = std::is_same_v<T, hlfir::MaxlocOp>;476 using Base = ReductionAsElementalConverter;477 478public:479 MinMaxlocAsElementalConverter(T op, mlir::PatternRewriter &rewriter)480 : Base{op.getOperation(), rewriter} {}481 482private:483 virtual mlir::Value getSource() const final { return getOp().getArray(); }484 virtual mlir::Value getDim() const final { return getOp().getDim(); }485 virtual mlir::Value getMask() const final { return getOp().getMask(); }486 virtual mlir::arith::FastMathFlags getFastMath() const final {487 return getOp().getFastmath();488 }489 490 virtual mlir::LogicalResult isConvertible() const final {491 if (getOp().getBack())492 return rewriter.notifyMatchFailure(493 getOp(), "BACK is not supported for MINLOC/MAXLOC inlining");494 if (mlir::isa<fir::CharacterType>(getSourceElementType()))495 return rewriter.notifyMatchFailure(496 getOp(),497 "CHARACTER type is not supported for MINLOC/MAXLOC inlining");498 return mlir::success();499 }500 501 // If the result is scalar, then DIM does not matter,502 // and this is a total reduction.503 // If DIM is not present, this is a total reduction.504 virtual bool isTotalReduction() const final {505 return getResultRank() == 0 || !getDim();506 }507 508 virtual llvm::SmallVector<mlir::Value> genReductionInitValues(509 mlir::ValueRange oneBasedIndices,510 const llvm::SmallVectorImpl<mlir::Value> &extents) final;511 virtual llvm::SmallVector<mlir::Value>512 reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> ¤tValue,513 hlfir::Entity array, mlir::ValueRange oneBasedIndices) final;514 virtual hlfir::Entity genFinalResult(515 const llvm::SmallVectorImpl<mlir::Value> &reductionResults) final;516 517private:518 T getOp() const { return mlir::cast<T>(op); }519 520 unsigned getNumCoors() const {521 return isTotalReduction() ? getSourceRank() : 1;522 }523 524 void525 checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {526 if (!useIsFirst())527 assert(reductions.size() == getNumCoors() + 1 &&528 "invalid number of reductions for MINLOC/MAXLOC");529 else530 assert(reductions.size() == getNumCoors() + 2 &&531 "invalid number of reductions for MINLOC/MAXLOC");532 }533 534 mlir::Value535 getCurrentMinMax(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {536 checkReductions(reductions);537 return reductions[getNumCoors()];538 }539 540 mlir::Value541 getIsFirst(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {542 checkReductions(reductions);543 assert(useIsFirst() && "IsFirst predicate must not be used");544 return reductions[getNumCoors() + 1];545 }546 547 // Return true iff the input can contain NaNs, and they should be548 // honored, such that all-NaNs input must produce the location549 // of the first unmasked NaN.550 bool honorNans() const {551 return !static_cast<bool>(getFastMath() & mlir::arith::FastMathFlags::nnan);552 }553 554 // Return true iff we have to use the loop-carried IsFirst predicate.555 // If there is no mask, we can initialize the reductions using556 // the first elements of the input.557 // If NaNs are not honored, we can initialize the starting MIN/MAX558 // value to +/-LARGEST; the coordinates are guaranteed to be updated559 // properly for non-empty input without NaNs.560 bool useIsFirst() const { return getMask() && honorNans(); }561};562 563template <typename T>564llvm::SmallVector<mlir::Value>565MinMaxlocAsElementalConverter<T>::genReductionInitValues(566 mlir::ValueRange oneBasedIndices,567 const llvm::SmallVectorImpl<mlir::Value> &extents) {568 fir::IfOp ifOp;569 if (!useIsFirst() && honorNans()) {570 // Check if we can load the value of the first element in the array571 // or its section (for partial reduction).572 assert(!getMask() && "cannot fetch first element when mask is present");573 assert(extents.size() == getNumCoors() &&574 "wrong number of extents for MINLOC/MAXLOC reduction");575 mlir::Value isNotEmpty = genIsNotEmptyArrayExtents(loc, builder, extents);576 577 llvm::SmallVector<mlir::Value> indices = genFirstElementIndicesForReduction(578 loc, builder, isTotalReduction(), getConstDim(), getSourceRank(),579 oneBasedIndices);580 581 llvm::SmallVector<mlir::Type> ifTypes(getNumCoors(),582 getResultElementType());583 ifTypes.push_back(getSourceElementType());584 ifOp = fir::IfOp::create(builder, loc, ifTypes, isNotEmpty,585 /*withElseRegion=*/true);586 builder.setInsertionPointToStart(&ifOp.getThenRegion().front());587 mlir::Value one =588 builder.createIntegerConstant(loc, getResultElementType(), 1);589 llvm::SmallVector<mlir::Value> results(getNumCoors(), one);590 mlir::Value minMaxFirst =591 hlfir::loadElementAt(loc, builder, hlfir::Entity{getSource()}, indices);592 results.push_back(minMaxFirst);593 fir::ResultOp::create(builder, loc, results);594 595 // In the 'else' block use default init values.596 builder.setInsertionPointToStart(&ifOp.getElseRegion().front());597 }598 599 // Initial value for the coordinate(s) is zero.600 mlir::Value zeroCoor =601 fir::factory::createZeroValue(builder, loc, getResultElementType());602 llvm::SmallVector<mlir::Value> result(getNumCoors(), zeroCoor);603 604 // Initial value for the MIN/MAX value.605 mlir::Value minMaxInit =606 genMinMaxInitValue<isMax>(loc, builder, getSourceElementType());607 result.push_back(minMaxInit);608 609 if (ifOp) {610 fir::ResultOp::create(builder, loc, result);611 builder.setInsertionPointAfter(ifOp);612 result = ifOp.getResults();613 } else if (useIsFirst()) {614 // Initial value for isFirst predicate. It is switched to false,615 // when the reduction update dynamically happens inside the reduction616 // loop.617 mlir::Value trueVal = builder.createBool(loc, true);618 result.push_back(trueVal);619 }620 621 return result;622}623 624template <typename T>625llvm::SmallVector<mlir::Value>626MinMaxlocAsElementalConverter<T>::reduceOneElement(627 const llvm::SmallVectorImpl<mlir::Value> ¤tValue, hlfir::Entity array,628 mlir::ValueRange oneBasedIndices) {629 checkReductions(currentValue);630 hlfir::Entity elementValue =631 hlfir::loadElementAt(loc, builder, array, oneBasedIndices);632 mlir::Value cmp = genMinMaxComparison<isMax>(loc, builder, elementValue,633 getCurrentMinMax(currentValue));634 if (useIsFirst()) {635 // If isFirst is true, then do the reduction update regardless636 // of the FP comparison.637 cmp =638 mlir::arith::OrIOp::create(builder, loc, cmp, getIsFirst(currentValue));639 }640 641 llvm::SmallVector<mlir::Value> newIndices;642 int64_t dim = 1;643 if (!isTotalReduction()) {644 auto dimVal = getConstDim();645 assert(mlir::succeeded(dimVal) &&646 "partial MINLOC/MAXLOC reduction with invalid DIM");647 dim = *dimVal;648 assert(getNumCoors() == 1 &&649 "partial MAXLOC/MINLOC reduction must compute one coordinate");650 }651 652 for (unsigned coorIdx = 0; coorIdx < getNumCoors(); ++coorIdx) {653 mlir::Value currentCoor = currentValue[coorIdx];654 mlir::Value newCoor = builder.createConvert(655 loc, currentCoor.getType(), oneBasedIndices[coorIdx + dim - 1]);656 mlir::Value update =657 mlir::arith::SelectOp::create(builder, loc, cmp, newCoor, currentCoor);658 newIndices.push_back(update);659 }660 661 mlir::Value newMinMax = mlir::arith::SelectOp::create(662 builder, loc, cmp, elementValue, getCurrentMinMax(currentValue));663 newIndices.push_back(newMinMax);664 665 if (useIsFirst()) {666 mlir::Value newIsFirst = builder.createBool(loc, false);667 newIndices.push_back(newIsFirst);668 }669 670 assert(currentValue.size() == newIndices.size() &&671 "invalid number of updated reductions");672 673 return newIndices;674}675 676template <typename T>677hlfir::Entity MinMaxlocAsElementalConverter<T>::genFinalResult(678 const llvm::SmallVectorImpl<mlir::Value> &reductionResults) {679 // Identification of the final result of MINLOC/MAXLOC:680 // * If DIM is absent, the result is rank-one array.681 // * If DIM is present:682 // - The result is scalar for rank-one input.683 // - The result is an array of rank RANK(ARRAY)-1.684 checkReductions(reductionResults);685 686 // 16.9.137 & 16.9.143:687 // The subscripts returned by MINLOC/MAXLOC are in the range688 // 1 to the extent of the corresponding dimension.689 mlir::Type indexType = builder.getIndexType();690 691 // For partial reductions, the final result of the reduction692 // loop is just a scalar - the coordinate within DIM dimension.693 if (getResultRank() == 0 || !isTotalReduction()) {694 // The result is a scalar, so just return the scalar.695 assert(getNumCoors() == 1 &&696 "unpexpected number of coordinates for scalar result");697 return hlfir::Entity{reductionResults[0]};698 }699 // This is a total reduction, and there is no wrapping hlfir.elemental.700 // We have to pack the reduced coordinates into a rank-one array.701 unsigned rank = getSourceRank();702 // TODO: in order to avoid introducing new memory effects703 // we should not use a temporary in memory.704 // We can use hlfir.elemental with a switch to pack all the coordinates705 // into an array expression, or we can have a dedicated HLFIR operation706 // for this.707 mlir::Value tempArray = builder.createTemporary(708 loc, fir::SequenceType::get(rank, getResultElementType()));709 for (unsigned i = 0; i < rank; ++i) {710 mlir::Value coor = reductionResults[i];711 mlir::Value idx = builder.createIntegerConstant(loc, indexType, i + 1);712 mlir::Value resultElement =713 hlfir::getElementAt(loc, builder, hlfir::Entity{tempArray}, {idx});714 hlfir::AssignOp::create(builder, loc, coor, resultElement);715 }716 mlir::Value tempExpr = hlfir::AsExprOp::create(717 builder, loc, tempArray, builder.createBool(loc, false));718 return hlfir::Entity{tempExpr};719}720 721/// Base class for numeric reductions like MAXVAl, MINVAL, SUM.722template <typename OpT>723class NumericReductionAsElementalConverterBase724 : public ReductionAsElementalConverter {725 using Base = ReductionAsElementalConverter;726 727protected:728 NumericReductionAsElementalConverterBase(OpT op,729 mlir::PatternRewriter &rewriter)730 : Base{op.getOperation(), rewriter} {}731 732 virtual mlir::Value getSource() const final { return getOp().getArray(); }733 virtual mlir::Value getDim() const final { return getOp().getDim(); }734 virtual mlir::Value getMask() const final { return getOp().getMask(); }735 virtual mlir::arith::FastMathFlags getFastMath() const final {736 return getOp().getFastmath();737 }738 739 OpT getOp() const { return mlir::cast<OpT>(op); }740 741 void checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) {742 assert(reductions.size() == 1 && "reduction must produce single value");743 }744};745 746/// Reduction converter for MAXMAL/MINVAL.747template <typename T>748class MinMaxvalAsElementalConverter749 : public NumericReductionAsElementalConverterBase<T> {750 static_assert(std::is_same_v<T, hlfir::MaxvalOp> ||751 std::is_same_v<T, hlfir::MinvalOp>);752 // We have two reduction values:753 // * The current MIN/MAX value.754 // * 1 boolean indicating whether it is the first time755 // the mask is true.756 //757 // The boolean flag is used to replace the initial value758 // with the first input element even if it is NaN.759 // If useIsFirst() returns false, then the boolean loop-carried760 // value is not used.761 static constexpr bool isMax = std::is_same_v<T, hlfir::MaxvalOp>;762 using Base = NumericReductionAsElementalConverterBase<T>;763 764public:765 MinMaxvalAsElementalConverter(T op, mlir::PatternRewriter &rewriter)766 : Base{op, rewriter} {}767 768private:769 virtual mlir::LogicalResult isConvertible() const final {770 if (mlir::isa<fir::CharacterType>(this->getSourceElementType()))771 return this->rewriter.notifyMatchFailure(772 this->getOp(),773 "CHARACTER type is not supported for MINVAL/MAXVAL inlining");774 return mlir::success();775 }776 777 virtual llvm::SmallVector<mlir::Value> genReductionInitValues(778 mlir::ValueRange oneBasedIndices,779 const llvm::SmallVectorImpl<mlir::Value> &extents) final;780 781 virtual llvm::SmallVector<mlir::Value>782 reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> ¤tValue,783 hlfir::Entity array,784 mlir::ValueRange oneBasedIndices) final {785 this->checkReductions(currentValue);786 llvm::SmallVector<mlir::Value> result;787 fir::FirOpBuilder &builder = this->builder;788 mlir::Location loc = this->loc;789 hlfir::Entity elementValue =790 hlfir::loadElementAt(loc, builder, array, oneBasedIndices);791 mlir::Value currentMinMax = getCurrentMinMax(currentValue);792 mlir::Value cmp =793 genMinMaxComparison<isMax>(loc, builder, elementValue, currentMinMax);794 if (useIsFirst())795 cmp = mlir::arith::OrIOp::create(builder, loc, cmp,796 getIsFirst(currentValue));797 mlir::Value newMinMax = mlir::arith::SelectOp::create(798 builder, loc, cmp, elementValue, currentMinMax);799 result.push_back(newMinMax);800 if (useIsFirst())801 result.push_back(builder.createBool(loc, false));802 return result;803 }804 805 virtual hlfir::Entity genFinalResult(806 const llvm::SmallVectorImpl<mlir::Value> &reductionResults) final {807 this->checkReductions(reductionResults);808 return hlfir::Entity{getCurrentMinMax(reductionResults)};809 }810 811 void812 checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {813 assert(reductions.size() == getNumReductions() &&814 "invalid number of reductions for MINVAL/MAXVAL");815 }816 817 mlir::Value818 getCurrentMinMax(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {819 this->checkReductions(reductions);820 return reductions[0];821 }822 823 mlir::Value824 getIsFirst(const llvm::SmallVectorImpl<mlir::Value> &reductions) const {825 this->checkReductions(reductions);826 assert(useIsFirst() && "IsFirst predicate must not be used");827 return reductions[1];828 }829 830 // Return true iff the input can contain NaNs, and they should be831 // honored, such that all-NaNs input must produce NaN result.832 bool honorNans() const {833 return !static_cast<bool>(this->getFastMath() &834 mlir::arith::FastMathFlags::nnan);835 }836 837 // Return true iff we have to use the loop-carried IsFirst predicate.838 // If there is no mask, we can initialize the reductions using839 // the first elements of the input.840 // If NaNs are not honored, we can initialize the starting MIN/MAX841 // value to +/-LARGEST.842 bool useIsFirst() const { return this->getMask() && honorNans(); }843 844 std::size_t getNumReductions() const { return useIsFirst() ? 2 : 1; }845};846 847template <typename T>848llvm::SmallVector<mlir::Value>849MinMaxvalAsElementalConverter<T>::genReductionInitValues(850 mlir::ValueRange oneBasedIndices,851 const llvm::SmallVectorImpl<mlir::Value> &extents) {852 llvm::SmallVector<mlir::Value> result;853 fir::FirOpBuilder &builder = this->builder;854 mlir::Location loc = this->loc;855 856 fir::IfOp ifOp;857 if (!useIsFirst() && honorNans()) {858 // Check if we can load the value of the first element in the array859 // or its section (for partial reduction).860 assert(!this->getMask() &&861 "cannot fetch first element when mask is present");862 assert(extents.size() ==863 (this->isTotalReduction() ? this->getSourceRank() : 1u) &&864 "wrong number of extents for MINVAL/MAXVAL reduction");865 mlir::Value isNotEmpty = genIsNotEmptyArrayExtents(loc, builder, extents);866 llvm::SmallVector<mlir::Value> indices = genFirstElementIndicesForReduction(867 loc, builder, this->isTotalReduction(), this->getConstDim(),868 this->getSourceRank(), oneBasedIndices);869 870 ifOp = fir::IfOp::create(builder, loc, this->getResultElementType(),871 isNotEmpty,872 /*withElseRegion=*/true);873 builder.setInsertionPointToStart(&ifOp.getThenRegion().front());874 mlir::Value minMaxFirst = hlfir::loadElementAt(875 loc, builder, hlfir::Entity{this->getSource()}, indices);876 fir::ResultOp::create(builder, loc, minMaxFirst);877 878 // In the 'else' block use default init values.879 builder.setInsertionPointToStart(&ifOp.getElseRegion().front());880 }881 882 mlir::Value init =883 genMinMaxInitValue<isMax>(loc, builder, this->getResultElementType());884 result.push_back(init);885 886 if (ifOp) {887 fir::ResultOp::create(builder, loc, result);888 builder.setInsertionPointAfter(ifOp);889 result = ifOp.getResults();890 } else if (useIsFirst()) {891 // Initial value for isFirst predicate. It is switched to false,892 // when the reduction update dynamically happens inside the reduction893 // loop.894 result.push_back(builder.createBool(loc, true));895 }896 897 return result;898}899 900/// Reduction converter for SUM.901class SumAsElementalConverter902 : public NumericReductionAsElementalConverterBase<hlfir::SumOp> {903 using Base = NumericReductionAsElementalConverterBase;904 905public:906 SumAsElementalConverter(hlfir::SumOp op, mlir::PatternRewriter &rewriter)907 : Base{op, rewriter} {}908 909private:910 virtual llvm::SmallVector<mlir::Value> genReductionInitValues(911 [[maybe_unused]] mlir::ValueRange oneBasedIndices,912 [[maybe_unused]] const llvm::SmallVectorImpl<mlir::Value> &extents)913 final {914 return {915 fir::factory::createZeroValue(builder, loc, getResultElementType())};916 }917 virtual llvm::SmallVector<mlir::Value>918 reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> ¤tValue,919 hlfir::Entity array,920 mlir::ValueRange oneBasedIndices) final {921 checkReductions(currentValue);922 hlfir::Entity elementValue =923 hlfir::loadElementAt(loc, builder, array, oneBasedIndices);924 // NOTE: we can use "Kahan summation" same way as the runtime925 // (e.g. when fast-math is not allowed), but let's start with926 // the simple version.927 return {genScalarAdd(currentValue[0], elementValue)};928 }929 930 // Generate scalar addition of the two values (of the same data type).931 mlir::Value genScalarAdd(mlir::Value value1, mlir::Value value2);932};933 934/// Base class for logical reductions like ALL, ANY, COUNT.935/// They do not have MASK and FastMathFlags.936template <typename OpT>937class LogicalReductionAsElementalConverterBase938 : public ReductionAsElementalConverter {939 using Base = ReductionAsElementalConverter;940 941public:942 LogicalReductionAsElementalConverterBase(OpT op,943 mlir::PatternRewriter &rewriter)944 : Base{op.getOperation(), rewriter} {}945 946protected:947 OpT getOp() const { return mlir::cast<OpT>(op); }948 949 void checkReductions(const llvm::SmallVectorImpl<mlir::Value> &reductions) {950 assert(reductions.size() == 1 && "reduction must produce single value");951 }952 953 virtual mlir::Value getSource() const final { return getOp().getMask(); }954 virtual mlir::Value getDim() const final { return getOp().getDim(); }955 956 virtual hlfir::Entity genFinalResult(957 const llvm::SmallVectorImpl<mlir::Value> &reductionResults) override {958 checkReductions(reductionResults);959 return hlfir::Entity{reductionResults[0]};960 }961};962 963/// Reduction converter for ALL/ANY.964template <typename T>965class AllAnyAsElementalConverter966 : public LogicalReductionAsElementalConverterBase<T> {967 static_assert(std::is_same_v<T, hlfir::AllOp> ||968 std::is_same_v<T, hlfir::AnyOp>);969 static constexpr bool isAll = std::is_same_v<T, hlfir::AllOp>;970 using Base = LogicalReductionAsElementalConverterBase<T>;971 972public:973 AllAnyAsElementalConverter(T op, mlir::PatternRewriter &rewriter)974 : Base{op, rewriter} {}975 976private:977 virtual llvm::SmallVector<mlir::Value> genReductionInitValues(978 [[maybe_unused]] mlir::ValueRange oneBasedIndices,979 [[maybe_unused]] const llvm::SmallVectorImpl<mlir::Value> &extents)980 final {981 return {this->builder.createBool(this->loc, isAll ? true : false)};982 }983 virtual llvm::SmallVector<mlir::Value>984 reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> ¤tValue,985 hlfir::Entity array,986 mlir::ValueRange oneBasedIndices) final {987 this->checkReductions(currentValue);988 fir::FirOpBuilder &builder = this->builder;989 mlir::Location loc = this->loc;990 hlfir::Entity elementValue =991 hlfir::loadElementAt(loc, builder, array, oneBasedIndices);992 mlir::Value mask =993 builder.createConvert(loc, builder.getI1Type(), elementValue);994 if constexpr (isAll)995 return {mlir::arith::AndIOp::create(builder, loc, mask, currentValue[0])};996 else997 return {mlir::arith::OrIOp::create(builder, loc, mask, currentValue[0])};998 }999 1000 virtual hlfir::Entity genFinalResult(1001 const llvm::SmallVectorImpl<mlir::Value> &reductionValues) final {1002 this->checkReductions(reductionValues);1003 return hlfir::Entity{this->builder.createConvert(1004 this->loc, this->getResultElementType(), reductionValues[0])};1005 }1006};1007 1008/// Reduction converter for COUNT.1009class CountAsElementalConverter1010 : public LogicalReductionAsElementalConverterBase<hlfir::CountOp> {1011 using Base = LogicalReductionAsElementalConverterBase<hlfir::CountOp>;1012 1013public:1014 CountAsElementalConverter(hlfir::CountOp op, mlir::PatternRewriter &rewriter)1015 : Base{op, rewriter} {}1016 1017private:1018 virtual llvm::SmallVector<mlir::Value> genReductionInitValues(1019 [[maybe_unused]] mlir::ValueRange oneBasedIndices,1020 [[maybe_unused]] const llvm::SmallVectorImpl<mlir::Value> &extents)1021 final {1022 return {1023 fir::factory::createZeroValue(builder, loc, getResultElementType())};1024 }1025 virtual llvm::SmallVector<mlir::Value>1026 reduceOneElement(const llvm::SmallVectorImpl<mlir::Value> ¤tValue,1027 hlfir::Entity array,1028 mlir::ValueRange oneBasedIndices) final {1029 checkReductions(currentValue);1030 hlfir::Entity elementValue =1031 hlfir::loadElementAt(loc, builder, array, oneBasedIndices);1032 mlir::Value cond =1033 builder.createConvert(loc, builder.getI1Type(), elementValue);1034 mlir::Value one =1035 builder.createIntegerConstant(loc, getResultElementType(), 1);1036 mlir::Value add1 =1037 mlir::arith::AddIOp::create(builder, loc, currentValue[0], one);1038 return {mlir::arith::SelectOp::create(builder, loc, cond, add1,1039 currentValue[0])};1040 }1041};1042 1043mlir::LogicalResult ReductionAsElementalConverter::convert() {1044 mlir::LogicalResult canConvert(isConvertible());1045 1046 if (mlir::failed(canConvert))1047 return canConvert;1048 1049 hlfir::Entity array = hlfir::Entity{getSource()};1050 bool isTotalReduce = isTotalReduction();1051 auto dimVal = getConstDim();1052 if (mlir::failed(dimVal))1053 return dimVal;1054 mlir::Value mask = getMask();1055 mlir::Value resultShape, dimExtent;1056 llvm::SmallVector<mlir::Value> arrayExtents;1057 if (isTotalReduce)1058 arrayExtents = hlfir::genExtentsVector(loc, builder, array);1059 else1060 std::tie(resultShape, dimExtent) =1061 genResultShapeForPartialReduction(array, *dimVal);1062 1063 // If the mask is present and is a scalar, then we'd better load its value1064 // outside of the reduction loop making the loop unswitching easier.1065 mlir::Value isPresentPred, maskValue;1066 if (mask) {1067 if (mlir::isa<fir::BaseBoxType>(mask.getType())) {1068 // MASK represented by a box might be dynamically optional,1069 // so we have to check for its presence before accessing it.1070 isPresentPred =1071 fir::IsPresentOp::create(builder, loc, builder.getI1Type(), mask);1072 }1073 1074 if (hlfir::Entity{mask}.isScalar())1075 maskValue = genMaskValue(mask, isPresentPred, {});1076 }1077 1078 auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,1079 mlir::ValueRange inputIndices) -> hlfir::Entity {1080 // Loop over all indices in the DIM dimension, and reduce all values.1081 // If DIM is not present, do total reduction.1082 1083 llvm::SmallVector<mlir::Value> extents;1084 if (isTotalReduce)1085 extents = arrayExtents;1086 else1087 extents.push_back(1088 builder.createConvert(loc, builder.getIndexType(), dimExtent));1089 1090 // Initial value for the reduction.1091 llvm::SmallVector<mlir::Value, 1> reductionInitValues =1092 genReductionInitValues(inputIndices, extents);1093 1094 auto genBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,1095 mlir::ValueRange oneBasedIndices,1096 mlir::ValueRange reductionArgs)1097 -> llvm::SmallVector<mlir::Value, 1> {1098 // Generate the reduction loop-nest body.1099 // The initial reduction value in the innermost loop1100 // is passed via reductionArgs[0].1101 llvm::SmallVector<mlir::Value> indices;1102 if (isTotalReduce) {1103 indices = oneBasedIndices;1104 } else {1105 indices = inputIndices;1106 indices.insert(indices.begin() + *dimVal - 1, oneBasedIndices[0]);1107 }1108 1109 llvm::SmallVector<mlir::Value, 1> reductionValues = reductionArgs;1110 llvm::SmallVector<mlir::Type, 1> reductionTypes;1111 llvm::transform(reductionValues, std::back_inserter(reductionTypes),1112 [](mlir::Value v) { return v.getType(); });1113 fir::IfOp ifOp;1114 if (mask) {1115 // Make the reduction value update conditional on the value1116 // of the mask.1117 if (!maskValue) {1118 // If the mask is an array, use the elemental and the loop indices1119 // to address the proper mask element.1120 maskValue = genMaskValue(mask, isPresentPred, indices);1121 }1122 mlir::Value isUnmasked = fir::ConvertOp::create(1123 builder, loc, builder.getI1Type(), maskValue);1124 ifOp = fir::IfOp::create(builder, loc, reductionTypes, isUnmasked,1125 /*withElseRegion=*/true);1126 // In the 'else' block return the current reduction value.1127 builder.setInsertionPointToStart(&ifOp.getElseRegion().front());1128 fir::ResultOp::create(builder, loc, reductionValues);1129 1130 // In the 'then' block do the actual addition.1131 builder.setInsertionPointToStart(&ifOp.getThenRegion().front());1132 }1133 reductionValues = reduceOneElement(reductionValues, array, indices);1134 if (ifOp) {1135 fir::ResultOp::create(builder, loc, reductionValues);1136 builder.setInsertionPointAfter(ifOp);1137 reductionValues = ifOp.getResults();1138 }1139 1140 return reductionValues;1141 };1142 1143 llvm::SmallVector<mlir::Value, 1> reductionFinalValues =1144 hlfir::genLoopNestWithReductions(1145 loc, builder, extents, reductionInitValues, genBody, isUnordered());1146 return genFinalResult(reductionFinalValues);1147 };1148 1149 if (isTotalReduce) {1150 hlfir::Entity result = genKernel(loc, builder, mlir::ValueRange{});1151 rewriter.replaceOp(op, result);1152 return mlir::success();1153 }1154 1155 hlfir::ElementalOp elementalOp = hlfir::genElementalOp(1156 loc, builder, getResultElementType(), resultShape, /*typeParams=*/{},1157 genKernel,1158 /*isUnordered=*/true, /*polymorphicMold=*/nullptr, getResultType());1159 1160 // it wouldn't be safe to replace block arguments with a different1161 // hlfir.expr type. Types can differ due to differing amounts of shape1162 // information1163 assert(elementalOp.getResult().getType() == op->getResult(0).getType());1164 1165 rewriter.replaceOp(op, elementalOp);1166 return mlir::success();1167}1168 1169std::tuple<mlir::Value, mlir::Value>1170ReductionAsElementalConverter::genResultShapeForPartialReduction(1171 hlfir::Entity array, int64_t dimVal) {1172 llvm::SmallVector<mlir::Value> inExtents =1173 hlfir::genExtentsVector(loc, builder, array);1174 assert(dimVal > 0 && dimVal <= static_cast<int64_t>(inExtents.size()) &&1175 "DIM must be present and a positive constant not exceeding "1176 "the array's rank");1177 1178 mlir::Value dimExtent = inExtents[dimVal - 1];1179 inExtents.erase(inExtents.begin() + dimVal - 1);1180 return {fir::ShapeOp::create(builder, loc, inExtents), dimExtent};1181}1182 1183mlir::Value SumAsElementalConverter::genScalarAdd(mlir::Value value1,1184 mlir::Value value2) {1185 mlir::Type ty = value1.getType();1186 assert(ty == value2.getType() && "reduction values' types do not match");1187 if (mlir::isa<mlir::FloatType>(ty))1188 return mlir::arith::AddFOp::create(builder, loc, value1, value2);1189 else if (mlir::isa<mlir::ComplexType>(ty))1190 return fir::AddcOp::create(builder, loc, value1, value2);1191 else if (mlir::isa<mlir::IntegerType>(ty))1192 return mlir::arith::AddIOp::create(builder, loc, value1, value2);1193 1194 llvm_unreachable("unsupported SUM reduction type");1195}1196 1197mlir::Value ReductionAsElementalConverter::genMaskValue(1198 mlir::Value mask, mlir::Value isPresentPred, mlir::ValueRange indices) {1199 mlir::OpBuilder::InsertionGuard guard(builder);1200 fir::IfOp ifOp;1201 mlir::Type maskType =1202 hlfir::getFortranElementType(fir::unwrapPassByRefType(mask.getType()));1203 if (isPresentPred) {1204 ifOp = fir::IfOp::create(builder, loc, maskType, isPresentPred,1205 /*withElseRegion=*/true);1206 1207 // Use 'true', if the mask is not present.1208 builder.setInsertionPointToStart(&ifOp.getElseRegion().front());1209 mlir::Value trueValue = builder.createBool(loc, true);1210 trueValue = builder.createConvert(loc, maskType, trueValue);1211 fir::ResultOp::create(builder, loc, trueValue);1212 1213 // Load the mask value, if the mask is present.1214 builder.setInsertionPointToStart(&ifOp.getThenRegion().front());1215 }1216 1217 hlfir::Entity maskVar{mask};1218 if (maskVar.isScalar()) {1219 if (mlir::isa<fir::BaseBoxType>(mask.getType())) {1220 // MASK may be a boxed scalar.1221 mlir::Value addr = hlfir::genVariableRawAddress(loc, builder, maskVar);1222 mask = fir::LoadOp::create(builder, loc, hlfir::Entity{addr});1223 } else {1224 mask = hlfir::loadTrivialScalar(loc, builder, maskVar);1225 }1226 } else {1227 // Load from the mask array.1228 assert(!indices.empty() && "no indices for addressing the mask array");1229 maskVar = hlfir::getElementAt(loc, builder, maskVar, indices);1230 mask = hlfir::loadTrivialScalar(loc, builder, maskVar);1231 }1232 1233 if (!isPresentPred)1234 return mask;1235 1236 fir::ResultOp::create(builder, loc, mask);1237 return ifOp.getResult(0);1238}1239 1240/// Convert an operation that is a partial or total reduction1241/// over an array of values into a reduction loop[-nest]1242/// optionally wrapped into hlfir.elemental.1243template <typename Op>1244class ReductionConversion : public mlir::OpRewritePattern<Op> {1245public:1246 using mlir::OpRewritePattern<Op>::OpRewritePattern;1247 1248 llvm::LogicalResult1249 matchAndRewrite(Op op, mlir::PatternRewriter &rewriter) const override {1250 if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||1251 std::is_same_v<Op, hlfir::MinlocOp>) {1252 MinMaxlocAsElementalConverter<Op> converter(op, rewriter);1253 return converter.convert();1254 } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||1255 std::is_same_v<Op, hlfir::MinvalOp>) {1256 MinMaxvalAsElementalConverter<Op> converter(op, rewriter);1257 return converter.convert();1258 } else if constexpr (std::is_same_v<Op, hlfir::CountOp>) {1259 CountAsElementalConverter converter(op, rewriter);1260 return converter.convert();1261 } else if constexpr (std::is_same_v<Op, hlfir::AllOp> ||1262 std::is_same_v<Op, hlfir::AnyOp>) {1263 AllAnyAsElementalConverter<Op> converter(op, rewriter);1264 return converter.convert();1265 } else if constexpr (std::is_same_v<Op, hlfir::SumOp>) {1266 SumAsElementalConverter converter{op, rewriter};1267 return converter.convert();1268 }1269 return rewriter.notifyMatchFailure(op, "unexpected reduction operation");1270 }1271};1272 1273template <typename Op>1274class ArrayShiftConversion : public mlir::OpRewritePattern<Op> {1275public:1276 // The implementation below only support CShiftOp and EOShiftOp.1277 static_assert(std::is_same_v<Op, hlfir::CShiftOp> ||1278 std::is_same_v<Op, hlfir::EOShiftOp>);1279 1280 using mlir::OpRewritePattern<Op>::OpRewritePattern;1281 1282 llvm::LogicalResult1283 matchAndRewrite(Op op, mlir::PatternRewriter &rewriter) const override {1284 1285 hlfir::ExprType expr = mlir::dyn_cast<hlfir::ExprType>(op.getType());1286 assert(expr &&1287 "expected an expression type for the result of the array shift");1288 unsigned arrayRank = expr.getRank();1289 // When it is a 1D CSHIFT/EOSHIFT, we may assume that the DIM argument1290 // (whether it is present or absent) is equal to 1, otherwise,1291 // the program is illegal.1292 int64_t dimVal = 1;1293 if (arrayRank != 1)1294 if (mlir::Value dim = op.getDim()) {1295 auto constDim = fir::getIntIfConstant(dim);1296 if (!constDim)1297 return rewriter.notifyMatchFailure(1298 op, "Nonconstant DIM for CSHIFT/EOSHIFT");1299 dimVal = *constDim;1300 }1301 1302 if (dimVal <= 0 || dimVal > arrayRank)1303 return rewriter.notifyMatchFailure(op, "Invalid DIM for CSHIFT/EOSHIFT");1304 1305 if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>) {1306 // TODO: the EOSHIFT inlining code is not ready to produce1307 // fir.if selecting between ARRAY and BOUNDARY (or the default1308 // boundary value), when they are expressions of type CHARACTER.1309 // This needs more work.1310 if (mlir::isa<fir::CharacterType>(expr.getEleTy())) {1311 if (!hlfir::Entity{op.getArray()}.isVariable())1312 return rewriter.notifyMatchFailure(1313 op, "EOSHIFT with ARRAY being CHARACTER expression");1314 if (op.getBoundary() && !hlfir::Entity{op.getBoundary()}.isVariable())1315 return rewriter.notifyMatchFailure(1316 op, "EOSHIFT with BOUNDARY being CHARACTER expression");1317 }1318 // TODO: selecting between ARRAY and BOUNDARY values with derived types1319 // need more work.1320 if (fir::isa_derived(expr.getEleTy()))1321 return rewriter.notifyMatchFailure(op, "EOSHIFT of derived type");1322 }1323 1324 // When DIM==1 and the contiguity of the input array is not statically1325 // known, try to exploit the fact that the leading dimension might be1326 // contiguous. We can do this now using hlfir.eval_in_mem with1327 // a dynamic check for the leading dimension contiguity.1328 // Otherwise, convert hlfir.cshift/eoshift to hlfir.elemental.1329 //1330 // Note that the hlfir.elemental can be inlined into other hlfir.elemental,1331 // while hlfir.eval_in_mem prevents this, and we will end up creating1332 // a temporary array for the result. We may need to come up with1333 // a more sophisticated logic for picking the most efficient1334 // representation.1335 hlfir::Entity array = hlfir::Entity{op.getArray()};1336 mlir::Type elementType = array.getFortranElementType();1337 if (dimVal == 1 && fir::isa_trivial(elementType) &&1338 // genInMemArrayShift() only works for variables currently.1339 array.isVariable())1340 rewriter.replaceOp(op, genInMemArrayShift(rewriter, op, dimVal));1341 else1342 rewriter.replaceOp(op, genElementalArrayShift(rewriter, op, dimVal));1343 return mlir::success();1344 }1345 1346private:1347 /// For CSHIFT, generate MODULO(\p shiftVal, \p extent).1348 /// For EOSHIFT, return \p shiftVal casted to \p calcType.1349 static mlir::Value normalizeShiftValue(mlir::Location loc,1350 fir::FirOpBuilder &builder,1351 mlir::Value shiftVal,1352 mlir::Value extent,1353 mlir::Type calcType) {1354 shiftVal = builder.createConvert(loc, calcType, shiftVal);1355 if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>)1356 return shiftVal;1357 1358 extent = builder.createConvert(loc, calcType, extent);1359 // Make sure that we do not divide by zero. When the dimension1360 // has zero size, turn the extent into 1. Note that the computed1361 // MODULO value won't be used in this case, so it does not matter1362 // which extent value we use.1363 mlir::Value zero = builder.createIntegerConstant(loc, calcType, 0);1364 mlir::Value one = builder.createIntegerConstant(loc, calcType, 1);1365 mlir::Value isZero = mlir::arith::CmpIOp::create(1366 builder, loc, mlir::arith::CmpIPredicate::eq, extent, zero);1367 extent = mlir::arith::SelectOp::create(builder, loc, isZero, one, extent);1368 shiftVal = fir::IntrinsicLibrary{builder, loc}.genModulo(1369 calcType, {shiftVal, extent});1370 return builder.createConvert(loc, calcType, shiftVal);1371 }1372 1373 /// The indices computations for the array shifts are done using I64 type.1374 /// For CSHIFT, all computations do not overflow signed and unsigned I64.1375 /// For EOSHIFT, some computations may involve negative shift values,1376 /// so using no-unsigned wrap flag would be incorrect.1377 static void setArithOverflowFlags(Op op, fir::FirOpBuilder &builder) {1378 if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>)1379 builder.setIntegerOverflowFlags(mlir::arith::IntegerOverflowFlags::nsw);1380 else1381 builder.setIntegerOverflowFlags(mlir::arith::IntegerOverflowFlags::nsw |1382 mlir::arith::IntegerOverflowFlags::nuw);1383 }1384 1385 /// Return the element type of the EOSHIFT boundary that may be omitted1386 /// statically or dynamically. This element type might be used1387 /// to generate MLIR where we have to select between the default1388 /// boundary value and the dynamically absent/present boundary value.1389 /// If the boundary has a type not defined in Table 16.4 in 16.9.771390 /// of F2023, then the return value is nullptr.1391 static mlir::Type getDefaultBoundaryValueType(mlir::Type elementType) {1392 // To be able to generate a "select" between the default boundary value1393 // and the dynamic boundary value, use BoxCharType for the CHARACTER1394 // cases. This might be a little bit inefficient, because we may1395 // create unnecessary tuples, but it simplifies the inlining code.1396 if (auto charTy = mlir::dyn_cast<fir::CharacterType>(elementType))1397 return fir::BoxCharType::get(charTy.getContext(), charTy.getFKind());1398 1399 if (mlir::isa<fir::LogicalType>(elementType) ||1400 fir::isa_integer(elementType) || fir::isa_real(elementType) ||1401 fir::isa_complex(elementType))1402 return elementType;1403 1404 return nullptr;1405 }1406 1407 /// Generate the default boundary value as defined in Table 16.4 in 16.9.771408 /// of F2023.1409 static mlir::Value genDefaultBoundary(mlir::Location loc,1410 fir::FirOpBuilder &builder,1411 mlir::Type elementType) {1412 assert(getDefaultBoundaryValueType(elementType) &&1413 "default boundary value cannot be computed for the given type");1414 if (mlir::isa<fir::CharacterType>(elementType)) {1415 // Create an empty CHARACTER of the same kind. The assignment1416 // of this empty CHARACTER into the result will add the padding1417 // if necessary.1418 fir::factory::CharacterExprHelper charHelper{builder, loc};1419 mlir::Value zeroLen = builder.createIntegerConstant(1420 loc, builder.getCharacterLengthType(), 0);1421 fir::CharBoxValue emptyCharTemp =1422 charHelper.createCharacterTemp(elementType, zeroLen);1423 return charHelper.createEmbox(emptyCharTemp);1424 }1425 1426 return fir::factory::createZeroValue(builder, loc, elementType);1427 }1428 1429 /// \p entity represents the boundary operand of hlfir.eoshift.1430 /// This method generates a scalar boundary value fetched1431 /// from the boundary entity using \p indices (which may be empty,1432 /// if the boundary operand is scalar).1433 static mlir::Value loadEoshiftVal(mlir::Location loc,1434 fir::FirOpBuilder &builder,1435 hlfir::Entity entity,1436 mlir::ValueRange indices = {}) {1437 hlfir::Entity boundaryVal =1438 hlfir::loadElementAt(loc, builder, entity, indices);1439 1440 mlir::Type boundaryValTy =1441 getDefaultBoundaryValueType(entity.getFortranElementType());1442 1443 // Boxed !fir.char<KIND,LEN> with known LEN are loaded1444 // as raw references to !fir.char<KIND,LEN>.1445 // We need to wrap them into the !fir.boxchar.1446 if (boundaryVal.isVariable() && boundaryValTy &&1447 mlir::isa<fir::BoxCharType>(boundaryValTy))1448 return hlfir::genVariableBoxChar(loc, builder, boundaryVal);1449 return boundaryVal;1450 }1451 1452 /// This method generates a scalar boundary value for the given hlfir.eoshift1453 /// \p op that can be used to initialize cells of the result1454 /// if the scalar/array boundary operand is statically or dynamically1455 /// absent. The first result is the scalar boundary value. The second result1456 /// is a dynamic predicate indicating whether the scalar boundary value1457 /// should actually be used.1458 [[maybe_unused]] static std::pair<mlir::Value, mlir::Value>1459 genScalarBoundaryForEOShift(mlir::Location loc, fir::FirOpBuilder &builder,1460 hlfir::EOShiftOp op) {1461 hlfir::Entity array{op.getArray()};1462 mlir::Type elementType = array.getFortranElementType();1463 1464 if (!op.getBoundary()) {1465 // Boundary operand is statically absent.1466 mlir::Value defaultVal = genDefaultBoundary(loc, builder, elementType);1467 mlir::Value boundaryIsScalarPred = builder.createBool(loc, true);1468 return {defaultVal, boundaryIsScalarPred};1469 }1470 1471 hlfir::Entity boundary{op.getBoundary()};1472 mlir::Type boundaryValTy = getDefaultBoundaryValueType(elementType);1473 1474 if (boundary.isScalar()) {1475 if (!boundaryValTy || !boundary.mayBeOptional()) {1476 // The boundary must be present.1477 mlir::Value boundaryVal = loadEoshiftVal(loc, builder, boundary);1478 mlir::Value boundaryIsScalarPred = builder.createBool(loc, true);1479 return {boundaryVal, boundaryIsScalarPred};1480 }1481 1482 // Boundary is a scalar that may be dynamically absent.1483 // If boundary is not present dynamically, we must use the default1484 // value.1485 assert(mlir::isa<fir::BaseBoxType>(boundary.getType()));1486 mlir::Value isPresentPred =1487 fir::IsPresentOp::create(builder, loc, builder.getI1Type(), boundary);1488 mlir::Value boundaryVal =1489 builder1490 .genIfOp(loc, {boundaryValTy}, isPresentPred,1491 /*withElseRegion=*/true)1492 .genThen([&]() {1493 mlir::Value boundaryVal =1494 loadEoshiftVal(loc, builder, boundary);1495 fir::ResultOp::create(builder, loc, boundaryVal);1496 })1497 .genElse([&]() {1498 mlir::Value defaultVal =1499 genDefaultBoundary(loc, builder, elementType);1500 fir::ResultOp::create(builder, loc, defaultVal);1501 })1502 .getResults()[0];1503 mlir::Value boundaryIsScalarPred = builder.createBool(loc, true);1504 return {boundaryVal, boundaryIsScalarPred};1505 }1506 if (!boundaryValTy || !boundary.mayBeOptional()) {1507 // The boundary must be present1508 mlir::Value boundaryIsScalarPred = builder.createBool(loc, false);1509 return {nullptr, boundaryIsScalarPred};1510 }1511 1512 // Boundary is an array that may be dynamically absent.1513 mlir::Value defaultVal = genDefaultBoundary(loc, builder, elementType);1514 mlir::Value isPresentPred =1515 fir::IsPresentOp::create(builder, loc, builder.getI1Type(), boundary);1516 // If the array is present, then boundaryIsScalarPred must be equal1517 // to false, otherwise, it should be true.1518 mlir::Value trueVal = builder.createBool(loc, true);1519 mlir::Value falseVal = builder.createBool(loc, false);1520 mlir::Value boundaryIsScalarPred = mlir::arith::SelectOp::create(1521 builder, loc, isPresentPred, falseVal, trueVal);1522 return {defaultVal, boundaryIsScalarPred};1523 }1524 1525 /// Generate code that produces the final boundary value to be assigned1526 /// to the result of hlfir.eoshift \p op. \p precomputedScalarBoundary1527 /// specifies the scalar boundary value pre-computed before the elemental1528 /// or the assignment loop. If it is nullptr, then the boundary operand1529 /// of \p op must be a present array. \p boundaryIsScalarPred is a dynamic1530 /// predicate that is true, when the pre-computed scalar value must be used.1531 /// \p oneBasedIndices specify the indices to address into the boundary1532 /// array - they may be empty, if the boundary is scalar.1533 [[maybe_unused]] static mlir::Value selectBoundaryValue(1534 mlir::Location loc, fir::FirOpBuilder &builder, hlfir::EOShiftOp op,1535 mlir::Value precomputedScalarBoundary, mlir::Value boundaryIsScalarPred,1536 mlir::ValueRange oneBasedIndices) {1537 // Boundary is statically absent: a default value has been precomputed.1538 if (!op.getBoundary())1539 return precomputedScalarBoundary;1540 1541 // Boundary is statically present and is a scalar: boundary does not depend1542 // upon the indices and so it has been precomputed.1543 hlfir::Entity boundary{op.getBoundary()};1544 if (boundary.isScalar())1545 return precomputedScalarBoundary;1546 1547 // Boundary is statically present and is an array: if the scalar1548 // boundary has not been precomputed, this means that the data type1549 // of the shifted values does not provide a way to compute1550 // the default boundary value, so the array boundary must be dynamically1551 // present, and we can load the boundary values from it.1552 bool mustBePresent = !precomputedScalarBoundary;1553 if (mustBePresent)1554 return loadEoshiftVal(loc, builder, boundary, oneBasedIndices);1555 1556 // The array boundary may be dynamically absent.1557 // In this case, precomputedScalarBoundary is a pre-computed scalar1558 // boundary value that has to be used if boundaryIsScalarPred1559 // is true, otherwise, the boundary value has to be loaded1560 // from the boundary array.1561 mlir::Type boundaryValTy = precomputedScalarBoundary.getType();1562 mlir::Value newBoundaryVal =1563 builder1564 .genIfOp(loc, {boundaryValTy}, boundaryIsScalarPred,1565 /*withElseRegion=*/true)1566 .genThen([&]() {1567 fir::ResultOp::create(builder, loc, precomputedScalarBoundary);1568 })1569 .genElse([&]() {1570 mlir::Value elem =1571 loadEoshiftVal(loc, builder, boundary, oneBasedIndices);1572 fir::ResultOp::create(builder, loc, elem);1573 })1574 .getResults()[0];1575 return newBoundaryVal;1576 }1577 1578 /// Convert \p op into an hlfir.elemental using1579 /// the pre-computed constant \p dimVal.1580 static mlir::Operation *1581 genElementalArrayShift(mlir::PatternRewriter &rewriter, Op op,1582 int64_t dimVal) {1583 using Fortran::common::maxRank;1584 hlfir::Entity shift = hlfir::Entity{op.getShift()};1585 hlfir::Entity array = hlfir::Entity{op.getArray()};1586 1587 mlir::Location loc = op.getLoc();1588 fir::FirOpBuilder builder{rewriter, op.getOperation()};1589 // The new index computation involves MODULO, which is not implemented1590 // for IndexType, so use I64 instead.1591 mlir::Type calcType = builder.getI64Type();1592 // Set the indices arithmetic overflow flags.1593 setArithOverflowFlags(op, builder);1594 1595 mlir::Value arrayShape = hlfir::genShape(loc, builder, array);1596 llvm::SmallVector<mlir::Value, maxRank> arrayExtents =1597 hlfir::getExplicitExtentsFromShape(arrayShape, builder);1598 llvm::SmallVector<mlir::Value, 1> typeParams;1599 hlfir::genLengthParameters(loc, builder, array, typeParams);1600 mlir::Value shiftDimExtent =1601 builder.createConvert(loc, calcType, arrayExtents[dimVal - 1]);1602 mlir::Value shiftVal;1603 if (shift.isScalar()) {1604 shiftVal = hlfir::loadTrivialScalar(loc, builder, shift);1605 shiftVal =1606 normalizeShiftValue(loc, builder, shiftVal, shiftDimExtent, calcType);1607 }1608 // The boundary operand of hlfir.eoshift may be statically or1609 // dynamically absent.1610 // In both cases, it is assumed to be a scalar with the value1611 // corresponding to the array element type.1612 // boundaryIsScalarPred is a dynamic predicate that identifies1613 // these cases. If boundaryIsScalarPred is dynamicaly false,1614 // then the boundary operand must be a present array.1615 mlir::Value boundaryVal, boundaryIsScalarPred;1616 if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>)1617 std::tie(boundaryVal, boundaryIsScalarPred) =1618 genScalarBoundaryForEOShift(loc, builder, op);1619 1620 auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,1621 mlir::ValueRange inputIndices) -> hlfir::Entity {1622 llvm::SmallVector<mlir::Value, maxRank> indices{inputIndices};1623 if (!shiftVal) {1624 // When the array is not a vector, section1625 // (s(1), s(2), ..., s(dim-1), :, s(dim+1), ..., s(n)1626 // of the result has a value equal to:1627 // CSHIFT(ARRAY(s(1), s(2), ..., s(dim-1), :, s(dim+1), ..., s(n)),1628 // SH, 1),1629 // where SH is either SHIFT (if scalar) or1630 // SHIFT(s(1), s(2), ..., s(dim-1), s(dim+1), ..., s(n)).1631 llvm::SmallVector<mlir::Value, maxRank> shiftIndices{indices};1632 shiftIndices.erase(shiftIndices.begin() + dimVal - 1);1633 hlfir::Entity shiftElement =1634 hlfir::getElementAt(loc, builder, shift, shiftIndices);1635 shiftVal = hlfir::loadTrivialScalar(loc, builder, shiftElement);1636 shiftVal = normalizeShiftValue(loc, builder, shiftVal, shiftDimExtent,1637 calcType);1638 }1639 if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>) {1640 llvm::SmallVector<mlir::Value, maxRank> boundaryIndices{indices};1641 boundaryIndices.erase(boundaryIndices.begin() + dimVal - 1);1642 boundaryVal =1643 selectBoundaryValue(loc, builder, op, boundaryVal,1644 boundaryIsScalarPred, boundaryIndices);1645 }1646 1647 if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>) {1648 // EOSHIFT:1649 // Element i of the result (1-based) is the element of the original1650 // array (or its section, when ARRAY is not a vector) with index1651 // (i + SH), if (1 <= i + SH <= SIZE(ARRAY,DIM)), otherwise1652 // it is the BOUNDARY value.1653 mlir::Value index =1654 builder.createConvert(loc, calcType, inputIndices[dimVal - 1]);1655 mlir::arith::IntegerOverflowFlags savedFlags =1656 builder.getIntegerOverflowFlags();1657 builder.setIntegerOverflowFlags(mlir::arith::IntegerOverflowFlags::nsw);1658 mlir::Value indexPlusShift =1659 mlir::arith::AddIOp::create(builder, loc, index, shiftVal);1660 builder.setIntegerOverflowFlags(savedFlags);1661 mlir::Value one = builder.createIntegerConstant(loc, calcType, 1);1662 mlir::Value cmp1 = mlir::arith::CmpIOp::create(1663 builder, loc, mlir::arith::CmpIPredicate::sge, indexPlusShift, one);1664 mlir::Value cmp2 = mlir::arith::CmpIOp::create(1665 builder, loc, mlir::arith::CmpIPredicate::sle, indexPlusShift,1666 shiftDimExtent);1667 mlir::Value loadFromArray =1668 mlir::arith::AndIOp::create(builder, loc, cmp1, cmp2);1669 mlir::Type boundaryValTy = boundaryVal.getType();1670 mlir::Value result =1671 builder1672 .genIfOp(loc, {boundaryValTy}, loadFromArray,1673 /*withElseRegion=*/true)1674 .genThen([&]() {1675 indices[dimVal - 1] = builder.createConvert(1676 loc, builder.getIndexType(), indexPlusShift);1677 ;1678 mlir::Value elem =1679 loadEoshiftVal(loc, builder, array, indices);1680 fir::ResultOp::create(builder, loc, elem);1681 })1682 .genElse(1683 [&]() { fir::ResultOp::create(builder, loc, boundaryVal); })1684 .getResults()[0];1685 return hlfir::Entity{result};1686 } else {1687 // CSHIFT:1688 // Element i of the result (1-based) is element1689 // 'MODULO(i + SH - 1, SIZE(ARRAY,DIM)) + 1' (1-based) of the original1690 // ARRAY (or its section, when ARRAY is not a vector).1691 1692 // Compute the index into the original array using the normalized1693 // shift value, which satisfies (SH >= 0 && SH < SIZE(ARRAY,DIM)):1694 // newIndex =1695 // i + ((i <= SIZE(ARRAY,DIM) - SH) ? SH : SH - SIZE(ARRAY,DIM))1696 //1697 // Such index computation allows for further loop vectorization1698 // in LLVM.1699 mlir::Value wrapBound =1700 mlir::arith::SubIOp::create(builder, loc, shiftDimExtent, shiftVal);1701 mlir::Value adjustedShiftVal =1702 mlir::arith::SubIOp::create(builder, loc, shiftVal, shiftDimExtent);1703 mlir::Value index =1704 builder.createConvert(loc, calcType, inputIndices[dimVal - 1]);1705 mlir::Value wrapCheck = mlir::arith::CmpIOp::create(1706 builder, loc, mlir::arith::CmpIPredicate::sle, index, wrapBound);1707 mlir::Value actualShift = mlir::arith::SelectOp::create(1708 builder, loc, wrapCheck, shiftVal, adjustedShiftVal);1709 mlir::Value newIndex =1710 mlir::arith::AddIOp::create(builder, loc, index, actualShift);1711 newIndex = builder.createConvert(loc, builder.getIndexType(), newIndex);1712 indices[dimVal - 1] = newIndex;1713 hlfir::Entity element =1714 hlfir::getElementAt(loc, builder, array, indices);1715 return hlfir::loadTrivialScalar(loc, builder, element);1716 }1717 };1718 1719 mlir::Type elementType = array.getFortranElementType();1720 hlfir::ElementalOp elementalOp = hlfir::genElementalOp(1721 loc, builder, elementType, arrayShape, typeParams, genKernel,1722 /*isUnordered=*/true,1723 array.isPolymorphic() ? static_cast<mlir::Value>(array) : nullptr,1724 op.getResult().getType());1725 return elementalOp.getOperation();1726 }1727 1728 /// Convert \p op into an hlfir.eval_in_mem using the pre-computed1729 /// constant \p dimVal.1730 /// The converted code for CSHIFT looks like this:1731 /// DEST_OFFSET = SIZE(ARRAY,DIM) - SH1732 /// COPY_END1 = SH1733 /// do i=1,COPY_END11734 /// result(i + DEST_OFFSET) = array(i)1735 /// end1736 /// SOURCE_OFFSET = SH1737 /// COPY_END2 = SIZE(ARRAY,DIM) - SH1738 /// do i=1,COPY_END21739 /// result(i) = array(i + SOURCE_OFFSET)1740 /// end1741 /// Where SH is the normalized shift value, which satisfies1742 /// (SH >= 0 && SH < SIZE(ARRAY,DIM)).1743 ///1744 /// The converted code for EOSHIFT looks like this:1745 /// EXTENT = SIZE(ARRAY,DIM)1746 /// DEST_OFFSET = SH < 0 ? -SH : 01747 /// SOURCE_OFFSET = SH < 0 ? 0 : SH1748 /// COPY_END = SH < 0 ?1749 /// (-EXTENT > SH ? 0 : EXTENT + SH) :1750 /// (EXTENT < SH ? 0 : EXTENT - SH)1751 /// do i=1,COPY_END1752 /// result(i + DEST_OFFSET) = array(i + SOURCE_OFFSET)1753 /// end1754 /// INIT_END = EXTENT - COPY_END1755 /// INIT_OFFSET = SH < 0 ? 0 : COPY_END1756 /// do i=1,INIT_END1757 /// result(i + INIT_OFFSET) = BOUNDARY1758 /// end1759 /// Where SH is the original shift value.1760 ///1761 /// When \p dimVal is 1, we generate the same code twice1762 /// under a dynamic check for the contiguity of the leading1763 /// dimension. In the code corresponding to the contiguous1764 /// leading dimension, the shift dimension is represented1765 /// as a contiguous slice of the original array.1766 /// This allows recognizing the above two loops as memcpy1767 /// loop idioms in LLVM.1768 static mlir::Operation *genInMemArrayShift(mlir::PatternRewriter &rewriter,1769 Op op, int64_t dimVal) {1770 using Fortran::common::maxRank;1771 hlfir::Entity shift = hlfir::Entity{op.getShift()};1772 hlfir::Entity array = hlfir::Entity{op.getArray()};1773 assert(array.isVariable() && "array must be a variable");1774 assert(!array.isPolymorphic() &&1775 "genInMemArrayShift does not support polymorphic types");1776 mlir::Location loc = op.getLoc();1777 fir::FirOpBuilder builder{rewriter, op.getOperation()};1778 // The new index computation involves MODULO, which is not implemented1779 // for IndexType, so use I64 instead.1780 mlir::Type calcType = builder.getI64Type();1781 // Set the indices arithmetic overflow flags.1782 setArithOverflowFlags(op, builder);1783 1784 mlir::Value arrayShape = hlfir::genShape(loc, builder, array);1785 llvm::SmallVector<mlir::Value, maxRank> arrayExtents =1786 hlfir::getExplicitExtentsFromShape(arrayShape, builder);1787 llvm::SmallVector<mlir::Value, 1> typeParams;1788 hlfir::genLengthParameters(loc, builder, array, typeParams);1789 mlir::Value shiftDimExtent =1790 builder.createConvert(loc, calcType, arrayExtents[dimVal - 1]);1791 mlir::Value shiftVal;1792 if (shift.isScalar()) {1793 shiftVal = hlfir::loadTrivialScalar(loc, builder, shift);1794 shiftVal =1795 normalizeShiftValue(loc, builder, shiftVal, shiftDimExtent, calcType);1796 }1797 // The boundary operand of hlfir.eoshift may be statically or1798 // dynamically absent.1799 // In both cases, it is assumed to be a scalar with the value1800 // corresponding to the array element type.1801 // boundaryIsScalarPred is a dynamic predicate that identifies1802 // these cases. If boundaryIsScalarPred is dynamicaly false,1803 // then the boundary operand must be a present array.1804 mlir::Value boundaryVal, boundaryIsScalarPred;1805 if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>)1806 std::tie(boundaryVal, boundaryIsScalarPred) =1807 genScalarBoundaryForEOShift(loc, builder, op);1808 1809 hlfir::EvaluateInMemoryOp evalOp = hlfir::EvaluateInMemoryOp::create(1810 builder, loc, mlir::cast<hlfir::ExprType>(op.getType()), arrayShape);1811 builder.setInsertionPointToStart(&evalOp.getBody().front());1812 1813 mlir::Value resultArray = evalOp.getMemory();1814 mlir::Type arrayType = fir::dyn_cast_ptrEleTy(resultArray.getType());1815 resultArray = builder.createBox(loc, fir::BoxType::get(arrayType),1816 resultArray, arrayShape, /*slice=*/nullptr,1817 typeParams, /*tdesc=*/nullptr);1818 1819 // This is a generator of the dimension shift code.1820 // The code is inserted inside a loop nest over the other dimensions1821 // (if any). If exposeContiguity is true, the array's section1822 // array(s(1), ..., s(dim-1), :, s(dim+1), ..., s(n)) is represented1823 // as a contiguous 1D array.1824 // For CSHIFT, shiftVal is the normalized shift value that satisfies1825 // (SH >= 0 && SH < SIZE(ARRAY,DIM)).1826 //1827 auto genDimensionShift = [&](mlir::Location loc, fir::FirOpBuilder &builder,1828 mlir::Value shiftVal, mlir::Value boundary,1829 bool exposeContiguity,1830 mlir::ValueRange oneBasedIndices)1831 -> llvm::SmallVector<mlir::Value, 0> {1832 // Create a vector of indices (s(1), ..., s(dim-1), nullptr, s(dim+1),1833 // ..., s(n)) so that we can update the dimVal index as needed.1834 llvm::SmallVector<mlir::Value, maxRank> srcIndices(1835 oneBasedIndices.begin(), oneBasedIndices.begin() + (dimVal - 1));1836 srcIndices.push_back(nullptr);1837 srcIndices.append(oneBasedIndices.begin() + (dimVal - 1),1838 oneBasedIndices.end());1839 llvm::SmallVector<mlir::Value, maxRank> dstIndices(srcIndices);1840 1841 hlfir::Entity srcArray = array;1842 if (exposeContiguity && mlir::isa<fir::BaseBoxType>(srcArray.getType())) {1843 assert(dimVal == 1 && "can expose contiguity only for dim 1");1844 llvm::SmallVector<mlir::Value, maxRank> arrayLbounds =1845 hlfir::genLowerbounds(loc, builder, arrayShape, array.getRank());1846 hlfir::Entity section =1847 hlfir::gen1DSection(loc, builder, srcArray, dimVal, arrayLbounds,1848 arrayExtents, oneBasedIndices, typeParams);1849 mlir::Value addr = hlfir::genVariableRawAddress(loc, builder, section);1850 mlir::Value shape = hlfir::genShape(loc, builder, section);1851 mlir::Type boxType = fir::wrapInClassOrBoxType(1852 hlfir::getFortranElementOrSequenceType(section.getType()),1853 section.isPolymorphic());1854 srcArray = hlfir::Entity{1855 builder.createBox(loc, boxType, addr, shape, /*slice=*/nullptr,1856 /*lengths=*/{}, /*tdesc=*/nullptr)};1857 // When shifting the dimension as a 1D section of the original1858 // array, we only need one index for addressing.1859 srcIndices.resize(1);1860 }1861 1862 // genCopy labda generates the body of a generic copy loop.1863 // do i=1,COPY_END1864 // result(i + DEST_OFFSET) = array(i + SOURCE_OFFSET)1865 // end1866 //1867 // It is parameterized by DEST_OFFSET and SOURCE_OFFSET.1868 mlir::Value dstOffset, srcOffset;1869 auto genCopy = [&](mlir::Location loc, fir::FirOpBuilder &builder,1870 mlir::ValueRange index, mlir::ValueRange reductionArgs)1871 -> llvm::SmallVector<mlir::Value, 0> {1872 assert(index.size() == 1 && "expected single loop");1873 mlir::Value srcIndex = builder.createConvert(loc, calcType, index[0]);1874 mlir::Value dstIndex = srcIndex;1875 if (srcOffset)1876 srcIndex =1877 mlir::arith::AddIOp::create(builder, loc, srcIndex, srcOffset);1878 srcIndices[dimVal - 1] = srcIndex;1879 hlfir::Entity srcElementValue =1880 hlfir::loadElementAt(loc, builder, srcArray, srcIndices);1881 if (dstOffset)1882 dstIndex =1883 mlir::arith::AddIOp::create(builder, loc, dstIndex, dstOffset);1884 dstIndices[dimVal - 1] = dstIndex;1885 hlfir::Entity dstElement = hlfir::getElementAt(1886 loc, builder, hlfir::Entity{resultArray}, dstIndices);1887 hlfir::AssignOp::create(builder, loc, srcElementValue, dstElement);1888 // Reset the external parameters' values to make sure1889 // they are properly updated between the labda calls.1890 // WARNING: if genLoopNestWithReductions() calls the lambda1891 // multiple times, this is going to be a problem.1892 dstOffset = nullptr;1893 srcOffset = nullptr;1894 return {};1895 };1896 1897 if constexpr (std::is_same_v<Op, hlfir::CShiftOp>) {1898 // Copy first portion of the array:1899 // DEST_OFFSET = SIZE(ARRAY,DIM) - SH1900 // COPY_END1 = SH1901 // do i=1,COPY_END11902 // result(i + DEST_OFFSET) = array(i)1903 // end1904 dstOffset =1905 mlir::arith::SubIOp::create(builder, loc, shiftDimExtent, shiftVal);1906 srcOffset = nullptr;1907 hlfir::genLoopNestWithReductions(loc, builder, {shiftVal},1908 /*reductionInits=*/{}, genCopy,1909 /*isUnordered=*/true);1910 1911 // Copy second portion of the array:1912 // SOURCE_OFFSET = SH1913 // COPY_END2 = SIZE(ARRAY,DIM) - SH1914 // do i=1,COPY_END21915 // result(i) = array(i + SOURCE_OFFSET)1916 // end1917 mlir::Value bound =1918 mlir::arith::SubIOp::create(builder, loc, shiftDimExtent, shiftVal);1919 dstOffset = nullptr;1920 srcOffset = shiftVal;1921 hlfir::genLoopNestWithReductions(loc, builder, {bound},1922 /*reductionInits=*/{}, genCopy,1923 /*isUnordered=*/true);1924 } else {1925 // Do the copy:1926 // EXTENT = SIZE(ARRAY,DIM)1927 // DEST_OFFSET = SH < 0 ? -SH : 01928 // SOURCE_OFFSET = SH < 0 ? 0 : SH1929 // COPY_END = SH < 0 ?1930 // (-EXTENT > SH ? 0 : EXTENT + SH) :1931 // (EXTENT < SH ? 0 : EXTENT - SH)1932 // do i=1,COPY_END1933 // result(i + DEST_OFFSET) = array(i + SOURCE_OFFSET)1934 // end1935 mlir::arith::IntegerOverflowFlags savedFlags =1936 builder.getIntegerOverflowFlags();1937 builder.setIntegerOverflowFlags(mlir::arith::IntegerOverflowFlags::nsw);1938 1939 mlir::Value zero = builder.createIntegerConstant(loc, calcType, 0);1940 mlir::Value isNegativeShift = mlir::arith::CmpIOp::create(1941 builder, loc, mlir::arith::CmpIPredicate::slt, shiftVal, zero);1942 mlir::Value shiftNeg =1943 mlir::arith::SubIOp::create(builder, loc, zero, shiftVal);1944 dstOffset = mlir::arith::SelectOp::create(builder, loc, isNegativeShift,1945 shiftNeg, zero);1946 srcOffset = mlir::arith::SelectOp::create(builder, loc, isNegativeShift,1947 zero, shiftVal);1948 mlir::Value extentNeg =1949 mlir::arith::SubIOp::create(builder, loc, zero, shiftDimExtent);1950 mlir::Value extentPlusShift =1951 mlir::arith::AddIOp::create(builder, loc, shiftDimExtent, shiftVal);1952 mlir::Value extentNegShiftCmp = mlir::arith::CmpIOp::create(1953 builder, loc, mlir::arith::CmpIPredicate::sgt, extentNeg, shiftVal);1954 mlir::Value negativeShiftBound = mlir::arith::SelectOp::create(1955 builder, loc, extentNegShiftCmp, zero, extentPlusShift);1956 mlir::Value extentMinusShift =1957 mlir::arith::SubIOp::create(builder, loc, shiftDimExtent, shiftVal);1958 mlir::Value extentShiftCmp = mlir::arith::CmpIOp::create(1959 builder, loc, mlir::arith::CmpIPredicate::slt, shiftDimExtent,1960 shiftVal);1961 mlir::Value positiveShiftBound = mlir::arith::SelectOp::create(1962 builder, loc, extentShiftCmp, zero, extentMinusShift);1963 mlir::Value copyEnd = mlir::arith::SelectOp::create(1964 builder, loc, isNegativeShift, negativeShiftBound,1965 positiveShiftBound);1966 hlfir::genLoopNestWithReductions(loc, builder, {copyEnd},1967 /*reductionInits=*/{}, genCopy,1968 /*isUnordered=*/true);1969 1970 // Do the init:1971 // INIT_END = EXTENT - COPY_END1972 // INIT_OFFSET = SH < 0 ? 0 : COPY_END1973 // do i=1,INIT_END1974 // result(i + INIT_OFFSET) = BOUNDARY1975 // end1976 assert(boundary && "boundary cannot be null");1977 mlir::Value initEnd =1978 mlir::arith::SubIOp::create(builder, loc, shiftDimExtent, copyEnd);1979 mlir::Value initOffset = mlir::arith::SelectOp::create(1980 builder, loc, isNegativeShift, zero, copyEnd);1981 auto genInit = [&](mlir::Location loc, fir::FirOpBuilder &builder,1982 mlir::ValueRange index,1983 mlir::ValueRange reductionArgs)1984 -> llvm::SmallVector<mlir::Value, 0> {1985 mlir::Value dstIndex = builder.createConvert(loc, calcType, index[0]);1986 dstIndex =1987 mlir::arith::AddIOp::create(builder, loc, dstIndex, initOffset);1988 dstIndices[dimVal - 1] = dstIndex;1989 hlfir::Entity dstElement = hlfir::getElementAt(1990 loc, builder, hlfir::Entity{resultArray}, dstIndices);1991 hlfir::AssignOp::create(builder, loc, boundary, dstElement);1992 return {};1993 };1994 hlfir::genLoopNestWithReductions(loc, builder, {initEnd},1995 /*reductionInits=*/{}, genInit,1996 /*isUnordered=*/true);1997 builder.setIntegerOverflowFlags(savedFlags);1998 }1999 return {};2000 };2001 2002 // A wrapper around genDimensionShift that computes the normalized2003 // shift value and manages the insertion of the multiple versions2004 // of the shift based on the dynamic check of the leading dimension's2005 // contiguity (when dimVal == 1).2006 auto genShiftBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,2007 mlir::ValueRange oneBasedIndices,2008 mlir::ValueRange reductionArgs)2009 -> llvm::SmallVector<mlir::Value, 0> {2010 // Copy the dimension with a shift:2011 // SH is either SHIFT (if scalar) or SHIFT(oneBasedIndices).2012 if (!shiftVal) {2013 assert(!oneBasedIndices.empty() && "scalar shift must be precomputed");2014 hlfir::Entity shiftElement =2015 hlfir::getElementAt(loc, builder, shift, oneBasedIndices);2016 shiftVal = hlfir::loadTrivialScalar(loc, builder, shiftElement);2017 shiftVal = normalizeShiftValue(loc, builder, shiftVal, shiftDimExtent,2018 calcType);2019 }2020 if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>)2021 boundaryVal =2022 selectBoundaryValue(loc, builder, op, boundaryVal,2023 boundaryIsScalarPred, oneBasedIndices);2024 2025 // If we can fetch the byte stride of the leading dimension,2026 // and the byte size of the element, then we can generate2027 // a dynamic contiguity check and expose the leading dimension's2028 // contiguity in FIR, making memcpy loop idiom recognition2029 // possible.2030 mlir::Value elemSize;2031 mlir::Value stride;2032 if (dimVal == 1 && mlir::isa<fir::BaseBoxType>(array.getType())) {2033 mlir::Type indexType = builder.getIndexType();2034 elemSize =2035 fir::BoxEleSizeOp::create(builder, loc, indexType, array.getBase());2036 mlir::Value dimIdx =2037 builder.createIntegerConstant(loc, indexType, dimVal - 1);2038 auto boxDim =2039 fir::BoxDimsOp::create(builder, loc, indexType, indexType,2040 indexType, array.getBase(), dimIdx);2041 stride = boxDim.getByteStride();2042 }2043 2044 if (array.isSimplyContiguous() || !elemSize || !stride) {2045 genDimensionShift(loc, builder, shiftVal, boundaryVal,2046 /*exposeContiguity=*/false, oneBasedIndices);2047 return {};2048 }2049 2050 mlir::Value isContiguous = mlir::arith::CmpIOp::create(2051 builder, loc, mlir::arith::CmpIPredicate::eq, elemSize, stride);2052 builder.genIfOp(loc, {}, isContiguous, /*withElseRegion=*/true)2053 .genThen([&]() {2054 genDimensionShift(loc, builder, shiftVal, boundaryVal,2055 /*exposeContiguity=*/true, oneBasedIndices);2056 })2057 .genElse([&]() {2058 genDimensionShift(loc, builder, shiftVal, boundaryVal,2059 /*exposeContiguity=*/false, oneBasedIndices);2060 });2061 2062 return {};2063 };2064 2065 // For 1D case, generate a single loop.2066 // For ND case, generate a loop nest over the other dimensions2067 // with a single loop inside (generated separately).2068 llvm::SmallVector<mlir::Value, maxRank> newExtents(arrayExtents);2069 newExtents.erase(newExtents.begin() + (dimVal - 1));2070 if (!newExtents.empty())2071 hlfir::genLoopNestWithReductions(loc, builder, newExtents,2072 /*reductionInits=*/{}, genShiftBody,2073 /*isUnordered=*/true);2074 else2075 genShiftBody(loc, builder, {}, {});2076 2077 return evalOp.getOperation();2078 }2079};2080 2081class CmpCharOpConversion : public mlir::OpRewritePattern<hlfir::CmpCharOp> {2082public:2083 using mlir::OpRewritePattern<hlfir::CmpCharOp>::OpRewritePattern;2084 2085 llvm::LogicalResult2086 matchAndRewrite(hlfir::CmpCharOp cmp,2087 mlir::PatternRewriter &rewriter) const override {2088 2089 fir::FirOpBuilder builder{rewriter, cmp.getOperation()};2090 const mlir::Location &loc = cmp->getLoc();2091 2092 auto toVariable =2093 [&builder,2094 &loc](mlir::Value val) -> std::pair<mlir::Value, hlfir::AssociateOp> {2095 mlir::Value opnd;2096 hlfir::AssociateOp associate;2097 if (mlir::isa<hlfir::ExprType>(val.getType())) {2098 hlfir::Entity entity{val};2099 mlir::NamedAttribute byRefAttr = fir::getAdaptToByRefAttr(builder);2100 associate = hlfir::genAssociateExpr(loc, builder, entity,2101 entity.getType(), "", byRefAttr);2102 opnd = associate.getBase();2103 } else {2104 opnd = val;2105 }2106 return {opnd, associate};2107 };2108 2109 auto [lhsOpnd, lhsAssociate] = toVariable(cmp.getLchr());2110 auto [rhsOpnd, rhsAssociate] = toVariable(cmp.getRchr());2111 2112 hlfir::Entity lhs{lhsOpnd};2113 hlfir::Entity rhs{rhsOpnd};2114 2115 auto charTy = mlir::cast<fir::CharacterType>(lhs.getFortranElementType());2116 unsigned kind = charTy.getFKind();2117 2118 auto bits = builder.getKindMap().getCharacterBitsize(kind);2119 auto intTy = builder.getIntegerType(bits);2120 2121 auto idxTy = builder.getIndexType();2122 auto charLen1Ty =2123 fir::CharacterType::getSingleton(builder.getContext(), kind);2124 mlir::Type designatorType =2125 fir::ReferenceType::get(charLen1Ty, fir::isa_volatile_type(charTy));2126 auto idxAttr = builder.getIntegerAttr(idxTy, 0);2127 2128 auto genExtractAndConvertToInt =2129 [&idxAttr, &intTy, &designatorType](2130 mlir::Location loc, fir::FirOpBuilder &builder,2131 hlfir::Entity &charStr, mlir::Value index, mlir::Value length) {2132 auto singleChr = hlfir::DesignateOp::create(2133 builder, loc, designatorType, charStr, /*component=*/{},2134 /*compShape=*/mlir::Value{}, hlfir::DesignateOp::Subscripts{},2135 /*substring=*/mlir::ValueRange{index, index},2136 /*complexPart=*/std::nullopt,2137 /*shape=*/mlir::Value{}, /*typeParams=*/mlir::ValueRange{length},2138 fir::FortranVariableFlagsAttr{});2139 auto chrVal = fir::LoadOp::create(builder, loc, singleChr);2140 mlir::Value intVal = fir::ExtractValueOp::create(2141 builder, loc, intTy, chrVal, builder.getArrayAttr(idxAttr));2142 return intVal;2143 };2144 2145 mlir::arith::CmpIPredicate predicate = cmp.getPredicate();2146 mlir::Value oneIdx = builder.createIntegerConstant(loc, idxTy, 1);2147 2148 mlir::Value lhsLen = builder.createConvert(2149 loc, idxTy, hlfir::genCharLength(loc, builder, lhs));2150 mlir::Value rhsLen = builder.createConvert(2151 loc, idxTy, hlfir::genCharLength(loc, builder, rhs));2152 2153 enum class GenCmp { LeftToRight, LeftToBlank, BlankToRight };2154 2155 mlir::Value zeroInt = builder.createIntegerConstant(loc, intTy, 0);2156 mlir::Value oneInt = builder.createIntegerConstant(loc, intTy, 1);2157 mlir::Value negOneInt = builder.createIntegerConstant(loc, intTy, -1);2158 mlir::Value blankInt = builder.createIntegerConstant(loc, intTy, ' ');2159 2160 auto step = GenCmp::LeftToRight;2161 auto genCmp = [&](mlir::Location loc, fir::FirOpBuilder &builder,2162 mlir::ValueRange index, mlir::ValueRange reductionArgs)2163 -> llvm::SmallVector<mlir::Value, 1> {2164 assert(index.size() == 1 && "expected single loop");2165 assert(reductionArgs.size() == 1 && "expected single reduction value");2166 mlir::Value inRes = reductionArgs[0];2167 auto accEQzero = mlir::arith::CmpIOp::create(2168 builder, loc, mlir::arith::CmpIPredicate::eq, inRes, zeroInt);2169 2170 mlir::Value res =2171 builder2172 .genIfOp(loc, {intTy}, accEQzero,2173 /*withElseRegion=*/true)2174 .genThen([&]() {2175 mlir::Value offset =2176 builder.createConvert(loc, idxTy, index[0]);2177 mlir::Value lhsInt;2178 mlir::Value rhsInt;2179 if (step == GenCmp::LeftToRight) {2180 lhsInt = genExtractAndConvertToInt(loc, builder, lhs, offset,2181 oneIdx);2182 rhsInt = genExtractAndConvertToInt(loc, builder, rhs, offset,2183 oneIdx);2184 } else if (step == GenCmp::LeftToBlank) {2185 // lhsLen > rhsLen2186 offset =2187 mlir::arith::AddIOp::create(builder, loc, rhsLen, offset);2188 2189 lhsInt = genExtractAndConvertToInt(loc, builder, lhs, offset,2190 oneIdx);2191 rhsInt = blankInt;2192 } else if (step == GenCmp::BlankToRight) {2193 // rhsLen > lhsLen2194 offset =2195 mlir::arith::AddIOp::create(builder, loc, lhsLen, offset);2196 2197 lhsInt = blankInt;2198 rhsInt = genExtractAndConvertToInt(loc, builder, rhs, offset,2199 oneIdx);2200 } else {2201 llvm_unreachable(2202 "unknown compare step for CmpCharOp lowering");2203 }2204 2205 mlir::Value newVal = mlir::arith::SelectOp::create(2206 builder, loc,2207 mlir::arith::CmpIOp::create(builder, loc,2208 mlir::arith::CmpIPredicate::ult,2209 lhsInt, rhsInt),2210 negOneInt, inRes);2211 newVal = mlir::arith::SelectOp::create(2212 builder, loc,2213 mlir::arith::CmpIOp::create(builder, loc,2214 mlir::arith::CmpIPredicate::ugt,2215 lhsInt, rhsInt),2216 oneInt, newVal);2217 fir::ResultOp::create(builder, loc, newVal);2218 })2219 .genElse([&]() { fir::ResultOp::create(builder, loc, inRes); })2220 .getResults()[0];2221 2222 return {res};2223 };2224 2225 // First generate comparison of two strings for the legth of the shorter2226 // one.2227 mlir::Value minLen = mlir::arith::SelectOp::create(2228 builder, loc,2229 mlir::arith::CmpIOp::create(2230 builder, loc, mlir::arith::CmpIPredicate::slt, lhsLen, rhsLen),2231 lhsLen, rhsLen);2232 2233 llvm::SmallVector<mlir::Value, 1> loopOut =2234 hlfir::genLoopNestWithReductions(loc, builder, {minLen},2235 /*reductionInits=*/{zeroInt}, genCmp,2236 /*isUnordered=*/false);2237 mlir::Value partRes = loopOut[0];2238 2239 auto lhsLonger = mlir::arith::CmpIOp::create(2240 builder, loc, mlir::arith::CmpIPredicate::sgt, lhsLen, rhsLen);2241 mlir::Value tempRes =2242 builder2243 .genIfOp(loc, {intTy}, lhsLonger,2244 /*withElseRegion=*/true)2245 .genThen([&]() {2246 // If left is the longer string generate compare left to blank.2247 step = GenCmp::LeftToBlank;2248 auto lenDiff =2249 mlir::arith::SubIOp::create(builder, loc, lhsLen, rhsLen);2250 2251 llvm::SmallVector<mlir::Value, 1> output =2252 hlfir::genLoopNestWithReductions(loc, builder, {lenDiff},2253 /*reductionInits=*/{partRes},2254 genCmp,2255 /*isUnordered=*/false);2256 mlir::Value res = output[0];2257 fir::ResultOp::create(builder, loc, res);2258 })2259 .genElse([&]() {2260 // If right is the longer string generate compare blank to2261 // right.2262 step = GenCmp::BlankToRight;2263 auto lenDiff =2264 mlir::arith::SubIOp::create(builder, loc, rhsLen, lhsLen);2265 llvm::SmallVector<mlir::Value, 1> output =2266 hlfir::genLoopNestWithReductions(loc, builder, {lenDiff},2267 /*reductionInits=*/{partRes},2268 genCmp,2269 /*isUnordered=*/false);2270 2271 mlir::Value res = output[0];2272 fir::ResultOp::create(builder, loc, res);2273 })2274 .getResults()[0];2275 if (lhsAssociate)2276 hlfir::EndAssociateOp::create(builder, loc, lhsAssociate);2277 if (rhsAssociate)2278 hlfir::EndAssociateOp::create(builder, loc, rhsAssociate);2279 2280 auto finalCmpResult =2281 mlir::arith::CmpIOp::create(builder, loc, predicate, tempRes, zeroInt);2282 rewriter.replaceOp(cmp, finalCmpResult);2283 return mlir::success();2284 }2285};2286 2287static std::pair<mlir::Value, hlfir::AssociateOp>2288getVariable(fir::FirOpBuilder &builder, mlir::Location loc, mlir::Value val) {2289 // If it is an expression - create a variable from it, or forward2290 // the value otherwise.2291 hlfir::AssociateOp associate;2292 if (!mlir::isa<hlfir::ExprType>(val.getType()))2293 return {val, associate};2294 hlfir::Entity entity{val};2295 mlir::NamedAttribute byRefAttr = fir::getAdaptToByRefAttr(builder);2296 associate = hlfir::genAssociateExpr(loc, builder, entity, entity.getType(),2297 "", byRefAttr);2298 return {associate.getBase(), associate};2299}2300 2301class IndexOpConversion : public mlir::OpRewritePattern<hlfir::IndexOp> {2302public:2303 using mlir::OpRewritePattern<hlfir::IndexOp>::OpRewritePattern;2304 2305 llvm::LogicalResult2306 matchAndRewrite(hlfir::IndexOp op,2307 mlir::PatternRewriter &rewriter) const override {2308 // We simplify only limited cases:2309 // 1) a substring length shall be known at compile time2310 // 2) if a substring length is 0 then replace with 1 for forward search,2311 // or otherwise with the string length + 1 (builder shall const-fold if2312 // lookup direction is known at compile time).2313 // 3) for known string length at compile time, if it is2314 // shorter than substring => replace with zero.2315 // 4) if a substring length is one => inline as simple search loop2316 // 5) for forward search with input strings of kind=1 runtime is faster.2317 // Do not simplify in all the other cases relying on a runtime call.2318 2319 fir::FirOpBuilder builder{rewriter, op.getOperation()};2320 const mlir::Location &loc = op->getLoc();2321 2322 auto resultTy = op.getType();2323 mlir::Value back = op.getBack();2324 auto substrLenCst =2325 hlfir::getCharLengthIfConst(hlfir::Entity{op.getSubstr()});2326 if (!substrLenCst) {2327 return rewriter.notifyMatchFailure(2328 op, "substring length unknown at compile time");2329 }2330 hlfir::Entity strEntity{op.getStr()};2331 auto i1Ty = builder.getI1Type();2332 auto idxTy = builder.getIndexType();2333 if (*substrLenCst == 0) {2334 mlir::Value oneIdx = builder.createIntegerConstant(loc, idxTy, 1);2335 // zero length substring. For back search replace with2336 // strLen+1, or otherwise with 1.2337 mlir::Value strLen = hlfir::genCharLength(loc, builder, strEntity);2338 mlir::Value strEnd = mlir::arith::AddIOp::create(2339 builder, loc, builder.createConvert(loc, idxTy, strLen), oneIdx);2340 if (back)2341 back = builder.createConvert(loc, i1Ty, back);2342 else2343 back = builder.createIntegerConstant(loc, i1Ty, 0);2344 mlir::Value result =2345 mlir::arith::SelectOp::create(builder, loc, back, strEnd, oneIdx);2346 2347 rewriter.replaceOp(op, builder.createConvert(loc, resultTy, result));2348 return mlir::success();2349 }2350 2351 if (auto strLenCst = hlfir::getCharLengthIfConst(strEntity)) {2352 if (*strLenCst < *substrLenCst) {2353 rewriter.replaceOp(op, builder.createIntegerConstant(loc, resultTy, 0));2354 return mlir::success();2355 }2356 if (*strLenCst == 0) {2357 // both strings have zero length2358 rewriter.replaceOp(op, builder.createIntegerConstant(loc, resultTy, 1));2359 return mlir::success();2360 }2361 }2362 if (*substrLenCst != 1) {2363 return rewriter.notifyMatchFailure(2364 op, "rely on runtime implementation if substring length > 1");2365 }2366 // For forward search and character kind=1 the runtime uses memchr2367 // which well optimized. But it looks like memchr idiom is not recognized2368 // in LLVM yet. On a micro-kernel test with strings of length 40 runtime2369 // had ~2x less execution time vs inlined code. For unknown search direction2370 // at compile time pessimistically assume "forward".2371 std::optional<bool> isBack;2372 if (back) {2373 if (auto backCst = fir::getIntIfConstant(back))2374 isBack = *backCst != 0;2375 } else {2376 isBack = false;2377 }2378 auto charTy = mlir::cast<fir::CharacterType>(2379 hlfir::getFortranElementType(op.getSubstr().getType()));2380 unsigned kind = charTy.getFKind();2381 if (kind == 1 && (!isBack || !*isBack)) {2382 return rewriter.notifyMatchFailure(2383 op, "rely on runtime implementation for character kind 1");2384 }2385 2386 // All checks are passed here. Generate single character search loop.2387 auto [strV, strAssociate] = getVariable(builder, loc, op.getStr());2388 auto [substrV, substrAssociate] = getVariable(builder, loc, op.getSubstr());2389 hlfir::Entity str{strV};2390 hlfir::Entity substr{substrV};2391 mlir::Value oneIdx = builder.createIntegerConstant(loc, idxTy, 1);2392 2393 auto genExtractAndConvertToInt = [&charTy, &idxTy, &oneIdx,2394 kind](mlir::Location loc,2395 fir::FirOpBuilder &builder,2396 hlfir::Entity &charStr,2397 mlir::Value index) {2398 auto bits = builder.getKindMap().getCharacterBitsize(kind);2399 auto intTy = builder.getIntegerType(bits);2400 auto charLen1Ty =2401 fir::CharacterType::getSingleton(builder.getContext(), kind);2402 mlir::Type designatorTy =2403 fir::ReferenceType::get(charLen1Ty, fir::isa_volatile_type(charTy));2404 auto idxAttr = builder.getIntegerAttr(idxTy, 0);2405 2406 auto singleChr = hlfir::DesignateOp::create(2407 builder, loc, designatorTy, charStr, /*component=*/{},2408 /*compShape=*/mlir::Value{}, hlfir::DesignateOp::Subscripts{},2409 /*substring=*/mlir::ValueRange{index, index},2410 /*complexPart=*/std::nullopt,2411 /*shape=*/mlir::Value{}, /*typeParams=*/mlir::ValueRange{oneIdx},2412 fir::FortranVariableFlagsAttr{});2413 auto chrVal = fir::LoadOp::create(builder, loc, singleChr);2414 mlir::Value intVal = fir::ExtractValueOp::create(2415 builder, loc, intTy, chrVal, builder.getArrayAttr(idxAttr));2416 return intVal;2417 };2418 2419 auto wantChar = genExtractAndConvertToInt(loc, builder, substr, oneIdx);2420 2421 // Generate search loop body with the following C equivalent:2422 // idx_t result = 0;2423 // idx_t end = strlen + 1;2424 // char want = substr[0];2425 // for (idx_t idx = 1; idx < end; ++idx) {2426 // if (result == 0) {2427 // idx_t at = back ? end - idx: idx;2428 // result = str[at-1] == want ? at : result;2429 // }2430 // }2431 mlir::Value strLen = hlfir::genCharLength(loc, builder, strEntity);2432 if (!back)2433 back = builder.createIntegerConstant(loc, i1Ty, 0);2434 else2435 back = builder.createConvert(loc, i1Ty, back);2436 mlir::Value strEnd = mlir::arith::AddIOp::create(2437 builder, loc, builder.createConvert(loc, idxTy, strLen), oneIdx);2438 mlir::Value zeroIdx = builder.createIntegerConstant(loc, idxTy, 0);2439 auto genSearchBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,2440 mlir::ValueRange index,2441 mlir::ValueRange reductionArgs)2442 -> llvm::SmallVector<mlir::Value, 1> {2443 assert(index.size() == 1 && "expected single loop");2444 assert(reductionArgs.size() == 1 && "expected single reduction value");2445 mlir::Value inRes = reductionArgs[0];2446 auto resEQzero = mlir::arith::CmpIOp::create(2447 builder, loc, mlir::arith::CmpIPredicate::eq, inRes, zeroIdx);2448 2449 mlir::Value res =2450 builder2451 .genIfOp(loc, {idxTy}, resEQzero,2452 /*withElseRegion=*/true)2453 .genThen([&]() {2454 mlir::Value idx = builder.createConvert(loc, idxTy, index[0]);2455 // offset = back ? end - idx : idx;2456 mlir::Value offset = mlir::arith::SelectOp::create(2457 builder, loc, back,2458 mlir::arith::SubIOp::create(builder, loc, strEnd, idx),2459 idx);2460 2461 auto haveChar =2462 genExtractAndConvertToInt(loc, builder, str, offset);2463 auto charsEQ = mlir::arith::CmpIOp::create(2464 builder, loc, mlir::arith::CmpIPredicate::eq, haveChar,2465 wantChar);2466 mlir::Value newVal = mlir::arith::SelectOp::create(2467 builder, loc, charsEQ, offset, inRes);2468 2469 fir::ResultOp::create(builder, loc, newVal);2470 })2471 .genElse([&]() { fir::ResultOp::create(builder, loc, inRes); })2472 .getResults()[0];2473 return {res};2474 };2475 2476 llvm::SmallVector<mlir::Value, 1> loopOut =2477 hlfir::genLoopNestWithReductions(loc, builder, {strLen},2478 /*reductionInits=*/{zeroIdx},2479 genSearchBody,2480 /*isUnordered=*/false);2481 mlir::Value result = builder.createConvert(loc, resultTy, loopOut[0]);2482 2483 if (strAssociate)2484 hlfir::EndAssociateOp::create(builder, loc, strAssociate);2485 if (substrAssociate)2486 hlfir::EndAssociateOp::create(builder, loc, substrAssociate);2487 2488 rewriter.replaceOp(op, result);2489 return mlir::success();2490 }2491};2492 2493template <typename Op>2494class MatmulConversion : public mlir::OpRewritePattern<Op> {2495public:2496 using mlir::OpRewritePattern<Op>::OpRewritePattern;2497 2498 llvm::LogicalResult2499 matchAndRewrite(Op matmul, mlir::PatternRewriter &rewriter) const override {2500 mlir::Location loc = matmul.getLoc();2501 fir::FirOpBuilder builder{rewriter, matmul.getOperation()};2502 hlfir::Entity lhs = hlfir::Entity{matmul.getLhs()};2503 hlfir::Entity rhs = hlfir::Entity{matmul.getRhs()};2504 mlir::Value resultShape, innerProductExtent;2505 std::tie(resultShape, innerProductExtent) =2506 genResultShape(loc, builder, lhs, rhs);2507 2508 if (forceMatmulAsElemental || isMatmulTranspose) {2509 // Generate hlfir.elemental that produces the result of2510 // MATMUL/MATMUL(TRANSPOSE).2511 // Note that this implementation is very suboptimal for MATMUL,2512 // but is quite good for MATMUL(TRANSPOSE), e.g.:2513 // R(1:N) = R(1:N) + MATMUL(TRANSPOSE(X(1:N,1:N)), Y(1:N))2514 // Inlining MATMUL(TRANSPOSE) as hlfir.elemental may result2515 // in merging the inner product computation with the elemental2516 // addition. Note that the inner product computation will2517 // benefit from processing the lowermost dimensions of X and Y,2518 // which may be the best when they are contiguous.2519 //2520 // This is why we always inline MATMUL(TRANSPOSE) as an elemental.2521 // MATMUL is inlined below by default unless forceMatmulAsElemental.2522 hlfir::ExprType resultType =2523 mlir::cast<hlfir::ExprType>(matmul.getType());2524 hlfir::ElementalOp newOp = genElementalMatmul(2525 loc, builder, resultType, resultShape, lhs, rhs, innerProductExtent);2526 rewriter.replaceOp(matmul, newOp);2527 return mlir::success();2528 }2529 2530 // Generate hlfir.eval_in_mem to mimic the MATMUL implementation2531 // from Fortran runtime. The implementation needs to operate2532 // with the result array as an in-memory object.2533 hlfir::EvaluateInMemoryOp evalOp = hlfir::EvaluateInMemoryOp::create(2534 builder, loc, mlir::cast<hlfir::ExprType>(matmul.getType()),2535 resultShape);2536 builder.setInsertionPointToStart(&evalOp.getBody().front());2537 2538 // Embox the raw array pointer to simplify designating it.2539 // TODO: this currently results in redundant lower bounds2540 // addition for the designator, but this should be fixed in2541 // hlfir::Entity::mayHaveNonDefaultLowerBounds().2542 mlir::Value resultArray = evalOp.getMemory();2543 mlir::Type arrayType = fir::dyn_cast_ptrEleTy(resultArray.getType());2544 resultArray = builder.createBox(loc, fir::BoxType::get(arrayType),2545 resultArray, resultShape, /*slice=*/nullptr,2546 /*lengths=*/{}, /*tdesc=*/nullptr);2547 2548 // The contiguous MATMUL version is best for the cases2549 // where the input arrays and (maybe) the result are contiguous2550 // in their lowermost dimensions.2551 // Especially, when LLVM can recognize the continuity2552 // and vectorize the loops properly.2553 // Note that the contiguous MATMUL inlining is correct2554 // even when the input arrays are not contiguous.2555 // TODO: we can try to recognize the cases when the continuity2556 // is not statically obvious and try to generate an explicitly2557 // continuous version under a dynamic check. This should allow2558 // LLVM to vectorize the loops better. Note that this can2559 // also be postponed up to the LoopVersioning pass.2560 // The fallback implementation may use genElementalMatmul() with2561 // an hlfir.assign into the result of eval_in_mem.2562 mlir::LogicalResult rewriteResult =2563 genContiguousMatmul(loc, builder, hlfir::Entity{resultArray},2564 resultShape, lhs, rhs, innerProductExtent);2565 2566 if (mlir::failed(rewriteResult)) {2567 // Erase the unclaimed eval_in_mem op.2568 rewriter.eraseOp(evalOp);2569 return rewriter.notifyMatchFailure(matmul,2570 "genContiguousMatmul() failed");2571 }2572 2573 rewriter.replaceOp(matmul, evalOp);2574 return mlir::success();2575 }2576 2577private:2578 static constexpr bool isMatmulTranspose =2579 std::is_same_v<Op, hlfir::MatmulTransposeOp>;2580 2581 // Return a tuple of:2582 // * A fir.shape operation representing the shape of the result2583 // of a MATMUL/MATMUL(TRANSPOSE).2584 // * An extent of the dimensions of the input array2585 // that are processed during the inner product computation.2586 static std::tuple<mlir::Value, mlir::Value>2587 genResultShape(mlir::Location loc, fir::FirOpBuilder &builder,2588 hlfir::Entity input1, hlfir::Entity input2) {2589 llvm::SmallVector<mlir::Value, 2> input1Extents =2590 hlfir::genExtentsVector(loc, builder, input1);2591 llvm::SmallVector<mlir::Value, 2> input2Extents =2592 hlfir::genExtentsVector(loc, builder, input2);2593 2594 llvm::SmallVector<mlir::Value, 2> newExtents;2595 mlir::Value innerProduct1Extent, innerProduct2Extent;2596 if (input1Extents.size() == 1) {2597 assert(!isMatmulTranspose &&2598 "hlfir.matmul_transpose's first operand must be rank-2 array");2599 assert(input2Extents.size() == 2 &&2600 "hlfir.matmul second argument must be rank-2 array");2601 newExtents.push_back(input2Extents[1]);2602 innerProduct1Extent = input1Extents[0];2603 innerProduct2Extent = input2Extents[0];2604 } else {2605 if (input2Extents.size() == 1) {2606 assert(input1Extents.size() == 2 &&2607 "hlfir.matmul first argument must be rank-2 array");2608 if constexpr (isMatmulTranspose)2609 newExtents.push_back(input1Extents[1]);2610 else2611 newExtents.push_back(input1Extents[0]);2612 } else {2613 assert(input1Extents.size() == 2 && input2Extents.size() == 2 &&2614 "hlfir.matmul arguments must be rank-2 arrays");2615 if constexpr (isMatmulTranspose)2616 newExtents.push_back(input1Extents[1]);2617 else2618 newExtents.push_back(input1Extents[0]);2619 2620 newExtents.push_back(input2Extents[1]);2621 }2622 if constexpr (isMatmulTranspose)2623 innerProduct1Extent = input1Extents[0];2624 else2625 innerProduct1Extent = input1Extents[1];2626 2627 innerProduct2Extent = input2Extents[0];2628 }2629 // The inner product dimensions of the input arrays2630 // must match. Pick the best (e.g. constant) out of them2631 // so that the inner product loop bound can be used in2632 // optimizations.2633 llvm::SmallVector<mlir::Value> innerProductExtent =2634 fir::factory::deduceOptimalExtents({innerProduct1Extent},2635 {innerProduct2Extent});2636 return {fir::ShapeOp::create(builder, loc, newExtents),2637 innerProductExtent[0]};2638 }2639 2640 static mlir::LogicalResult2641 genContiguousMatmul(mlir::Location loc, fir::FirOpBuilder &builder,2642 hlfir::Entity result, mlir::Value resultShape,2643 hlfir::Entity lhs, hlfir::Entity rhs,2644 mlir::Value innerProductExtent) {2645 // This code does not support MATMUL(TRANSPOSE), and it is supposed2646 // to be inlined as hlfir.elemental.2647 if constexpr (isMatmulTranspose)2648 return mlir::failure();2649 2650 mlir::OpBuilder::InsertionGuard guard(builder);2651 mlir::Type resultElementType = result.getFortranElementType();2652 llvm::SmallVector<mlir::Value, 2> resultExtents =2653 mlir::cast<fir::ShapeOp>(resultShape.getDefiningOp()).getExtents();2654 2655 // The inner product loop may be unordered if FastMathFlags::reassoc2656 // transformations are allowed. The integer/logical inner product is2657 // always unordered.2658 // Note that isUnordered is currently applied to all loops2659 // in the loop nests generated below, while it has to be applied2660 // only to one.2661 bool isUnordered = mlir::isa<mlir::IntegerType>(resultElementType) ||2662 mlir::isa<fir::LogicalType>(resultElementType) ||2663 static_cast<bool>(builder.getFastMathFlags() &2664 mlir::arith::FastMathFlags::reassoc);2665 2666 // Insert the initialization loop nest that fills the whole result with2667 // zeroes.2668 mlir::Value initValue =2669 fir::factory::createZeroValue(builder, loc, resultElementType);2670 auto genInitBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,2671 mlir::ValueRange oneBasedIndices,2672 mlir::ValueRange reductionArgs)2673 -> llvm::SmallVector<mlir::Value, 0> {2674 hlfir::Entity resultElement =2675 hlfir::getElementAt(loc, builder, result, oneBasedIndices);2676 hlfir::AssignOp::create(builder, loc, initValue, resultElement);2677 return {};2678 };2679 2680 hlfir::genLoopNestWithReductions(loc, builder, resultExtents,2681 /*reductionInits=*/{}, genInitBody,2682 /*isUnordered=*/true);2683 2684 if (lhs.getRank() == 2 && rhs.getRank() == 2) {2685 // LHS(NROWS,N) * RHS(N,NCOLS) -> RESULT(NROWS,NCOLS)2686 //2687 // Insert the computation loop nest:2688 // DO 2 K = 1, N2689 // DO 2 J = 1, NCOLS2690 // DO 2 I = 1, NROWS2691 // 2 RESULT(I,J) = RESULT(I,J) + LHS(I,K)*RHS(K,J)2692 auto genMatrixMatrix = [&](mlir::Location loc, fir::FirOpBuilder &builder,2693 mlir::ValueRange oneBasedIndices,2694 mlir::ValueRange reductionArgs)2695 -> llvm::SmallVector<mlir::Value, 0> {2696 mlir::Value I = oneBasedIndices[0];2697 mlir::Value J = oneBasedIndices[1];2698 mlir::Value K = oneBasedIndices[2];2699 hlfir::Entity resultElement =2700 hlfir::getElementAt(loc, builder, result, {I, J});2701 hlfir::Entity resultElementValue =2702 hlfir::loadTrivialScalar(loc, builder, resultElement);2703 hlfir::Entity lhsElementValue =2704 hlfir::loadElementAt(loc, builder, lhs, {I, K});2705 hlfir::Entity rhsElementValue =2706 hlfir::loadElementAt(loc, builder, rhs, {K, J});2707 mlir::Value productValue =2708 ProductFactory{loc, builder}.genAccumulateProduct(2709 resultElementValue, lhsElementValue, rhsElementValue);2710 hlfir::AssignOp::create(builder, loc, productValue, resultElement);2711 return {};2712 };2713 2714 // Note that the loops are inserted in reverse order,2715 // so innerProductExtent should be passed as the last extent.2716 hlfir::genLoopNestWithReductions(2717 loc, builder,2718 {resultExtents[0], resultExtents[1], innerProductExtent},2719 /*reductionInits=*/{}, genMatrixMatrix, isUnordered);2720 return mlir::success();2721 }2722 2723 if (lhs.getRank() == 2 && rhs.getRank() == 1) {2724 // LHS(NROWS,N) * RHS(N) -> RESULT(NROWS)2725 //2726 // Insert the computation loop nest:2727 // DO 2 K = 1, N2728 // DO 2 J = 1, NROWS2729 // 2 RES(J) = RES(J) + LHS(J,K)*RHS(K)2730 auto genMatrixVector = [&](mlir::Location loc, fir::FirOpBuilder &builder,2731 mlir::ValueRange oneBasedIndices,2732 mlir::ValueRange reductionArgs)2733 -> llvm::SmallVector<mlir::Value, 0> {2734 mlir::Value J = oneBasedIndices[0];2735 mlir::Value K = oneBasedIndices[1];2736 hlfir::Entity resultElement =2737 hlfir::getElementAt(loc, builder, result, {J});2738 hlfir::Entity resultElementValue =2739 hlfir::loadTrivialScalar(loc, builder, resultElement);2740 hlfir::Entity lhsElementValue =2741 hlfir::loadElementAt(loc, builder, lhs, {J, K});2742 hlfir::Entity rhsElementValue =2743 hlfir::loadElementAt(loc, builder, rhs, {K});2744 mlir::Value productValue =2745 ProductFactory{loc, builder}.genAccumulateProduct(2746 resultElementValue, lhsElementValue, rhsElementValue);2747 hlfir::AssignOp::create(builder, loc, productValue, resultElement);2748 return {};2749 };2750 hlfir::genLoopNestWithReductions(2751 loc, builder, {resultExtents[0], innerProductExtent},2752 /*reductionInits=*/{}, genMatrixVector, isUnordered);2753 return mlir::success();2754 }2755 if (lhs.getRank() == 1 && rhs.getRank() == 2) {2756 // LHS(N) * RHS(N,NCOLS) -> RESULT(NCOLS)2757 //2758 // Insert the computation loop nest:2759 // DO 2 K = 1, N2760 // DO 2 J = 1, NCOLS2761 // 2 RES(J) = RES(J) + LHS(K)*RHS(K,J)2762 auto genVectorMatrix = [&](mlir::Location loc, fir::FirOpBuilder &builder,2763 mlir::ValueRange oneBasedIndices,2764 mlir::ValueRange reductionArgs)2765 -> llvm::SmallVector<mlir::Value, 0> {2766 mlir::Value J = oneBasedIndices[0];2767 mlir::Value K = oneBasedIndices[1];2768 hlfir::Entity resultElement =2769 hlfir::getElementAt(loc, builder, result, {J});2770 hlfir::Entity resultElementValue =2771 hlfir::loadTrivialScalar(loc, builder, resultElement);2772 hlfir::Entity lhsElementValue =2773 hlfir::loadElementAt(loc, builder, lhs, {K});2774 hlfir::Entity rhsElementValue =2775 hlfir::loadElementAt(loc, builder, rhs, {K, J});2776 mlir::Value productValue =2777 ProductFactory{loc, builder}.genAccumulateProduct(2778 resultElementValue, lhsElementValue, rhsElementValue);2779 hlfir::AssignOp::create(builder, loc, productValue, resultElement);2780 return {};2781 };2782 hlfir::genLoopNestWithReductions(2783 loc, builder, {resultExtents[0], innerProductExtent},2784 /*reductionInits=*/{}, genVectorMatrix, isUnordered);2785 return mlir::success();2786 }2787 2788 llvm_unreachable("unsupported MATMUL arguments' ranks");2789 }2790 2791 static hlfir::ElementalOp2792 genElementalMatmul(mlir::Location loc, fir::FirOpBuilder &builder,2793 hlfir::ExprType resultType, mlir::Value resultShape,2794 hlfir::Entity lhs, hlfir::Entity rhs,2795 mlir::Value innerProductExtent) {2796 mlir::OpBuilder::InsertionGuard guard(builder);2797 mlir::Type resultElementType = resultType.getElementType();2798 auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,2799 mlir::ValueRange resultIndices) -> hlfir::Entity {2800 mlir::Value initValue =2801 fir::factory::createZeroValue(builder, loc, resultElementType);2802 // The inner product loop may be unordered if FastMathFlags::reassoc2803 // transformations are allowed. The integer/logical inner product is2804 // always unordered.2805 bool isUnordered = mlir::isa<mlir::IntegerType>(resultElementType) ||2806 mlir::isa<fir::LogicalType>(resultElementType) ||2807 static_cast<bool>(builder.getFastMathFlags() &2808 mlir::arith::FastMathFlags::reassoc);2809 2810 auto genBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,2811 mlir::ValueRange oneBasedIndices,2812 mlir::ValueRange reductionArgs)2813 -> llvm::SmallVector<mlir::Value, 1> {2814 llvm::SmallVector<mlir::Value, 2> lhsIndices;2815 llvm::SmallVector<mlir::Value, 2> rhsIndices;2816 // MATMUL:2817 // LHS(NROWS,N) * RHS(N,NCOLS) -> RESULT(NROWS,NCOLS)2818 // LHS(NROWS,N) * RHS(N) -> RESULT(NROWS)2819 // LHS(N) * RHS(N,NCOLS) -> RESULT(NCOLS)2820 //2821 // MATMUL(TRANSPOSE):2822 // TRANSPOSE(LHS(N,NROWS)) * RHS(N,NCOLS) -> RESULT(NROWS,NCOLS)2823 // TRANSPOSE(LHS(N,NROWS)) * RHS(N) -> RESULT(NROWS)2824 //2825 // The resultIndices iterate over (NROWS[,NCOLS]).2826 // The oneBasedIndices iterate over (N).2827 if (lhs.getRank() > 1)2828 lhsIndices.push_back(resultIndices[0]);2829 lhsIndices.push_back(oneBasedIndices[0]);2830 2831 if constexpr (isMatmulTranspose) {2832 // Swap the LHS indices for TRANSPOSE.2833 std::swap(lhsIndices[0], lhsIndices[1]);2834 }2835 2836 rhsIndices.push_back(oneBasedIndices[0]);2837 if (rhs.getRank() > 1)2838 rhsIndices.push_back(resultIndices.back());2839 2840 hlfir::Entity lhsElementValue =2841 hlfir::loadElementAt(loc, builder, lhs, lhsIndices);2842 hlfir::Entity rhsElementValue =2843 hlfir::loadElementAt(loc, builder, rhs, rhsIndices);2844 mlir::Value productValue =2845 ProductFactory{loc, builder}.genAccumulateProduct(2846 reductionArgs[0], lhsElementValue, rhsElementValue);2847 return {productValue};2848 };2849 llvm::SmallVector<mlir::Value, 1> innerProductValue =2850 hlfir::genLoopNestWithReductions(loc, builder, {innerProductExtent},2851 {initValue}, genBody, isUnordered);2852 return hlfir::Entity{innerProductValue[0]};2853 };2854 hlfir::ElementalOp elementalOp = hlfir::genElementalOp(2855 loc, builder, resultElementType, resultShape, /*typeParams=*/{},2856 genKernel,2857 /*isUnordered=*/true, /*polymorphicMold=*/nullptr, resultType);2858 2859 return elementalOp;2860 }2861};2862 2863class DotProductConversion2864 : public mlir::OpRewritePattern<hlfir::DotProductOp> {2865public:2866 using mlir::OpRewritePattern<hlfir::DotProductOp>::OpRewritePattern;2867 2868 llvm::LogicalResult2869 matchAndRewrite(hlfir::DotProductOp product,2870 mlir::PatternRewriter &rewriter) const override {2871 hlfir::Entity op = hlfir::Entity{product};2872 if (!op.isScalar())2873 return rewriter.notifyMatchFailure(product, "produces non-scalar result");2874 2875 mlir::Location loc = product.getLoc();2876 fir::FirOpBuilder builder{rewriter, product.getOperation()};2877 hlfir::Entity lhs = hlfir::Entity{product.getLhs()};2878 hlfir::Entity rhs = hlfir::Entity{product.getRhs()};2879 mlir::Type resultElementType = product.getType();2880 bool isUnordered = mlir::isa<mlir::IntegerType>(resultElementType) ||2881 mlir::isa<fir::LogicalType>(resultElementType) ||2882 static_cast<bool>(builder.getFastMathFlags() &2883 mlir::arith::FastMathFlags::reassoc);2884 2885 mlir::Value extent = genProductExtent(loc, builder, lhs, rhs);2886 2887 auto genBody = [&](mlir::Location loc, fir::FirOpBuilder &builder,2888 mlir::ValueRange oneBasedIndices,2889 mlir::ValueRange reductionArgs)2890 -> llvm::SmallVector<mlir::Value, 1> {2891 hlfir::Entity lhsElementValue =2892 hlfir::loadElementAt(loc, builder, lhs, oneBasedIndices);2893 hlfir::Entity rhsElementValue =2894 hlfir::loadElementAt(loc, builder, rhs, oneBasedIndices);2895 mlir::Value productValue =2896 ProductFactory{loc, builder}.genAccumulateProduct</*CONJ=*/true>(2897 reductionArgs[0], lhsElementValue, rhsElementValue);2898 return {productValue};2899 };2900 2901 mlir::Value initValue =2902 fir::factory::createZeroValue(builder, loc, resultElementType);2903 2904 llvm::SmallVector<mlir::Value, 1> result = hlfir::genLoopNestWithReductions(2905 loc, builder, {extent},2906 /*reductionInits=*/{initValue}, genBody, isUnordered);2907 2908 rewriter.replaceOp(product, result[0]);2909 return mlir::success();2910 }2911 2912private:2913 static mlir::Value genProductExtent(mlir::Location loc,2914 fir::FirOpBuilder &builder,2915 hlfir::Entity input1,2916 hlfir::Entity input2) {2917 llvm::SmallVector<mlir::Value, 1> input1Extents =2918 hlfir::genExtentsVector(loc, builder, input1);2919 llvm::SmallVector<mlir::Value, 1> input2Extents =2920 hlfir::genExtentsVector(loc, builder, input2);2921 2922 assert(input1Extents.size() == 1 && input2Extents.size() == 1 &&2923 "hlfir.dot_product arguments must be vectors");2924 llvm::SmallVector<mlir::Value, 1> extent =2925 fir::factory::deduceOptimalExtents(input1Extents, input2Extents);2926 return extent[0];2927 }2928};2929 2930class ReshapeAsElementalConversion2931 : public mlir::OpRewritePattern<hlfir::ReshapeOp> {2932public:2933 using mlir::OpRewritePattern<hlfir::ReshapeOp>::OpRewritePattern;2934 2935 llvm::LogicalResult2936 matchAndRewrite(hlfir::ReshapeOp reshape,2937 mlir::PatternRewriter &rewriter) const override {2938 // Do not inline RESHAPE with ORDER yet. The runtime implementation2939 // may be good enough, unless the temporary creation overhead2940 // is high.2941 // TODO: If ORDER is constant, then we can still easily inline.2942 // TODO: If the result's rank is 1, then we can assume ORDER == (/1/).2943 if (reshape.getOrder())2944 return rewriter.notifyMatchFailure(reshape,2945 "RESHAPE with ORDER argument");2946 2947 // Verify that the element types of ARRAY, PAD and the result2948 // match before doing any transformations. For example,2949 // the character types of different lengths may appear in the dead2950 // code, and it just does not make sense to inline hlfir.reshape2951 // in this case (a runtime call might have less code size footprint).2952 hlfir::Entity result = hlfir::Entity{reshape};2953 hlfir::Entity array = hlfir::Entity{reshape.getArray()};2954 mlir::Type elementType = array.getFortranElementType();2955 if (result.getFortranElementType() != elementType)2956 return rewriter.notifyMatchFailure(2957 reshape, "ARRAY and result have different types");2958 mlir::Value pad = reshape.getPad();2959 if (pad && hlfir::getFortranElementType(pad.getType()) != elementType)2960 return rewriter.notifyMatchFailure(reshape,2961 "ARRAY and PAD have different types");2962 2963 // TODO: selecting between ARRAY and PAD of non-trivial element types2964 // requires more work. We have to select between two references2965 // to elements in ARRAY and PAD. This requires conditional2966 // bufferization of the element, if ARRAY/PAD is an expression.2967 if (pad && !fir::isa_trivial(elementType))2968 return rewriter.notifyMatchFailure(reshape,2969 "PAD present with non-trivial type");2970 2971 mlir::Location loc = reshape.getLoc();2972 fir::FirOpBuilder builder{rewriter, reshape.getOperation()};2973 // Assume that all the indices arithmetic does not overflow2974 // the IndexType.2975 builder.setIntegerOverflowFlags(mlir::arith::IntegerOverflowFlags::nuw);2976 2977 llvm::SmallVector<mlir::Value, 1> typeParams;2978 hlfir::genLengthParameters(loc, builder, array, typeParams);2979 2980 // Fetch the extents of ARRAY, PAD and result beforehand.2981 llvm::SmallVector<mlir::Value, Fortran::common::maxRank> arrayExtents =2982 hlfir::genExtentsVector(loc, builder, array);2983 2984 // If PAD is present, we have to use array size to start taking2985 // elements from the PAD array.2986 mlir::Value arraySize =2987 pad ? computeArraySize(loc, builder, arrayExtents) : nullptr;2988 hlfir::Entity shape = hlfir::Entity{reshape.getShape()};2989 llvm::SmallVector<mlir::Value, Fortran::common::maxRank> resultExtents;2990 mlir::Type indexType = builder.getIndexType();2991 for (int idx = 0; idx < result.getRank(); ++idx)2992 resultExtents.push_back(hlfir::loadElementAt(2993 loc, builder, shape,2994 builder.createIntegerConstant(loc, indexType, idx + 1)));2995 auto resultShape = fir::ShapeOp::create(builder, loc, resultExtents);2996 2997 auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,2998 mlir::ValueRange inputIndices) -> hlfir::Entity {2999 mlir::Value linearIndex =3000 computeLinearIndex(loc, builder, resultExtents, inputIndices);3001 fir::IfOp ifOp;3002 if (pad) {3003 // PAD is present. Check if this element comes from the PAD array.3004 mlir::Value isInsideArray = mlir::arith::CmpIOp::create(3005 builder, loc, mlir::arith::CmpIPredicate::ult, linearIndex,3006 arraySize);3007 ifOp = fir::IfOp::create(builder, loc, elementType, isInsideArray,3008 /*withElseRegion=*/true);3009 3010 // In the 'else' block, return an element from the PAD.3011 builder.setInsertionPointToStart(&ifOp.getElseRegion().front());3012 // PAD is dynamically optional, but we can unconditionally access it3013 // in the 'else' block. If we have to start taking elements from it,3014 // then it must be present in a valid program.3015 llvm::SmallVector<mlir::Value, Fortran::common::maxRank> padExtents =3016 hlfir::genExtentsVector(loc, builder, hlfir::Entity{pad});3017 // Subtract the ARRAY size from the zero-based linear index3018 // to get the zero-based linear index into PAD.3019 mlir::Value padLinearIndex =3020 mlir::arith::SubIOp::create(builder, loc, linearIndex, arraySize);3021 llvm::SmallVector<mlir::Value, Fortran::common::maxRank> padIndices =3022 delinearizeIndex(loc, builder, padExtents, padLinearIndex,3023 /*wrapAround=*/true);3024 mlir::Value padElement =3025 hlfir::loadElementAt(loc, builder, hlfir::Entity{pad}, padIndices);3026 fir::ResultOp::create(builder, loc, padElement);3027 3028 // In the 'then' block, return an element from the ARRAY.3029 builder.setInsertionPointToStart(&ifOp.getThenRegion().front());3030 }3031 3032 llvm::SmallVector<mlir::Value, Fortran::common::maxRank> arrayIndices =3033 delinearizeIndex(loc, builder, arrayExtents, linearIndex,3034 /*wrapAround=*/false);3035 mlir::Value arrayElement =3036 hlfir::loadElementAt(loc, builder, array, arrayIndices);3037 3038 if (ifOp) {3039 fir::ResultOp::create(builder, loc, arrayElement);3040 builder.setInsertionPointAfter(ifOp);3041 arrayElement = ifOp.getResult(0);3042 }3043 3044 return hlfir::Entity{arrayElement};3045 };3046 hlfir::ElementalOp elementalOp = hlfir::genElementalOp(3047 loc, builder, elementType, resultShape, typeParams, genKernel,3048 /*isUnordered=*/true,3049 /*polymorphicMold=*/result.isPolymorphic() ? array : mlir::Value{},3050 reshape.getResult().getType());3051 assert(elementalOp.getResult().getType() == reshape.getResult().getType());3052 rewriter.replaceOp(reshape, elementalOp);3053 return mlir::success();3054 }3055 3056private:3057 /// Compute zero-based linear index given an array extents3058 /// and one-based indices:3059 /// \p extents: [e0, e1, ..., en]3060 /// \p indices: [i0, i1, ..., in]3061 ///3062 /// linear-index :=3063 /// (...((in-1)*e(n-1)+(i(n-1)-1))*e(n-2)+...)*e0+(i0-1)3064 static mlir::Value computeLinearIndex(mlir::Location loc,3065 fir::FirOpBuilder &builder,3066 mlir::ValueRange extents,3067 mlir::ValueRange indices) {3068 std::size_t rank = extents.size();3069 assert(rank == indices.size());3070 mlir::Type indexType = builder.getIndexType();3071 mlir::Value zero = builder.createIntegerConstant(loc, indexType, 0);3072 mlir::Value one = builder.createIntegerConstant(loc, indexType, 1);3073 mlir::Value linearIndex = zero;3074 std::size_t idx = 0;3075 for (auto index : llvm::reverse(indices)) {3076 mlir::Value tmp = mlir::arith::SubIOp::create(3077 builder, loc, builder.createConvert(loc, indexType, index), one);3078 tmp = mlir::arith::AddIOp::create(builder, loc, linearIndex, tmp);3079 if (idx + 1 < rank)3080 tmp = mlir::arith::MulIOp::create(3081 builder, loc, tmp,3082 builder.createConvert(loc, indexType, extents[rank - idx - 2]));3083 3084 linearIndex = tmp;3085 ++idx;3086 }3087 return linearIndex;3088 }3089 3090 /// Compute one-based array indices from the given zero-based \p linearIndex3091 /// and the array \p extents [e0, e1, ..., en].3092 /// i0 := linearIndex % e0 + 13093 /// linearIndex := linearIndex / e03094 /// i1 := linearIndex % e1 + 13095 /// linearIndex := linearIndex / e13096 /// ...3097 /// i(n-1) := linearIndex % e(n-1) + 13098 /// linearIndex := linearIndex / e(n-1)3099 /// if (wrapAround) {3100 /// // If the index is allowed to wrap around, then3101 /// // we need to modulo it by the last dimension's extent.3102 /// in := linearIndex % en + 13103 /// } else {3104 /// in := linearIndex + 13105 /// }3106 static llvm::SmallVector<mlir::Value, Fortran::common::maxRank>3107 delinearizeIndex(mlir::Location loc, fir::FirOpBuilder &builder,3108 mlir::ValueRange extents, mlir::Value linearIndex,3109 bool wrapAround) {3110 llvm::SmallVector<mlir::Value, Fortran::common::maxRank> indices;3111 mlir::Type indexType = builder.getIndexType();3112 mlir::Value one = builder.createIntegerConstant(loc, indexType, 1);3113 linearIndex = builder.createConvert(loc, indexType, linearIndex);3114 3115 for (std::size_t dim = 0; dim < extents.size(); ++dim) {3116 mlir::Value extent = builder.createConvert(loc, indexType, extents[dim]);3117 // Avoid the modulo for the last index, unless wrap around is allowed.3118 mlir::Value currentIndex = linearIndex;3119 if (dim != extents.size() - 1 || wrapAround)3120 currentIndex =3121 mlir::arith::RemUIOp::create(builder, loc, linearIndex, extent);3122 // The result of the last division is unused, so it will be DCEd.3123 linearIndex =3124 mlir::arith::DivUIOp::create(builder, loc, linearIndex, extent);3125 indices.push_back(3126 mlir::arith::AddIOp::create(builder, loc, currentIndex, one));3127 }3128 return indices;3129 }3130 3131 /// Return size of an array given its extents.3132 static mlir::Value computeArraySize(mlir::Location loc,3133 fir::FirOpBuilder &builder,3134 mlir::ValueRange extents) {3135 mlir::Type indexType = builder.getIndexType();3136 mlir::Value size = builder.createIntegerConstant(loc, indexType, 1);3137 for (auto extent : extents)3138 size = mlir::arith::MulIOp::create(3139 builder, loc, size, builder.createConvert(loc, indexType, extent));3140 return size;3141 }3142};3143 3144class SimplifyHLFIRIntrinsics3145 : public hlfir::impl::SimplifyHLFIRIntrinsicsBase<SimplifyHLFIRIntrinsics> {3146public:3147 using SimplifyHLFIRIntrinsicsBase<3148 SimplifyHLFIRIntrinsics>::SimplifyHLFIRIntrinsicsBase;3149 3150 void runOnOperation() override {3151 mlir::MLIRContext *context = &getContext();3152 3153 mlir::GreedyRewriteConfig config;3154 // Prevent the pattern driver from merging blocks3155 config.setRegionSimplificationLevel(3156 mlir::GreedySimplifyRegionLevel::Disabled);3157 3158 mlir::RewritePatternSet patterns(context);3159 patterns.insert<TransposeAsElementalConversion>(context);3160 patterns.insert<ReductionConversion<hlfir::SumOp>>(context);3161 patterns.insert<ArrayShiftConversion<hlfir::CShiftOp>>(context);3162 patterns.insert<ArrayShiftConversion<hlfir::EOShiftOp>>(context);3163 patterns.insert<CmpCharOpConversion>(context);3164 patterns.insert<IndexOpConversion>(context);3165 patterns.insert<MatmulConversion<hlfir::MatmulTransposeOp>>(context);3166 patterns.insert<ReductionConversion<hlfir::CountOp>>(context);3167 patterns.insert<ReductionConversion<hlfir::AnyOp>>(context);3168 patterns.insert<ReductionConversion<hlfir::AllOp>>(context);3169 patterns.insert<ReductionConversion<hlfir::MaxlocOp>>(context);3170 patterns.insert<ReductionConversion<hlfir::MinlocOp>>(context);3171 patterns.insert<ReductionConversion<hlfir::MaxvalOp>>(context);3172 patterns.insert<ReductionConversion<hlfir::MinvalOp>>(context);3173 3174 // If forceMatmulAsElemental is false, then hlfir.matmul inlining3175 // will introduce hlfir.eval_in_mem operation with new memory side3176 // effects. This conflicts with CSE and optimized bufferization, e.g.:3177 // A(1:N,1:N) = A(1:N,1:N) - MATMUL(...)3178 // If we introduce hlfir.eval_in_mem before CSE, then the current3179 // MLIR CSE won't be able to optimize the trivial loads of 'N' value3180 // that happen before and after hlfir.matmul.3181 // If 'N' loads are not optimized, then the optimized bufferization3182 // won't be able to prove that the slices of A are identical3183 // on both sides of the assignment.3184 // This is actually the CSE problem, but we can work it around3185 // for the time being.3186 if (forceMatmulAsElemental || this->allowNewSideEffects)3187 patterns.insert<MatmulConversion<hlfir::MatmulOp>>(context);3188 3189 patterns.insert<DotProductConversion>(context);3190 patterns.insert<ReshapeAsElementalConversion>(context);3191 3192 if (mlir::failed(mlir::applyPatternsGreedily(3193 getOperation(), std::move(patterns), config))) {3194 mlir::emitError(getOperation()->getLoc(),3195 "failure in HLFIR intrinsic simplification");3196 signalPassFailure();3197 }3198 }3199};3200} // namespace3201