884 lines · cpp
1//===-- ReductionProcessor.cpp ----------------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/10//11//===----------------------------------------------------------------------===//12 13#include "flang/Lower/Support/ReductionProcessor.h"14 15#include "flang/Lower/AbstractConverter.h"16#include "flang/Lower/ConvertType.h"17#include "flang/Lower/OpenMP/Clauses.h"18#include "flang/Lower/Support/PrivateReductionUtils.h"19#include "flang/Lower/SymbolMap.h"20#include "flang/Optimizer/Builder/Complex.h"21#include "flang/Optimizer/Builder/HLFIRTools.h"22#include "flang/Optimizer/Builder/Todo.h"23#include "flang/Optimizer/Dialect/FIRType.h"24#include "flang/Optimizer/HLFIR/HLFIROps.h"25#include "mlir/Dialect/OpenMP/OpenMPDialect.h"26#include "llvm/Support/CommandLine.h"27#include <type_traits>28 29static llvm::cl::opt<bool> forceByrefReduction(30 "force-byref-reduction",31 llvm::cl::desc("Pass all reduction arguments by reference"),32 llvm::cl::Hidden);33 34using ReductionModifier =35 Fortran::lower::omp::clause::Reduction::ReductionModifier;36 37namespace Fortran {38namespace lower {39namespace omp {40 41// explicit template declarations42template bool ReductionProcessor::processReductionArguments<43 mlir::omp::DeclareReductionOp, omp::clause::ReductionOperatorList>(44 mlir::Location currentLocation, lower::AbstractConverter &converter,45 const omp::clause::ReductionOperatorList &redOperatorList,46 llvm::SmallVectorImpl<mlir::Value> &reductionVars,47 llvm::SmallVectorImpl<bool> &reduceVarByRef,48 llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,49 const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols);50 51template bool ReductionProcessor::processReductionArguments<52 fir::DeclareReductionOp, llvm::SmallVector<fir::ReduceOperationEnum>>(53 mlir::Location currentLocation, lower::AbstractConverter &converter,54 const llvm::SmallVector<fir::ReduceOperationEnum> &redOperatorList,55 llvm::SmallVectorImpl<mlir::Value> &reductionVars,56 llvm::SmallVectorImpl<bool> &reduceVarByRef,57 llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,58 const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols);59 60template mlir::omp::DeclareReductionOp61ReductionProcessor::createDeclareReduction<mlir::omp::DeclareReductionOp>(62 AbstractConverter &converter, llvm::StringRef reductionOpName,63 const ReductionIdentifier redId, mlir::Type type, mlir::Location loc,64 bool isByRef);65 66template fir::DeclareReductionOp67ReductionProcessor::createDeclareReduction<fir::DeclareReductionOp>(68 AbstractConverter &converter, llvm::StringRef reductionOpName,69 const ReductionIdentifier redId, mlir::Type type, mlir::Location loc,70 bool isByRef);71 72ReductionProcessor::ReductionIdentifier ReductionProcessor::getReductionType(73 const omp::clause::ProcedureDesignator &pd) {74 auto redType = llvm::StringSwitch<std::optional<ReductionIdentifier>>(75 getRealName(pd.v.sym()).ToString())76 .Case("max", ReductionIdentifier::MAX)77 .Case("min", ReductionIdentifier::MIN)78 .Case("iand", ReductionIdentifier::IAND)79 .Case("ior", ReductionIdentifier::IOR)80 .Case("ieor", ReductionIdentifier::IEOR)81 .Default(std::nullopt);82 assert(redType && "Invalid Reduction");83 return *redType;84}85 86ReductionProcessor::ReductionIdentifier ReductionProcessor::getReductionType(87 omp::clause::DefinedOperator::IntrinsicOperator intrinsicOp) {88 switch (intrinsicOp) {89 case omp::clause::DefinedOperator::IntrinsicOperator::Add:90 return ReductionIdentifier::ADD;91 case omp::clause::DefinedOperator::IntrinsicOperator::Subtract:92 return ReductionIdentifier::SUBTRACT;93 case omp::clause::DefinedOperator::IntrinsicOperator::Multiply:94 return ReductionIdentifier::MULTIPLY;95 case omp::clause::DefinedOperator::IntrinsicOperator::AND:96 return ReductionIdentifier::AND;97 case omp::clause::DefinedOperator::IntrinsicOperator::EQV:98 return ReductionIdentifier::EQV;99 case omp::clause::DefinedOperator::IntrinsicOperator::OR:100 return ReductionIdentifier::OR;101 case omp::clause::DefinedOperator::IntrinsicOperator::NEQV:102 return ReductionIdentifier::NEQV;103 default:104 llvm_unreachable("unexpected intrinsic operator in reduction");105 }106}107 108ReductionProcessor::ReductionIdentifier109ReductionProcessor::getReductionType(const fir::ReduceOperationEnum &redOp) {110 switch (redOp) {111 case fir::ReduceOperationEnum::Add:112 return ReductionIdentifier::ADD;113 case fir::ReduceOperationEnum::Multiply:114 return ReductionIdentifier::MULTIPLY;115 116 case fir::ReduceOperationEnum::AND:117 return ReductionIdentifier::AND;118 case fir::ReduceOperationEnum::OR:119 return ReductionIdentifier::OR;120 121 case fir::ReduceOperationEnum::EQV:122 return ReductionIdentifier::EQV;123 case fir::ReduceOperationEnum::NEQV:124 return ReductionIdentifier::NEQV;125 126 case fir::ReduceOperationEnum::IAND:127 return ReductionIdentifier::IAND;128 case fir::ReduceOperationEnum::IEOR:129 return ReductionIdentifier::IEOR;130 case fir::ReduceOperationEnum::IOR:131 return ReductionIdentifier::IOR;132 case fir::ReduceOperationEnum::MAX:133 return ReductionIdentifier::MAX;134 case fir::ReduceOperationEnum::MIN:135 return ReductionIdentifier::MIN;136 }137 llvm_unreachable("Unhandled ReductionIdentifier case");138}139 140bool ReductionProcessor::supportedIntrinsicProcReduction(141 const omp::clause::ProcedureDesignator &pd) {142 semantics::Symbol *sym = pd.v.sym();143 if (!sym->GetUltimate().attrs().test(semantics::Attr::INTRINSIC))144 return false;145 auto redType = llvm::StringSwitch<bool>(getRealName(sym).ToString())146 .Case("max", true)147 .Case("min", true)148 .Case("iand", true)149 .Case("ior", true)150 .Case("ieor", true)151 .Default(false);152 return redType;153}154 155std::string156ReductionProcessor::getReductionName(llvm::StringRef name,157 const fir::KindMapping &kindMap,158 mlir::Type ty, bool isByRef) {159 ty = fir::unwrapRefType(ty);160 161 // extra string to distinguish reduction functions for variables passed by162 // reference163 llvm::StringRef byrefAddition{""};164 if (isByRef)165 byrefAddition = "_byref";166 167 return fir::getTypeAsString(ty, kindMap, (name + byrefAddition).str());168}169 170std::string171ReductionProcessor::getReductionName(ReductionIdentifier redId,172 const fir::KindMapping &kindMap,173 mlir::Type ty, bool isByRef) {174 std::string reductionName;175 176 switch (redId) {177 case ReductionIdentifier::ADD:178 reductionName = "add_reduction";179 break;180 case ReductionIdentifier::MULTIPLY:181 reductionName = "multiply_reduction";182 break;183 case ReductionIdentifier::AND:184 reductionName = "and_reduction";185 break;186 case ReductionIdentifier::EQV:187 reductionName = "eqv_reduction";188 break;189 case ReductionIdentifier::OR:190 reductionName = "or_reduction";191 break;192 case ReductionIdentifier::NEQV:193 reductionName = "neqv_reduction";194 break;195 default:196 reductionName = "other_reduction";197 break;198 }199 200 return getReductionName(reductionName, kindMap, ty, isByRef);201}202 203mlir::Value204ReductionProcessor::getReductionInitValue(mlir::Location loc, mlir::Type type,205 ReductionIdentifier redId,206 fir::FirOpBuilder &builder) {207 type = fir::unwrapRefType(type);208 if (!fir::isa_integer(type) && !fir::isa_real(type) &&209 !fir::isa_complex(type) && !mlir::isa<fir::LogicalType>(type))210 TODO(loc, "Reduction of some types is not supported");211 switch (redId) {212 case ReductionIdentifier::MAX: {213 if (auto ty = mlir::dyn_cast<mlir::FloatType>(type)) {214 const llvm::fltSemantics &sem = ty.getFloatSemantics();215 return builder.createRealConstant(216 loc, type, llvm::APFloat::getLargest(sem, /*Negative=*/true));217 }218 unsigned bits = type.getIntOrFloatBitWidth();219 int64_t minInt = llvm::APInt::getSignedMinValue(bits).getSExtValue();220 return builder.createIntegerConstant(loc, type, minInt);221 }222 case ReductionIdentifier::MIN: {223 if (auto ty = mlir::dyn_cast<mlir::FloatType>(type)) {224 const llvm::fltSemantics &sem = ty.getFloatSemantics();225 return builder.createRealConstant(226 loc, type, llvm::APFloat::getLargest(sem, /*Negative=*/false));227 }228 unsigned bits = type.getIntOrFloatBitWidth();229 int64_t maxInt = llvm::APInt::getSignedMaxValue(bits).getSExtValue();230 return builder.createIntegerConstant(loc, type, maxInt);231 }232 case ReductionIdentifier::IOR: {233 unsigned bits = type.getIntOrFloatBitWidth();234 int64_t zeroInt = llvm::APInt::getZero(bits).getSExtValue();235 return builder.createIntegerConstant(loc, type, zeroInt);236 }237 case ReductionIdentifier::IEOR: {238 unsigned bits = type.getIntOrFloatBitWidth();239 int64_t zeroInt = llvm::APInt::getZero(bits).getSExtValue();240 return builder.createIntegerConstant(loc, type, zeroInt);241 }242 case ReductionIdentifier::IAND: {243 unsigned bits = type.getIntOrFloatBitWidth();244 int64_t allOnInt = llvm::APInt::getAllOnes(bits).getSExtValue();245 return builder.createIntegerConstant(loc, type, allOnInt);246 }247 case ReductionIdentifier::ADD:248 case ReductionIdentifier::MULTIPLY:249 case ReductionIdentifier::AND:250 case ReductionIdentifier::OR:251 case ReductionIdentifier::EQV:252 case ReductionIdentifier::NEQV:253 if (auto cplxTy = mlir::dyn_cast<mlir::ComplexType>(type)) {254 mlir::Type realTy = cplxTy.getElementType();255 mlir::Value initRe = builder.createRealConstant(256 loc, realTy, getOperationIdentity(redId, loc));257 mlir::Value initIm = builder.createRealConstant(loc, realTy, 0);258 259 return fir::factory::Complex{builder, loc}.createComplex(type, initRe,260 initIm);261 }262 if (mlir::isa<mlir::FloatType>(type))263 return mlir::arith::ConstantOp::create(264 builder, loc, type,265 builder.getFloatAttr(type, (double)getOperationIdentity(redId, loc)));266 267 if (mlir::isa<fir::LogicalType>(type)) {268 mlir::Value intConst = mlir::arith::ConstantOp::create(269 builder, loc, builder.getI1Type(),270 builder.getIntegerAttr(builder.getI1Type(),271 getOperationIdentity(redId, loc)));272 return builder.createConvert(loc, type, intConst);273 }274 275 return mlir::arith::ConstantOp::create(276 builder, loc, type,277 builder.getIntegerAttr(type, getOperationIdentity(redId, loc)));278 case ReductionIdentifier::ID:279 case ReductionIdentifier::USER_DEF_OP:280 case ReductionIdentifier::SUBTRACT:281 TODO(loc, "Reduction of some identifier types is not supported");282 }283 llvm_unreachable("Unhandled Reduction identifier : getReductionInitValue");284}285 286mlir::Value ReductionProcessor::createScalarCombiner(287 fir::FirOpBuilder &builder, mlir::Location loc, ReductionIdentifier redId,288 mlir::Type type, mlir::Value op1, mlir::Value op2) {289 mlir::Value reductionOp;290 type = fir::unwrapRefType(type);291 switch (redId) {292 case ReductionIdentifier::MAX:293 reductionOp =294 getReductionOperation<mlir::arith::MaxNumFOp, mlir::arith::MaxSIOp>(295 builder, type, loc, op1, op2);296 break;297 case ReductionIdentifier::MIN:298 reductionOp =299 getReductionOperation<mlir::arith::MinNumFOp, mlir::arith::MinSIOp>(300 builder, type, loc, op1, op2);301 break;302 case ReductionIdentifier::IOR:303 assert((type.isIntOrIndex()) && "only integer is expected");304 reductionOp = mlir::arith::OrIOp::create(builder, loc, op1, op2);305 break;306 case ReductionIdentifier::IEOR:307 assert((type.isIntOrIndex()) && "only integer is expected");308 reductionOp = mlir::arith::XOrIOp::create(builder, loc, op1, op2);309 break;310 case ReductionIdentifier::IAND:311 assert((type.isIntOrIndex()) && "only integer is expected");312 reductionOp = mlir::arith::AndIOp::create(builder, loc, op1, op2);313 break;314 case ReductionIdentifier::ADD:315 reductionOp =316 getReductionOperation<mlir::arith::AddFOp, mlir::arith::AddIOp,317 fir::AddcOp>(builder, type, loc, op1, op2);318 break;319 case ReductionIdentifier::MULTIPLY:320 reductionOp =321 getReductionOperation<mlir::arith::MulFOp, mlir::arith::MulIOp,322 fir::MulcOp>(builder, type, loc, op1, op2);323 break;324 case ReductionIdentifier::AND: {325 mlir::Value op1I1 = builder.createConvert(loc, builder.getI1Type(), op1);326 mlir::Value op2I1 = builder.createConvert(loc, builder.getI1Type(), op2);327 328 mlir::Value andiOp =329 mlir::arith::AndIOp::create(builder, loc, op1I1, op2I1);330 331 reductionOp = builder.createConvert(loc, type, andiOp);332 break;333 }334 case ReductionIdentifier::OR: {335 mlir::Value op1I1 = builder.createConvert(loc, builder.getI1Type(), op1);336 mlir::Value op2I1 = builder.createConvert(loc, builder.getI1Type(), op2);337 338 mlir::Value oriOp = mlir::arith::OrIOp::create(builder, loc, op1I1, op2I1);339 340 reductionOp = builder.createConvert(loc, type, oriOp);341 break;342 }343 case ReductionIdentifier::EQV: {344 mlir::Value op1I1 = builder.createConvert(loc, builder.getI1Type(), op1);345 mlir::Value op2I1 = builder.createConvert(loc, builder.getI1Type(), op2);346 347 mlir::Value cmpiOp = mlir::arith::CmpIOp::create(348 builder, loc, mlir::arith::CmpIPredicate::eq, op1I1, op2I1);349 350 reductionOp = builder.createConvert(loc, type, cmpiOp);351 break;352 }353 case ReductionIdentifier::NEQV: {354 mlir::Value op1I1 = builder.createConvert(loc, builder.getI1Type(), op1);355 mlir::Value op2I1 = builder.createConvert(loc, builder.getI1Type(), op2);356 357 mlir::Value cmpiOp = mlir::arith::CmpIOp::create(358 builder, loc, mlir::arith::CmpIPredicate::ne, op1I1, op2I1);359 360 reductionOp = builder.createConvert(loc, type, cmpiOp);361 break;362 }363 default:364 TODO(loc, "Reduction of some intrinsic operators is not supported");365 }366 367 return reductionOp;368}369 370template <typename ParentDeclOpType>371static void genYield(fir::FirOpBuilder &builder, mlir::Location loc,372 mlir::Value yieldedValue) {373 if constexpr (std::is_same_v<ParentDeclOpType, mlir::omp::DeclareReductionOp>)374 mlir::omp::YieldOp::create(builder, loc, yieldedValue);375 else376 fir::YieldOp::create(builder, loc, yieldedValue);377}378 379/// Create reduction combiner region for reduction variables which are boxed380/// arrays381template <typename DeclRedOpType>382static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,383 ReductionProcessor::ReductionIdentifier redId,384 fir::BaseBoxType boxTy, mlir::Value lhs,385 mlir::Value rhs) {386 fir::SequenceType seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(387 fir::unwrapRefType(boxTy.getEleTy()));388 fir::HeapType heapTy =389 mlir::dyn_cast_or_null<fir::HeapType>(boxTy.getEleTy());390 fir::PointerType ptrTy =391 mlir::dyn_cast_or_null<fir::PointerType>(boxTy.getEleTy());392 if ((!seqTy || seqTy.hasUnknownShape()) && !heapTy && !ptrTy)393 TODO(loc, "Unsupported boxed type in OpenMP reduction");394 395 // load fir.ref<fir.box<...>>396 mlir::Value lhsAddr = lhs;397 lhs = fir::LoadOp::create(builder, loc, lhs);398 rhs = fir::LoadOp::create(builder, loc, rhs);399 400 if ((heapTy || ptrTy) && !seqTy) {401 // get box contents (heap pointers)402 lhs = fir::BoxAddrOp::create(builder, loc, lhs);403 rhs = fir::BoxAddrOp::create(builder, loc, rhs);404 mlir::Value lhsValAddr = lhs;405 406 // load heap pointers407 lhs = fir::LoadOp::create(builder, loc, lhs);408 rhs = fir::LoadOp::create(builder, loc, rhs);409 410 mlir::Type eleTy = heapTy ? heapTy.getEleTy() : ptrTy.getEleTy();411 412 mlir::Value result = ReductionProcessor::createScalarCombiner(413 builder, loc, redId, eleTy, lhs, rhs);414 fir::StoreOp::create(builder, loc, result, lhsValAddr);415 genYield<DeclRedOpType>(builder, loc, lhsAddr);416 return;417 }418 419 // Get ShapeShift with default lower bounds. This makes it possible to use420 // unmodified LoopNest's indices with ArrayCoorOp.421 fir::ShapeShiftOp shapeShift =422 getShapeShift(builder, loc, lhs,423 /*cannotHaveNonDefaultLowerBounds=*/false,424 /*useDefaultLowerBounds=*/true);425 426 // Iterate over array elements, applying the equivalent scalar reduction:427 428 // F2018 5.4.10.2: Unallocated allocatable variables may not be referenced429 // and so no null check is needed here before indexing into the (possibly430 // allocatable) arrays.431 432 // A hlfir::elemental here gets inlined with a temporary so create the433 // loop nest directly.434 // This function already controls all of the code in this region so we435 // know this won't miss any opportuinties for clever elemental inlining436 hlfir::LoopNest nest = hlfir::genLoopNest(437 loc, builder, shapeShift.getExtents(), /*isUnordered=*/true);438 builder.setInsertionPointToStart(nest.body);439 const bool seqIsVolatile = fir::isa_volatile_type(seqTy.getEleTy());440 mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy(), seqIsVolatile);441 auto lhsEleAddr = fir::ArrayCoorOp::create(442 builder, loc, refTy, lhs, shapeShift, /*slice=*/mlir::Value{},443 nest.oneBasedIndices, /*typeparms=*/mlir::ValueRange{});444 auto rhsEleAddr = fir::ArrayCoorOp::create(445 builder, loc, refTy, rhs, shapeShift, /*slice=*/mlir::Value{},446 nest.oneBasedIndices, /*typeparms=*/mlir::ValueRange{});447 auto lhsEle = fir::LoadOp::create(builder, loc, lhsEleAddr);448 auto rhsEle = fir::LoadOp::create(builder, loc, rhsEleAddr);449 mlir::Value scalarReduction = ReductionProcessor::createScalarCombiner(450 builder, loc, redId, refTy, lhsEle, rhsEle);451 fir::StoreOp::create(builder, loc, scalarReduction, lhsEleAddr);452 453 builder.setInsertionPointAfter(nest.outerOp);454 genYield<DeclRedOpType>(builder, loc, lhsAddr);455}456 457// generate combiner region for reduction operations458template <typename DeclRedOpType>459static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,460 ReductionProcessor::ReductionIdentifier redId,461 mlir::Type ty, mlir::Value lhs, mlir::Value rhs,462 bool isByRef) {463 ty = fir::unwrapRefType(ty);464 465 if (fir::isa_trivial(ty)) {466 mlir::Value lhsLoaded = builder.loadIfRef(loc, lhs);467 mlir::Value rhsLoaded = builder.loadIfRef(loc, rhs);468 469 mlir::Value result = ReductionProcessor::createScalarCombiner(470 builder, loc, redId, ty, lhsLoaded, rhsLoaded);471 if (isByRef) {472 fir::StoreOp::create(builder, loc, result, lhs);473 genYield<DeclRedOpType>(builder, loc, lhs);474 } else {475 genYield<DeclRedOpType>(builder, loc, result);476 }477 return;478 }479 // all arrays should have been boxed480 if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(ty)) {481 genBoxCombiner<DeclRedOpType>(builder, loc, redId, boxTy, lhs, rhs);482 return;483 }484 485 TODO(loc, "OpenMP genCombiner for unsupported reduction variable type");486}487 488// like fir::unwrapSeqOrBoxedSeqType except it also works for non-sequence boxes489static mlir::Type unwrapSeqOrBoxedType(mlir::Type ty) {490 if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(ty))491 return seqTy.getEleTy();492 if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(ty)) {493 auto eleTy = fir::unwrapRefType(boxTy.getEleTy());494 if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(eleTy))495 return seqTy.getEleTy();496 return eleTy;497 }498 return ty;499}500 501template <typename OpType>502static void createReductionAllocAndInitRegions(503 AbstractConverter &converter, mlir::Location loc, OpType &reductionDecl,504 ReductionProcessor::GenInitValueCBTy genInitValueCB, mlir::Type type,505 bool isByRef) {506 fir::FirOpBuilder &builder = converter.getFirOpBuilder();507 auto yield = [&](mlir::Value ret) { genYield<OpType>(builder, loc, ret); };508 509 mlir::Block *allocBlock = nullptr;510 mlir::Block *initBlock = nullptr;511 if (isByRef) {512 allocBlock =513 builder.createBlock(&reductionDecl.getAllocRegion(),514 reductionDecl.getAllocRegion().end(), {}, {});515 initBlock = builder.createBlock(&reductionDecl.getInitializerRegion(),516 reductionDecl.getInitializerRegion().end(),517 {type, type}, {loc, loc});518 } else {519 initBlock = builder.createBlock(&reductionDecl.getInitializerRegion(),520 reductionDecl.getInitializerRegion().end(),521 {type}, {loc});522 }523 524 mlir::Type ty = fir::unwrapRefType(type);525 builder.setInsertionPointToEnd(initBlock);526 mlir::Value initValue =527 genInitValueCB(builder, loc, ty, initBlock->getArgument(0));528 if (isByRef) {529 populateByRefInitAndCleanupRegions(530 converter, loc, type, initValue, initBlock,531 reductionDecl.getInitializerAllocArg(),532 reductionDecl.getInitializerMoldArg(), reductionDecl.getCleanupRegion(),533 DeclOperationKind::Reduction, /*sym=*/nullptr,534 /*cannotHaveLowerBounds=*/false,535 /*isDoConcurrent*/ std::is_same_v<OpType, fir::DeclareReductionOp>);536 }537 538 if (fir::isa_trivial(ty) || fir::isa_derived(ty)) {539 if (isByRef) {540 // alloc region541 builder.setInsertionPointToEnd(allocBlock);542 mlir::Value alloca = fir::AllocaOp::create(builder, loc, ty);543 yield(alloca);544 return;545 }546 // by val547 yield(initValue);548 return;549 }550 assert(isByRef && "passing non-trivial types by val is unsupported");551 552 // alloc region553 builder.setInsertionPointToEnd(allocBlock);554 mlir::Value boxAlloca = fir::AllocaOp::create(builder, loc, ty);555 yield(boxAlloca);556}557 558template <typename DeclareRedType>559DeclareRedType ReductionProcessor::createDeclareReductionHelper(560 AbstractConverter &converter, llvm::StringRef reductionOpName,561 mlir::Type type, mlir::Location loc, bool isByRef,562 GenCombinerCBTy genCombinerCB, GenInitValueCBTy genInitValueCB) {563 fir::FirOpBuilder &builder = converter.getFirOpBuilder();564 mlir::OpBuilder::InsertionGuard guard(builder);565 mlir::ModuleOp module = builder.getModule();566 567 assert(!reductionOpName.empty());568 569 auto decl = module.lookupSymbol<DeclareRedType>(reductionOpName);570 if (decl)571 return decl;572 573 mlir::OpBuilder modBuilder(module.getBodyRegion());574 mlir::Type valTy = fir::unwrapRefType(type);575 576 // For by-ref reductions, we want to keep track of the577 // boxed/referenced/allocated type. For example, for a `real, allocatable`578 // variable, `real` should be stored.579 mlir::TypeAttr boxedTyAttr{};580 mlir::Type boxedTy;581 582 if (isByRef) {583 boxedTy = fir::unwrapPassByRefType(valTy);584 boxedTyAttr = mlir::TypeAttr::get(boxedTy);585 } else586 type = valTy;587 588 decl = DeclareRedType::create(modBuilder, loc, reductionOpName, type,589 boxedTyAttr);590 createReductionAllocAndInitRegions(converter, loc, decl, genInitValueCB, type,591 isByRef);592 builder.createBlock(&decl.getReductionRegion(),593 decl.getReductionRegion().end(), {type, type},594 {loc, loc});595 builder.setInsertionPointToEnd(&decl.getReductionRegion().back());596 mlir::Value op1 = decl.getReductionRegion().front().getArgument(0);597 mlir::Value op2 = decl.getReductionRegion().front().getArgument(1);598 genCombinerCB(builder, loc, type, op1, op2, isByRef);599 600 if (isByRef && fir::isa_box_type(valTy)) {601 bool isBoxReductionSupported = [&]() {602 auto offloadMod = llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(603 *builder.getModule());604 605 // This check tests the implementation status on the GPU. Box reductions606 // are fully supported on the CPU.607 if (!offloadMod.getIsGPU())608 return true;609 610 auto seqTy = mlir::dyn_cast<fir::SequenceType>(boxedTy);611 612 // Dynamically-shaped arrays are not supported yet on the GPU.613 return !seqTy || !fir::sequenceWithNonConstantShape(seqTy);614 }();615 616 if (!isBoxReductionSupported) {617 TODO(loc, "Reduction of dynamically-shaped arrays are not supported yet "618 "on the GPU.");619 }620 621 mlir::Region &dataPtrPtrRegion = decl.getDataPtrPtrRegion();622 mlir::Block &dataAddrBlock = *builder.createBlock(623 &dataPtrPtrRegion, dataPtrPtrRegion.end(), {type}, {loc});624 builder.setInsertionPointToEnd(&dataAddrBlock);625 mlir::Value boxRefOperand = dataAddrBlock.getArgument(0);626 mlir::Value baseAddrOffset = fir::BoxOffsetOp::create(627 builder, loc, boxRefOperand, fir::BoxFieldAttr::base_addr);628 genYield<DeclareRedType>(builder, loc, baseAddrOffset);629 }630 631 return decl;632}633 634template <typename OpType>635OpType ReductionProcessor::createDeclareReduction(636 AbstractConverter &converter, llvm::StringRef reductionOpName,637 const ReductionIdentifier redId, mlir::Type type, mlir::Location loc,638 bool isByRef) {639 auto genInitValueCB = [&](fir::FirOpBuilder &builder, mlir::Location loc,640 mlir::Type type, mlir::Value val) {641 mlir::Type ty = fir::unwrapRefType(type);642 mlir::Value initValue = ReductionProcessor::getReductionInitValue(643 loc, unwrapSeqOrBoxedType(ty), redId, builder);644 return initValue;645 };646 auto genCombinerCB = [&](fir::FirOpBuilder &builder, mlir::Location loc,647 mlir::Type type, mlir::Value op1, mlir::Value op2,648 bool isByRef) {649 genCombiner<OpType>(builder, loc, redId, type, op1, op2, isByRef);650 };651 652 return createDeclareReductionHelper<OpType>(converter, reductionOpName, type,653 loc, isByRef, genCombinerCB,654 genInitValueCB);655}656 657bool ReductionProcessor::doReductionByRef(mlir::Type reductionType) {658 if (forceByrefReduction)659 return true;660 661 if (!fir::isa_trivial(fir::unwrapRefType(reductionType)) &&662 !fir::isa_derived(fir::unwrapRefType(reductionType)))663 return true;664 665 return false;666}667 668bool ReductionProcessor::doReductionByRef(mlir::Value reductionVar) {669 if (forceByrefReduction)670 return true;671 672 if (auto declare =673 mlir::dyn_cast<hlfir::DeclareOp>(reductionVar.getDefiningOp()))674 reductionVar = declare.getMemref();675 676 return doReductionByRef(reductionVar.getType());677}678 679template <typename OpType, typename RedOperatorListTy>680bool ReductionProcessor::processReductionArguments(681 mlir::Location currentLocation, lower::AbstractConverter &converter,682 const RedOperatorListTy &redOperatorList,683 llvm::SmallVectorImpl<mlir::Value> &reductionVars,684 llvm::SmallVectorImpl<bool> &reduceVarByRef,685 llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,686 const llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSymbols) {687 fir::FirOpBuilder &builder = converter.getFirOpBuilder();688 689 if constexpr (std::is_same_v<RedOperatorListTy,690 omp::clause::ReductionOperatorList>) {691 // For OpenMP reduction clauses, check if the reduction operator is692 // supported.693 assert(redOperatorList.size() == 1 && "Expecting single operator");694 const Fortran::lower::omp::clause::ReductionOperator &redOperator =695 redOperatorList.front();696 697 if (!std::holds_alternative<omp::clause::DefinedOperator>(redOperator.u)) {698 if (const auto *reductionIntrinsic =699 std::get_if<omp::clause::ProcedureDesignator>(&redOperator.u)) {700 if (!ReductionProcessor::supportedIntrinsicProcReduction(701 *reductionIntrinsic)) {702 // If not an intrinsic is has to be a custom reduction op, and should703 // be available in the module.704 semantics::Symbol *sym = reductionIntrinsic->v.sym();705 mlir::ModuleOp module = builder.getModule();706 auto decl = module.lookupSymbol<OpType>(getRealName(sym).ToString());707 if (!decl)708 return false;709 }710 } else {711 return false;712 }713 }714 }715 716 // Reduction variable processing common to both intrinsic operators and717 // procedure designators718 mlir::OpBuilder::InsertPoint dcIP;719 constexpr bool isDoConcurrent =720 std::is_same_v<OpType, fir::DeclareReductionOp>;721 722 if (isDoConcurrent) {723 dcIP = builder.saveInsertionPoint();724 builder.setInsertionPoint(725 builder.getRegion().getParentOfType<fir::DoConcurrentOp>());726 }727 728 for (const semantics::Symbol *symbol : reductionSymbols) {729 mlir::Value symVal = converter.getSymbolAddress(*symbol);730 731 if (auto declOp = symVal.getDefiningOp<hlfir::DeclareOp>())732 symVal = declOp.getBase();733 734 mlir::Type eleType;735 auto refType = mlir::dyn_cast_or_null<fir::ReferenceType>(symVal.getType());736 if (refType)737 eleType = refType.getEleTy();738 else739 eleType = symVal.getType();740 741 // all arrays must be boxed so that we have convenient access to all the742 // information needed to iterate over the array743 if (mlir::isa<fir::SequenceType>(eleType)) {744 // For Host associated symbols, use `SymbolBox` instead745 lower::SymbolBox symBox = converter.lookupOneLevelUpSymbol(*symbol);746 hlfir::Entity entity{symBox.getAddr()};747 entity = genVariableBox(currentLocation, builder, entity);748 mlir::Value box = entity.getBase();749 750 // Always pass the box by reference so that the OpenMP dialect751 // verifiers don't need to know anything about fir.box752 auto alloca =753 fir::AllocaOp::create(builder, currentLocation, box.getType());754 fir::StoreOp::create(builder, currentLocation, box, alloca);755 756 symVal = alloca;757 } else if (mlir::isa<fir::BaseBoxType>(symVal.getType())) {758 // boxed arrays are passed as values not by reference. Unfortunately,759 // we can't pass a box by value to omp.redution_declare, so turn it760 // into a reference761 auto oldIP = builder.saveInsertionPoint();762 builder.setInsertionPointToStart(builder.getAllocaBlock());763 auto alloca =764 fir::AllocaOp::create(builder, currentLocation, symVal.getType());765 builder.restoreInsertionPoint(oldIP);766 fir::StoreOp::create(builder, currentLocation, symVal, alloca);767 symVal = alloca;768 }769 770 // this isn't the same as the by-val and by-ref passing later in the771 // pipeline. Both styles assume that the variable is a reference at772 // this point773 assert(fir::isa_ref_type(symVal.getType()) &&774 "reduction input var is passed by reference");775 mlir::Type elementType = fir::dyn_cast_ptrEleTy(symVal.getType());776 const bool symIsVolatile = fir::isa_volatile_type(symVal.getType());777 mlir::Type refTy = fir::ReferenceType::get(elementType, symIsVolatile);778 779 reductionVars.push_back(780 builder.createConvert(currentLocation, refTy, symVal));781 reduceVarByRef.push_back(doReductionByRef(symVal));782 }783 784 unsigned idx = 0;785 for (auto [symVal, isByRef] : llvm::zip(reductionVars, reduceVarByRef)) {786 auto redType = mlir::cast<fir::ReferenceType>(symVal.getType());787 const auto &kindMap = builder.getKindMap();788 std::string reductionName;789 ReductionIdentifier redId;790 791 if constexpr (std::is_same_v<RedOperatorListTy,792 omp::clause::ReductionOperatorList>) {793 const Fortran::lower::omp::clause::ReductionOperator &redOperator =794 redOperatorList.front();795 if (const auto &redDefinedOp =796 std::get_if<omp::clause::DefinedOperator>(&redOperator.u)) {797 const auto &intrinsicOp{798 std::get<omp::clause::DefinedOperator::IntrinsicOperator>(799 redDefinedOp->u)};800 redId = getReductionType(intrinsicOp);801 switch (redId) {802 case ReductionIdentifier::ADD:803 case ReductionIdentifier::MULTIPLY:804 case ReductionIdentifier::AND:805 case ReductionIdentifier::EQV:806 case ReductionIdentifier::OR:807 case ReductionIdentifier::NEQV:808 break;809 default:810 TODO(currentLocation,811 "Reduction of some intrinsic operators is not supported");812 break;813 }814 815 reductionName = getReductionName(redId, kindMap, redType, isByRef);816 } else if (const auto *reductionIntrinsic =817 std::get_if<omp::clause::ProcedureDesignator>(818 &redOperator.u)) {819 if (!ReductionProcessor::supportedIntrinsicProcReduction(820 *reductionIntrinsic)) {821 // Custom reductions we can just add to the symbols without822 // generating the declare reduction op.823 semantics::Symbol *sym = reductionIntrinsic->v.sym();824 reductionDeclSymbols.push_back(mlir::SymbolRefAttr::get(825 builder.getContext(), sym->name().ToString()));826 ++idx;827 continue;828 }829 redId = getReductionType(*reductionIntrinsic);830 reductionName =831 getReductionName(getRealName(*reductionIntrinsic).ToString(),832 kindMap, redType, isByRef);833 } else {834 TODO(currentLocation, "Unexpected reduction type");835 }836 } else {837 // `do concurrent` reductions838 redId = getReductionType(redOperatorList[idx]);839 reductionName = getReductionName(redId, kindMap, redType, isByRef);840 }841 842 OpType decl = createDeclareReduction<OpType>(843 converter, reductionName, redId, redType, currentLocation, isByRef);844 reductionDeclSymbols.push_back(845 mlir::SymbolRefAttr::get(builder.getContext(), decl.getSymName()));846 ++idx;847 }848 849 if (isDoConcurrent)850 builder.restoreInsertionPoint(dcIP);851 852 return true;853}854 855const semantics::SourceName856ReductionProcessor::getRealName(const semantics::Symbol *symbol) {857 return symbol->GetUltimate().name();858}859 860const semantics::SourceName861ReductionProcessor::getRealName(const omp::clause::ProcedureDesignator &pd) {862 return getRealName(pd.v.sym());863}864 865int ReductionProcessor::getOperationIdentity(ReductionIdentifier redId,866 mlir::Location loc) {867 switch (redId) {868 case ReductionIdentifier::ADD:869 case ReductionIdentifier::OR:870 case ReductionIdentifier::NEQV:871 return 0;872 case ReductionIdentifier::MULTIPLY:873 case ReductionIdentifier::AND:874 case ReductionIdentifier::EQV:875 return 1;876 default:877 TODO(loc, "Reduction of some intrinsic operators is not supported");878 }879}880 881} // namespace omp882} // namespace lower883} // namespace Fortran884