2456 lines · cpp
1//===- ModuleTranslation.cpp - MLIR to LLVM conversion --------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the translation between an MLIR LLVM dialect module and10// the corresponding LLVMIR module. It only handles core LLVM IR operations.11//12//===----------------------------------------------------------------------===//13 14#include "mlir/Target/LLVMIR/ModuleTranslation.h"15 16#include "AttrKindDetail.h"17#include "DebugTranslation.h"18#include "LoopAnnotationTranslation.h"19#include "mlir/Analysis/TopologicalSortUtils.h"20#include "mlir/Dialect/DLTI/DLTI.h"21#include "mlir/Dialect/LLVMIR/LLVMDialect.h"22#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h"23#include "mlir/Dialect/LLVMIR/Transforms/DIExpressionLegalization.h"24#include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h"25#include "mlir/IR/AttrTypeSubElements.h"26#include "mlir/IR/Attributes.h"27#include "mlir/IR/BuiltinOps.h"28#include "mlir/IR/BuiltinTypes.h"29#include "mlir/IR/DialectResourceBlobManager.h"30#include "mlir/Support/LLVM.h"31#include "mlir/Target/LLVMIR/LLVMTranslationInterface.h"32#include "mlir/Target/LLVMIR/TypeToLLVM.h"33 34#include "llvm/ADT/STLExtras.h"35#include "llvm/ADT/StringExtras.h"36#include "llvm/ADT/TypeSwitch.h"37#include "llvm/Analysis/TargetFolder.h"38#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"39#include "llvm/IR/BasicBlock.h"40#include "llvm/IR/CFG.h"41#include "llvm/IR/Constants.h"42#include "llvm/IR/DerivedTypes.h"43#include "llvm/IR/IRBuilder.h"44#include "llvm/IR/InlineAsm.h"45#include "llvm/IR/LLVMContext.h"46#include "llvm/IR/MDBuilder.h"47#include "llvm/IR/Module.h"48#include "llvm/IR/Verifier.h"49#include "llvm/Support/Debug.h"50#include "llvm/Support/ErrorHandling.h"51#include "llvm/Support/raw_ostream.h"52#include "llvm/Transforms/Utils/BasicBlockUtils.h"53#include "llvm/Transforms/Utils/Cloning.h"54#include "llvm/Transforms/Utils/ModuleUtils.h"55#include <numeric>56#include <optional>57 58#define DEBUG_TYPE "llvm-dialect-to-llvm-ir"59 60using namespace mlir;61using namespace mlir::LLVM;62using namespace mlir::LLVM::detail;63 64#include "mlir/Dialect/LLVMIR/LLVMConversionEnumsToLLVM.inc"65 66namespace {67/// A customized inserter for LLVM's IRBuilder that captures all LLVM IR68/// instructions that are created for future reference.69///70/// This is intended to be used with the `CollectionScope` RAII object:71///72/// llvm::IRBuilder<..., InstructionCapturingInserter> builder;73/// {74/// InstructionCapturingInserter::CollectionScope scope(builder);75/// // Call IRBuilder methods as usual.76///77/// // This will return a list of all instructions created by the builder,78/// // in order of creation.79/// builder.getInserter().getCapturedInstructions();80/// }81/// // This will return an empty list.82/// builder.getInserter().getCapturedInstructions();83///84/// The capturing functionality is _disabled_ by default for performance85/// consideration. It needs to be explicitly enabled, which is achieved by86/// creating a `CollectionScope`.87class InstructionCapturingInserter : public llvm::IRBuilderCallbackInserter {88public:89 /// Constructs the inserter.90 InstructionCapturingInserter()91 : llvm::IRBuilderCallbackInserter([this](llvm::Instruction *instruction) {92 if (LLVM_LIKELY(enabled))93 capturedInstructions.push_back(instruction);94 }) {}95 96 /// Returns the list of LLVM IR instructions captured since the last cleanup.97 ArrayRef<llvm::Instruction *> getCapturedInstructions() const {98 return capturedInstructions;99 }100 101 /// Clears the list of captured LLVM IR instructions.102 void clearCapturedInstructions() { capturedInstructions.clear(); }103 104 /// RAII object enabling the capture of created LLVM IR instructions.105 class CollectionScope {106 public:107 /// Creates the scope for the given inserter.108 CollectionScope(llvm::IRBuilderBase &irBuilder, bool isBuilderCapturing);109 110 /// Ends the scope.111 ~CollectionScope();112 113 ArrayRef<llvm::Instruction *> getCapturedInstructions() {114 if (!inserter)115 return {};116 return inserter->getCapturedInstructions();117 }118 119 private:120 /// Back reference to the inserter.121 InstructionCapturingInserter *inserter = nullptr;122 123 /// List of instructions in the inserter prior to this scope.124 SmallVector<llvm::Instruction *> previouslyCollectedInstructions;125 126 /// Whether the inserter was enabled prior to this scope.127 bool wasEnabled;128 };129 130 /// Enable or disable the capturing mechanism.131 void setEnabled(bool enabled = true) { this->enabled = enabled; }132 133private:134 /// List of captured instructions.135 SmallVector<llvm::Instruction *> capturedInstructions;136 137 /// Whether the collection is enabled.138 bool enabled = false;139};140 141using CapturingIRBuilder =142 llvm::IRBuilder<llvm::TargetFolder, InstructionCapturingInserter>;143} // namespace144 145InstructionCapturingInserter::CollectionScope::CollectionScope(146 llvm::IRBuilderBase &irBuilder, bool isBuilderCapturing) {147 148 if (!isBuilderCapturing)149 return;150 151 auto &capturingIRBuilder = static_cast<CapturingIRBuilder &>(irBuilder);152 inserter = &capturingIRBuilder.getInserter();153 wasEnabled = inserter->enabled;154 if (wasEnabled)155 previouslyCollectedInstructions.swap(inserter->capturedInstructions);156 inserter->setEnabled(true);157}158 159InstructionCapturingInserter::CollectionScope::~CollectionScope() {160 if (!inserter)161 return;162 163 previouslyCollectedInstructions.swap(inserter->capturedInstructions);164 // If collection was enabled (likely in another, surrounding scope), keep165 // the instructions collected in this scope.166 if (wasEnabled) {167 llvm::append_range(inserter->capturedInstructions,168 previouslyCollectedInstructions);169 }170 inserter->setEnabled(wasEnabled);171}172 173/// Translates the given data layout spec attribute to the LLVM IR data layout.174/// Only integer, float, pointer and endianness entries are currently supported.175static FailureOr<llvm::DataLayout>176translateDataLayout(DataLayoutSpecInterface attribute,177 const DataLayout &dataLayout,178 std::optional<Location> loc = std::nullopt) {179 if (!loc)180 loc = UnknownLoc::get(attribute.getContext());181 182 // Translate the endianness attribute.183 std::string llvmDataLayout;184 llvm::raw_string_ostream layoutStream(llvmDataLayout);185 for (DataLayoutEntryInterface entry : attribute.getEntries()) {186 auto key = llvm::dyn_cast_if_present<StringAttr>(entry.getKey());187 if (!key)188 continue;189 if (key.getValue() == DLTIDialect::kDataLayoutEndiannessKey) {190 auto value = cast<StringAttr>(entry.getValue());191 bool isLittleEndian =192 value.getValue() == DLTIDialect::kDataLayoutEndiannessLittle;193 layoutStream << "-" << (isLittleEndian ? "e" : "E");194 continue;195 }196 if (key.getValue() == DLTIDialect::kDataLayoutManglingModeKey) {197 auto value = cast<StringAttr>(entry.getValue());198 layoutStream << "-m:" << value.getValue();199 continue;200 }201 if (key.getValue() == DLTIDialect::kDataLayoutProgramMemorySpaceKey) {202 auto value = cast<IntegerAttr>(entry.getValue());203 uint64_t space = value.getValue().getZExtValue();204 // Skip the default address space.205 if (space == 0)206 continue;207 layoutStream << "-P" << space;208 continue;209 }210 if (key.getValue() == DLTIDialect::kDataLayoutGlobalMemorySpaceKey) {211 auto value = cast<IntegerAttr>(entry.getValue());212 uint64_t space = value.getValue().getZExtValue();213 // Skip the default address space.214 if (space == 0)215 continue;216 layoutStream << "-G" << space;217 continue;218 }219 if (key.getValue() == DLTIDialect::kDataLayoutAllocaMemorySpaceKey) {220 auto value = cast<IntegerAttr>(entry.getValue());221 uint64_t space = value.getValue().getZExtValue();222 // Skip the default address space.223 if (space == 0)224 continue;225 layoutStream << "-A" << space;226 continue;227 }228 if (key.getValue() == DLTIDialect::kDataLayoutStackAlignmentKey) {229 auto value = cast<IntegerAttr>(entry.getValue());230 uint64_t alignment = value.getValue().getZExtValue();231 // Skip the default stack alignment.232 if (alignment == 0)233 continue;234 layoutStream << "-S" << alignment;235 continue;236 }237 if (key.getValue() == DLTIDialect::kDataLayoutFunctionPointerAlignmentKey) {238 auto value = cast<FunctionPointerAlignmentAttr>(entry.getValue());239 uint64_t alignment = value.getAlignment();240 // Skip the default function pointer alignment.241 if (alignment == 0)242 continue;243 layoutStream << "-F" << (value.getFunctionDependent() ? "n" : "i")244 << alignment;245 continue;246 }247 if (key.getValue() == DLTIDialect::kDataLayoutLegalIntWidthsKey) {248 layoutStream << "-n";249 llvm::interleave(250 cast<DenseI32ArrayAttr>(entry.getValue()).asArrayRef(), layoutStream,251 [&](int32_t val) { layoutStream << val; }, ":");252 continue;253 }254 emitError(*loc) << "unsupported data layout key " << key;255 return failure();256 }257 258 // Go through the list of entries to check which types are explicitly259 // specified in entries. Where possible, data layout queries are used instead260 // of directly inspecting the entries.261 for (DataLayoutEntryInterface entry : attribute.getEntries()) {262 auto type = llvm::dyn_cast_if_present<Type>(entry.getKey());263 if (!type)264 continue;265 // Data layout for the index type is irrelevant at this point.266 if (isa<IndexType>(type))267 continue;268 layoutStream << "-";269 LogicalResult result =270 llvm::TypeSwitch<Type, LogicalResult>(type)271 .Case<IntegerType, Float16Type, Float32Type, Float64Type,272 Float80Type, Float128Type>([&](Type type) -> LogicalResult {273 if (auto intType = dyn_cast<IntegerType>(type)) {274 if (intType.getSignedness() != IntegerType::Signless)275 return emitError(*loc)276 << "unsupported data layout for non-signless integer "277 << intType;278 layoutStream << "i";279 } else {280 layoutStream << "f";281 }282 uint64_t size = dataLayout.getTypeSizeInBits(type);283 uint64_t abi = dataLayout.getTypeABIAlignment(type) * 8u;284 uint64_t preferred =285 dataLayout.getTypePreferredAlignment(type) * 8u;286 layoutStream << size << ":" << abi;287 if (abi != preferred)288 layoutStream << ":" << preferred;289 return success();290 })291 .Case([&](LLVMPointerType type) {292 layoutStream << "p" << type.getAddressSpace() << ":";293 uint64_t size = dataLayout.getTypeSizeInBits(type);294 uint64_t abi = dataLayout.getTypeABIAlignment(type) * 8u;295 uint64_t preferred =296 dataLayout.getTypePreferredAlignment(type) * 8u;297 uint64_t index = *dataLayout.getTypeIndexBitwidth(type);298 layoutStream << size << ":" << abi << ":" << preferred << ":"299 << index;300 return success();301 })302 .Default([loc](Type type) {303 return emitError(*loc)304 << "unsupported type in data layout: " << type;305 });306 if (failed(result))307 return failure();308 }309 StringRef layoutSpec(llvmDataLayout);310 layoutSpec.consume_front("-");311 312 return llvm::DataLayout(layoutSpec);313}314 315/// Builds a constant of a sequential LLVM type `type`, potentially containing316/// other sequential types recursively, from the individual constant values317/// provided in `constants`. `shape` contains the number of elements in nested318/// sequential types. Reports errors at `loc` and returns nullptr on error.319static llvm::Constant *320buildSequentialConstant(ArrayRef<llvm::Constant *> &constants,321 ArrayRef<int64_t> shape, llvm::Type *type,322 Location loc) {323 if (shape.empty()) {324 llvm::Constant *result = constants.front();325 constants = constants.drop_front();326 return result;327 }328 329 llvm::Type *elementType;330 if (auto *arrayTy = dyn_cast<llvm::ArrayType>(type)) {331 elementType = arrayTy->getElementType();332 } else if (auto *vectorTy = dyn_cast<llvm::VectorType>(type)) {333 elementType = vectorTy->getElementType();334 } else {335 emitError(loc) << "expected sequential LLVM types wrapping a scalar";336 return nullptr;337 }338 339 SmallVector<llvm::Constant *, 8> nested;340 nested.reserve(shape.front());341 for (int64_t i = 0; i < shape.front(); ++i) {342 nested.push_back(buildSequentialConstant(constants, shape.drop_front(),343 elementType, loc));344 if (!nested.back())345 return nullptr;346 }347 348 if (shape.size() == 1 && type->isVectorTy())349 return llvm::ConstantVector::get(nested);350 return llvm::ConstantArray::get(351 llvm::ArrayType::get(elementType, shape.front()), nested);352}353 354/// Returns the first non-sequential type nested in sequential types.355static llvm::Type *getInnermostElementType(llvm::Type *type) {356 do {357 if (auto *arrayTy = dyn_cast<llvm::ArrayType>(type)) {358 type = arrayTy->getElementType();359 } else if (auto *vectorTy = dyn_cast<llvm::VectorType>(type)) {360 type = vectorTy->getElementType();361 } else {362 return type;363 }364 } while (true);365}366 367/// Convert a dense elements attribute to an LLVM IR constant using its raw data368/// storage if possible. This supports elements attributes of tensor or vector369/// type and avoids constructing separate objects for individual values of the370/// innermost dimension. Constants for other dimensions are still constructed371/// recursively. Returns null if constructing from raw data is not supported for372/// this type, e.g., element type is not a power-of-two-sized primitive. Reports373/// other errors at `loc`.374static llvm::Constant *375convertDenseElementsAttr(Location loc, DenseElementsAttr denseElementsAttr,376 llvm::Type *llvmType,377 const ModuleTranslation &moduleTranslation) {378 if (!denseElementsAttr)379 return nullptr;380 381 llvm::Type *innermostLLVMType = getInnermostElementType(llvmType);382 if (!llvm::ConstantDataSequential::isElementTypeCompatible(innermostLLVMType))383 return nullptr;384 385 ShapedType type = denseElementsAttr.getType();386 if (type.getNumElements() == 0)387 return nullptr;388 389 // Check that the raw data size matches what is expected for the scalar size.390 // TODO: in theory, we could repack the data here to keep constructing from391 // raw data.392 // TODO: we may also need to consider endianness when cross-compiling to an393 // architecture where it is different.394 int64_t elementByteSize = denseElementsAttr.getRawData().size() /395 denseElementsAttr.getNumElements();396 if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits())397 return nullptr;398 399 // Compute the shape of all dimensions but the innermost. Note that the400 // innermost dimension may be that of the vector element type.401 bool hasVectorElementType = isa<VectorType>(type.getElementType());402 int64_t numAggregates =403 denseElementsAttr.getNumElements() /404 (hasVectorElementType ? 1405 : denseElementsAttr.getType().getShape().back());406 ArrayRef<int64_t> outerShape = type.getShape();407 if (!hasVectorElementType)408 outerShape = outerShape.drop_back();409 410 // Handle the case of vector splat, LLVM has special support for it.411 if (denseElementsAttr.isSplat() &&412 (isa<VectorType>(type) || hasVectorElementType)) {413 llvm::Constant *splatValue = LLVM::detail::getLLVMConstant(414 innermostLLVMType, denseElementsAttr.getSplatValue<Attribute>(), loc,415 moduleTranslation);416 llvm::Constant *splatVector =417 llvm::ConstantDataVector::getSplat(0, splatValue);418 SmallVector<llvm::Constant *> constants(numAggregates, splatVector);419 ArrayRef<llvm::Constant *> constantsRef = constants;420 return buildSequentialConstant(constantsRef, outerShape, llvmType, loc);421 }422 if (denseElementsAttr.isSplat())423 return nullptr;424 425 // In case of non-splat, create a constructor for the innermost constant from426 // a piece of raw data.427 std::function<llvm::Constant *(StringRef)> buildCstData;428 if (isa<TensorType>(type)) {429 auto vectorElementType = dyn_cast<VectorType>(type.getElementType());430 if (vectorElementType && vectorElementType.getRank() == 1) {431 buildCstData = [&](StringRef data) {432 return llvm::ConstantDataVector::getRaw(433 data, vectorElementType.getShape().back(), innermostLLVMType);434 };435 } else if (!vectorElementType) {436 buildCstData = [&](StringRef data) {437 return llvm::ConstantDataArray::getRaw(data, type.getShape().back(),438 innermostLLVMType);439 };440 }441 } else if (isa<VectorType>(type)) {442 buildCstData = [&](StringRef data) {443 return llvm::ConstantDataVector::getRaw(data, type.getShape().back(),444 innermostLLVMType);445 };446 }447 if (!buildCstData)448 return nullptr;449 450 // Create innermost constants and defer to the default constant creation451 // mechanism for other dimensions.452 SmallVector<llvm::Constant *> constants;453 int64_t aggregateSize = denseElementsAttr.getType().getShape().back() *454 (innermostLLVMType->getScalarSizeInBits() / 8);455 constants.reserve(numAggregates);456 for (unsigned i = 0; i < numAggregates; ++i) {457 StringRef data(denseElementsAttr.getRawData().data() + i * aggregateSize,458 aggregateSize);459 constants.push_back(buildCstData(data));460 }461 462 ArrayRef<llvm::Constant *> constantsRef = constants;463 return buildSequentialConstant(constantsRef, outerShape, llvmType, loc);464}465 466/// Convert a dense resource elements attribute to an LLVM IR constant using its467/// raw data storage if possible. This supports elements attributes of tensor or468/// vector type and avoids constructing separate objects for individual values469/// of the innermost dimension. Constants for other dimensions are still470/// constructed recursively. Returns nullptr on failure and emits errors at471/// `loc`.472static llvm::Constant *convertDenseResourceElementsAttr(473 Location loc, DenseResourceElementsAttr denseResourceAttr,474 llvm::Type *llvmType, const ModuleTranslation &moduleTranslation) {475 assert(denseResourceAttr && "expected non-null attribute");476 477 llvm::Type *innermostLLVMType = getInnermostElementType(llvmType);478 if (!llvm::ConstantDataSequential::isElementTypeCompatible(479 innermostLLVMType)) {480 emitError(loc, "no known conversion for innermost element type");481 return nullptr;482 }483 484 ShapedType type = denseResourceAttr.getType();485 assert(type.getNumElements() > 0 && "Expected non-empty elements attribute");486 487 AsmResourceBlob *blob = denseResourceAttr.getRawHandle().getBlob();488 if (!blob) {489 emitError(loc, "resource does not exist");490 return nullptr;491 }492 493 ArrayRef<char> rawData = blob->getData();494 495 // Check that the raw data size matches what is expected for the scalar size.496 // TODO: in theory, we could repack the data here to keep constructing from497 // raw data.498 // TODO: we may also need to consider endianness when cross-compiling to an499 // architecture where it is different.500 int64_t numElements = denseResourceAttr.getType().getNumElements();501 int64_t elementByteSize = rawData.size() / numElements;502 if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits()) {503 emitError(loc, "raw data size does not match element type size");504 return nullptr;505 }506 507 // Compute the shape of all dimensions but the innermost. Note that the508 // innermost dimension may be that of the vector element type.509 bool hasVectorElementType = isa<VectorType>(type.getElementType());510 int64_t numAggregates =511 numElements / (hasVectorElementType512 ? 1513 : denseResourceAttr.getType().getShape().back());514 ArrayRef<int64_t> outerShape = type.getShape();515 if (!hasVectorElementType)516 outerShape = outerShape.drop_back();517 518 // Create a constructor for the innermost constant from a piece of raw data.519 std::function<llvm::Constant *(StringRef)> buildCstData;520 if (isa<TensorType>(type)) {521 auto vectorElementType = dyn_cast<VectorType>(type.getElementType());522 if (vectorElementType && vectorElementType.getRank() == 1) {523 buildCstData = [&](StringRef data) {524 return llvm::ConstantDataVector::getRaw(525 data, vectorElementType.getShape().back(), innermostLLVMType);526 };527 } else if (!vectorElementType) {528 buildCstData = [&](StringRef data) {529 return llvm::ConstantDataArray::getRaw(data, type.getShape().back(),530 innermostLLVMType);531 };532 }533 } else if (isa<VectorType>(type)) {534 buildCstData = [&](StringRef data) {535 return llvm::ConstantDataVector::getRaw(data, type.getShape().back(),536 innermostLLVMType);537 };538 }539 if (!buildCstData) {540 emitError(loc, "unsupported dense_resource type");541 return nullptr;542 }543 544 // Create innermost constants and defer to the default constant creation545 // mechanism for other dimensions.546 SmallVector<llvm::Constant *> constants;547 int64_t aggregateSize = denseResourceAttr.getType().getShape().back() *548 (innermostLLVMType->getScalarSizeInBits() / 8);549 constants.reserve(numAggregates);550 for (unsigned i = 0; i < numAggregates; ++i) {551 StringRef data(rawData.data() + i * aggregateSize, aggregateSize);552 constants.push_back(buildCstData(data));553 }554 555 ArrayRef<llvm::Constant *> constantsRef = constants;556 return buildSequentialConstant(constantsRef, outerShape, llvmType, loc);557}558 559/// Create an LLVM IR constant of `llvmType` from the MLIR attribute `attr`.560/// This currently supports integer, floating point, splat and dense element561/// attributes and combinations thereof. Also, an array attribute with two562/// elements is supported to represent a complex constant. In case of error,563/// report it to `loc` and return nullptr.564llvm::Constant *mlir::LLVM::detail::getLLVMConstant(565 llvm::Type *llvmType, Attribute attr, Location loc,566 const ModuleTranslation &moduleTranslation) {567 if (!attr || isa<UndefAttr>(attr))568 return llvm::UndefValue::get(llvmType);569 if (isa<ZeroAttr>(attr))570 return llvm::Constant::getNullValue(llvmType);571 if (auto *structType = dyn_cast<::llvm::StructType>(llvmType)) {572 auto arrayAttr = dyn_cast<ArrayAttr>(attr);573 if (!arrayAttr) {574 emitError(loc, "expected an array attribute for a struct constant");575 return nullptr;576 }577 SmallVector<llvm::Constant *> structElements;578 structElements.reserve(structType->getNumElements());579 for (auto [elemType, elemAttr] :580 zip_equal(structType->elements(), arrayAttr)) {581 llvm::Constant *element =582 getLLVMConstant(elemType, elemAttr, loc, moduleTranslation);583 if (!element)584 return nullptr;585 structElements.push_back(element);586 }587 return llvm::ConstantStruct::get(structType, structElements);588 }589 // For integer types, we allow a mismatch in sizes as the index type in590 // MLIR might have a different size than the index type in the LLVM module.591 if (auto intAttr = dyn_cast<IntegerAttr>(attr))592 return llvm::ConstantInt::get(593 llvmType,594 intAttr.getValue().sextOrTrunc(llvmType->getIntegerBitWidth()));595 if (auto floatAttr = dyn_cast<FloatAttr>(attr)) {596 const llvm::fltSemantics &sem = floatAttr.getValue().getSemantics();597 // Special case for 8-bit floats, which are represented by integers due to598 // the lack of native fp8 types in LLVM at the moment. Additionally, handle599 // targets (like AMDGPU) that don't implement bfloat and convert all bfloats600 // to i16.601 unsigned floatWidth = APFloat::getSizeInBits(sem);602 if (llvmType->isIntegerTy(floatWidth))603 return llvm::ConstantInt::get(llvmType,604 floatAttr.getValue().bitcastToAPInt());605 if (llvmType !=606 llvm::Type::getFloatingPointTy(llvmType->getContext(),607 floatAttr.getValue().getSemantics())) {608 emitError(loc, "FloatAttr does not match expected type of the constant");609 return nullptr;610 }611 return llvm::ConstantFP::get(llvmType, floatAttr.getValue());612 }613 if (auto funcAttr = dyn_cast<FlatSymbolRefAttr>(attr))614 return llvm::ConstantExpr::getBitCast(615 moduleTranslation.lookupFunction(funcAttr.getValue()), llvmType);616 if (auto splatAttr = dyn_cast<SplatElementsAttr>(attr)) {617 llvm::Type *elementType;618 uint64_t numElements;619 bool isScalable = false;620 if (auto *arrayTy = dyn_cast<llvm::ArrayType>(llvmType)) {621 elementType = arrayTy->getElementType();622 numElements = arrayTy->getNumElements();623 } else if (auto *fVectorTy = dyn_cast<llvm::FixedVectorType>(llvmType)) {624 elementType = fVectorTy->getElementType();625 numElements = fVectorTy->getNumElements();626 } else if (auto *sVectorTy = dyn_cast<llvm::ScalableVectorType>(llvmType)) {627 elementType = sVectorTy->getElementType();628 numElements = sVectorTy->getMinNumElements();629 isScalable = true;630 } else {631 llvm_unreachable("unrecognized constant vector type");632 }633 // Splat value is a scalar. Extract it only if the element type is not634 // another sequence type. The recursion terminates because each step removes635 // one outer sequential type.636 bool elementTypeSequential =637 isa<llvm::ArrayType, llvm::VectorType>(elementType);638 llvm::Constant *child = getLLVMConstant(639 elementType,640 elementTypeSequential ? splatAttr641 : splatAttr.getSplatValue<Attribute>(),642 loc, moduleTranslation);643 if (!child)644 return nullptr;645 if (llvmType->isVectorTy())646 return llvm::ConstantVector::getSplat(647 llvm::ElementCount::get(numElements, /*Scalable=*/isScalable), child);648 if (llvmType->isArrayTy()) {649 auto *arrayType = llvm::ArrayType::get(elementType, numElements);650 if (child->isZeroValue() && !elementType->isFPOrFPVectorTy()) {651 return llvm::ConstantAggregateZero::get(arrayType);652 }653 if (llvm::ConstantDataSequential::isElementTypeCompatible(elementType)) {654 // TODO: Handle all compatible types. This code only handles integer.655 if (isa<llvm::IntegerType>(elementType)) {656 if (llvm::ConstantInt *ci = dyn_cast<llvm::ConstantInt>(child)) {657 if (ci->getBitWidth() == 8) {658 SmallVector<int8_t> constants(numElements, ci->getZExtValue());659 return llvm::ConstantDataArray::get(elementType->getContext(),660 constants);661 }662 if (ci->getBitWidth() == 16) {663 SmallVector<int16_t> constants(numElements, ci->getZExtValue());664 return llvm::ConstantDataArray::get(elementType->getContext(),665 constants);666 }667 if (ci->getBitWidth() == 32) {668 SmallVector<int32_t> constants(numElements, ci->getZExtValue());669 return llvm::ConstantDataArray::get(elementType->getContext(),670 constants);671 }672 if (ci->getBitWidth() == 64) {673 SmallVector<int64_t> constants(numElements, ci->getZExtValue());674 return llvm::ConstantDataArray::get(elementType->getContext(),675 constants);676 }677 }678 }679 }680 // std::vector is used here to accomodate large number of elements that681 // exceed SmallVector capacity.682 std::vector<llvm::Constant *> constants(numElements, child);683 return llvm::ConstantArray::get(arrayType, constants);684 }685 }686 687 // Try using raw elements data if possible.688 if (llvm::Constant *result =689 convertDenseElementsAttr(loc, dyn_cast<DenseElementsAttr>(attr),690 llvmType, moduleTranslation)) {691 return result;692 }693 694 if (auto denseResourceAttr = dyn_cast<DenseResourceElementsAttr>(attr)) {695 return convertDenseResourceElementsAttr(loc, denseResourceAttr, llvmType,696 moduleTranslation);697 }698 699 // Fall back to element-by-element construction otherwise.700 if (auto elementsAttr = dyn_cast<ElementsAttr>(attr)) {701 assert(elementsAttr.getShapedType().hasStaticShape());702 assert(!elementsAttr.getShapedType().getShape().empty() &&703 "unexpected empty elements attribute shape");704 705 SmallVector<llvm::Constant *, 8> constants;706 constants.reserve(elementsAttr.getNumElements());707 llvm::Type *innermostType = getInnermostElementType(llvmType);708 for (auto n : elementsAttr.getValues<Attribute>()) {709 constants.push_back(710 getLLVMConstant(innermostType, n, loc, moduleTranslation));711 if (!constants.back())712 return nullptr;713 }714 ArrayRef<llvm::Constant *> constantsRef = constants;715 llvm::Constant *result = buildSequentialConstant(716 constantsRef, elementsAttr.getShapedType().getShape(), llvmType, loc);717 assert(constantsRef.empty() && "did not consume all elemental constants");718 return result;719 }720 721 if (auto stringAttr = dyn_cast<StringAttr>(attr)) {722 return llvm::ConstantDataArray::get(moduleTranslation.getLLVMContext(),723 ArrayRef<char>{stringAttr.getValue()});724 }725 726 // Handle arrays of structs that cannot be represented as DenseElementsAttr727 // in MLIR.728 if (auto arrayAttr = dyn_cast<ArrayAttr>(attr)) {729 if (auto *arrayTy = dyn_cast<llvm::ArrayType>(llvmType)) {730 llvm::Type *elementType = arrayTy->getElementType();731 Attribute previousElementAttr;732 llvm::Constant *elementCst = nullptr;733 SmallVector<llvm::Constant *> constants;734 constants.reserve(arrayTy->getNumElements());735 for (Attribute elementAttr : arrayAttr) {736 // Arrays with a single value or with repeating values are quite common.737 // Short-circuit the translation when the element value is the same as738 // the previous one.739 if (!previousElementAttr || previousElementAttr != elementAttr) {740 previousElementAttr = elementAttr;741 elementCst =742 getLLVMConstant(elementType, elementAttr, loc, moduleTranslation);743 if (!elementCst)744 return nullptr;745 }746 constants.push_back(elementCst);747 }748 return llvm::ConstantArray::get(arrayTy, constants);749 }750 }751 752 emitError(loc, "unsupported constant value");753 return nullptr;754}755 756ModuleTranslation::ModuleTranslation(Operation *module,757 std::unique_ptr<llvm::Module> llvmModule)758 : mlirModule(module), llvmModule(std::move(llvmModule)),759 debugTranslation(760 std::make_unique<DebugTranslation>(module, *this->llvmModule)),761 loopAnnotationTranslation(std::make_unique<LoopAnnotationTranslation>(762 *this, *this->llvmModule)),763 typeTranslator(this->llvmModule->getContext()),764 iface(module->getContext()) {765 assert(satisfiesLLVMModule(mlirModule) &&766 "mlirModule should honor LLVM's module semantics.");767}768 769ModuleTranslation::~ModuleTranslation() {770 if (ompBuilder && !ompBuilder->isFinalized())771 ompBuilder->finalize();772}773 774void ModuleTranslation::forgetMapping(Region ®ion) {775 SmallVector<Region *> toProcess;776 toProcess.push_back(®ion);777 while (!toProcess.empty()) {778 Region *current = toProcess.pop_back_val();779 for (Block &block : *current) {780 blockMapping.erase(&block);781 for (Value arg : block.getArguments())782 valueMapping.erase(arg);783 for (Operation &op : block) {784 for (Value value : op.getResults())785 valueMapping.erase(value);786 if (op.hasSuccessors())787 branchMapping.erase(&op);788 if (isa<LLVM::GlobalOp>(op))789 globalsMapping.erase(&op);790 if (isa<LLVM::AliasOp>(op))791 aliasesMapping.erase(&op);792 if (isa<LLVM::IFuncOp>(op))793 ifuncMapping.erase(&op);794 if (isa<LLVM::CallOp>(op))795 callMapping.erase(&op);796 llvm::append_range(797 toProcess,798 llvm::map_range(op.getRegions(), [](Region &r) { return &r; }));799 }800 }801 }802}803 804/// Get the SSA value passed to the current block from the terminator operation805/// of its predecessor.806static Value getPHISourceValue(Block *current, Block *pred,807 unsigned numArguments, unsigned index) {808 Operation &terminator = *pred->getTerminator();809 if (isa<LLVM::BrOp>(terminator))810 return terminator.getOperand(index);811 812#ifndef NDEBUG813 llvm::SmallPtrSet<Block *, 4> seenSuccessors;814 for (unsigned i = 0, e = terminator.getNumSuccessors(); i < e; ++i) {815 Block *successor = terminator.getSuccessor(i);816 auto branch = cast<BranchOpInterface>(terminator);817 SuccessorOperands successorOperands = branch.getSuccessorOperands(i);818 assert(819 (!seenSuccessors.contains(successor) || successorOperands.empty()) &&820 "successors with arguments in LLVM branches must be different blocks");821 seenSuccessors.insert(successor);822 }823#endif824 825 // For instructions that branch based on a condition value, we need to take826 // the operands for the branch that was taken.827 if (auto condBranchOp = dyn_cast<LLVM::CondBrOp>(terminator)) {828 // For conditional branches, we take the operands from either the "true" or829 // the "false" branch.830 return condBranchOp.getSuccessor(0) == current831 ? condBranchOp.getTrueDestOperands()[index]832 : condBranchOp.getFalseDestOperands()[index];833 }834 835 if (auto switchOp = dyn_cast<LLVM::SwitchOp>(terminator)) {836 // For switches, we take the operands from either the default case, or from837 // the case branch that was taken.838 if (switchOp.getDefaultDestination() == current)839 return switchOp.getDefaultOperands()[index];840 for (const auto &i : llvm::enumerate(switchOp.getCaseDestinations()))841 if (i.value() == current)842 return switchOp.getCaseOperands(i.index())[index];843 }844 845 if (auto indBrOp = dyn_cast<LLVM::IndirectBrOp>(terminator)) {846 // For indirect branches we take operands for each successor.847 for (const auto &i : llvm::enumerate(indBrOp->getSuccessors())) {848 if (indBrOp->getSuccessor(i.index()) == current)849 return indBrOp.getSuccessorOperands(i.index())[index];850 }851 }852 853 if (auto invokeOp = dyn_cast<LLVM::InvokeOp>(terminator)) {854 return invokeOp.getNormalDest() == current855 ? invokeOp.getNormalDestOperands()[index]856 : invokeOp.getUnwindDestOperands()[index];857 }858 859 llvm_unreachable(860 "only branch, switch or invoke operations can be terminators "861 "of a block that has successors");862}863 864/// Connect the PHI nodes to the results of preceding blocks.865void mlir::LLVM::detail::connectPHINodes(Region ®ion,866 const ModuleTranslation &state) {867 // Skip the first block, it cannot be branched to and its arguments correspond868 // to the arguments of the LLVM function.869 for (Block &bb : llvm::drop_begin(region)) {870 llvm::BasicBlock *llvmBB = state.lookupBlock(&bb);871 auto phis = llvmBB->phis();872 auto numArguments = bb.getNumArguments();873 assert(numArguments == std::distance(phis.begin(), phis.end()));874 for (auto [index, phiNode] : llvm::enumerate(phis)) {875 for (auto *pred : bb.getPredecessors()) {876 // Find the LLVM IR block that contains the converted terminator877 // instruction and use it in the PHI node. Note that this block is not878 // necessarily the same as state.lookupBlock(pred), some operations879 // (in particular, OpenMP operations using OpenMPIRBuilder) may have880 // split the blocks.881 llvm::Instruction *terminator =882 state.lookupBranch(pred->getTerminator());883 assert(terminator && "missing the mapping for a terminator");884 phiNode.addIncoming(state.lookupValue(getPHISourceValue(885 &bb, pred, numArguments, index)),886 terminator->getParent());887 }888 }889 }890}891 892llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(893 llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic,894 ArrayRef<llvm::Value *> args, ArrayRef<llvm::Type *> tys) {895 return builder.CreateIntrinsic(intrinsic, tys, args);896}897 898llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(899 llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic,900 llvm::Type *retTy, ArrayRef<llvm::Value *> args) {901 return builder.CreateIntrinsic(retTy, intrinsic, args);902}903 904llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(905 llvm::IRBuilderBase &builder, ModuleTranslation &moduleTranslation,906 Operation *intrOp, llvm::Intrinsic::ID intrinsic, unsigned numResults,907 ArrayRef<unsigned> overloadedResults, ArrayRef<unsigned> overloadedOperands,908 ArrayRef<unsigned> immArgPositions,909 ArrayRef<StringLiteral> immArgAttrNames) {910 assert(immArgPositions.size() == immArgAttrNames.size() &&911 "LLVM `immArgPositions` and MLIR `immArgAttrNames` should have equal "912 "length");913 914 SmallVector<llvm::OperandBundleDef> opBundles;915 size_t numOpBundleOperands = 0;916 auto opBundleSizesAttr = cast_if_present<DenseI32ArrayAttr>(917 intrOp->getAttr(LLVMDialect::getOpBundleSizesAttrName()));918 auto opBundleTagsAttr = cast_if_present<ArrayAttr>(919 intrOp->getAttr(LLVMDialect::getOpBundleTagsAttrName()));920 921 if (opBundleSizesAttr && opBundleTagsAttr) {922 ArrayRef<int> opBundleSizes = opBundleSizesAttr.asArrayRef();923 assert(opBundleSizes.size() == opBundleTagsAttr.size() &&924 "operand bundles and tags do not match");925 926 numOpBundleOperands = llvm::sum_of(opBundleSizes);927 assert(numOpBundleOperands <= intrOp->getNumOperands() &&928 "operand bundle operands is more than the number of operands");929 930 ValueRange operands = intrOp->getOperands().take_back(numOpBundleOperands);931 size_t nextOperandIdx = 0;932 opBundles.reserve(opBundleSizesAttr.size());933 934 for (auto [opBundleTagAttr, bundleSize] :935 llvm::zip(opBundleTagsAttr, opBundleSizes)) {936 auto bundleTag = cast<StringAttr>(opBundleTagAttr).str();937 auto bundleOperands = moduleTranslation.lookupValues(938 operands.slice(nextOperandIdx, bundleSize));939 opBundles.emplace_back(std::move(bundleTag), std::move(bundleOperands));940 nextOperandIdx += bundleSize;941 }942 }943 944 // Map operands and attributes to LLVM values.945 auto opOperands = intrOp->getOperands().drop_back(numOpBundleOperands);946 auto operands = moduleTranslation.lookupValues(opOperands);947 SmallVector<llvm::Value *> args(immArgPositions.size() + operands.size());948 for (auto [immArgPos, immArgName] :949 llvm::zip(immArgPositions, immArgAttrNames)) {950 auto attr = llvm::cast<TypedAttr>(intrOp->getAttr(immArgName));951 assert(attr.getType().isIntOrFloat() && "expected int or float immarg");952 auto *type = moduleTranslation.convertType(attr.getType());953 args[immArgPos] = LLVM::detail::getLLVMConstant(954 type, attr, intrOp->getLoc(), moduleTranslation);955 }956 unsigned opArg = 0;957 for (auto &arg : args) {958 if (!arg)959 arg = operands[opArg++];960 }961 962 // Resolve overloaded intrinsic declaration.963 SmallVector<llvm::Type *> overloadedTypes;964 for (unsigned overloadedResultIdx : overloadedResults) {965 if (numResults > 1) {966 // More than one result is mapped to an LLVM struct.967 overloadedTypes.push_back(moduleTranslation.convertType(968 llvm::cast<LLVM::LLVMStructType>(intrOp->getResult(0).getType())969 .getBody()[overloadedResultIdx]));970 } else {971 overloadedTypes.push_back(972 moduleTranslation.convertType(intrOp->getResult(0).getType()));973 }974 }975 for (unsigned overloadedOperandIdx : overloadedOperands)976 overloadedTypes.push_back(args[overloadedOperandIdx]->getType());977 llvm::Module *module = builder.GetInsertBlock()->getModule();978 llvm::Function *llvmIntr = llvm::Intrinsic::getOrInsertDeclaration(979 module, intrinsic, overloadedTypes);980 981 return builder.CreateCall(llvmIntr, args, opBundles);982}983 984/// Given a single MLIR operation, create the corresponding LLVM IR operation985/// using the `builder`.986LogicalResult ModuleTranslation::convertOperation(Operation &op,987 llvm::IRBuilderBase &builder,988 bool recordInsertions) {989 const LLVMTranslationDialectInterface *opIface = iface.getInterfaceFor(&op);990 if (!opIface)991 return op.emitError("cannot be converted to LLVM IR: missing "992 "`LLVMTranslationDialectInterface` registration for "993 "dialect for op: ")994 << op.getName();995 996 InstructionCapturingInserter::CollectionScope scope(builder,997 recordInsertions);998 if (failed(opIface->convertOperation(&op, builder, *this)))999 return op.emitError("LLVM Translation failed for operation: ")1000 << op.getName();1001 1002 return convertDialectAttributes(&op, scope.getCapturedInstructions());1003}1004 1005/// Convert block to LLVM IR. Unless `ignoreArguments` is set, emit PHI nodes1006/// to define values corresponding to the MLIR block arguments. These nodes1007/// are not connected to the source basic blocks, which may not exist yet. Uses1008/// `builder` to construct the LLVM IR. Expects the LLVM IR basic block to have1009/// been created for `bb` and included in the block mapping. Inserts new1010/// instructions at the end of the block and leaves `builder` in a state1011/// suitable for further insertion into the end of the block.1012LogicalResult ModuleTranslation::convertBlockImpl(Block &bb,1013 bool ignoreArguments,1014 llvm::IRBuilderBase &builder,1015 bool recordInsertions) {1016 builder.SetInsertPoint(lookupBlock(&bb));1017 auto *subprogram = builder.GetInsertBlock()->getParent()->getSubprogram();1018 1019 // Before traversing operations, make block arguments available through1020 // value remapping and PHI nodes, but do not add incoming edges for the PHI1021 // nodes just yet: those values may be defined by this or following blocks.1022 // This step is omitted if "ignoreArguments" is set. The arguments of the1023 // first block have been already made available through the remapping of1024 // LLVM function arguments.1025 if (!ignoreArguments) {1026 auto predecessors = bb.getPredecessors();1027 unsigned numPredecessors =1028 std::distance(predecessors.begin(), predecessors.end());1029 for (auto arg : bb.getArguments()) {1030 auto wrappedType = arg.getType();1031 if (!isCompatibleType(wrappedType))1032 return emitError(bb.front().getLoc(),1033 "block argument does not have an LLVM type");1034 builder.SetCurrentDebugLocation(1035 debugTranslation->translateLoc(arg.getLoc(), subprogram));1036 llvm::Type *type = convertType(wrappedType);1037 llvm::PHINode *phi = builder.CreatePHI(type, numPredecessors);1038 mapValue(arg, phi);1039 }1040 }1041 1042 // Traverse operations.1043 for (auto &op : bb) {1044 // Set the current debug location within the builder.1045 builder.SetCurrentDebugLocation(1046 debugTranslation->translateLoc(op.getLoc(), subprogram));1047 1048 if (failed(convertOperation(op, builder, recordInsertions)))1049 return failure();1050 1051 // Set the branch weight metadata on the translated instruction.1052 if (auto iface = dyn_cast<WeightedBranchOpInterface>(op))1053 setBranchWeightsMetadata(iface);1054 }1055 1056 return success();1057}1058 1059/// A helper method to get the single Block in an operation honoring LLVM's1060/// module requirements.1061static Block &getModuleBody(Operation *module) {1062 return module->getRegion(0).front();1063}1064 1065/// A helper method to decide if a constant must not be set as a global variable1066/// initializer. For an external linkage variable, the variable with an1067/// initializer is considered externally visible and defined in this module, the1068/// variable without an initializer is externally available and is defined1069/// elsewhere.1070static bool shouldDropGlobalInitializer(llvm::GlobalValue::LinkageTypes linkage,1071 llvm::Constant *cst) {1072 return (linkage == llvm::GlobalVariable::ExternalLinkage && !cst) ||1073 linkage == llvm::GlobalVariable::ExternalWeakLinkage;1074}1075 1076/// Sets the runtime preemption specifier of `gv` to dso_local if1077/// `dsoLocalRequested` is true, otherwise it is left unchanged.1078static void addRuntimePreemptionSpecifier(bool dsoLocalRequested,1079 llvm::GlobalValue *gv) {1080 if (dsoLocalRequested)1081 gv->setDSOLocal(true);1082}1083 1084/// Attempts to translate an MLIR attribute identified by `key`, optionally with1085/// the given `value`, into an LLVM IR attribute. Reports errors at `loc` if1086/// any. If the attribute name corresponds to a known LLVM IR attribute kind,1087/// creates the LLVM attribute of that kind; otherwise, keeps it as a string1088/// attribute. Performs additional checks for attributes known to have or not1089/// have a value in order to avoid assertions inside LLVM upon construction.1090static FailureOr<llvm::Attribute>1091convertMLIRAttributeToLLVM(Location loc, llvm::LLVMContext &ctx, StringRef key,1092 StringRef value = StringRef()) {1093 auto kind = llvm::Attribute::getAttrKindFromName(key);1094 if (kind == llvm::Attribute::None)1095 return llvm::Attribute::get(ctx, key, value);1096 1097 if (llvm::Attribute::isIntAttrKind(kind)) {1098 if (value.empty())1099 return emitError(loc) << "LLVM attribute '" << key << "' expects a value";1100 1101 int64_t result;1102 if (!value.getAsInteger(/*Radix=*/0, result))1103 return llvm::Attribute::get(ctx, kind, result);1104 return llvm::Attribute::get(ctx, key, value);1105 }1106 1107 if (!value.empty())1108 return emitError(loc) << "LLVM attribute '" << key1109 << "' does not expect a value, found '" << value1110 << "'";1111 1112 return llvm::Attribute::get(ctx, kind);1113}1114 1115/// Converts the MLIR attributes listed in the given array attribute into LLVM1116/// attributes. Returns an `AttrBuilder` containing the converted attributes.1117/// Reports error to `loc` if any and returns immediately. Expects `arrayAttr`1118/// to contain either string attributes, treated as value-less LLVM attributes,1119/// or array attributes containing two string attributes, with the first string1120/// being the name of the corresponding LLVM attribute and the second string1121/// beings its value. Note that even integer attributes are expected to have1122/// their values expressed as strings.1123static FailureOr<llvm::AttrBuilder>1124convertMLIRAttributesToLLVM(Location loc, llvm::LLVMContext &ctx,1125 ArrayAttr arrayAttr, StringRef arrayAttrName) {1126 llvm::AttrBuilder attrBuilder(ctx);1127 if (!arrayAttr)1128 return attrBuilder;1129 1130 for (Attribute attr : arrayAttr) {1131 if (auto stringAttr = dyn_cast<StringAttr>(attr)) {1132 FailureOr<llvm::Attribute> llvmAttr =1133 convertMLIRAttributeToLLVM(loc, ctx, stringAttr.getValue());1134 if (failed(llvmAttr))1135 return failure();1136 attrBuilder.addAttribute(*llvmAttr);1137 continue;1138 }1139 1140 auto arrayAttr = dyn_cast<ArrayAttr>(attr);1141 if (!arrayAttr || arrayAttr.size() != 2)1142 return emitError(loc) << "expected '" << arrayAttrName1143 << "' to contain string or array attributes";1144 1145 auto keyAttr = dyn_cast<StringAttr>(arrayAttr[0]);1146 auto valueAttr = dyn_cast<StringAttr>(arrayAttr[1]);1147 if (!keyAttr || !valueAttr)1148 return emitError(loc) << "expected arrays within '" << arrayAttrName1149 << "' to contain two strings";1150 1151 FailureOr<llvm::Attribute> llvmAttr = convertMLIRAttributeToLLVM(1152 loc, ctx, keyAttr.getValue(), valueAttr.getValue());1153 if (failed(llvmAttr))1154 return failure();1155 attrBuilder.addAttribute(*llvmAttr);1156 }1157 1158 return attrBuilder;1159}1160 1161LogicalResult ModuleTranslation::convertGlobalsAndAliases() {1162 // Mapping from compile unit to its respective set of global variables.1163 DenseMap<llvm::DICompileUnit *, SmallVector<llvm::Metadata *>> allGVars;1164 1165 // First, create all global variables and global aliases in LLVM IR. A global1166 // or alias body may refer to another global/alias or itself, so all the1167 // mapping needs to happen prior to body conversion.1168 1169 // Create all llvm::GlobalVariable1170 for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) {1171 llvm::Type *type = convertType(op.getType());1172 llvm::Constant *cst = nullptr;1173 if (op.getValueOrNull()) {1174 // String attributes are treated separately because they cannot appear as1175 // in-function constants and are thus not supported by getLLVMConstant.1176 if (auto strAttr = dyn_cast_or_null<StringAttr>(op.getValueOrNull())) {1177 cst = llvm::ConstantDataArray::getString(1178 llvmModule->getContext(), strAttr.getValue(), /*AddNull=*/false);1179 type = cst->getType();1180 } else if (!(cst = getLLVMConstant(type, op.getValueOrNull(), op.getLoc(),1181 *this))) {1182 return failure();1183 }1184 }1185 1186 auto linkage = convertLinkageToLLVM(op.getLinkage());1187 1188 // LLVM IR requires constant with linkage other than external or weak1189 // external to have initializers. If MLIR does not provide an initializer,1190 // default to undef.1191 bool dropInitializer = shouldDropGlobalInitializer(linkage, cst);1192 if (!dropInitializer && !cst)1193 cst = llvm::UndefValue::get(type);1194 else if (dropInitializer && cst)1195 cst = nullptr;1196 1197 auto *var = new llvm::GlobalVariable(1198 *llvmModule, type, op.getConstant(), linkage, cst, op.getSymName(),1199 /*InsertBefore=*/nullptr,1200 op.getThreadLocal_() ? llvm::GlobalValue::GeneralDynamicTLSModel1201 : llvm::GlobalValue::NotThreadLocal,1202 op.getAddrSpace(), op.getExternallyInitialized());1203 1204 if (std::optional<mlir::SymbolRefAttr> comdat = op.getComdat()) {1205 auto selectorOp = cast<ComdatSelectorOp>(1206 SymbolTable::lookupNearestSymbolFrom(op, *comdat));1207 var->setComdat(comdatMapping.lookup(selectorOp));1208 }1209 1210 if (op.getUnnamedAddr().has_value())1211 var->setUnnamedAddr(convertUnnamedAddrToLLVM(*op.getUnnamedAddr()));1212 1213 if (op.getSection().has_value())1214 var->setSection(*op.getSection());1215 1216 addRuntimePreemptionSpecifier(op.getDsoLocal(), var);1217 1218 std::optional<uint64_t> alignment = op.getAlignment();1219 if (alignment.has_value())1220 var->setAlignment(llvm::MaybeAlign(alignment.value()));1221 1222 var->setVisibility(convertVisibilityToLLVM(op.getVisibility_()));1223 1224 globalsMapping.try_emplace(op, var);1225 1226 // Add debug information if present.1227 if (op.getDbgExprs()) {1228 for (auto exprAttr :1229 op.getDbgExprs()->getAsRange<DIGlobalVariableExpressionAttr>()) {1230 llvm::DIGlobalVariableExpression *diGlobalExpr =1231 debugTranslation->translateGlobalVariableExpression(exprAttr);1232 llvm::DIGlobalVariable *diGlobalVar = diGlobalExpr->getVariable();1233 var->addDebugInfo(diGlobalExpr);1234 1235 // There is no `globals` field in DICompileUnitAttr which can be1236 // directly assigned to DICompileUnit. We have to build the list by1237 // looking at the dbgExpr of all the GlobalOps. The scope of the1238 // variable is used to get the DICompileUnit in which to add it. But1239 // there are cases where the scope of a global does not directly point1240 // to the DICompileUnit and we have to do a bit more work to get to1241 // it. Some of those cases are:1242 //1243 // 1. For the languages that support modules, the scope hierarchy can1244 // be variable -> DIModule -> DICompileUnit1245 //1246 // 2. For the Fortran common block variable, the scope hierarchy can1247 // be variable -> DICommonBlock -> DISubprogram -> DICompileUnit1248 //1249 // 3. For entities like static local variables in C or variable with1250 // SAVE attribute in Fortran, the scope hierarchy can be1251 // variable -> DISubprogram -> DICompileUnit1252 llvm::DIScope *scope = diGlobalVar->getScope();1253 if (auto *mod = dyn_cast_if_present<llvm::DIModule>(scope))1254 scope = mod->getScope();1255 else if (auto *cb = dyn_cast_if_present<llvm::DICommonBlock>(scope)) {1256 if (auto *sp =1257 dyn_cast_if_present<llvm::DISubprogram>(cb->getScope()))1258 scope = sp->getUnit();1259 } else if (auto *sp = dyn_cast_if_present<llvm::DISubprogram>(scope))1260 scope = sp->getUnit();1261 1262 // Get the compile unit (scope) of the the global variable.1263 if (llvm::DICompileUnit *compileUnit =1264 dyn_cast_if_present<llvm::DICompileUnit>(scope)) {1265 // Update the compile unit with this incoming global variable1266 // expression during the finalizing step later.1267 allGVars[compileUnit].push_back(diGlobalExpr);1268 }1269 }1270 }1271 1272 // Forward the target-specific attributes to LLVM.1273 FailureOr<llvm::AttrBuilder> convertedTargetSpecificAttrs =1274 convertMLIRAttributesToLLVM(op.getLoc(), var->getContext(),1275 op.getTargetSpecificAttrsAttr(),1276 op.getTargetSpecificAttrsAttrName());1277 if (failed(convertedTargetSpecificAttrs))1278 return failure();1279 var->addAttributes(*convertedTargetSpecificAttrs);1280 }1281 1282 // Create all llvm::GlobalAlias1283 for (auto op : getModuleBody(mlirModule).getOps<LLVM::AliasOp>()) {1284 llvm::Type *type = convertType(op.getType());1285 llvm::Constant *cst = nullptr;1286 llvm::GlobalValue::LinkageTypes linkage =1287 convertLinkageToLLVM(op.getLinkage());1288 llvm::Module &llvmMod = *llvmModule;1289 1290 // Note address space and aliasee info isn't set just yet.1291 llvm::GlobalAlias *var = llvm::GlobalAlias::create(1292 type, op.getAddrSpace(), linkage, op.getSymName(), /*placeholder*/ cst,1293 &llvmMod);1294 1295 var->setThreadLocalMode(op.getThreadLocal_()1296 ? llvm::GlobalAlias::GeneralDynamicTLSModel1297 : llvm::GlobalAlias::NotThreadLocal);1298 1299 // Note there is no need to setup the comdat because GlobalAlias calls into1300 // the aliasee comdat information automatically.1301 1302 if (op.getUnnamedAddr().has_value())1303 var->setUnnamedAddr(convertUnnamedAddrToLLVM(*op.getUnnamedAddr()));1304 1305 var->setVisibility(convertVisibilityToLLVM(op.getVisibility_()));1306 1307 aliasesMapping.try_emplace(op, var);1308 }1309 1310 // Convert global variable bodies.1311 for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) {1312 if (Block *initializer = op.getInitializerBlock()) {1313 llvm::IRBuilder<llvm::TargetFolder> builder(1314 llvmModule->getContext(),1315 llvm::TargetFolder(llvmModule->getDataLayout()));1316 1317 [[maybe_unused]] int numConstantsHit = 0;1318 [[maybe_unused]] int numConstantsErased = 0;1319 DenseMap<llvm::ConstantAggregate *, int> constantAggregateUseMap;1320 1321 for (auto &op : initializer->without_terminator()) {1322 if (failed(convertOperation(op, builder)))1323 return emitError(op.getLoc(), "fail to convert global initializer");1324 auto *cst = dyn_cast<llvm::Constant>(lookupValue(op.getResult(0)));1325 if (!cst)1326 return emitError(op.getLoc(), "unemittable constant value");1327 1328 // When emitting an LLVM constant, a new constant is created and the old1329 // constant may become dangling and take space. We should remove the1330 // dangling constants to avoid memory explosion especially for constant1331 // arrays whose number of elements is large.1332 // Because multiple operations may refer to the same constant, we need1333 // to count the number of uses of each constant array and remove it only1334 // when the count becomes zero.1335 if (auto *agg = dyn_cast<llvm::ConstantAggregate>(cst)) {1336 numConstantsHit++;1337 Value result = op.getResult(0);1338 int numUsers = std::distance(result.use_begin(), result.use_end());1339 auto [iterator, inserted] =1340 constantAggregateUseMap.try_emplace(agg, numUsers);1341 if (!inserted) {1342 // Key already exists, update the value1343 iterator->second += numUsers;1344 }1345 }1346 // Scan the operands of the operation to decrement the use count of1347 // constants. Erase the constant if the use count becomes zero.1348 for (Value v : op.getOperands()) {1349 auto cst = dyn_cast<llvm::ConstantAggregate>(lookupValue(v));1350 if (!cst)1351 continue;1352 auto iter = constantAggregateUseMap.find(cst);1353 assert(iter != constantAggregateUseMap.end() && "constant not found");1354 iter->second--;1355 if (iter->second == 0) {1356 // NOTE: cannot call removeDeadConstantUsers() here because it1357 // may remove the constant which has uses not be converted yet.1358 if (cst->user_empty()) {1359 cst->destroyConstant();1360 numConstantsErased++;1361 }1362 constantAggregateUseMap.erase(iter);1363 }1364 }1365 }1366 1367 ReturnOp ret = cast<ReturnOp>(initializer->getTerminator());1368 llvm::Constant *cst =1369 cast<llvm::Constant>(lookupValue(ret.getOperand(0)));1370 auto *global = cast<llvm::GlobalVariable>(lookupGlobal(op));1371 if (!shouldDropGlobalInitializer(global->getLinkage(), cst))1372 global->setInitializer(cst);1373 1374 // Try to remove the dangling constants again after all operations are1375 // converted.1376 for (auto it : constantAggregateUseMap) {1377 auto cst = it.first;1378 cst->removeDeadConstantUsers();1379 if (cst->user_empty()) {1380 cst->destroyConstant();1381 numConstantsErased++;1382 }1383 }1384 1385 LLVM_DEBUG(llvm::dbgs()1386 << "Convert initializer for " << op.getName() << "\n";1387 llvm::dbgs() << numConstantsHit << " new constants hit\n";1388 llvm::dbgs()1389 << numConstantsErased << " dangling constants erased\n";);1390 }1391 }1392 1393 // Convert llvm.mlir.global_ctors and dtors.1394 for (Operation &op : getModuleBody(mlirModule)) {1395 auto ctorOp = dyn_cast<GlobalCtorsOp>(op);1396 auto dtorOp = dyn_cast<GlobalDtorsOp>(op);1397 if (!ctorOp && !dtorOp)1398 continue;1399 1400 // The empty / zero initialized version of llvm.global_(c|d)tors cannot be1401 // handled by appendGlobalFn logic below, which just ignores empty (c|d)tor1402 // lists. Make sure it gets emitted.1403 if ((ctorOp && ctorOp.getCtors().empty()) ||1404 (dtorOp && dtorOp.getDtors().empty())) {1405 llvm::IRBuilder<llvm::TargetFolder> builder(1406 llvmModule->getContext(),1407 llvm::TargetFolder(llvmModule->getDataLayout()));1408 llvm::Type *eltTy = llvm::StructType::get(1409 builder.getInt32Ty(), builder.getPtrTy(), builder.getPtrTy());1410 llvm::ArrayType *at = llvm::ArrayType::get(eltTy, 0);1411 llvm::Constant *zeroInit = llvm::Constant::getNullValue(at);1412 (void)new llvm::GlobalVariable(1413 *llvmModule, zeroInit->getType(), false,1414 llvm::GlobalValue::AppendingLinkage, zeroInit,1415 ctorOp ? "llvm.global_ctors" : "llvm.global_dtors");1416 } else {1417 auto range = ctorOp1418 ? llvm::zip(ctorOp.getCtors(), ctorOp.getPriorities())1419 : llvm::zip(dtorOp.getDtors(), dtorOp.getPriorities());1420 auto appendGlobalFn =1421 ctorOp ? llvm::appendToGlobalCtors : llvm::appendToGlobalDtors;1422 for (const auto &[sym, prio] : range) {1423 llvm::Function *f =1424 lookupFunction(cast<FlatSymbolRefAttr>(sym).getValue());1425 appendGlobalFn(*llvmModule, f, cast<IntegerAttr>(prio).getInt(),1426 /*Data=*/nullptr);1427 }1428 }1429 }1430 1431 for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>())1432 if (failed(convertDialectAttributes(op, {})))1433 return failure();1434 1435 // Finally, update the compile units their respective sets of global variables1436 // created earlier.1437 for (const auto &[compileUnit, globals] : allGVars) {1438 compileUnit->replaceGlobalVariables(1439 llvm::MDTuple::get(getLLVMContext(), globals));1440 }1441 1442 // Convert global alias bodies.1443 for (auto op : getModuleBody(mlirModule).getOps<LLVM::AliasOp>()) {1444 Block &initializer = op.getInitializerBlock();1445 llvm::IRBuilder<llvm::TargetFolder> builder(1446 llvmModule->getContext(),1447 llvm::TargetFolder(llvmModule->getDataLayout()));1448 1449 for (mlir::Operation &op : initializer.without_terminator()) {1450 if (failed(convertOperation(op, builder)))1451 return emitError(op.getLoc(), "fail to convert alias initializer");1452 if (!isa<llvm::Constant>(lookupValue(op.getResult(0))))1453 return emitError(op.getLoc(), "unemittable constant value");1454 }1455 1456 auto ret = cast<ReturnOp>(initializer.getTerminator());1457 auto *cst = cast<llvm::Constant>(lookupValue(ret.getOperand(0)));1458 assert(aliasesMapping.count(op));1459 auto *alias = cast<llvm::GlobalAlias>(aliasesMapping[op]);1460 alias->setAliasee(cst);1461 }1462 1463 for (auto op : getModuleBody(mlirModule).getOps<LLVM::AliasOp>())1464 if (failed(convertDialectAttributes(op, {})))1465 return failure();1466 1467 return success();1468}1469 1470/// Return a representation of `value` as metadata.1471static llvm::Metadata *convertIntegerToMetadata(llvm::LLVMContext &context,1472 const llvm::APInt &value) {1473 llvm::Constant *constant = llvm::ConstantInt::get(context, value);1474 return llvm::ConstantAsMetadata::get(constant);1475}1476 1477/// Return a representation of `value` as an MDNode.1478static llvm::MDNode *convertIntegerToMDNode(llvm::LLVMContext &context,1479 const llvm::APInt &value) {1480 return llvm::MDNode::get(context, convertIntegerToMetadata(context, value));1481}1482 1483/// Return an MDNode encoding `vec_type_hint` metadata.1484static llvm::MDNode *convertVecTypeHintToMDNode(llvm::LLVMContext &context,1485 llvm::Type *type,1486 bool isSigned) {1487 llvm::Metadata *typeMD =1488 llvm::ConstantAsMetadata::get(llvm::UndefValue::get(type));1489 llvm::Metadata *isSignedMD =1490 convertIntegerToMetadata(context, llvm::APInt(32, isSigned ? 1 : 0));1491 return llvm::MDNode::get(context, {typeMD, isSignedMD});1492}1493 1494/// Return an MDNode with a tuple given by the values in `values`.1495static llvm::MDNode *convertIntegerArrayToMDNode(llvm::LLVMContext &context,1496 ArrayRef<int32_t> values) {1497 SmallVector<llvm::Metadata *> mdValues;1498 llvm::transform(1499 values, std::back_inserter(mdValues), [&context](int32_t value) {1500 return convertIntegerToMetadata(context, llvm::APInt(32, value));1501 });1502 return llvm::MDNode::get(context, mdValues);1503}1504 1505LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {1506 // Clear the block, branch value mappings, they are only relevant within one1507 // function.1508 blockMapping.clear();1509 valueMapping.clear();1510 branchMapping.clear();1511 llvm::Function *llvmFunc = lookupFunction(func.getName());1512 1513 // Add function arguments to the value remapping table.1514 for (auto [mlirArg, llvmArg] :1515 llvm::zip(func.getArguments(), llvmFunc->args()))1516 mapValue(mlirArg, &llvmArg);1517 1518 // Check the personality and set it.1519 if (func.getPersonality()) {1520 llvm::Type *ty = llvm::PointerType::getUnqual(llvmFunc->getContext());1521 if (llvm::Constant *pfunc = getLLVMConstant(ty, func.getPersonalityAttr(),1522 func.getLoc(), *this))1523 llvmFunc->setPersonalityFn(pfunc);1524 }1525 1526 if (std::optional<StringRef> section = func.getSection())1527 llvmFunc->setSection(*section);1528 1529 if (func.getArmStreaming())1530 llvmFunc->addFnAttr("aarch64_pstate_sm_enabled");1531 else if (func.getArmLocallyStreaming())1532 llvmFunc->addFnAttr("aarch64_pstate_sm_body");1533 else if (func.getArmStreamingCompatible())1534 llvmFunc->addFnAttr("aarch64_pstate_sm_compatible");1535 1536 if (func.getArmNewZa())1537 llvmFunc->addFnAttr("aarch64_new_za");1538 else if (func.getArmInZa())1539 llvmFunc->addFnAttr("aarch64_in_za");1540 else if (func.getArmOutZa())1541 llvmFunc->addFnAttr("aarch64_out_za");1542 else if (func.getArmInoutZa())1543 llvmFunc->addFnAttr("aarch64_inout_za");1544 else if (func.getArmPreservesZa())1545 llvmFunc->addFnAttr("aarch64_preserves_za");1546 1547 if (auto targetCpu = func.getTargetCpu())1548 llvmFunc->addFnAttr("target-cpu", *targetCpu);1549 1550 if (auto tuneCpu = func.getTuneCpu())1551 llvmFunc->addFnAttr("tune-cpu", *tuneCpu);1552 1553 if (auto reciprocalEstimates = func.getReciprocalEstimates())1554 llvmFunc->addFnAttr("reciprocal-estimates", *reciprocalEstimates);1555 1556 if (auto preferVectorWidth = func.getPreferVectorWidth())1557 llvmFunc->addFnAttr("prefer-vector-width", *preferVectorWidth);1558 1559 if (auto attr = func.getVscaleRange())1560 llvmFunc->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(1561 getLLVMContext(), attr->getMinRange().getInt(),1562 attr->getMaxRange().getInt()));1563 1564 if (auto noInfsFpMath = func.getNoInfsFpMath())1565 llvmFunc->addFnAttr("no-infs-fp-math", llvm::toStringRef(*noInfsFpMath));1566 1567 if (auto noNansFpMath = func.getNoNansFpMath())1568 llvmFunc->addFnAttr("no-nans-fp-math", llvm::toStringRef(*noNansFpMath));1569 1570 if (auto noSignedZerosFpMath = func.getNoSignedZerosFpMath())1571 llvmFunc->addFnAttr("no-signed-zeros-fp-math",1572 llvm::toStringRef(*noSignedZerosFpMath));1573 1574 if (auto denormalFpMath = func.getDenormalFpMath())1575 llvmFunc->addFnAttr("denormal-fp-math", *denormalFpMath);1576 1577 if (auto denormalFpMathF32 = func.getDenormalFpMathF32())1578 llvmFunc->addFnAttr("denormal-fp-math-f32", *denormalFpMathF32);1579 1580 if (auto fpContract = func.getFpContract())1581 llvmFunc->addFnAttr("fp-contract", *fpContract);1582 1583 if (auto instrumentFunctionEntry = func.getInstrumentFunctionEntry())1584 llvmFunc->addFnAttr("instrument-function-entry", *instrumentFunctionEntry);1585 1586 if (auto instrumentFunctionExit = func.getInstrumentFunctionExit())1587 llvmFunc->addFnAttr("instrument-function-exit", *instrumentFunctionExit);1588 1589 // First, create all blocks so we can jump to them.1590 llvm::LLVMContext &llvmContext = llvmFunc->getContext();1591 for (auto &bb : func) {1592 auto *llvmBB = llvm::BasicBlock::Create(llvmContext);1593 llvmBB->insertInto(llvmFunc);1594 mapBlock(&bb, llvmBB);1595 }1596 1597 // Then, convert blocks one by one in topological order to ensure defs are1598 // converted before uses.1599 auto blocks = getBlocksSortedByDominance(func.getBody());1600 for (Block *bb : blocks) {1601 CapturingIRBuilder builder(llvmContext,1602 llvm::TargetFolder(llvmModule->getDataLayout()));1603 if (failed(convertBlockImpl(*bb, bb->isEntryBlock(), builder,1604 /*recordInsertions=*/true)))1605 return failure();1606 }1607 1608 // After all blocks have been traversed and values mapped, connect the PHI1609 // nodes to the results of preceding blocks.1610 detail::connectPHINodes(func.getBody(), *this);1611 1612 // Finally, convert dialect attributes attached to the function.1613 return convertDialectAttributes(func, {});1614}1615 1616LogicalResult ModuleTranslation::convertDialectAttributes(1617 Operation *op, ArrayRef<llvm::Instruction *> instructions) {1618 for (NamedAttribute attribute : op->getDialectAttrs())1619 if (failed(iface.amendOperation(op, instructions, attribute, *this)))1620 return failure();1621 return success();1622}1623 1624/// Converts memory effect attributes from `func` and attaches them to1625/// `llvmFunc`.1626static void convertFunctionMemoryAttributes(LLVMFuncOp func,1627 llvm::Function *llvmFunc) {1628 if (!func.getMemoryEffects())1629 return;1630 1631 MemoryEffectsAttr memEffects = func.getMemoryEffectsAttr();1632 1633 // Add memory effects incrementally.1634 llvm::MemoryEffects newMemEffects =1635 llvm::MemoryEffects(llvm::MemoryEffects::Location::ArgMem,1636 convertModRefInfoToLLVM(memEffects.getArgMem()));1637 newMemEffects |= llvm::MemoryEffects(1638 llvm::MemoryEffects::Location::InaccessibleMem,1639 convertModRefInfoToLLVM(memEffects.getInaccessibleMem()));1640 newMemEffects |=1641 llvm::MemoryEffects(llvm::MemoryEffects::Location::Other,1642 convertModRefInfoToLLVM(memEffects.getOther()));1643 newMemEffects |=1644 llvm::MemoryEffects(llvm::MemoryEffects::Location::ErrnoMem,1645 convertModRefInfoToLLVM(memEffects.getErrnoMem()));1646 newMemEffects |=1647 llvm::MemoryEffects(llvm::MemoryEffects::Location::TargetMem0,1648 convertModRefInfoToLLVM(memEffects.getTargetMem0()));1649 newMemEffects |=1650 llvm::MemoryEffects(llvm::MemoryEffects::Location::TargetMem1,1651 convertModRefInfoToLLVM(memEffects.getTargetMem1()));1652 llvmFunc->setMemoryEffects(newMemEffects);1653}1654 1655/// Converts function attributes from `func` and attaches them to `llvmFunc`.1656static void convertFunctionAttributes(LLVMFuncOp func,1657 llvm::Function *llvmFunc) {1658 if (func.getNoInlineAttr())1659 llvmFunc->addFnAttr(llvm::Attribute::NoInline);1660 if (func.getAlwaysInlineAttr())1661 llvmFunc->addFnAttr(llvm::Attribute::AlwaysInline);1662 if (func.getInlineHintAttr())1663 llvmFunc->addFnAttr(llvm::Attribute::InlineHint);1664 if (func.getOptimizeNoneAttr())1665 llvmFunc->addFnAttr(llvm::Attribute::OptimizeNone);1666 if (func.getConvergentAttr())1667 llvmFunc->addFnAttr(llvm::Attribute::Convergent);1668 if (func.getNoUnwindAttr())1669 llvmFunc->addFnAttr(llvm::Attribute::NoUnwind);1670 if (func.getWillReturnAttr())1671 llvmFunc->addFnAttr(llvm::Attribute::WillReturn);1672 if (TargetFeaturesAttr targetFeatAttr = func.getTargetFeaturesAttr())1673 llvmFunc->addFnAttr("target-features", targetFeatAttr.getFeaturesString());1674 if (FramePointerKindAttr fpAttr = func.getFramePointerAttr())1675 llvmFunc->addFnAttr("frame-pointer", stringifyFramePointerKind(1676 fpAttr.getFramePointerKind()));1677 if (UWTableKindAttr uwTableKindAttr = func.getUwtableKindAttr())1678 llvmFunc->setUWTableKind(1679 convertUWTableKindToLLVM(uwTableKindAttr.getUwtableKind()));1680 convertFunctionMemoryAttributes(func, llvmFunc);1681}1682 1683/// Converts function attributes from `func` and attaches them to `llvmFunc`.1684static void convertFunctionKernelAttributes(LLVMFuncOp func,1685 llvm::Function *llvmFunc,1686 ModuleTranslation &translation) {1687 llvm::LLVMContext &llvmContext = llvmFunc->getContext();1688 1689 if (VecTypeHintAttr vecTypeHint = func.getVecTypeHintAttr()) {1690 Type type = vecTypeHint.getHint().getValue();1691 llvm::Type *llvmType = translation.convertType(type);1692 bool isSigned = vecTypeHint.getIsSigned();1693 llvmFunc->setMetadata(1694 func.getVecTypeHintAttrName(),1695 convertVecTypeHintToMDNode(llvmContext, llvmType, isSigned));1696 }1697 1698 if (std::optional<ArrayRef<int32_t>> workGroupSizeHint =1699 func.getWorkGroupSizeHint()) {1700 llvmFunc->setMetadata(1701 func.getWorkGroupSizeHintAttrName(),1702 convertIntegerArrayToMDNode(llvmContext, *workGroupSizeHint));1703 }1704 1705 if (std::optional<ArrayRef<int32_t>> reqdWorkGroupSize =1706 func.getReqdWorkGroupSize()) {1707 llvmFunc->setMetadata(1708 func.getReqdWorkGroupSizeAttrName(),1709 convertIntegerArrayToMDNode(llvmContext, *reqdWorkGroupSize));1710 }1711 1712 if (std::optional<uint32_t> intelReqdSubGroupSize =1713 func.getIntelReqdSubGroupSize()) {1714 llvmFunc->setMetadata(1715 func.getIntelReqdSubGroupSizeAttrName(),1716 convertIntegerToMDNode(llvmContext,1717 llvm::APInt(32, *intelReqdSubGroupSize)));1718 }1719}1720 1721static LogicalResult convertParameterAttr(llvm::AttrBuilder &attrBuilder,1722 llvm::Attribute::AttrKind llvmKind,1723 NamedAttribute namedAttr,1724 ModuleTranslation &moduleTranslation,1725 Location loc) {1726 return llvm::TypeSwitch<Attribute, LogicalResult>(namedAttr.getValue())1727 .Case<TypeAttr>([&](auto typeAttr) {1728 attrBuilder.addTypeAttr(1729 llvmKind, moduleTranslation.convertType(typeAttr.getValue()));1730 return success();1731 })1732 .Case<IntegerAttr>([&](auto intAttr) {1733 attrBuilder.addRawIntAttr(llvmKind, intAttr.getInt());1734 return success();1735 })1736 .Case<UnitAttr>([&](auto) {1737 attrBuilder.addAttribute(llvmKind);1738 return success();1739 })1740 .Case<LLVM::ConstantRangeAttr>([&](auto rangeAttr) {1741 attrBuilder.addConstantRangeAttr(1742 llvmKind,1743 llvm::ConstantRange(rangeAttr.getLower(), rangeAttr.getUpper()));1744 return success();1745 })1746 .Default([loc](auto) {1747 return emitError(loc, "unsupported parameter attribute type");1748 });1749}1750 1751FailureOr<llvm::AttrBuilder>1752ModuleTranslation::convertParameterAttrs(LLVMFuncOp func, int argIdx,1753 DictionaryAttr paramAttrs) {1754 llvm::AttrBuilder attrBuilder(llvmModule->getContext());1755 auto attrNameToKindMapping = getAttrNameToKindMapping();1756 Location loc = func.getLoc();1757 1758 for (auto namedAttr : paramAttrs) {1759 auto it = attrNameToKindMapping.find(namedAttr.getName());1760 if (it != attrNameToKindMapping.end()) {1761 llvm::Attribute::AttrKind llvmKind = it->second;1762 if (failed(convertParameterAttr(attrBuilder, llvmKind, namedAttr, *this,1763 loc)))1764 return failure();1765 } else if (namedAttr.getNameDialect()) {1766 if (failed(iface.convertParameterAttr(func, argIdx, namedAttr, *this)))1767 return failure();1768 }1769 }1770 1771 return attrBuilder;1772}1773 1774LogicalResult ModuleTranslation::convertArgAndResultAttrs(1775 ArgAndResultAttrsOpInterface attrsOp, llvm::CallBase *call,1776 ArrayRef<unsigned> immArgPositions) {1777 // Convert the argument attributes.1778 if (ArrayAttr argAttrsArray = attrsOp.getArgAttrsAttr()) {1779 unsigned argAttrIdx = 0;1780 llvm::SmallDenseSet<unsigned> immArgPositionsSet(immArgPositions.begin(),1781 immArgPositions.end());1782 for (unsigned argIdx : llvm::seq<unsigned>(call->arg_size())) {1783 if (argAttrIdx >= argAttrsArray.size())1784 break;1785 // Skip immediate arguments (they have no entries in argAttrsArray).1786 if (immArgPositionsSet.contains(argIdx))1787 continue;1788 // Skip empty argument attributes.1789 auto argAttrs = cast<DictionaryAttr>(argAttrsArray[argAttrIdx++]);1790 if (argAttrs.empty())1791 continue;1792 // Convert and add attributes to the call instruction.1793 FailureOr<llvm::AttrBuilder> attrBuilder =1794 convertParameterAttrs(attrsOp->getLoc(), argAttrs);1795 if (failed(attrBuilder))1796 return failure();1797 call->addParamAttrs(argIdx, *attrBuilder);1798 }1799 }1800 1801 // Convert the result attributes.1802 if (ArrayAttr resAttrsArray = attrsOp.getResAttrsAttr()) {1803 if (!resAttrsArray.empty()) {1804 auto resAttrs = cast<DictionaryAttr>(resAttrsArray[0]);1805 FailureOr<llvm::AttrBuilder> attrBuilder =1806 convertParameterAttrs(attrsOp->getLoc(), resAttrs);1807 if (failed(attrBuilder))1808 return failure();1809 call->addRetAttrs(*attrBuilder);1810 }1811 }1812 1813 return success();1814}1815 1816FailureOr<llvm::AttrBuilder>1817ModuleTranslation::convertParameterAttrs(Location loc,1818 DictionaryAttr paramAttrs) {1819 llvm::AttrBuilder attrBuilder(llvmModule->getContext());1820 auto attrNameToKindMapping = getAttrNameToKindMapping();1821 1822 for (auto namedAttr : paramAttrs) {1823 auto it = attrNameToKindMapping.find(namedAttr.getName());1824 if (it != attrNameToKindMapping.end()) {1825 llvm::Attribute::AttrKind llvmKind = it->second;1826 if (failed(convertParameterAttr(attrBuilder, llvmKind, namedAttr, *this,1827 loc)))1828 return failure();1829 }1830 }1831 1832 return attrBuilder;1833}1834 1835LogicalResult ModuleTranslation::convertFunctionSignatures() {1836 // Declare all functions first because there may be function calls that form a1837 // call graph with cycles, or global initializers that reference functions.1838 for (auto function : getModuleBody(mlirModule).getOps<LLVMFuncOp>()) {1839 llvm::FunctionCallee llvmFuncCst = llvmModule->getOrInsertFunction(1840 function.getName(),1841 cast<llvm::FunctionType>(convertType(function.getFunctionType())));1842 llvm::Function *llvmFunc = cast<llvm::Function>(llvmFuncCst.getCallee());1843 llvmFunc->setLinkage(convertLinkageToLLVM(function.getLinkage()));1844 llvmFunc->setCallingConv(convertCConvToLLVM(function.getCConv()));1845 mapFunction(function.getName(), llvmFunc);1846 addRuntimePreemptionSpecifier(function.getDsoLocal(), llvmFunc);1847 1848 // Convert function attributes.1849 convertFunctionAttributes(function, llvmFunc);1850 1851 // Convert function kernel attributes to metadata.1852 convertFunctionKernelAttributes(function, llvmFunc, *this);1853 1854 // Convert function_entry_count attribute to metadata.1855 if (std::optional<uint64_t> entryCount = function.getFunctionEntryCount())1856 llvmFunc->setEntryCount(entryCount.value());1857 1858 // Convert result attributes.1859 if (ArrayAttr allResultAttrs = function.getAllResultAttrs()) {1860 DictionaryAttr resultAttrs = cast<DictionaryAttr>(allResultAttrs[0]);1861 FailureOr<llvm::AttrBuilder> attrBuilder =1862 convertParameterAttrs(function, -1, resultAttrs);1863 if (failed(attrBuilder))1864 return failure();1865 llvmFunc->addRetAttrs(*attrBuilder);1866 }1867 1868 // Convert argument attributes.1869 for (auto [argIdx, llvmArg] : llvm::enumerate(llvmFunc->args())) {1870 if (DictionaryAttr argAttrs = function.getArgAttrDict(argIdx)) {1871 FailureOr<llvm::AttrBuilder> attrBuilder =1872 convertParameterAttrs(function, argIdx, argAttrs);1873 if (failed(attrBuilder))1874 return failure();1875 llvmArg.addAttrs(*attrBuilder);1876 }1877 }1878 1879 // Forward the pass-through attributes to LLVM.1880 FailureOr<llvm::AttrBuilder> convertedPassthroughAttrs =1881 convertMLIRAttributesToLLVM(function.getLoc(), llvmFunc->getContext(),1882 function.getPassthroughAttr(),1883 function.getPassthroughAttrName());1884 if (failed(convertedPassthroughAttrs))1885 return failure();1886 llvmFunc->addFnAttrs(*convertedPassthroughAttrs);1887 1888 // Convert visibility attribute.1889 llvmFunc->setVisibility(convertVisibilityToLLVM(function.getVisibility_()));1890 1891 // Convert the comdat attribute.1892 if (std::optional<mlir::SymbolRefAttr> comdat = function.getComdat()) {1893 auto selectorOp = cast<ComdatSelectorOp>(1894 SymbolTable::lookupNearestSymbolFrom(function, *comdat));1895 llvmFunc->setComdat(comdatMapping.lookup(selectorOp));1896 }1897 1898 if (auto gc = function.getGarbageCollector())1899 llvmFunc->setGC(gc->str());1900 1901 if (auto unnamedAddr = function.getUnnamedAddr())1902 llvmFunc->setUnnamedAddr(convertUnnamedAddrToLLVM(*unnamedAddr));1903 1904 if (auto alignment = function.getAlignment())1905 llvmFunc->setAlignment(llvm::MaybeAlign(*alignment));1906 1907 // Translate the debug information for this function.1908 debugTranslation->translate(function, *llvmFunc);1909 }1910 1911 return success();1912}1913 1914LogicalResult ModuleTranslation::convertFunctions() {1915 // Convert functions.1916 for (auto function : getModuleBody(mlirModule).getOps<LLVMFuncOp>()) {1917 // Do not convert external functions, but do process dialect attributes1918 // attached to them.1919 if (function.isExternal()) {1920 if (failed(convertDialectAttributes(function, {})))1921 return failure();1922 continue;1923 }1924 1925 if (failed(convertOneFunction(function)))1926 return failure();1927 }1928 1929 return success();1930}1931 1932LogicalResult ModuleTranslation::convertIFuncs() {1933 for (auto op : getModuleBody(mlirModule).getOps<IFuncOp>()) {1934 llvm::Type *type = convertType(op.getIFuncType());1935 llvm::GlobalValue::LinkageTypes linkage =1936 convertLinkageToLLVM(op.getLinkage());1937 llvm::Constant *resolver;1938 if (auto *resolverFn = lookupFunction(op.getResolver())) {1939 resolver = cast<llvm::Constant>(resolverFn);1940 } else {1941 Operation *aliasOp = symbolTable().lookupSymbolIn(parentLLVMModule(op),1942 op.getResolverAttr());1943 resolver = cast<llvm::Constant>(lookupAlias(aliasOp));1944 }1945 1946 auto *ifunc =1947 llvm::GlobalIFunc::create(type, op.getAddressSpace(), linkage,1948 op.getSymName(), resolver, llvmModule.get());1949 addRuntimePreemptionSpecifier(op.getDsoLocal(), ifunc);1950 ifunc->setUnnamedAddr(convertUnnamedAddrToLLVM(op.getUnnamedAddr()));1951 ifunc->setVisibility(convertVisibilityToLLVM(op.getVisibility_()));1952 1953 ifuncMapping.try_emplace(op, ifunc);1954 }1955 1956 return success();1957}1958 1959LogicalResult ModuleTranslation::convertComdats() {1960 for (auto comdatOp : getModuleBody(mlirModule).getOps<ComdatOp>()) {1961 for (auto selectorOp : comdatOp.getOps<ComdatSelectorOp>()) {1962 llvm::Module *module = getLLVMModule();1963 if (module->getComdatSymbolTable().contains(selectorOp.getSymName()))1964 return emitError(selectorOp.getLoc())1965 << "comdat selection symbols must be unique even in different "1966 "comdat regions";1967 llvm::Comdat *comdat = module->getOrInsertComdat(selectorOp.getSymName());1968 comdat->setSelectionKind(convertComdatToLLVM(selectorOp.getComdat()));1969 comdatMapping.try_emplace(selectorOp, comdat);1970 }1971 }1972 return success();1973}1974 1975LogicalResult ModuleTranslation::convertUnresolvedBlockAddress() {1976 for (auto &[blockAddressOp, llvmCst] : unresolvedBlockAddressMapping) {1977 BlockAddressAttr blockAddressAttr = blockAddressOp.getBlockAddr();1978 llvm::BasicBlock *llvmBlock = lookupBlockAddress(blockAddressAttr);1979 assert(llvmBlock && "expected LLVM blocks to be already translated");1980 1981 // Update mapping with new block address constant.1982 auto *llvmBlockAddr = llvm::BlockAddress::get(1983 lookupFunction(blockAddressAttr.getFunction().getValue()), llvmBlock);1984 llvmCst->replaceAllUsesWith(llvmBlockAddr);1985 assert(llvmCst->use_empty() && "expected all uses to be replaced");1986 cast<llvm::GlobalVariable>(llvmCst)->eraseFromParent();1987 }1988 unresolvedBlockAddressMapping.clear();1989 return success();1990}1991 1992void ModuleTranslation::setAccessGroupsMetadata(AccessGroupOpInterface op,1993 llvm::Instruction *inst) {1994 if (llvm::MDNode *node = loopAnnotationTranslation->getAccessGroups(op))1995 inst->setMetadata(llvm::LLVMContext::MD_access_group, node);1996}1997 1998llvm::MDNode *1999ModuleTranslation::getOrCreateAliasScope(AliasScopeAttr aliasScopeAttr) {2000 auto [scopeIt, scopeInserted] =2001 aliasScopeMetadataMapping.try_emplace(aliasScopeAttr, nullptr);2002 if (!scopeInserted)2003 return scopeIt->second;2004 llvm::LLVMContext &ctx = llvmModule->getContext();2005 auto dummy = llvm::MDNode::getTemporary(ctx, {});2006 // Convert the domain metadata node if necessary.2007 auto [domainIt, insertedDomain] = aliasDomainMetadataMapping.try_emplace(2008 aliasScopeAttr.getDomain(), nullptr);2009 if (insertedDomain) {2010 llvm::SmallVector<llvm::Metadata *, 2> operands;2011 // Placeholder for potential self-reference.2012 operands.push_back(dummy.get());2013 if (StringAttr description = aliasScopeAttr.getDomain().getDescription())2014 operands.push_back(llvm::MDString::get(ctx, description));2015 domainIt->second = llvm::MDNode::get(ctx, operands);2016 // Self-reference for uniqueness.2017 llvm::Metadata *replacement;2018 if (auto stringAttr =2019 dyn_cast<StringAttr>(aliasScopeAttr.getDomain().getId()))2020 replacement = llvm::MDString::get(ctx, stringAttr.getValue());2021 else2022 replacement = domainIt->second;2023 domainIt->second->replaceOperandWith(0, replacement);2024 }2025 // Convert the scope metadata node.2026 assert(domainIt->second && "Scope's domain should already be valid");2027 llvm::SmallVector<llvm::Metadata *, 3> operands;2028 // Placeholder for potential self-reference.2029 operands.push_back(dummy.get());2030 operands.push_back(domainIt->second);2031 if (StringAttr description = aliasScopeAttr.getDescription())2032 operands.push_back(llvm::MDString::get(ctx, description));2033 scopeIt->second = llvm::MDNode::get(ctx, operands);2034 // Self-reference for uniqueness.2035 llvm::Metadata *replacement;2036 if (auto stringAttr = dyn_cast<StringAttr>(aliasScopeAttr.getId()))2037 replacement = llvm::MDString::get(ctx, stringAttr.getValue());2038 else2039 replacement = scopeIt->second;2040 scopeIt->second->replaceOperandWith(0, replacement);2041 return scopeIt->second;2042}2043 2044llvm::MDNode *ModuleTranslation::getOrCreateAliasScopes(2045 ArrayRef<AliasScopeAttr> aliasScopeAttrs) {2046 SmallVector<llvm::Metadata *> nodes;2047 nodes.reserve(aliasScopeAttrs.size());2048 for (AliasScopeAttr aliasScopeAttr : aliasScopeAttrs)2049 nodes.push_back(getOrCreateAliasScope(aliasScopeAttr));2050 return llvm::MDNode::get(getLLVMContext(), nodes);2051}2052 2053void ModuleTranslation::setAliasScopeMetadata(AliasAnalysisOpInterface op,2054 llvm::Instruction *inst) {2055 auto populateScopeMetadata = [&](ArrayAttr aliasScopeAttrs, unsigned kind) {2056 if (!aliasScopeAttrs || aliasScopeAttrs.empty())2057 return;2058 llvm::MDNode *node = getOrCreateAliasScopes(2059 llvm::to_vector(aliasScopeAttrs.getAsRange<AliasScopeAttr>()));2060 inst->setMetadata(kind, node);2061 };2062 2063 populateScopeMetadata(op.getAliasScopesOrNull(),2064 llvm::LLVMContext::MD_alias_scope);2065 populateScopeMetadata(op.getNoAliasScopesOrNull(),2066 llvm::LLVMContext::MD_noalias);2067}2068 2069llvm::MDNode *ModuleTranslation::getTBAANode(TBAATagAttr tbaaAttr) const {2070 return tbaaMetadataMapping.lookup(tbaaAttr);2071}2072 2073void ModuleTranslation::setTBAAMetadata(AliasAnalysisOpInterface op,2074 llvm::Instruction *inst) {2075 ArrayAttr tagRefs = op.getTBAATagsOrNull();2076 if (!tagRefs || tagRefs.empty())2077 return;2078 2079 // LLVM IR currently does not support attaching more than one TBAA access tag2080 // to a memory accessing instruction. It may be useful to support this in2081 // future, but for the time being just ignore the metadata if MLIR operation2082 // has multiple access tags.2083 if (tagRefs.size() > 1) {2084 op.emitWarning() << "TBAA access tags were not translated, because LLVM "2085 "IR only supports a single tag per instruction";2086 return;2087 }2088 2089 llvm::MDNode *node = getTBAANode(cast<TBAATagAttr>(tagRefs[0]));2090 inst->setMetadata(llvm::LLVMContext::MD_tbaa, node);2091}2092 2093void ModuleTranslation::setDereferenceableMetadata(2094 DereferenceableOpInterface op, llvm::Instruction *inst) {2095 DereferenceableAttr derefAttr = op.getDereferenceableOrNull();2096 if (!derefAttr)2097 return;2098 2099 llvm::MDNode *derefSizeNode = llvm::MDNode::get(2100 getLLVMContext(),2101 llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(2102 llvm::IntegerType::get(getLLVMContext(), 64), derefAttr.getBytes())));2103 unsigned kindId = derefAttr.getMayBeNull()2104 ? llvm::LLVMContext::MD_dereferenceable_or_null2105 : llvm::LLVMContext::MD_dereferenceable;2106 inst->setMetadata(kindId, derefSizeNode);2107}2108 2109void ModuleTranslation::setBranchWeightsMetadata(WeightedBranchOpInterface op) {2110 SmallVector<uint32_t> weights;2111 llvm::transform(op.getWeights(), std::back_inserter(weights),2112 [](int32_t value) { return static_cast<uint32_t>(value); });2113 if (weights.empty())2114 return;2115 2116 llvm::Instruction *inst = isa<CallOp>(op) ? lookupCall(op) : lookupBranch(op);2117 assert(inst && "expected the operation to have a mapping to an instruction");2118 inst->setMetadata(2119 llvm::LLVMContext::MD_prof,2120 llvm::MDBuilder(getLLVMContext()).createBranchWeights(weights));2121}2122 2123LogicalResult ModuleTranslation::createTBAAMetadata() {2124 llvm::LLVMContext &ctx = llvmModule->getContext();2125 llvm::IntegerType *offsetTy = llvm::IntegerType::get(ctx, 64);2126 2127 // Walk the entire module and create all metadata nodes for the TBAA2128 // attributes. The code below relies on two invariants of the2129 // `AttrTypeWalker`:2130 // 1. Attributes are visited in post-order: Since the attributes create a DAG,2131 // this ensures that any lookups into `tbaaMetadataMapping` for child2132 // attributes succeed.2133 // 2. Attributes are only ever visited once: This way we don't leak any2134 // LLVM metadata instances.2135 AttrTypeWalker walker;2136 walker.addWalk([&](TBAARootAttr root) {2137 llvm::MDNode *node;2138 if (StringAttr id = root.getId()) {2139 node = llvm::MDNode::get(ctx, llvm::MDString::get(ctx, id));2140 } else {2141 // Anonymous root nodes are self-referencing.2142 auto selfRef = llvm::MDNode::getTemporary(ctx, {});2143 node = llvm::MDNode::get(ctx, {selfRef.get()});2144 node->replaceOperandWith(0, node);2145 }2146 tbaaMetadataMapping.insert({root, node});2147 });2148 2149 walker.addWalk([&](TBAATypeDescriptorAttr descriptor) {2150 SmallVector<llvm::Metadata *> operands;2151 operands.push_back(llvm::MDString::get(ctx, descriptor.getId()));2152 for (TBAAMemberAttr member : descriptor.getMembers()) {2153 operands.push_back(tbaaMetadataMapping.lookup(member.getTypeDesc()));2154 operands.push_back(llvm::ConstantAsMetadata::get(2155 llvm::ConstantInt::get(offsetTy, member.getOffset())));2156 }2157 2158 tbaaMetadataMapping.insert({descriptor, llvm::MDNode::get(ctx, operands)});2159 });2160 2161 walker.addWalk([&](TBAATagAttr tag) {2162 SmallVector<llvm::Metadata *> operands;2163 2164 operands.push_back(tbaaMetadataMapping.lookup(tag.getBaseType()));2165 operands.push_back(tbaaMetadataMapping.lookup(tag.getAccessType()));2166 2167 operands.push_back(llvm::ConstantAsMetadata::get(2168 llvm::ConstantInt::get(offsetTy, tag.getOffset())));2169 if (tag.getConstant())2170 operands.push_back(2171 llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(offsetTy, 1)));2172 2173 tbaaMetadataMapping.insert({tag, llvm::MDNode::get(ctx, operands)});2174 });2175 2176 mlirModule->walk([&](AliasAnalysisOpInterface analysisOpInterface) {2177 if (auto attr = analysisOpInterface.getTBAATagsOrNull())2178 walker.walk(attr);2179 });2180 2181 return success();2182}2183 2184LogicalResult ModuleTranslation::createIdentMetadata() {2185 if (auto attr = mlirModule->getAttrOfType<StringAttr>(2186 LLVMDialect::getIdentAttrName())) {2187 StringRef ident = attr;2188 llvm::LLVMContext &ctx = llvmModule->getContext();2189 llvm::NamedMDNode *namedMd =2190 llvmModule->getOrInsertNamedMetadata(LLVMDialect::getIdentAttrName());2191 llvm::MDNode *md = llvm::MDNode::get(ctx, llvm::MDString::get(ctx, ident));2192 namedMd->addOperand(md);2193 }2194 2195 return success();2196}2197 2198LogicalResult ModuleTranslation::createCommandlineMetadata() {2199 if (auto attr = mlirModule->getAttrOfType<StringAttr>(2200 LLVMDialect::getCommandlineAttrName())) {2201 StringRef cmdLine = attr;2202 llvm::LLVMContext &ctx = llvmModule->getContext();2203 llvm::NamedMDNode *nmd = llvmModule->getOrInsertNamedMetadata(2204 LLVMDialect::getCommandlineAttrName());2205 llvm::MDNode *md =2206 llvm::MDNode::get(ctx, llvm::MDString::get(ctx, cmdLine));2207 nmd->addOperand(md);2208 }2209 2210 return success();2211}2212 2213LogicalResult ModuleTranslation::createDependentLibrariesMetadata() {2214 if (auto dependentLibrariesAttr = mlirModule->getDiscardableAttr(2215 LLVM::LLVMDialect::getDependentLibrariesAttrName())) {2216 auto *nmd =2217 llvmModule->getOrInsertNamedMetadata("llvm.dependent-libraries");2218 llvm::LLVMContext &ctx = llvmModule->getContext();2219 for (auto libAttr :2220 cast<ArrayAttr>(dependentLibrariesAttr).getAsRange<StringAttr>()) {2221 auto *md =2222 llvm::MDNode::get(ctx, llvm::MDString::get(ctx, libAttr.getValue()));2223 nmd->addOperand(md);2224 }2225 }2226 return success();2227}2228 2229void ModuleTranslation::setLoopMetadata(Operation *op,2230 llvm::Instruction *inst) {2231 LoopAnnotationAttr attr =2232 TypeSwitch<Operation *, LoopAnnotationAttr>(op)2233 .Case<LLVM::BrOp, LLVM::CondBrOp>(2234 [](auto branchOp) { return branchOp.getLoopAnnotationAttr(); });2235 if (!attr)2236 return;2237 llvm::MDNode *loopMD =2238 loopAnnotationTranslation->translateLoopAnnotation(attr, op);2239 inst->setMetadata(llvm::LLVMContext::MD_loop, loopMD);2240}2241 2242void ModuleTranslation::setDisjointFlag(Operation *op, llvm::Value *value) {2243 auto iface = cast<DisjointFlagInterface>(op);2244 // We do a dyn_cast here in case the value got folded into a constant.2245 if (auto disjointInst = dyn_cast<llvm::PossiblyDisjointInst>(value))2246 disjointInst->setIsDisjoint(iface.getIsDisjoint());2247}2248 2249llvm::Type *ModuleTranslation::convertType(Type type) {2250 return typeTranslator.translateType(type);2251}2252 2253/// A helper to look up remapped operands in the value remapping table.2254SmallVector<llvm::Value *> ModuleTranslation::lookupValues(ValueRange values) {2255 SmallVector<llvm::Value *> remapped;2256 remapped.reserve(values.size());2257 for (Value v : values)2258 remapped.push_back(lookupValue(v));2259 return remapped;2260}2261 2262llvm::OpenMPIRBuilder *ModuleTranslation::getOpenMPBuilder() {2263 if (!ompBuilder) {2264 ompBuilder = std::make_unique<llvm::OpenMPIRBuilder>(*llvmModule);2265 2266 // Flags represented as top-level OpenMP dialect attributes are set in2267 // `OpenMPDialectLLVMIRTranslationInterface::amendOperation()`. Here we set2268 // the default configuration.2269 llvm::OpenMPIRBuilderConfig config(2270 /* IsTargetDevice = */ false, /* IsGPU = */ false,2271 /* OpenMPOffloadMandatory = */ false,2272 /* HasRequiresReverseOffload = */ false,2273 /* HasRequiresUnifiedAddress = */ false,2274 /* HasRequiresUnifiedSharedMemory = */ false,2275 /* HasRequiresDynamicAllocators = */ false);2276 unsigned int defaultAS =2277 llvmModule->getDataLayout().getProgramAddressSpace();2278 config.setDefaultTargetAS(defaultAS);2279 config.setRuntimeCC(llvmModule->getTargetTriple().isSPIRV()2280 ? llvm::CallingConv::SPIR_FUNC2281 : llvm::CallingConv::C);2282 ompBuilder->setConfig(std::move(config));2283 ompBuilder->initialize();2284 }2285 return ompBuilder.get();2286}2287 2288llvm::DILocation *ModuleTranslation::translateLoc(Location loc,2289 llvm::DILocalScope *scope) {2290 return debugTranslation->translateLoc(loc, scope);2291}2292 2293llvm::DIExpression *2294ModuleTranslation::translateExpression(LLVM::DIExpressionAttr attr) {2295 return debugTranslation->translateExpression(attr);2296}2297 2298llvm::DIGlobalVariableExpression *2299ModuleTranslation::translateGlobalVariableExpression(2300 LLVM::DIGlobalVariableExpressionAttr attr) {2301 return debugTranslation->translateGlobalVariableExpression(attr);2302}2303 2304llvm::Metadata *ModuleTranslation::translateDebugInfo(LLVM::DINodeAttr attr) {2305 return debugTranslation->translate(attr);2306}2307 2308llvm::RoundingMode2309ModuleTranslation::translateRoundingMode(LLVM::RoundingMode rounding) {2310 return convertRoundingModeToLLVM(rounding);2311}2312 2313llvm::fp::ExceptionBehavior ModuleTranslation::translateFPExceptionBehavior(2314 LLVM::FPExceptionBehavior exceptionBehavior) {2315 return convertFPExceptionBehaviorToLLVM(exceptionBehavior);2316}2317 2318llvm::NamedMDNode *2319ModuleTranslation::getOrInsertNamedModuleMetadata(StringRef name) {2320 return llvmModule->getOrInsertNamedMetadata(name);2321}2322 2323static std::unique_ptr<llvm::Module>2324prepareLLVMModule(Operation *m, llvm::LLVMContext &llvmContext,2325 StringRef name) {2326 m->getContext()->getOrLoadDialect<LLVM::LLVMDialect>();2327 auto llvmModule = std::make_unique<llvm::Module>(name, llvmContext);2328 if (auto dataLayoutAttr =2329 m->getDiscardableAttr(LLVM::LLVMDialect::getDataLayoutAttrName())) {2330 llvmModule->setDataLayout(cast<StringAttr>(dataLayoutAttr).getValue());2331 } else {2332 FailureOr<llvm::DataLayout> llvmDataLayout(llvm::DataLayout(""));2333 if (auto iface = dyn_cast<DataLayoutOpInterface>(m)) {2334 if (DataLayoutSpecInterface spec = iface.getDataLayoutSpec()) {2335 llvmDataLayout =2336 translateDataLayout(spec, DataLayout(iface), m->getLoc());2337 }2338 } else if (auto mod = dyn_cast<ModuleOp>(m)) {2339 if (DataLayoutSpecInterface spec = mod.getDataLayoutSpec()) {2340 llvmDataLayout =2341 translateDataLayout(spec, DataLayout(mod), m->getLoc());2342 }2343 }2344 if (failed(llvmDataLayout))2345 return nullptr;2346 llvmModule->setDataLayout(*llvmDataLayout);2347 }2348 if (auto targetTripleAttr =2349 m->getDiscardableAttr(LLVM::LLVMDialect::getTargetTripleAttrName()))2350 llvmModule->setTargetTriple(2351 llvm::Triple(cast<StringAttr>(targetTripleAttr).getValue()));2352 2353 if (auto asmAttr = m->getDiscardableAttr(2354 LLVM::LLVMDialect::getModuleLevelAsmAttrName())) {2355 auto asmArrayAttr = dyn_cast<ArrayAttr>(asmAttr);2356 if (!asmArrayAttr) {2357 m->emitError("expected an array attribute for a module level asm");2358 return nullptr;2359 }2360 2361 for (Attribute elt : asmArrayAttr) {2362 auto asmStrAttr = dyn_cast<StringAttr>(elt);2363 if (!asmStrAttr) {2364 m->emitError(2365 "expected a string attribute for each entry of a module level asm");2366 return nullptr;2367 }2368 llvmModule->appendModuleInlineAsm(asmStrAttr.getValue());2369 }2370 }2371 2372 return llvmModule;2373}2374 2375std::unique_ptr<llvm::Module>2376mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext,2377 StringRef name, bool disableVerification) {2378 if (!satisfiesLLVMModule(module)) {2379 module->emitOpError("can not be translated to an LLVMIR module");2380 return nullptr;2381 }2382 2383 std::unique_ptr<llvm::Module> llvmModule =2384 prepareLLVMModule(module, llvmContext, name);2385 if (!llvmModule)2386 return nullptr;2387 2388 LLVM::ensureDistinctSuccessors(module);2389 LLVM::legalizeDIExpressionsRecursively(module);2390 2391 ModuleTranslation translator(module, std::move(llvmModule));2392 llvm::IRBuilder<llvm::TargetFolder> llvmBuilder(2393 llvmContext,2394 llvm::TargetFolder(translator.getLLVMModule()->getDataLayout()));2395 2396 // Convert module before functions and operations inside, so dialect2397 // attributes can be used to change dialect-specific global configurations via2398 // `amendOperation()`. These configurations can then influence the translation2399 // of operations afterwards.2400 if (failed(translator.convertOperation(*module, llvmBuilder)))2401 return nullptr;2402 2403 if (failed(translator.convertComdats()))2404 return nullptr;2405 if (failed(translator.convertFunctionSignatures()))2406 return nullptr;2407 if (failed(translator.convertGlobalsAndAliases()))2408 return nullptr;2409 if (failed(translator.convertIFuncs()))2410 return nullptr;2411 if (failed(translator.createTBAAMetadata()))2412 return nullptr;2413 if (failed(translator.createIdentMetadata()))2414 return nullptr;2415 if (failed(translator.createCommandlineMetadata()))2416 return nullptr;2417 if (failed(translator.createDependentLibrariesMetadata()))2418 return nullptr;2419 2420 // Convert other top-level operations if possible.2421 for (Operation &o : getModuleBody(module).getOperations()) {2422 if (!isa<LLVM::LLVMFuncOp, LLVM::AliasOp, LLVM::GlobalOp,2423 LLVM::GlobalCtorsOp, LLVM::GlobalDtorsOp, LLVM::ComdatOp,2424 LLVM::IFuncOp>(&o) &&2425 !o.hasTrait<OpTrait::IsTerminator>() &&2426 failed(translator.convertOperation(o, llvmBuilder))) {2427 return nullptr;2428 }2429 }2430 2431 // Operations in function bodies with symbolic references must be converted2432 // after the top-level operations they refer to are declared, so we do it2433 // last.2434 if (failed(translator.convertFunctions()))2435 return nullptr;2436 2437 // Now that all MLIR blocks are resolved into LLVM ones, patch block address2438 // constants to point to the correct blocks.2439 if (failed(translator.convertUnresolvedBlockAddress()))2440 return nullptr;2441 2442 // Add the necessary debug info module flags, if they were not encoded in MLIR2443 // beforehand.2444 translator.debugTranslation->addModuleFlagsIfNotPresent();2445 2446 // Call the OpenMP IR Builder callbacks prior to verifying the module2447 if (auto *ompBuilder = translator.getOpenMPBuilder())2448 ompBuilder->finalize();2449 2450 if (!disableVerification &&2451 llvm::verifyModule(*translator.llvmModule, &llvm::errs()))2452 return nullptr;2453 2454 return std::move(translator.llvmModule);2455}2456