947 lines · cpp
1//===- DataLayoutInterfaces.cpp - Data Layout Interface Implementation ----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "mlir/Interfaces/DataLayoutInterfaces.h"10#include "mlir/IR/BuiltinDialect.h"11#include "mlir/IR/BuiltinOps.h"12#include "mlir/IR/BuiltinTypes.h"13#include "mlir/IR/Operation.h"14 15#include "llvm/ADT/TypeSwitch.h"16#include "llvm/Support/MathExtras.h"17 18using namespace mlir;19 20//===----------------------------------------------------------------------===//21// Default implementations22//===----------------------------------------------------------------------===//23 24/// Reports that the given type is missing the data layout information and25/// exits.26[[noreturn]] static void reportMissingDataLayout(Type type) {27 std::string message;28 llvm::raw_string_ostream os(message);29 os << "neither the scoping op nor the type class provide data layout "30 "information for "31 << type;32 llvm::report_fatal_error(Twine(message));33}34 35/// Returns the bitwidth of the index type if specified in the param list.36/// Assumes 64-bit index otherwise.37static uint64_t getIndexBitwidth(DataLayoutEntryListRef params) {38 if (params.empty())39 return 64;40 auto attr = cast<IntegerAttr>(params.front().getValue());41 return attr.getValue().getZExtValue();42}43 44llvm::TypeSize45mlir::detail::getDefaultTypeSize(Type type, const DataLayout &dataLayout,46 ArrayRef<DataLayoutEntryInterface> params) {47 llvm::TypeSize bits = getDefaultTypeSizeInBits(type, dataLayout, params);48 return divideCeil(bits, 8);49}50 51llvm::TypeSize52mlir::detail::getDefaultTypeSizeInBits(Type type, const DataLayout &dataLayout,53 DataLayoutEntryListRef params) {54 if (type.isIntOrFloat())55 return llvm::TypeSize::getFixed(type.getIntOrFloatBitWidth());56 57 if (auto ctype = dyn_cast<ComplexType>(type)) {58 Type et = ctype.getElementType();59 uint64_t innerAlignment =60 getDefaultPreferredAlignment(et, dataLayout, params) * 8;61 llvm::TypeSize innerSize = getDefaultTypeSizeInBits(et, dataLayout, params);62 63 // Include padding required to align the imaginary value in the complex64 // type.65 return llvm::alignTo(innerSize, innerAlignment) + innerSize;66 }67 68 // Index is an integer of some bitwidth.69 if (isa<IndexType>(type))70 return dataLayout.getTypeSizeInBits(71 IntegerType::get(type.getContext(), getIndexBitwidth(params)));72 73 // Sizes of vector types are rounded up to those of types with closest74 // power-of-two number of elements in the innermost dimension. We also assume75 // there is no bit-packing at the moment element sizes are taken in bytes and76 // multiplied with 8 bits.77 // TODO: make this extensible.78 if (auto vecType = dyn_cast<VectorType>(type)) {79 uint64_t baseSize = vecType.getNumElements() / vecType.getShape().back() *80 llvm::PowerOf2Ceil(vecType.getShape().back()) *81 dataLayout.getTypeSize(vecType.getElementType()) * 8;82 return llvm::TypeSize::get(baseSize, vecType.isScalable());83 }84 85 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))86 return typeInterface.getTypeSizeInBits(dataLayout, params);87 88 reportMissingDataLayout(type);89}90 91static DataLayoutEntryInterface92findEntryForIntegerType(IntegerType intType,93 ArrayRef<DataLayoutEntryInterface> params) {94 assert(!params.empty() && "expected non-empty parameter list");95 std::map<unsigned, DataLayoutEntryInterface> sortedParams;96 for (DataLayoutEntryInterface entry : params) {97 sortedParams.insert(std::make_pair(98 cast<Type>(entry.getKey()).getIntOrFloatBitWidth(), entry));99 }100 auto iter = sortedParams.lower_bound(intType.getWidth());101 if (iter == sortedParams.end())102 iter = std::prev(iter);103 104 return iter->second;105}106 107constexpr const static uint64_t kDefaultBitsInByte = 8u;108 109static uint64_t extractABIAlignment(DataLayoutEntryInterface entry) {110 auto values =111 cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();112 return static_cast<uint64_t>(*values.begin()) / kDefaultBitsInByte;113}114 115static uint64_t116getIntegerTypeABIAlignment(IntegerType intType,117 ArrayRef<DataLayoutEntryInterface> params) {118 constexpr uint64_t kDefaultSmallIntAlignment = 4u;119 constexpr unsigned kSmallIntSize = 64;120 if (params.empty()) {121 return intType.getWidth() < kSmallIntSize122 ? llvm::PowerOf2Ceil(123 llvm::divideCeil(intType.getWidth(), kDefaultBitsInByte))124 : kDefaultSmallIntAlignment;125 }126 127 return extractABIAlignment(findEntryForIntegerType(intType, params));128}129 130static uint64_t131getFloatTypeABIAlignment(FloatType fltType, const DataLayout &dataLayout,132 ArrayRef<DataLayoutEntryInterface> params) {133 assert(params.size() <= 1 && "at most one data layout entry is expected for "134 "the singleton floating-point type");135 if (params.empty())136 return llvm::PowerOf2Ceil(dataLayout.getTypeSize(fltType).getFixedValue());137 return extractABIAlignment(params[0]);138}139 140uint64_t mlir::detail::getDefaultABIAlignment(141 Type type, const DataLayout &dataLayout,142 ArrayRef<DataLayoutEntryInterface> params) {143 // Natural alignment is the closest power-of-two number above. For scalable144 // vectors, aligning them to the same as the base vector is sufficient.145 if (isa<VectorType>(type))146 return llvm::PowerOf2Ceil(dataLayout.getTypeSize(type).getKnownMinValue());147 148 if (auto fltType = dyn_cast<FloatType>(type))149 return getFloatTypeABIAlignment(fltType, dataLayout, params);150 151 // Index is an integer of some bitwidth.152 if (isa<IndexType>(type))153 return dataLayout.getTypeABIAlignment(154 IntegerType::get(type.getContext(), getIndexBitwidth(params)));155 156 if (auto intType = dyn_cast<IntegerType>(type))157 return getIntegerTypeABIAlignment(intType, params);158 159 if (auto ctype = dyn_cast<ComplexType>(type))160 return getDefaultABIAlignment(ctype.getElementType(), dataLayout, params);161 162 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))163 return typeInterface.getABIAlignment(dataLayout, params);164 165 reportMissingDataLayout(type);166}167 168static uint64_t extractPreferredAlignment(DataLayoutEntryInterface entry) {169 auto values =170 cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();171 return *std::next(values.begin(), values.size() - 1) / kDefaultBitsInByte;172}173 174static uint64_t175getIntegerTypePreferredAlignment(IntegerType intType,176 const DataLayout &dataLayout,177 ArrayRef<DataLayoutEntryInterface> params) {178 if (params.empty())179 return llvm::PowerOf2Ceil(dataLayout.getTypeSize(intType).getFixedValue());180 181 return extractPreferredAlignment(findEntryForIntegerType(intType, params));182}183 184static uint64_t185getFloatTypePreferredAlignment(FloatType fltType, const DataLayout &dataLayout,186 ArrayRef<DataLayoutEntryInterface> params) {187 assert(params.size() <= 1 && "at most one data layout entry is expected for "188 "the singleton floating-point type");189 if (params.empty())190 return dataLayout.getTypeABIAlignment(fltType);191 return extractPreferredAlignment(params[0]);192}193 194uint64_t mlir::detail::getDefaultPreferredAlignment(195 Type type, const DataLayout &dataLayout,196 ArrayRef<DataLayoutEntryInterface> params) {197 // Preferred alignment is same as natural for floats and vectors.198 if (isa<VectorType>(type))199 return dataLayout.getTypeABIAlignment(type);200 201 if (auto fltType = dyn_cast<FloatType>(type))202 return getFloatTypePreferredAlignment(fltType, dataLayout, params);203 204 // Preferred alignment is the closest power-of-two number above for integers205 // (ABI alignment may be smaller).206 if (auto intType = dyn_cast<IntegerType>(type))207 return getIntegerTypePreferredAlignment(intType, dataLayout, params);208 209 if (isa<IndexType>(type)) {210 return dataLayout.getTypePreferredAlignment(211 IntegerType::get(type.getContext(), getIndexBitwidth(params)));212 }213 214 if (auto ctype = dyn_cast<ComplexType>(type))215 return getDefaultPreferredAlignment(ctype.getElementType(), dataLayout,216 params);217 218 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))219 return typeInterface.getPreferredAlignment(dataLayout, params);220 221 reportMissingDataLayout(type);222}223 224std::optional<uint64_t> mlir::detail::getDefaultIndexBitwidth(225 Type type, const DataLayout &dataLayout,226 ArrayRef<DataLayoutEntryInterface> params) {227 if (isa<IndexType>(type))228 return getIndexBitwidth(params);229 230 if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))231 if (std::optional<uint64_t> indexBitwidth =232 typeInterface.getIndexBitwidth(dataLayout, params))233 return *indexBitwidth;234 235 // Return std::nullopt for all other types, which are assumed to be non236 // pointer-like types.237 return std::nullopt;238}239 240// Returns the endianness if specified in the given entry. If the entry is empty241// the default endianness represented by an empty attribute is returned.242Attribute mlir::detail::getDefaultEndianness(DataLayoutEntryInterface entry) {243 if (entry == DataLayoutEntryInterface())244 return Attribute();245 246 return entry.getValue();247}248 249// Returns the default memory space if specified in the given entry. If the250// entry is empty the default memory space represented by an empty attribute is251// returned.252Attribute mlir::detail::getDefaultMemorySpace(DataLayoutEntryInterface entry) {253 if (!entry)254 return Attribute();255 256 return entry.getValue();257}258 259// Returns the memory space used for alloca operations if specified in the260// given entry. If the entry is empty the default memory space represented by261// an empty attribute is returned.262Attribute263mlir::detail::getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry) {264 if (entry == DataLayoutEntryInterface()) {265 return Attribute();266 }267 268 return entry.getValue();269}270 271// Returns the mangling mode if specified in the given entry.272// If the entry is empty, an empty attribute is returned.273Attribute mlir::detail::getDefaultManglingMode(DataLayoutEntryInterface entry) {274 if (entry == DataLayoutEntryInterface())275 return Attribute();276 277 return entry.getValue();278}279 280// Returns the memory space used for the program memory space. if281// specified in the given entry. If the entry is empty the default282// memory space represented by an empty attribute is returned.283Attribute284mlir::detail::getDefaultProgramMemorySpace(DataLayoutEntryInterface entry) {285 if (entry == DataLayoutEntryInterface()) {286 return Attribute();287 }288 289 return entry.getValue();290}291 292// Returns the memory space used for global the global memory space. if293// specified in the given entry. If the entry is empty the default memory294// space represented by an empty attribute is returned.295Attribute296mlir::detail::getDefaultGlobalMemorySpace(DataLayoutEntryInterface entry) {297 if (entry == DataLayoutEntryInterface()) {298 return Attribute();299 }300 301 return entry.getValue();302}303 304// Returns the stack alignment if specified in the given entry. If the entry is305// empty the default alignment zero is returned.306uint64_t307mlir::detail::getDefaultStackAlignment(DataLayoutEntryInterface entry) {308 if (entry == DataLayoutEntryInterface())309 return 0;310 311 auto value = cast<IntegerAttr>(entry.getValue());312 return value.getValue().getZExtValue();313}314 315// Returns the function pointer alignment if specified in the given entry. If316// the entry is empty the default alignment zero is returned.317Attribute mlir::detail::getDefaultFunctionPointerAlignment(318 DataLayoutEntryInterface entry) {319 if (entry == DataLayoutEntryInterface())320 return Attribute();321 return entry.getValue();322}323 324// Returns the legal int widths if specified in the given entry. If the entry is325// empty the default legal int widths represented by an empty attribute is326// returned.327Attribute328mlir::detail::getDefaultLegalIntWidths(DataLayoutEntryInterface entry) {329 if (entry == DataLayoutEntryInterface())330 return Attribute();331 return entry.getValue();332}333 334std::optional<Attribute>335mlir::detail::getDevicePropertyValue(DataLayoutEntryInterface entry) {336 if (entry == DataLayoutEntryInterface())337 return std::nullopt;338 339 return entry.getValue();340}341 342DataLayoutEntryList343mlir::detail::filterEntriesForType(DataLayoutEntryListRef entries,344 TypeID typeID) {345 return llvm::filter_to_vector<4>(346 entries, [typeID](DataLayoutEntryInterface entry) {347 auto type = llvm::dyn_cast_if_present<Type>(entry.getKey());348 return type && type.getTypeID() == typeID;349 });350}351 352DataLayoutEntryInterface353mlir::detail::filterEntryForIdentifier(DataLayoutEntryListRef entries,354 StringAttr id) {355 const auto *it = llvm::find_if(entries, [id](DataLayoutEntryInterface entry) {356 if (auto attr = dyn_cast<StringAttr>(entry.getKey()))357 return attr == id;358 return false;359 });360 return it == entries.end() ? DataLayoutEntryInterface() : *it;361}362 363static DataLayoutSpecInterface getSpec(Operation *operation) {364 return llvm::TypeSwitch<Operation *, DataLayoutSpecInterface>(operation)365 .Case<ModuleOp, DataLayoutOpInterface>(366 [&](auto op) { return op.getDataLayoutSpec(); })367 .DefaultUnreachable("expected an op with data layout spec");368}369 370static TargetSystemSpecInterface getTargetSystemSpec(Operation *operation) {371 if (operation) {372 ModuleOp moduleOp = dyn_cast<ModuleOp>(operation);373 if (!moduleOp)374 moduleOp = operation->getParentOfType<ModuleOp>();375 return moduleOp.getTargetSystemSpec();376 }377 return TargetSystemSpecInterface();378}379 380/// Populates `opsWithLayout` with the list of proper ancestors of `leaf` that381/// are either modules or implement the `DataLayoutOpInterface`.382static void383collectParentLayouts(Operation *leaf,384 SmallVectorImpl<DataLayoutSpecInterface> &specs,385 SmallVectorImpl<Location> *opLocations = nullptr) {386 if (!leaf)387 return;388 389 for (Operation *parent = leaf->getParentOp(); parent != nullptr;390 parent = parent->getParentOp()) {391 llvm::TypeSwitch<Operation *>(parent)392 .Case<ModuleOp>([&](ModuleOp op) {393 // Skip top-level module op unless it has a layout. Top-level module394 // without layout is most likely the one implicitly added by the395 // parser and it doesn't have location. Top-level null specification396 // would have had the same effect as not having a specification at all397 // (using type defaults).398 if (!op->getParentOp() && !op.getDataLayoutSpec())399 return;400 specs.push_back(op.getDataLayoutSpec());401 if (opLocations)402 opLocations->push_back(op.getLoc());403 })404 .Case<DataLayoutOpInterface>([&](DataLayoutOpInterface op) {405 specs.push_back(op.getDataLayoutSpec());406 if (opLocations)407 opLocations->push_back(op.getLoc());408 });409 }410}411 412/// Returns a layout spec that is a combination of the layout specs attached413/// to the given operation and all its ancestors.414static DataLayoutSpecInterface getCombinedDataLayout(Operation *leaf) {415 if (!leaf)416 return {};417 418 assert((isa<ModuleOp, DataLayoutOpInterface>(leaf)) &&419 "expected an op with data layout spec");420 421 SmallVector<DataLayoutSpecInterface> specs;422 collectParentLayouts(leaf, specs);423 424 // Fast track if there are no ancestors.425 if (specs.empty())426 return getSpec(leaf);427 428 // Create the list of non-null specs (null/missing specs can be safely429 // ignored) from the outermost to the innermost.430 auto nonNullSpecs = llvm::filter_to_vector<2>(431 llvm::reverse(specs),432 [](DataLayoutSpecInterface iface) { return iface != nullptr; });433 434 // Combine the specs using the innermost as anchor.435 if (DataLayoutSpecInterface current = getSpec(leaf))436 return current.combineWith(nonNullSpecs);437 if (nonNullSpecs.empty())438 return {};439 return nonNullSpecs.back().combineWith(440 llvm::ArrayRef(nonNullSpecs).drop_back());441}442 443LogicalResult mlir::detail::verifyDataLayoutOp(Operation *op) {444 DataLayoutSpecInterface spec = getSpec(op);445 // The layout specification may be missing and it's fine.446 if (!spec)447 return success();448 449 if (failed(spec.verifySpec(op->getLoc())))450 return failure();451 if (!getCombinedDataLayout(op)) {452 InFlightDiagnostic diag =453 op->emitError()454 << "data layout does not combine with layouts of enclosing ops";455 SmallVector<DataLayoutSpecInterface> specs;456 SmallVector<Location> opLocations;457 collectParentLayouts(op, specs, &opLocations);458 for (Location loc : opLocations)459 diag.attachNote(loc) << "enclosing op with data layout";460 return diag;461 }462 return success();463}464 465llvm::TypeSize mlir::detail::divideCeil(llvm::TypeSize numerator,466 uint64_t denominator) {467 uint64_t divided =468 llvm::divideCeil(numerator.getKnownMinValue(), denominator);469 return llvm::TypeSize::get(divided, numerator.isScalable());470}471 472//===----------------------------------------------------------------------===//473// DataLayout474//===----------------------------------------------------------------------===//475 476template <typename OpTy>477void checkMissingLayout(DataLayoutSpecInterface originalLayout, OpTy op) {478 if (!originalLayout) {479 assert((!op || !op.getDataLayoutSpec()) &&480 "could not compute layout information for an op (failed to "481 "combine attributes?)");482 }483}484 485mlir::DataLayout::DataLayout() : DataLayout(ModuleOp()) {}486 487mlir::DataLayout::DataLayout(DataLayoutOpInterface op)488 : originalLayout(getCombinedDataLayout(op)),489 originalTargetSystemDesc(getTargetSystemSpec(op)), scope(op),490 allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),491 globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {492#if LLVM_ENABLE_ABI_BREAKING_CHECKS493 checkMissingLayout(originalLayout, op);494 collectParentLayouts(op, layoutStack);495#endif496}497 498mlir::DataLayout::DataLayout(ModuleOp op)499 : originalLayout(getCombinedDataLayout(op)),500 originalTargetSystemDesc(getTargetSystemSpec(op)), scope(op),501 allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),502 globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {503#if LLVM_ENABLE_ABI_BREAKING_CHECKS504 checkMissingLayout(originalLayout, op);505 collectParentLayouts(op, layoutStack);506#endif507}508 509mlir::DataLayout mlir::DataLayout::closest(Operation *op) {510 // Search the closest parent either being a module operation or implementing511 // the data layout interface.512 while (op) {513 if (auto module = dyn_cast<ModuleOp>(op))514 return DataLayout(module);515 if (auto iface = dyn_cast<DataLayoutOpInterface>(op))516 return DataLayout(iface);517 op = op->getParentOp();518 }519 return DataLayout();520}521 522void mlir::DataLayout::checkValid() const {523#if LLVM_ENABLE_ABI_BREAKING_CHECKS524 SmallVector<DataLayoutSpecInterface> specs;525 collectParentLayouts(scope, specs);526 assert(specs.size() == layoutStack.size() &&527 "data layout object used, but no longer valid due to the change in "528 "number of nested layouts");529 for (auto pair : llvm::zip(specs, layoutStack)) {530 Attribute newLayout = std::get<0>(pair);531 Attribute origLayout = std::get<1>(pair);532 assert(newLayout == origLayout &&533 "data layout object used, but no longer valid "534 "due to the change in layout attributes");535 }536#endif537 assert(((!scope && !this->originalLayout) ||538 (scope && this->originalLayout == getCombinedDataLayout(scope))) &&539 "data layout object used, but no longer valid due to the change in "540 "layout spec");541}542 543/// Looks up the value for the given type key in the given cache. If there is no544/// such value in the cache, compute it using the given callback and put it in545/// the cache before returning.546template <typename T>547static T cachedLookup(Type t, DenseMap<Type, T> &cache,548 function_ref<T(Type)> compute) {549 auto it = cache.find(t);550 if (it != cache.end())551 return it->second;552 553 auto result = cache.try_emplace(t, compute(t));554 return result.first->second;555}556 557llvm::TypeSize mlir::DataLayout::getTypeSize(Type t) const {558 checkValid();559 return cachedLookup<llvm::TypeSize>(t, sizes, [&](Type ty) {560 DataLayoutEntryList list;561 if (originalLayout)562 list = originalLayout.getSpecForType(ty.getTypeID());563 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))564 return iface.getTypeSize(ty, *this, list);565 return detail::getDefaultTypeSize(ty, *this, list);566 });567}568 569llvm::TypeSize mlir::DataLayout::getTypeSizeInBits(Type t) const {570 checkValid();571 return cachedLookup<llvm::TypeSize>(t, bitsizes, [&](Type ty) {572 DataLayoutEntryList list;573 if (originalLayout)574 list = originalLayout.getSpecForType(ty.getTypeID());575 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))576 return iface.getTypeSizeInBits(ty, *this, list);577 return detail::getDefaultTypeSizeInBits(ty, *this, list);578 });579}580 581uint64_t mlir::DataLayout::getTypeABIAlignment(Type t) const {582 checkValid();583 return cachedLookup<uint64_t>(t, abiAlignments, [&](Type ty) {584 DataLayoutEntryList list;585 if (originalLayout)586 list = originalLayout.getSpecForType(ty.getTypeID());587 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))588 return iface.getTypeABIAlignment(ty, *this, list);589 return detail::getDefaultABIAlignment(ty, *this, list);590 });591}592 593uint64_t mlir::DataLayout::getTypePreferredAlignment(Type t) const {594 checkValid();595 return cachedLookup<uint64_t>(t, preferredAlignments, [&](Type ty) {596 DataLayoutEntryList list;597 if (originalLayout)598 list = originalLayout.getSpecForType(ty.getTypeID());599 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))600 return iface.getTypePreferredAlignment(ty, *this, list);601 return detail::getDefaultPreferredAlignment(ty, *this, list);602 });603}604 605std::optional<uint64_t> mlir::DataLayout::getTypeIndexBitwidth(Type t) const {606 checkValid();607 return cachedLookup<std::optional<uint64_t>>(t, indexBitwidths, [&](Type ty) {608 DataLayoutEntryList list;609 if (originalLayout)610 list = originalLayout.getSpecForType(ty.getTypeID());611 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))612 return iface.getIndexBitwidth(ty, *this, list);613 return detail::getDefaultIndexBitwidth(ty, *this, list);614 });615}616 617mlir::Attribute mlir::DataLayout::getEndianness() const {618 checkValid();619 if (endianness)620 return *endianness;621 DataLayoutEntryInterface entry;622 if (originalLayout)623 entry = originalLayout.getSpecForIdentifier(624 originalLayout.getEndiannessIdentifier(originalLayout.getContext()));625 626 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))627 endianness = iface.getEndianness(entry);628 else629 endianness = detail::getDefaultEndianness(entry);630 return *endianness;631}632 633mlir::Attribute mlir::DataLayout::getDefaultMemorySpace() const {634 checkValid();635 if (defaultMemorySpace)636 return *defaultMemorySpace;637 DataLayoutEntryInterface entry;638 if (originalLayout)639 entry = originalLayout.getSpecForIdentifier(640 originalLayout.getDefaultMemorySpaceIdentifier(641 originalLayout.getContext()));642 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))643 defaultMemorySpace = iface.getDefaultMemorySpace(entry);644 else645 defaultMemorySpace = detail::getDefaultMemorySpace(entry);646 return *defaultMemorySpace;647}648 649mlir::Attribute mlir::DataLayout::getAllocaMemorySpace() const {650 checkValid();651 if (allocaMemorySpace)652 return *allocaMemorySpace;653 DataLayoutEntryInterface entry;654 if (originalLayout)655 entry = originalLayout.getSpecForIdentifier(656 originalLayout.getAllocaMemorySpaceIdentifier(657 originalLayout.getContext()));658 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))659 allocaMemorySpace = iface.getAllocaMemorySpace(entry);660 else661 allocaMemorySpace = detail::getDefaultAllocaMemorySpace(entry);662 return *allocaMemorySpace;663}664 665mlir::Attribute mlir::DataLayout::getManglingMode() const {666 checkValid();667 if (manglingMode)668 return *manglingMode;669 DataLayoutEntryInterface entry;670 if (originalLayout)671 entry = originalLayout.getSpecForIdentifier(672 originalLayout.getManglingModeIdentifier(originalLayout.getContext()));673 674 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))675 manglingMode = iface.getManglingMode(entry);676 else677 manglingMode = detail::getDefaultManglingMode(entry);678 return *manglingMode;679}680 681mlir::Attribute mlir::DataLayout::getProgramMemorySpace() const {682 checkValid();683 if (programMemorySpace)684 return *programMemorySpace;685 DataLayoutEntryInterface entry;686 if (originalLayout)687 entry = originalLayout.getSpecForIdentifier(688 originalLayout.getProgramMemorySpaceIdentifier(689 originalLayout.getContext()));690 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))691 programMemorySpace = iface.getProgramMemorySpace(entry);692 else693 programMemorySpace = detail::getDefaultProgramMemorySpace(entry);694 return *programMemorySpace;695}696 697mlir::Attribute mlir::DataLayout::getGlobalMemorySpace() const {698 checkValid();699 if (globalMemorySpace)700 return *globalMemorySpace;701 DataLayoutEntryInterface entry;702 if (originalLayout)703 entry = originalLayout.getSpecForIdentifier(704 originalLayout.getGlobalMemorySpaceIdentifier(705 originalLayout.getContext()));706 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))707 globalMemorySpace = iface.getGlobalMemorySpace(entry);708 else709 globalMemorySpace = detail::getDefaultGlobalMemorySpace(entry);710 return *globalMemorySpace;711}712 713uint64_t mlir::DataLayout::getStackAlignment() const {714 checkValid();715 if (stackAlignment)716 return *stackAlignment;717 DataLayoutEntryInterface entry;718 if (originalLayout)719 entry = originalLayout.getSpecForIdentifier(720 originalLayout.getStackAlignmentIdentifier(721 originalLayout.getContext()));722 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))723 stackAlignment = iface.getStackAlignment(entry);724 else725 stackAlignment = detail::getDefaultStackAlignment(entry);726 return *stackAlignment;727}728 729Attribute mlir::DataLayout::getFunctionPointerAlignment() const {730 checkValid();731 if (functionPointerAlignment)732 return *functionPointerAlignment;733 DataLayoutEntryInterface entry;734 if (originalLayout)735 entry = originalLayout.getSpecForIdentifier(736 originalLayout.getFunctionPointerAlignmentIdentifier(737 originalLayout.getContext()));738 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))739 functionPointerAlignment = iface.getFunctionPointerAlignment(entry);740 else741 functionPointerAlignment =742 detail::getDefaultFunctionPointerAlignment(entry);743 return *functionPointerAlignment;744}745 746Attribute mlir::DataLayout::getLegalIntWidths() const {747 checkValid();748 if (legalIntWidths)749 return *legalIntWidths;750 DataLayoutEntryInterface entry;751 if (originalLayout)752 entry = originalLayout.getSpecForIdentifier(753 originalLayout.getLegalIntWidthsIdentifier(754 originalLayout.getContext()));755 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))756 legalIntWidths = iface.getLegalIntWidths(entry);757 else758 legalIntWidths = detail::getDefaultLegalIntWidths(entry);759 return *legalIntWidths;760}761 762std::optional<Attribute> mlir::DataLayout::getDevicePropertyValue(763 TargetSystemSpecInterface::DeviceID deviceID,764 StringAttr propertyName) const {765 checkValid();766 DataLayoutEntryInterface entry;767 if (originalTargetSystemDesc) {768 if (std::optional<TargetDeviceSpecInterface> device =769 originalTargetSystemDesc.getDeviceSpecForDeviceID(deviceID))770 entry = device->getSpecForIdentifier(propertyName);771 }772 // Currently I am not caching the results because we do not return773 // default values of these properties. Instead if the property is774 // missing, we return std::nullopt so that the users can resort to775 // the default value however they want.776 if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))777 return iface.getDevicePropertyValue(entry);778 else779 return detail::getDevicePropertyValue(entry);780}781 782//===----------------------------------------------------------------------===//783// DataLayoutSpecInterface784//===----------------------------------------------------------------------===//785 786void DataLayoutSpecInterface::bucketEntriesByType(787 llvm::MapVector<TypeID, DataLayoutEntryList> &types,788 llvm::MapVector<StringAttr, DataLayoutEntryInterface> &ids) {789 for (DataLayoutEntryInterface entry : getEntries()) {790 if (auto type = llvm::dyn_cast_if_present<Type>(entry.getKey()))791 types[type.getTypeID()].push_back(entry);792 else793 ids[llvm::cast<StringAttr>(entry.getKey())] = entry;794 }795}796 797LogicalResult mlir::detail::verifyDataLayoutSpec(DataLayoutSpecInterface spec,798 Location loc) {799 // First, verify individual entries.800 for (DataLayoutEntryInterface entry : spec.getEntries())801 if (failed(entry.verifyEntry(loc)))802 return failure();803 804 // Second, dispatch verifications of entry groups to types or dialects they805 // are associated with.806 llvm::MapVector<TypeID, DataLayoutEntryList> types;807 llvm::MapVector<StringAttr, DataLayoutEntryInterface> ids;808 spec.bucketEntriesByType(types, ids);809 810 for (const auto &kvp : types) {811 auto sampleType = cast<Type>(kvp.second.front().getKey());812 if (isa<IndexType>(sampleType)) {813 assert(kvp.second.size() == 1 &&814 "expected one data layout entry for non-parametric 'index' type");815 if (!isa<IntegerAttr>(kvp.second.front().getValue()))816 return emitError(loc)817 << "expected integer attribute in the data layout entry for "818 << sampleType;819 continue;820 }821 822 if (sampleType.isIntOrFloat()) {823 for (DataLayoutEntryInterface entry : kvp.second) {824 auto value = dyn_cast<DenseIntElementsAttr>(entry.getValue());825 if (!value || !value.getElementType().isSignlessInteger(64)) {826 emitError(loc) << "expected a dense i64 elements attribute in the "827 "data layout entry "828 << entry;829 return failure();830 }831 832 auto elements = llvm::to_vector<2>(value.getValues<uint64_t>());833 unsigned numElements = elements.size();834 if (numElements < 1 || numElements > 2) {835 emitError(loc) << "expected 1 or 2 elements in the data layout entry "836 << entry;837 return failure();838 }839 840 uint64_t abi = elements[0];841 uint64_t preferred = numElements == 2 ? elements[1] : abi;842 if (preferred < abi) {843 emitError(loc)844 << "preferred alignment is expected to be greater than or equal "845 "to the abi alignment in data layout entry "846 << entry;847 return failure();848 }849 }850 continue;851 }852 853 if (isa<BuiltinDialect>(&sampleType.getDialect()))854 return emitError(loc) << "unexpected data layout for a built-in type";855 856 auto dlType = dyn_cast<DataLayoutTypeInterface>(sampleType);857 if (!dlType)858 return emitError(loc)859 << "data layout specified for a type that does not support it";860 if (failed(dlType.verifyEntries(kvp.second, loc)))861 return failure();862 }863 864 for (const auto &kvp : ids) {865 StringAttr identifier = cast<StringAttr>(kvp.second.getKey());866 Dialect *dialect = identifier.getReferencedDialect();867 868 // Ignore attributes that belong to an unknown dialect, the dialect may869 // actually implement the relevant interface but we don't know about that.870 if (!dialect)871 continue;872 873 const auto *iface = dyn_cast<DataLayoutDialectInterface>(dialect);874 if (!iface) {875 return emitError(loc)876 << "the '" << dialect->getNamespace()877 << "' dialect does not support identifier data layout entries";878 }879 if (failed(iface->verifyEntry(kvp.second, loc)))880 return failure();881 }882 883 return success();884}885 886LogicalResult887mlir::detail::verifyTargetSystemSpec(TargetSystemSpecInterface spec,888 Location loc) {889 DenseMap<StringAttr, DataLayoutEntryInterface> deviceDescKeys;890 DenseSet<TargetSystemSpecInterface::DeviceID> deviceIDs;891 for (const auto &entry : spec.getEntries()) {892 auto targetDeviceSpec =893 dyn_cast<TargetDeviceSpecInterface>(entry.getValue());894 895 if (!targetDeviceSpec)896 return failure();897 898 // First, verify individual target device desc specs.899 if (failed(targetDeviceSpec.verifyEntry(loc)))900 return failure();901 902 // Check that device IDs are unique across all entries.903 auto deviceID =904 llvm::dyn_cast<TargetSystemSpecInterface::DeviceID>(entry.getKey());905 if (!deviceID)906 return failure();907 908 if (!deviceIDs.insert(deviceID).second) {909 return failure();910 }911 912 // collect all the keys used by all the target device specs.913 for (DataLayoutEntryInterface entry : targetDeviceSpec.getEntries()) {914 if (auto type = llvm::dyn_cast_if_present<Type>(entry.getKey())) {915 // targetDeviceSpec does not support Type as a key.916 return failure();917 } else {918 deviceDescKeys[cast<StringAttr>(entry.getKey())] = entry;919 }920 }921 }922 923 for (const auto &[keyName, keyVal] : deviceDescKeys) {924 Dialect *dialect = keyName.getReferencedDialect();925 926 // Ignore attributes that belong to an unknown dialect, the dialect may927 // actually implement the relevant interface but we don't know about that.928 if (!dialect)929 return failure();930 931 const auto *iface = dyn_cast<DataLayoutDialectInterface>(dialect);932 if (!iface) {933 return emitError(loc)934 << "the '" << dialect->getNamespace()935 << "' dialect does not support identifier data layout entries";936 }937 if (failed(iface->verifyEntry(keyVal, loc)))938 return failure();939 }940 941 return success();942}943 944#include "mlir/Interfaces/DataLayoutAttrInterface.cpp.inc"945#include "mlir/Interfaces/DataLayoutOpInterface.cpp.inc"946#include "mlir/Interfaces/DataLayoutTypeInterface.cpp.inc"947