1062 lines · cpp
1//===- IntrinsicEmitter.cpp - Generate intrinsic information --------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This tablegen backend emits information about intrinsic functions.10//11//===----------------------------------------------------------------------===//12 13#include "CodeGenIntrinsics.h"14#include "SequenceToOffsetTable.h"15#include "llvm/ADT/STLExtras.h"16#include "llvm/ADT/SmallVector.h"17#include "llvm/ADT/StringRef.h"18#include "llvm/ADT/Twine.h"19#include "llvm/Support/CommandLine.h"20#include "llvm/Support/ErrorHandling.h"21#include "llvm/Support/FormatVariadic.h"22#include "llvm/Support/ModRef.h"23#include "llvm/Support/SourceMgr.h"24#include "llvm/Support/raw_ostream.h"25#include "llvm/TableGen/Error.h"26#include "llvm/TableGen/Record.h"27#include "llvm/TableGen/StringToOffsetTable.h"28#include "llvm/TableGen/TableGenBackend.h"29#include <algorithm>30#include <array>31#include <cassert>32#include <cctype>33#include <map>34#include <optional>35#include <string>36#include <utility>37#include <vector>38using namespace llvm;39 40static cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums");41static cl::opt<std::string>42 IntrinsicPrefix("intrinsic-prefix",43 cl::desc("Generate intrinsics with this target prefix"),44 cl::value_desc("target prefix"), cl::cat(GenIntrinsicCat));45 46namespace {47class IntrinsicEmitter {48 const RecordKeeper &Records;49 50public:51 IntrinsicEmitter(const RecordKeeper &R) : Records(R) {}52 53 void run(raw_ostream &OS, bool Enums);54 55 void EmitEnumInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);56 void EmitArgKind(raw_ostream &OS);57 void EmitIITInfo(raw_ostream &OS);58 void EmitTargetInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);59 void EmitIntrinsicToNameTable(const CodeGenIntrinsicTable &Ints,60 raw_ostream &OS);61 void EmitIntrinsicToOverloadTable(const CodeGenIntrinsicTable &Ints,62 raw_ostream &OS);63 void EmitIntrinsicToPrettyPrintTable(const CodeGenIntrinsicTable &Ints,64 raw_ostream &OS);65 void EmitIntrinsicBitTable(66 const CodeGenIntrinsicTable &Ints, raw_ostream &OS, StringRef Guard,67 StringRef TableName, StringRef Comment,68 function_ref<bool(const CodeGenIntrinsic &Int)> GetProperty);69 void EmitGenerator(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);70 void EmitAttributes(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);71 void EmitPrettyPrintArguments(const CodeGenIntrinsicTable &Ints,72 raw_ostream &OS);73 void EmitIntrinsicToBuiltinMap(const CodeGenIntrinsicTable &Ints,74 bool IsClang, raw_ostream &OS);75};76 77// Helper class to use with `TableGen::Emitter::OptClass`.78template <bool Enums> class IntrinsicEmitterOpt : public IntrinsicEmitter {79public:80 IntrinsicEmitterOpt(const RecordKeeper &R) : IntrinsicEmitter(R) {}81 void run(raw_ostream &OS) { IntrinsicEmitter::run(OS, Enums); }82};83 84} // End anonymous namespace85 86//===----------------------------------------------------------------------===//87// IntrinsicEmitter Implementation88//===----------------------------------------------------------------------===//89 90void IntrinsicEmitter::run(raw_ostream &OS, bool Enums) {91 emitSourceFileHeader("Intrinsic Function Source Fragment", OS);92 93 CodeGenIntrinsicTable Ints(Records);94 95 if (Enums) {96 // Emit the enum information.97 EmitEnumInfo(Ints, OS);98 99 // Emit ArgKind for Intrinsics.h.100 EmitArgKind(OS);101 } else {102 // Emit IIT_Info constants.103 EmitIITInfo(OS);104 105 // Emit the target metadata.106 EmitTargetInfo(Ints, OS);107 108 // Emit the intrinsic ID -> name table.109 EmitIntrinsicToNameTable(Ints, OS);110 111 // Emit the intrinsic ID -> overload table.112 EmitIntrinsicToOverloadTable(Ints, OS);113 114 // Emit the intrinsic declaration generator.115 EmitGenerator(Ints, OS);116 117 // Emit the intrinsic parameter attributes.118 EmitAttributes(Ints, OS);119 120 // Emit the intrinsic ID -> pretty print table.121 EmitIntrinsicToPrettyPrintTable(Ints, OS);122 123 // Emit Pretty Print attribute.124 EmitPrettyPrintArguments(Ints, OS);125 126 // Emit code to translate Clang builtins into LLVM intrinsics.127 EmitIntrinsicToBuiltinMap(Ints, true, OS);128 129 // Emit code to translate MS builtins into LLVM intrinsics.130 EmitIntrinsicToBuiltinMap(Ints, false, OS);131 }132}133 134void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,135 raw_ostream &OS) {136 // Find the TargetSet for which to generate enums. There will be an initial137 // set with an empty target prefix which will include target independent138 // intrinsics like dbg.value.139 using TargetSet = CodeGenIntrinsicTable::TargetSet;140 const TargetSet *Set = nullptr;141 for (const auto &Target : Ints.getTargets()) {142 if (Target.Name == IntrinsicPrefix) {143 Set = &Target;144 break;145 }146 }147 if (!Set) {148 // The first entry is for target independent intrinsics, so drop it.149 auto KnowTargets = Ints.getTargets().drop_front();150 PrintFatalError([KnowTargets](raw_ostream &OS) {151 OS << "tried to generate intrinsics for unknown target "152 << IntrinsicPrefix << "\nKnown targets are: ";153 interleaveComma(KnowTargets, OS,154 [&OS](const TargetSet &Target) { OS << Target.Name; });155 OS << '\n';156 });157 }158 159 // Generate a complete header for target specific intrinsics.160 if (IntrinsicPrefix.empty()) {161 OS << "#ifdef GET_INTRINSIC_ENUM_VALUES\n";162 } else {163 std::string UpperPrefix = StringRef(IntrinsicPrefix).upper();164 OS << formatv("#ifndef LLVM_IR_INTRINSIC_{}_ENUMS_H\n", UpperPrefix);165 OS << formatv("#define LLVM_IR_INTRINSIC_{}_ENUMS_H\n", UpperPrefix);166 OS << "namespace llvm::Intrinsic {\n";167 OS << formatv("enum {}Intrinsics : unsigned {{\n", UpperPrefix);168 }169 170 OS << "// Enum values for intrinsics.\n";171 bool First = true;172 for (const CodeGenIntrinsic &Int : Ints[*Set]) {173 OS << " " << Int.EnumName;174 175 // Assign a value to the first intrinsic in this target set so that all176 // intrinsic ids are distinct.177 if (First) {178 OS << " = " << Set->Offset + 1;179 First = false;180 }181 182 OS << ", ";183 if (Int.EnumName.size() < 40)184 OS.indent(40 - Int.EnumName.size());185 OS << formatv(186 " // {} ({})\n", Int.Name,187 SrcMgr.getFormattedLocationNoOffset(Int.TheDef->getLoc().front()));188 }189 190 // Emit num_intrinsics into the target neutral enum.191 if (IntrinsicPrefix.empty()) {192 OS << formatv(" num_intrinsics = {}\n", Ints.size() + 1);193 OS << "#endif\n\n";194 } else {195 OS << R"(}; // enum196} // namespace llvm::Intrinsic197#endif198 199)";200 }201}202 203void IntrinsicEmitter::EmitArgKind(raw_ostream &OS) {204 if (!IntrinsicPrefix.empty())205 return;206 OS << "// llvm::Intrinsic::IITDescriptor::ArgKind.\n";207 OS << "#ifdef GET_INTRINSIC_ARGKIND\n";208 if (const auto RecArgKind = Records.getDef("ArgKind")) {209 for (const auto &RV : RecArgKind->getValues())210 OS << " AK_" << RV.getName() << " = " << *RV.getValue() << ",\n";211 } else {212 OS << "#error \"ArgKind is not defined\"\n";213 }214 OS << "#endif\n\n";215}216 217void IntrinsicEmitter::EmitIITInfo(raw_ostream &OS) {218 OS << "#ifdef GET_INTRINSIC_IITINFO\n";219 std::array<StringRef, 256> RecsByNumber;220 auto IIT_Base = Records.getAllDerivedDefinitionsIfDefined("IIT_Base");221 for (const Record *Rec : IIT_Base) {222 auto Number = Rec->getValueAsInt("Number");223 assert(0 <= Number && Number < (int)RecsByNumber.size() &&224 "IIT_Info.Number should be uint8_t");225 assert(RecsByNumber[Number].empty() && "Duplicate IIT_Info.Number");226 RecsByNumber[Number] = Rec->getName();227 }228 if (IIT_Base.size() > 0) {229 for (unsigned I = 0, E = RecsByNumber.size(); I < E; ++I)230 if (!RecsByNumber[I].empty())231 OS << " " << RecsByNumber[I] << " = " << I << ",\n";232 } else {233 OS << "#error \"class IIT_Base is not defined\"\n";234 }235 OS << "#endif\n\n";236}237 238void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints,239 raw_ostream &OS) {240 OS << R"(// Target mapping.241#ifdef GET_INTRINSIC_TARGET_DATA242struct IntrinsicTargetInfo {243 StringLiteral Name;244 size_t Offset;245 size_t Count;246};247static constexpr IntrinsicTargetInfo TargetInfos[] = {248)";249 for (const auto [Name, Offset, Count] : Ints.getTargets())250 OS << formatv(" {{\"{}\", {}, {}},\n", Name, Offset, Count);251 OS << R"(};252#endif253 254)";255}256 257/// Helper function to emit a bit table for intrinsic properties.258/// This is used for both overload and pretty print bit tables.259void IntrinsicEmitter::EmitIntrinsicBitTable(260 const CodeGenIntrinsicTable &Ints, raw_ostream &OS, StringRef Guard,261 StringRef TableName, StringRef Comment,262 function_ref<bool(const CodeGenIntrinsic &Int)> GetProperty) {263 OS << formatv("// {}\n", Comment);264 OS << formatv("#ifdef {}\n", Guard);265 OS << formatv("static constexpr uint8_t {}[] = {{\n", TableName);266 OS << " 0\n ";267 for (auto [I, Int] : enumerate(Ints)) {268 // Add one to the index so we emit a null bit for the invalid #0 intrinsic.269 size_t Idx = I + 1;270 if (Idx % 8 == 0)271 OS << ",\n 0";272 if (GetProperty(Int))273 OS << " | (1<<" << Idx % 8 << ')';274 }275 OS << "\n};\n\n";276 OS << formatv("return ({}[id/8] & (1 << (id%8))) != 0;\n", TableName);277 OS << formatv("#endif // {}\n\n", Guard);278}279 280void IntrinsicEmitter::EmitIntrinsicToNameTable(281 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {282 // Built up a table of the intrinsic names.283 constexpr StringLiteral NotIntrinsic = "not_intrinsic";284 StringToOffsetTable Table;285 Table.GetOrAddStringOffset(NotIntrinsic);286 for (const auto &Int : Ints)287 Table.GetOrAddStringOffset(Int.Name);288 289 OS << R"(// Intrinsic ID to name table.290#ifdef GET_INTRINSIC_NAME_TABLE291// Note that entry #0 is the invalid intrinsic!292 293)";294 295 Table.EmitStringTableDef(OS, "IntrinsicNameTable");296 297 OS << R"(298static constexpr unsigned IntrinsicNameOffsetTable[] = {299)";300 301 OS << formatv(" {}, // {}\n", Table.GetStringOffset(NotIntrinsic),302 NotIntrinsic);303 for (const auto &Int : Ints)304 OS << formatv(" {}, // {}\n", Table.GetStringOffset(Int.Name), Int.Name);305 306 OS << R"(307}; // IntrinsicNameOffsetTable308 309#endif310 311)";312}313 314void IntrinsicEmitter::EmitIntrinsicToOverloadTable(315 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {316 EmitIntrinsicBitTable(317 Ints, OS, "GET_INTRINSIC_OVERLOAD_TABLE", "OTable",318 "Intrinsic ID to overload bitset.",319 [](const CodeGenIntrinsic &Int) { return Int.isOverloaded; });320}321 322using TypeSigTy = SmallVector<unsigned char>;323 324/// Computes type signature of the intrinsic \p Int.325static TypeSigTy ComputeTypeSignature(const CodeGenIntrinsic &Int) {326 TypeSigTy TypeSig;327 const Record *TypeInfo = Int.TheDef->getValueAsDef("TypeInfo");328 const ListInit *TypeList = TypeInfo->getValueAsListInit("TypeSig");329 330 for (const auto *TypeListEntry : TypeList->getElements())331 TypeSig.emplace_back(cast<IntInit>(TypeListEntry)->getValue());332 return TypeSig;333}334 335// Pack the type signature into 32-bit fixed encoding word.336static std::optional<uint32_t> encodePacked(const TypeSigTy &TypeSig) {337 if (TypeSig.size() > 8)338 return std::nullopt;339 340 uint32_t Result = 0;341 for (unsigned char C : reverse(TypeSig)) {342 if (C > 15)343 return std::nullopt;344 Result = (Result << 4) | C;345 }346 return Result;347}348 349void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints,350 raw_ostream &OS) {351 // Note: the code below can be switched to use 32-bit fixed encoding by352 // flipping the flag below.353 constexpr bool Use16BitFixedEncoding = true;354 using FixedEncodingTy =355 std::conditional_t<Use16BitFixedEncoding, uint16_t, uint32_t>;356 constexpr unsigned FixedEncodingBits = sizeof(FixedEncodingTy) * CHAR_BIT;357 // Mask with all bits 1 except the most significant bit.358 const unsigned Mask = (1U << (FixedEncodingBits - 1)) - 1;359 const unsigned MSBPostion = FixedEncodingBits - 1;360 StringRef FixedEncodingTypeName =361 Use16BitFixedEncoding ? "uint16_t" : "uint32_t";362 363 // If we can compute a 16/32-bit fixed encoding for this intrinsic, do so and364 // capture it in this vector, otherwise store a ~0U.365 std::vector<FixedEncodingTy> FixedEncodings;366 SequenceToOffsetTable<TypeSigTy> LongEncodingTable;367 368 FixedEncodings.reserve(Ints.size());369 370 // Compute the unique argument type info.371 for (const CodeGenIntrinsic &Int : Ints) {372 // Get the signature for the intrinsic.373 TypeSigTy TypeSig = ComputeTypeSignature(Int);374 375 // Check to see if we can encode it into a 16/32 bit word.376 std::optional<uint32_t> Result = encodePacked(TypeSig);377 if (Result && (*Result & Mask) == Result) {378 FixedEncodings.push_back(static_cast<FixedEncodingTy>(*Result));379 continue;380 }381 382 LongEncodingTable.add(TypeSig);383 384 // This is a placehold that we'll replace after the table is laid out.385 FixedEncodings.push_back(static_cast<FixedEncodingTy>(~0U));386 }387 388 LongEncodingTable.layout();389 390 OS << formatv(R"(// Global intrinsic function declaration type table.391#ifdef GET_INTRINSIC_GENERATOR_GLOBAL392static constexpr {} IIT_Table[] = {{393 )",394 FixedEncodingTypeName);395 396 unsigned MaxOffset = 0;397 for (auto [Idx, FixedEncoding, Int] : enumerate(FixedEncodings, Ints)) {398 if ((Idx & 7) == 7)399 OS << "\n ";400 401 // If the entry fit in the table, just emit it.402 if ((FixedEncoding & Mask) == FixedEncoding) {403 OS << "0x" << Twine::utohexstr(FixedEncoding) << ", ";404 continue;405 }406 407 TypeSigTy TypeSig = ComputeTypeSignature(Int);408 unsigned Offset = LongEncodingTable.get(TypeSig);409 MaxOffset = std::max(MaxOffset, Offset);410 411 // Otherwise, emit the offset into the long encoding table. We emit it this412 // way so that it is easier to read the offset in the .def file.413 OS << formatv("(1U<<{}) | {}, ", MSBPostion, Offset);414 }415 416 OS << "0\n};\n\n";417 418 // verify that all offsets will fit in 16/32 bits.419 if ((MaxOffset & Mask) != MaxOffset)420 PrintFatalError("Offset of long encoding table exceeds encoding bits");421 422 // Emit the shared table of register lists.423 OS << "static constexpr unsigned char IIT_LongEncodingTable[] = {\n";424 if (!LongEncodingTable.empty())425 LongEncodingTable.emit(426 OS, [](raw_ostream &OS, unsigned char C) { OS << (unsigned)C; });427 OS << " 255\n};\n";428 OS << "#endif\n\n"; // End of GET_INTRINSIC_GENERATOR_GLOBAL429}430 431/// Returns the effective MemoryEffects for intrinsic \p Int.432static MemoryEffects getEffectiveME(const CodeGenIntrinsic &Int) {433 MemoryEffects ME = Int.ME;434 // TODO: IntrHasSideEffects should affect not only readnone intrinsics.435 if (ME.doesNotAccessMemory() && Int.hasSideEffects)436 ME = MemoryEffects::unknown();437 return ME;438}439 440static bool compareFnAttributes(const CodeGenIntrinsic *L,441 const CodeGenIntrinsic *R) {442 auto TieBoolAttributes = [](const CodeGenIntrinsic *I) -> auto {443 // Sort throwing intrinsics after non-throwing intrinsics.444 return std::tie(I->canThrow, I->isNoDuplicate, I->isNoMerge, I->isNoReturn,445 I->isNoCallback, I->isNoSync, I->isNoFree, I->isWillReturn,446 I->isCold, I->isConvergent, I->isSpeculatable,447 I->hasSideEffects, I->isStrictFP,448 I->isNoCreateUndefOrPoison);449 };450 451 auto TieL = TieBoolAttributes(L);452 auto TieR = TieBoolAttributes(R);453 454 if (TieL != TieR)455 return TieL < TieR;456 457 // Try to order by readonly/readnone attribute.458 uint32_t LME = getEffectiveME(*L).toIntValue();459 uint32_t RME = getEffectiveME(*R).toIntValue();460 if (LME != RME)461 return LME > RME;462 463 return false;464}465 466/// Returns true if \p Int has a non-empty set of function attributes. Note that467/// NoUnwind = !canThrow, so we need to negate it's sense to test if the468// intrinsic has NoUnwind attribute.469static bool hasFnAttributes(const CodeGenIntrinsic &Int) {470 return !Int.canThrow || Int.isNoReturn || Int.isNoCallback || Int.isNoSync ||471 Int.isNoFree || Int.isWillReturn || Int.isCold || Int.isNoDuplicate ||472 Int.isNoMerge || Int.isConvergent || Int.isSpeculatable ||473 Int.isStrictFP || Int.isNoCreateUndefOrPoison ||474 getEffectiveME(Int) != MemoryEffects::unknown();475}476 477namespace {478struct FnAttributeComparator {479 bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const {480 return compareFnAttributes(L, R);481 }482};483 484struct AttributeComparator {485 bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const {486 // This comparator is used to unique just the argument attributes of an487 // intrinsic without considering any function attributes.488 return L->ArgumentAttributes < R->ArgumentAttributes;489 }490};491} // End anonymous namespace492 493/// Returns the name of the IR enum for argument attribute kind \p Kind.494static StringRef getArgAttrEnumName(CodeGenIntrinsic::ArgAttrKind Kind) {495 switch (Kind) {496 case CodeGenIntrinsic::NoCapture:497 llvm_unreachable("Handled separately");498 case CodeGenIntrinsic::NoAlias:499 return "NoAlias";500 case CodeGenIntrinsic::NoUndef:501 return "NoUndef";502 case CodeGenIntrinsic::NonNull:503 return "NonNull";504 case CodeGenIntrinsic::Returned:505 return "Returned";506 case CodeGenIntrinsic::ReadOnly:507 return "ReadOnly";508 case CodeGenIntrinsic::WriteOnly:509 return "WriteOnly";510 case CodeGenIntrinsic::ReadNone:511 return "ReadNone";512 case CodeGenIntrinsic::ImmArg:513 return "ImmArg";514 case CodeGenIntrinsic::Alignment:515 return "Alignment";516 case CodeGenIntrinsic::Dereferenceable:517 return "Dereferenceable";518 case CodeGenIntrinsic::Range:519 return "Range";520 }521 llvm_unreachable("Unknown CodeGenIntrinsic::ArgAttrKind enum");522}523 524/// EmitAttributes - This emits the Intrinsic::getAttributes method.525void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,526 raw_ostream &OS) {527 OS << R"(// Add parameter attributes that are not common to all intrinsics.528#ifdef GET_INTRINSIC_ATTRIBUTES529static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID,530 Type *ArgType) {531 unsigned BitWidth = ArgType->getScalarSizeInBits();532 switch (ID) {533 default: llvm_unreachable("Invalid attribute set number");)";534 // Compute unique argument attribute sets.535 std::map<SmallVector<CodeGenIntrinsic::ArgAttribute, 0>, unsigned>536 UniqArgAttributes;537 for (const CodeGenIntrinsic &Int : Ints) {538 for (auto &Attrs : Int.ArgumentAttributes) {539 if (Attrs.empty())540 continue;541 542 unsigned ID = UniqArgAttributes.size();543 if (!UniqArgAttributes.try_emplace(Attrs, ID).second)544 continue;545 546 assert(is_sorted(Attrs) && "Argument attributes are not sorted");547 548 OS << formatv(R"(549 case {}:550 return AttributeSet::get(C, {{551)",552 ID);553 for (const CodeGenIntrinsic::ArgAttribute &Attr : Attrs) {554 if (Attr.Kind == CodeGenIntrinsic::NoCapture) {555 OS << " Attribute::getWithCaptureInfo(C, "556 "CaptureInfo::none()),\n";557 continue;558 }559 StringRef AttrName = getArgAttrEnumName(Attr.Kind);560 if (Attr.Kind == CodeGenIntrinsic::Alignment ||561 Attr.Kind == CodeGenIntrinsic::Dereferenceable)562 OS << formatv(" Attribute::get(C, Attribute::{}, {}),\n",563 AttrName, Attr.Value);564 else if (Attr.Kind == CodeGenIntrinsic::Range)565 // This allows implicitTrunc because the range may only fit the566 // type based on rules implemented in the IR verifier. E.g. the567 // [-1, 1] range for ucmp/scmp intrinsics requires a minimum i2 type.568 // Give the verifier a chance to diagnose this instead of asserting569 // here.570 OS << formatv(" Attribute::get(C, Attribute::{}, "571 "ConstantRange(APInt(BitWidth, {}, /*isSigned=*/true, "572 "/*implicitTrunc=*/true), APInt(BitWidth, {}, "573 "/*isSigned=*/true, /*implicitTrunc=*/true))),\n",574 AttrName, (int64_t)Attr.Value, (int64_t)Attr.Value2);575 else576 OS << formatv(" Attribute::get(C, Attribute::{}),\n", AttrName);577 }578 OS << " });";579 }580 }581 OS << R"(582 }583} // getIntrinsicArgAttributeSet584)";585 586 // Compute unique function attribute sets. Note that ID 255 will be used for587 // intrinsics with no function attributes.588 std::map<const CodeGenIntrinsic *, unsigned, FnAttributeComparator>589 UniqFnAttributes;590 OS << R"(591static AttributeSet getIntrinsicFnAttributeSet(LLVMContext &C, unsigned ID) {592 switch (ID) {593 default: llvm_unreachable("Invalid attribute set number");)";594 595 for (const CodeGenIntrinsic &Int : Ints) {596 if (!hasFnAttributes(Int))597 continue;598 unsigned ID = UniqFnAttributes.size();599 if (!UniqFnAttributes.try_emplace(&Int, ID).second)600 continue;601 OS << formatv(R"(602 case {}: // {}603 return AttributeSet::get(C, {{604)",605 ID, Int.Name);606 auto addAttribute = [&OS](StringRef Attr) {607 OS << formatv(" Attribute::get(C, Attribute::{}),\n", Attr);608 };609 if (!Int.canThrow)610 addAttribute("NoUnwind");611 if (Int.isNoReturn)612 addAttribute("NoReturn");613 if (Int.isNoCallback)614 addAttribute("NoCallback");615 if (Int.isNoSync)616 addAttribute("NoSync");617 if (Int.isNoFree)618 addAttribute("NoFree");619 if (Int.isWillReturn)620 addAttribute("WillReturn");621 if (Int.isCold)622 addAttribute("Cold");623 if (Int.isNoDuplicate)624 addAttribute("NoDuplicate");625 if (Int.isNoMerge)626 addAttribute("NoMerge");627 if (Int.isConvergent)628 addAttribute("Convergent");629 if (Int.isSpeculatable)630 addAttribute("Speculatable");631 if (Int.isStrictFP)632 addAttribute("StrictFP");633 if (Int.isNoCreateUndefOrPoison)634 addAttribute("NoCreateUndefOrPoison");635 636 const MemoryEffects ME = getEffectiveME(Int);637 if (ME != MemoryEffects::unknown()) {638 OS << formatv(" // {}\n", ME);639 OS << formatv(" Attribute::getWithMemoryEffects(C, "640 "MemoryEffects::createFromIntValue({})),\n",641 ME.toIntValue());642 }643 OS << " });";644 }645 OS << R"(646 }647} // getIntrinsicFnAttributeSet)";648 649 // Compute unique argument attributes.650 std::map<const CodeGenIntrinsic *, unsigned, AttributeComparator>651 UniqAttributes;652 for (const CodeGenIntrinsic &Int : Ints) {653 unsigned ID = UniqAttributes.size();654 UniqAttributes.try_emplace(&Int, ID);655 }656 657 const uint8_t UniqAttributesBitSize = Log2_32_Ceil(UniqAttributes.size());658 // Note, max value is used to indicate no function attributes.659 const uint8_t UniqFnAttributesBitSize =660 Log2_32_Ceil(UniqFnAttributes.size() + 1);661 const uint32_t NoFunctionAttrsID =662 maskTrailingOnes<uint32_t>(UniqFnAttributesBitSize);663 uint8_t AttributesMapDataBitSize =664 PowerOf2Ceil(UniqAttributesBitSize + UniqFnAttributesBitSize);665 if (AttributesMapDataBitSize < 8)666 AttributesMapDataBitSize = 8;667 else if (AttributesMapDataBitSize > 64)668 PrintFatalError("Packed ID of IntrinsicsToAttributesMap exceeds 64b!");669 else if (AttributesMapDataBitSize > 16)670 PrintWarning("Packed ID of IntrinsicsToAttributesMap exceeds 16b, "671 "this may cause performance drop (pr106809), "672 "please consider redesigning intrinsic sets!");673 674 // Assign a packed ID for each intrinsic. The lower bits will be its675 // "argument attribute ID" (index in UniqAttributes) and upper bits will be676 // its "function attribute ID" (index in UniqFnAttributes).677 OS << formatv("\nstatic constexpr uint{}_t IntrinsicsToAttributesMap[] = {{",678 AttributesMapDataBitSize);679 for (const CodeGenIntrinsic &Int : Ints) {680 uint32_t FnAttrIndex =681 hasFnAttributes(Int) ? UniqFnAttributes[&Int] : NoFunctionAttrsID;682 OS << formatv("\n {} << {} | {}, // {}", FnAttrIndex,683 UniqAttributesBitSize, UniqAttributes[&Int], Int.Name);684 }685 686 OS << R"(687}; // IntrinsicsToAttributesMap688)";689 690 // For a given intrinsic, its attributes are constructed by populating the691 // local array `AS` below with its non-empty argument attributes followed by692 // function attributes if any. Each argument attribute is constructed as:693 //694 // getIntrinsicArgAttributeSet(C, ArgAttrID, FT->getContainedType(ArgNo));695 //696 // Create a table that records, for each argument attributes, the list of697 // <ArgNo, ArgAttrID> pairs that are needed to construct its argument698 // attributes. These tables for all intrinsics will be concatenated into one699 // large table and then for each intrinsic, we remember the Staring index and700 // number of size of its slice of entries (i.e., number of arguments with701 // non-empty attributes), so that we can build the attribute list for an702 // intrinsic without using a switch-case.703 704 using ArgNoAttrIDPair = std::pair<uint16_t, uint16_t>;705 706 // Emit the table of concatenated <ArgNo, AttrId> using SequenceToOffsetTable707 // so that entries can be reused if possible. Individual sequences in this708 // table do not have any terminator.709 using ArgAttrIDSubTable = SmallVector<ArgNoAttrIDPair>;710 SequenceToOffsetTable<ArgAttrIDSubTable> ArgAttrIdSequenceTable(std::nullopt);711 SmallVector<ArgAttrIDSubTable> ArgAttrIdSubTables(712 UniqAttributes.size()); // Indexed by UniqueID.713 714 // Find the max number of attributes to create the local array.715 unsigned MaxNumAttrs = 0;716 for (const auto [IntPtr, UniqueID] : UniqAttributes) {717 const CodeGenIntrinsic &Int = *IntPtr;718 ArgAttrIDSubTable SubTable;719 720 for (const auto &[ArgNo, Attrs] : enumerate(Int.ArgumentAttributes)) {721 if (Attrs.empty())722 continue;723 724 uint16_t ArgAttrID = UniqArgAttributes.find(Attrs)->second;725 SubTable.emplace_back((uint16_t)ArgNo, ArgAttrID);726 }727 ArgAttrIdSubTables[UniqueID] = SubTable;728 if (!SubTable.empty())729 ArgAttrIdSequenceTable.add(SubTable);730 unsigned NumAttrs = SubTable.size() + hasFnAttributes(Int);731 MaxNumAttrs = std::max(MaxNumAttrs, NumAttrs);732 }733 734 ArgAttrIdSequenceTable.layout();735 736 if (ArgAttrIdSequenceTable.size() >= std::numeric_limits<uint16_t>::max())737 PrintFatalError("Size of ArgAttrIdTable exceeds supported limit");738 739 // Emit the 2 tables (flattened ArgNo, ArgAttrID) and ArgAttributesInfoTable.740 OS << formatv(R"(741namespace {{742struct ArgNoAttrIDPair {{743 uint16_t ArgNo, ArgAttrID;744};745} // namespace746 747// Number of entries: {}748static constexpr ArgNoAttrIDPair ArgAttrIdTable[] = {{749)",750 ArgAttrIdSequenceTable.size());751 752 ArgAttrIdSequenceTable.emit(OS, [](raw_ostream &OS, ArgNoAttrIDPair Elem) {753 OS << formatv("{{{}, {}}", Elem.first, Elem.second);754 });755 756 OS << formatv(R"(}; // ArgAttrIdTable757 758namespace {{759struct ArgAttributesInfo {{760 uint16_t StartIndex;761 uint16_t NumAttrs;762};763} // namespace764 765// Number of entries: {}766static constexpr ArgAttributesInfo ArgAttributesInfoTable[] = {{767)",768 ArgAttrIdSubTables.size());769 770 for (const auto &SubTable : ArgAttrIdSubTables) {771 unsigned NumAttrs = SubTable.size();772 unsigned StartIndex = NumAttrs ? ArgAttrIdSequenceTable.get(SubTable) : 0;773 OS << formatv(" {{{}, {}},\n", StartIndex, NumAttrs);774 }775 OS << "}; // ArgAttributesInfoTable\n";776 777 // Now emit the Intrinsic::getAttributes function. This will first map778 // from intrinsic ID -> unique arg/function attr ID (using the779 // IntrinsicsToAttributesMap) table. Then it will use the unique arg ID to780 // construct all the argument attributes (using the ArgAttributesInfoTable and781 // ArgAttrIdTable) and then add on the function attributes if any.782 OS << formatv(R"(783 784template <typename IDTy>785inline std::pair<uint32_t, uint32_t> unpackID(const IDTy PackedID) {{786 constexpr uint8_t UniqAttributesBitSize = {};787 const uint32_t FnAttrID = PackedID >> UniqAttributesBitSize;788 const uint32_t ArgAttrID = PackedID &789 maskTrailingOnes<uint32_t>(UniqAttributesBitSize);790 return {{FnAttrID, ArgAttrID};791}792 793AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id,794 FunctionType *FT) {{795 if (id == 0)796 return AttributeList();797 auto [FnAttrID, ArgAttrID] = unpackID(IntrinsicsToAttributesMap[id - 1]);798 using PairTy = std::pair<unsigned, AttributeSet>;799 alignas(PairTy) char ASStorage[sizeof(PairTy) * {}];800 PairTy *AS = reinterpret_cast<PairTy *>(ASStorage);801 802 // Construct an ArrayRef for easier range checking.803 ArrayRef<ArgAttributesInfo> ArgAttributesInfoTableAR(ArgAttributesInfoTable);804 if (ArgAttrID >= ArgAttributesInfoTableAR.size())805 llvm_unreachable("Invalid arguments attribute ID");806 807 auto [StartIndex, NumAttrs] = ArgAttributesInfoTableAR[ArgAttrID];808 for (unsigned Idx = 0; Idx < NumAttrs; ++Idx) {{809 auto [ArgNo, ArgAttrID] = ArgAttrIdTable[StartIndex + Idx];810 AS[Idx] = {{ArgNo,811 getIntrinsicArgAttributeSet(C, ArgAttrID, FT->getContainedType(ArgNo))};812 }813 if (FnAttrID != {}) {814 AS[NumAttrs++] = {{AttributeList::FunctionIndex,815 getIntrinsicFnAttributeSet(C, FnAttrID)};816 }817 return AttributeList::get(C, ArrayRef(AS, NumAttrs));818}819 820AttributeSet Intrinsic::getFnAttributes(LLVMContext &C, ID id) {{821 if (id == 0)822 return AttributeSet();823 auto [FnAttrID, _] = unpackID(IntrinsicsToAttributesMap[id - 1]);824 if (FnAttrID == {})825 return AttributeSet();826 return getIntrinsicFnAttributeSet(C, FnAttrID);827}828#endif // GET_INTRINSIC_ATTRIBUTES829 830)",831 UniqAttributesBitSize, MaxNumAttrs, NoFunctionAttrsID,832 NoFunctionAttrsID);833}834 835void IntrinsicEmitter::EmitIntrinsicToPrettyPrintTable(836 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {837 EmitIntrinsicBitTable(Ints, OS, "GET_INTRINSIC_PRETTY_PRINT_TABLE", "PPTable",838 "Intrinsic ID to pretty print bitset.",839 [](const CodeGenIntrinsic &Int) {840 return !Int.PrettyPrintFunctions.empty();841 });842}843 844void IntrinsicEmitter::EmitPrettyPrintArguments(845 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {846 OS << R"(847#ifdef GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS848void Intrinsic::printImmArg(ID IID, unsigned ArgIdx, raw_ostream &OS, const Constant *ImmArgVal) {849 using namespace Intrinsic;850 switch (IID) {851)";852 853 for (const CodeGenIntrinsic &Int : Ints) {854 if (Int.PrettyPrintFunctions.empty())855 continue;856 857 OS << " case " << Int.EnumName << ":\n";858 OS << " switch (ArgIdx) {\n";859 for (const auto [ArgIdx, ArgName, FuncName] : Int.PrettyPrintFunctions) {860 OS << " case " << ArgIdx << ":\n";861 OS << " OS << \"" << ArgName << "=\";\n";862 if (!FuncName.empty()) {863 OS << " ";864 if (!Int.TargetPrefix.empty())865 OS << Int.TargetPrefix << "::";866 OS << FuncName << "(OS, ImmArgVal);\n";867 }868 OS << " return;\n";869 }870 OS << " }\n";871 OS << " break;\n";872 }873 OS << R"( default:874 break;875 }876}877#endif // GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS878)";879}880 881void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(882 const CodeGenIntrinsicTable &Ints, bool IsClang, raw_ostream &OS) {883 StringRef CompilerName = IsClang ? "Clang" : "MS";884 StringRef UpperCompilerName = IsClang ? "CLANG" : "MS";885 886 // map<TargetPrefix, pair<map<BuiltinName, EnumName>, CommonPrefix>.887 // Note that we iterate over both the maps in the code below and both888 // iterations need to iterate in sorted key order. For the inner map, entries889 // need to be emitted in the sorted order of `BuiltinName` with `CommonPrefix`890 // rempved, because we use std::lower_bound to search these entries. For the891 // outer map as well, entries need to be emitted in sorter order of892 // `TargetPrefix` as we use std::lower_bound to search these entries.893 using BIMEntryTy =894 std::pair<std::map<StringRef, StringRef>, std::optional<StringRef>>;895 std::map<StringRef, BIMEntryTy> BuiltinMap;896 897 for (const CodeGenIntrinsic &Int : Ints) {898 StringRef BuiltinName = IsClang ? Int.ClangBuiltinName : Int.MSBuiltinName;899 if (BuiltinName.empty())900 continue;901 // Get the map for this target prefix.902 auto &[Map, CommonPrefix] = BuiltinMap[Int.TargetPrefix];903 904 if (!Map.try_emplace(BuiltinName, Int.EnumName).second)905 PrintFatalError(Int.TheDef->getLoc(),906 "Intrinsic '" + Int.TheDef->getName() + "': duplicate " +907 CompilerName + " builtin name!");908 909 // Update common prefix.910 if (!CommonPrefix) {911 // For the first builtin for this target, initialize the common prefix.912 CommonPrefix = BuiltinName;913 continue;914 }915 916 // Update the common prefix. Note that this assumes that `take_front` will917 // never set the `Data` pointer in CommonPrefix to nullptr.918 const char *Mismatch = mismatch(*CommonPrefix, BuiltinName).first;919 *CommonPrefix = CommonPrefix->take_front(Mismatch - CommonPrefix->begin());920 }921 922 // Populate the string table with the names of all the builtins after923 // removing this common prefix.924 StringToOffsetTable Table;925 for (const auto &[TargetPrefix, Entry] : BuiltinMap) {926 auto &[Map, CommonPrefix] = Entry;927 for (auto &[BuiltinName, EnumName] : Map) {928 StringRef Suffix = BuiltinName.substr(CommonPrefix->size());929 Table.GetOrAddStringOffset(Suffix);930 }931 }932 933 OS << formatv(R"(934// Get the LLVM intrinsic that corresponds to a builtin. This is used by the935// C front-end. The builtin name is passed in as BuiltinName, and a target936// prefix (e.g. 'ppc') is passed in as TargetPrefix.937#ifdef GET_LLVM_INTRINSIC_FOR_{}_BUILTIN938Intrinsic::ID939Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix, 940 StringRef BuiltinName) {{941 using namespace Intrinsic;942)",943 UpperCompilerName, CompilerName);944 945 if (BuiltinMap.empty()) {946 OS << formatv(R"(947 return not_intrinsic;948 }949#endif // GET_LLVM_INTRINSIC_FOR_{}_BUILTIN950)",951 UpperCompilerName);952 return;953 }954 955 if (!Table.empty()) {956 Table.EmitStringTableDef(OS, "BuiltinNames");957 958 OS << R"(959 struct BuiltinEntry {960 ID IntrinsicID;961 unsigned StrTabOffset;962 const char *getName() const { return BuiltinNames[StrTabOffset].data(); }963 bool operator<(StringRef RHS) const {964 return strncmp(getName(), RHS.data(), RHS.size()) < 0;965 }966 };967 968)";969 }970 971 // Emit a per target table of bultin names.972 bool HasTargetIndependentBuiltins = false;973 StringRef TargetIndepndentCommonPrefix;974 for (const auto &[TargetPrefix, Entry] : BuiltinMap) {975 const auto &[Map, CommonPrefix] = Entry;976 if (!TargetPrefix.empty()) {977 OS << formatv(" // Builtins for {0}.\n", TargetPrefix);978 } else {979 OS << " // Target independent builtins.\n";980 HasTargetIndependentBuiltins = true;981 TargetIndepndentCommonPrefix = *CommonPrefix;982 }983 984 // Emit the builtin table for this target prefix.985 OS << formatv(" static constexpr BuiltinEntry {}Names[] = {{\n",986 TargetPrefix);987 for (const auto &[BuiltinName, EnumName] : Map) {988 StringRef Suffix = BuiltinName.substr(CommonPrefix->size());989 OS << formatv(" {{{}, {}}, // {}\n", EnumName,990 *Table.GetStringOffset(Suffix), BuiltinName);991 }992 OS << formatv(" }; // {}Names\n\n", TargetPrefix);993 }994 995 // After emitting the builtin tables for all targets, emit a lookup table for996 // all targets. We will use binary search, similar to the table for builtin997 // names to lookup into this table.998 OS << R"(999 struct TargetEntry {1000 StringLiteral TargetPrefix;1001 ArrayRef<BuiltinEntry> Names;1002 StringLiteral CommonPrefix;1003 bool operator<(StringRef RHS) const {1004 return TargetPrefix < RHS;1005 };1006 };1007 static constexpr TargetEntry TargetTable[] = {1008)";1009 1010 for (const auto &[TargetPrefix, Entry] : BuiltinMap) {1011 const auto &[Map, CommonPrefix] = Entry;1012 if (TargetPrefix.empty())1013 continue;1014 OS << formatv(R"( {{"{0}", {0}Names, "{1}"},)", TargetPrefix,1015 CommonPrefix)1016 << "\n";1017 }1018 OS << " };\n";1019 1020 // Now for the actual lookup, first check the target independent table if1021 // we emitted one.1022 if (HasTargetIndependentBuiltins) {1023 OS << formatv(R"(1024 // Check if it's a target independent builtin.1025 // Copy the builtin name so we can use it in consume_front without clobbering1026 // if for the lookup in the target specific table.1027 StringRef Suffix = BuiltinName;1028 if (Suffix.consume_front("{}")) {{1029 auto II = lower_bound(Names, Suffix);1030 if (II != std::end(Names) && II->getName() == Suffix)1031 return II->IntrinsicID;1032 }1033)",1034 TargetIndepndentCommonPrefix);1035 }1036 1037 // If a target independent builtin was not found, lookup the target specific.1038 OS << formatv(R"(1039 auto TI = lower_bound(TargetTable, TargetPrefix);1040 if (TI == std::end(TargetTable) || TI->TargetPrefix != TargetPrefix)1041 return not_intrinsic;1042 // This is the last use of BuiltinName, so no need to copy before using it in1043 // consume_front.1044 if (!BuiltinName.consume_front(TI->CommonPrefix))1045 return not_intrinsic;1046 auto II = lower_bound(TI->Names, BuiltinName);1047 if (II == std::end(TI->Names) || II->getName() != BuiltinName)1048 return not_intrinsic;1049 return II->IntrinsicID;1050}1051#endif // GET_LLVM_INTRINSIC_FOR_{}_BUILTIN1052 1053)",1054 UpperCompilerName);1055}1056 1057static TableGen::Emitter::OptClass<IntrinsicEmitterOpt</*Enums=*/true>>1058 X("gen-intrinsic-enums", "Generate intrinsic enums");1059 1060static TableGen::Emitter::OptClass<IntrinsicEmitterOpt</*Enums=*/false>>1061 Y("gen-intrinsic-impl", "Generate intrinsic implementation code");1062