544 lines · cpp
1//===- CodeGenIntrinsics.cpp - Intrinsic Class Wrapper --------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file defines a wrapper class for the 'Intrinsic' TableGen class.10//11//===----------------------------------------------------------------------===//12 13#include "CodeGenIntrinsics.h"14#include "llvm/ADT/ArrayRef.h"15#include "llvm/ADT/STLExtras.h"16#include "llvm/ADT/StringSwitch.h"17#include "llvm/ADT/Twine.h"18#include "llvm/Support/ErrorHandling.h"19#include "llvm/TableGen/Error.h"20#include "llvm/TableGen/Record.h"21#include <algorithm>22#include <cassert>23using namespace llvm;24 25// As the type of more than one return values is represented as an anonymous26// struct, which is encoded with `IIT_STRUCT` followed by a byte specifying27// the number of return values, starting from 2 (encoded as 0) to 25728// (encoded as 255). So, the maximum number of values that an intrinsic can29// return is 257.30static constexpr unsigned MaxNumReturn = 257;31 32//===----------------------------------------------------------------------===//33// CodeGenIntrinsic Implementation34//===----------------------------------------------------------------------===//35 36CodeGenIntrinsicContext::CodeGenIntrinsicContext(const RecordKeeper &RC) {37 for (const Record *Rec : RC.getAllDerivedDefinitions("IntrinsicProperty"))38 if (Rec->getValueAsBit("IsDefault"))39 DefaultProperties.push_back(Rec);40}41 42CodeGenIntrinsicTable::CodeGenIntrinsicTable(const RecordKeeper &RC) {43 CodeGenIntrinsicContext Ctx(RC);44 45 ArrayRef<const Record *> Defs = RC.getAllDerivedDefinitions("Intrinsic");46 Intrinsics.reserve(Defs.size());47 48 for (const Record *Def : Defs)49 Intrinsics.emplace_back(CodeGenIntrinsic(Def, Ctx));50 51 llvm::sort(Intrinsics,52 [](const CodeGenIntrinsic &LHS, const CodeGenIntrinsic &RHS) {53 // Order target independent intrinsics before target dependent54 // ones.55 bool LHSHasTarget = !LHS.TargetPrefix.empty();56 bool RHSHasTarget = !RHS.TargetPrefix.empty();57 58 // To ensure deterministic sorted order when duplicates are59 // present, use record ID as a tie-breaker similar to60 // sortAndReportDuplicates in Utils.cpp.61 unsigned LhsID = LHS.TheDef->getID();62 unsigned RhsID = RHS.TheDef->getID();63 64 return std::tie(LHSHasTarget, LHS.Name, LhsID) <65 std::tie(RHSHasTarget, RHS.Name, RhsID);66 });67 68 Targets.push_back({"", 0, 0});69 for (size_t I = 0, E = Intrinsics.size(); I < E; ++I)70 if (Intrinsics[I].TargetPrefix != Targets.back().Name) {71 Targets.back().Count = I - Targets.back().Offset;72 Targets.push_back({Intrinsics[I].TargetPrefix, I, 0});73 }74 Targets.back().Count = Intrinsics.size() - Targets.back().Offset;75 76 CheckDuplicateIntrinsics();77 CheckTargetIndependentIntrinsics();78 CheckOverloadSuffixConflicts();79}80 81// Check for duplicate intrinsic names.82void CodeGenIntrinsicTable::CheckDuplicateIntrinsics() const {83 // Since the Intrinsics vector is already sorted by name, if there are 2 or84 // more intrinsics with duplicate names, they will appear adjacent in sorted85 // order. Note that if the intrinsic name was derived from the record name86 // there cannot be be duplicate as TableGen parser would have flagged that.87 // However, if the name was specified in the intrinsic definition, then its88 // possible to have duplicate names.89 auto I = std::adjacent_find(90 Intrinsics.begin(), Intrinsics.end(),91 [](const CodeGenIntrinsic &Int1, const CodeGenIntrinsic &Int2) {92 return Int1.Name == Int2.Name;93 });94 if (I == Intrinsics.end())95 return;96 97 // Found a duplicate intrinsics.98 const CodeGenIntrinsic &First = *I;99 const CodeGenIntrinsic &Second = *(I + 1);100 PrintError(Second.TheDef,101 Twine("Intrinsic `") + First.Name + "` is already defined");102 PrintFatalNote(First.TheDef, "Previous definition here");103}104 105// For target independent intrinsics, check that their second dotted component106// does not match any target name.107void CodeGenIntrinsicTable::CheckTargetIndependentIntrinsics() const {108 SmallDenseSet<StringRef> TargetNames;109 for (const auto &Target : ArrayRef(Targets).drop_front())110 TargetNames.insert(Target.Name);111 112 // Set of target independent intrinsics.113 const auto &Set = Targets[0];114 for (const auto &Int : ArrayRef(&Intrinsics[Set.Offset], Set.Count)) {115 StringRef Name = Int.Name;116 StringRef Prefix = Name.drop_front(5).split('.').first;117 if (!TargetNames.contains(Prefix))118 continue;119 PrintFatalError(Int.TheDef,120 "target independent intrinsic `" + Name +121 "' has prefix `llvm." + Prefix +122 "` that conflicts with intrinsics for target `" +123 Prefix + "`");124 }125}126 127// Return true if the given Suffix looks like a mangled type. Note that this128// check is conservative, but allows all existing LLVM intrinsic suffixes to be129// considered as not looking like a mangling suffix.130static bool doesSuffixLookLikeMangledType(StringRef Suffix) {131 // Try to match against possible mangling suffixes for various types.132 // See getMangledTypeStr() for the mangling suffixes possible. It includes133 // pointer : p[0-9]+134 // array : a[0-9]+.+135 // struct: : s_/sl_.+136 // function : f_.+137 // vector : v/nxv[0-9]+.+138 // target type : t.+139 // integer : i[0-9]+140 // named types : See `NamedTypes` below.141 142 // Match anything with an _, so match function and struct types.143 if (Suffix.contains('_'))144 return true;145 146 // [av][0-9]+.+, simplified to [av][0-9].+147 if (Suffix.size() >= 2 && is_contained("av", Suffix[0]) && isDigit(Suffix[1]))148 return true;149 150 // nxv[0-9]+.+, simplified to nxv[0-9].+151 if (Suffix.size() >= 4 && Suffix.starts_with("nxv") && isDigit(Suffix[3]))152 return true;153 154 // t.+155 if (Suffix.size() > 1 && Suffix.starts_with('t'))156 return false;157 158 // [pi][0-9]+159 if (Suffix.size() > 1 && is_contained("pi", Suffix[0]) &&160 all_of(Suffix.drop_front(), isDigit))161 return true;162 163 // Match one of the named types.164 static constexpr StringLiteral NamedTypes[] = {165 "isVoid", "Metadata", "f16", "f32", "f64",166 "f80", "f128", "bf16", "ppcf128", "x86amx"};167 return is_contained(NamedTypes, Suffix);168}169 170// Check for conflicts with overloaded intrinsics. If there exists an overloaded171// intrinsic with base name `llvm.target.foo`, LLVM will add a mangling suffix172// to it to encode the overload types. This mangling suffix is 1 or more .173// prefixed mangled type string as defined in `getMangledTypeStr`. If there174// exists another intrinsic `llvm.target.foo[.<suffixN>]+`, which has the same175// prefix as the overloaded intrinsic, its possible that there may be a name176// conflict with the overloaded intrinsic and either one may interfere with name177// lookup for the other, leading to wrong intrinsic ID being assigned.178//179// The actual name lookup in the intrinsic name table is done by a search180// on each successive '.' separted component of the intrinsic name (see181// `lookupLLVMIntrinsicByName`). Consider first the case where there exists a182// non-overloaded intrinsic `llvm.target.foo[.suffix]+`. For the non-overloaded183// intrinsics, the name lookup is an exact match, so the presence of the184// overloaded intrinsic with the same prefix will not interfere with the185// search. However, a lookup intended to match the overloaded intrinsic might be186// affected by the presence of another entry in the name table with the same187// prefix.188//189// Since LLVM's name lookup first selects the target specific (or target190// independent) slice of the name table to look into, intrinsics in 2 different191// targets cannot conflict with each other. Within a specific target,192// if we have an overloaded intrinsic with name `llvm.target.foo` and another193// one with same prefix and one or more suffixes `llvm.target.foo[.<suffixN>]+`,194// then the name search will try to first match against suffix0, then suffix1195// etc. If suffix0 can match a mangled type, then the search for an196// `llvm.target.foo` with a mangling suffix can match against suffix0,197// preventing a match with `llvm.target.foo`. If suffix0 cannot match a mangled198// type, then that cannot happen, so we do not need to check for later suffixes.199//200// Generalizing, the `llvm.target.foo[.suffixN]+` will cause a conflict if the201// first suffix (.suffix0) can match a mangled type (and then we do not need to202// check later suffixes) and will not cause a conflict if it cannot (and then203// again, we do not need to check for later suffixes).204void CodeGenIntrinsicTable::CheckOverloadSuffixConflicts() const {205 for (const TargetSet &Set : Targets) {206 const CodeGenIntrinsic *Overloaded = nullptr;207 for (const CodeGenIntrinsic &Int : (*this)[Set]) {208 // If we do not have an overloaded intrinsic to check against, nothing209 // to do except potentially identifying this as a candidate for checking210 // against in future iteration.211 if (!Overloaded) {212 if (Int.isOverloaded)213 Overloaded = ∬214 continue;215 }216 217 StringRef Name = Int.Name;218 StringRef OverloadName = Overloaded->Name;219 // If we have an overloaded intrinsic to check again, check if its name is220 // a proper prefix of this intrinsic.221 if (Name.starts_with(OverloadName) && Name[OverloadName.size()] == '.') {222 // If yes, verify suffixes and flag an error.223 StringRef Suffixes = Name.drop_front(OverloadName.size() + 1);224 225 // Only need to look at the first suffix.226 StringRef Suffix0 = Suffixes.split('.').first;227 228 if (!doesSuffixLookLikeMangledType(Suffix0))229 continue;230 231 unsigned SuffixSize = OverloadName.size() + 1 + Suffix0.size();232 // If suffix looks like mangling suffix, flag it as an error.233 PrintError(Int.TheDef->getLoc(),234 "intrinsic `" + Name + "` cannot share prefix `" +235 Name.take_front(SuffixSize) +236 "` with another overloaded intrinsic `" + OverloadName +237 "`");238 PrintNote(Overloaded->TheDef->getLoc(),239 "Overloaded intrinsic `" + OverloadName + "` defined here");240 continue;241 }242 243 // If we find an intrinsic that is not a proper prefix, any later244 // intrinsic is also not going to be a proper prefix, so invalidate the245 // overloaded to check against.246 Overloaded = nullptr;247 }248 }249}250 251const CodeGenIntrinsic &CodeGenIntrinsicMap::operator[](const Record *Record) {252 if (!Record->isSubClassOf("Intrinsic"))253 PrintFatalError("Intrinsic defs should be subclass of 'Intrinsic' class");254 255 auto [Iter, Inserted] = Map.try_emplace(Record);256 if (Inserted)257 Iter->second = std::make_unique<CodeGenIntrinsic>(Record, Ctx);258 return *Iter->second;259}260 261CodeGenIntrinsic::CodeGenIntrinsic(const Record *R,262 const CodeGenIntrinsicContext &Ctx)263 : TheDef(R) {264 StringRef DefName = TheDef->getName();265 ArrayRef<SMLoc> DefLoc = R->getLoc();266 267 if (!DefName.starts_with("int_"))268 PrintFatalError(DefLoc,269 "Intrinsic '" + DefName + "' does not start with 'int_'!");270 271 EnumName = DefName.substr(4);272 273 // Ignore a missing ClangBuiltinName field.274 ClangBuiltinName =275 R->getValueAsOptionalString("ClangBuiltinName").value_or("");276 // Ignore a missing MSBuiltinName field.277 MSBuiltinName = R->getValueAsOptionalString("MSBuiltinName").value_or("");278 279 TargetPrefix = R->getValueAsString("TargetPrefix");280 Name = R->getValueAsString("LLVMName").str();281 282 std::string DefaultName = "llvm." + EnumName.str();283 llvm::replace(DefaultName, '_', '.');284 285 if (Name == "") {286 // If an explicit name isn't specified, derive one from the DefName.287 Name = std::move(DefaultName);288 } else {289 // Verify it starts with "llvm.".290 if (!StringRef(Name).starts_with("llvm."))291 PrintFatalError(DefLoc, "Intrinsic '" + DefName +292 "'s name does not start with 'llvm.'!");293 294 if (Name == DefaultName)295 PrintNote(DefLoc, "Explicitly specified name matches default name, "296 "consider dropping it");297 }298 299 // If TargetPrefix is specified, make sure that Name starts with300 // "llvm.<targetprefix>.".301 if (!TargetPrefix.empty()) {302 StringRef Prefix = StringRef(Name).drop_front(5); // Drop llvm.303 if (!Prefix.consume_front(TargetPrefix) || !Prefix.starts_with('.'))304 PrintFatalError(DefLoc, "Intrinsic '" + DefName +305 "' does not start with 'llvm." +306 TargetPrefix + ".'!");307 }308 309 unsigned NumRet = R->getValueAsListInit("RetTypes")->size();310 if (NumRet > MaxNumReturn)311 PrintFatalError(DefLoc, "intrinsics can only return upto " +312 Twine(MaxNumReturn) + " values, '" + DefName +313 "' returns " + Twine(NumRet) + " values");314 315 const Record *TypeInfo = R->getValueAsDef("TypeInfo");316 if (!TypeInfo->isSubClassOf("TypeInfoGen"))317 PrintFatalError(DefLoc, "TypeInfo field in " + DefName +318 " should be of subclass of TypeInfoGen!");319 320 isOverloaded = TypeInfo->getValueAsBit("isOverloaded");321 const ListInit *TypeList = TypeInfo->getValueAsListInit("Types");322 323 // Types field is a concatenation of Return types followed by Param types.324 unsigned Idx = 0;325 for (; Idx < NumRet; ++Idx)326 IS.RetTys.push_back(TypeList->getElementAsRecord(Idx));327 328 for (unsigned E = TypeList->size(); Idx < E; ++Idx)329 IS.ParamTys.push_back(TypeList->getElementAsRecord(Idx));330 331 // Parse the intrinsic properties.332 const ListInit *PropList = R->getValueAsListInit("IntrProperties");333 for (unsigned i = 0, e = PropList->size(); i != e; ++i) {334 const Record *Property = PropList->getElementAsRecord(i);335 assert(Property->isSubClassOf("IntrinsicProperty") &&336 "Expected a property!");337 338 setProperty(Property);339 }340 341 // Set default properties to true.342 setDefaultProperties(Ctx.DefaultProperties);343 344 // Also record the SDPatternOperator Properties.345 Properties = parseSDPatternOperatorProperties(R);346 347 // Sort the argument attributes for later benefit.348 for (auto &Attrs : ArgumentAttributes)349 llvm::sort(Attrs);350}351 352void CodeGenIntrinsic::setDefaultProperties(353 ArrayRef<const Record *> DefaultProperties) {354 // opt-out of using default attributes.355 if (TheDef->getValueAsBit("DisableDefaultAttributes"))356 return;357 358 for (const Record *Rec : DefaultProperties)359 setProperty(Rec);360}361 362void CodeGenIntrinsic::setProperty(const Record *R) {363 if (R->getName() == "IntrNoMem")364 ME = MemoryEffects::none();365 else if (R->getName() == "IntrReadMem") {366 if (ME.onlyWritesMemory())367 PrintFatalError(TheDef->getLoc(),368 Twine("IntrReadMem cannot be used after IntrNoMem or "369 "IntrWriteMem. Default is ReadWrite"));370 ME &= MemoryEffects::readOnly();371 } else if (R->getName() == "IntrWriteMem") {372 if (ME.onlyReadsMemory())373 PrintFatalError(TheDef->getLoc(),374 Twine("IntrWriteMem cannot be used after IntrNoMem or "375 "IntrReadMem. Default is ReadWrite"));376 ME &= MemoryEffects::writeOnly();377 } else if (R->getName() == "IntrArgMemOnly")378 ME &= MemoryEffects::argMemOnly();379 else if (R->getName() == "IntrInaccessibleMemOnly")380 ME &= MemoryEffects::inaccessibleMemOnly();381 else if (R->isSubClassOf("IntrRead")) {382 MemoryEffects ReadMask = MemoryEffects::writeOnly();383 for (const Record *RLoc : R->getValueAsListOfDefs("MemLoc"))384 ReadMask = ReadMask.getWithModRef(getValueAsIRMemLocation(RLoc),385 ModRefInfo::ModRef);386 ME &= ReadMask;387 } else if (R->isSubClassOf("IntrWrite")) {388 MemoryEffects WriteMask = MemoryEffects::readOnly();389 for (const Record *WLoc : R->getValueAsListOfDefs("MemLoc"))390 WriteMask = WriteMask.getWithModRef(getValueAsIRMemLocation(WLoc),391 ModRefInfo::ModRef);392 ME &= WriteMask;393 } else if (R->getName() == "IntrInaccessibleMemOrArgMemOnly")394 ME &= MemoryEffects::inaccessibleOrArgMemOnly();395 else if (R->getName() == "Commutative")396 isCommutative = true;397 else if (R->getName() == "Throws")398 canThrow = true;399 else if (R->getName() == "IntrNoDuplicate")400 isNoDuplicate = true;401 else if (R->getName() == "IntrNoMerge")402 isNoMerge = true;403 else if (R->getName() == "IntrConvergent")404 isConvergent = true;405 else if (R->getName() == "IntrNoReturn")406 isNoReturn = true;407 else if (R->getName() == "IntrNoCallback")408 isNoCallback = true;409 else if (R->getName() == "IntrNoSync")410 isNoSync = true;411 else if (R->getName() == "IntrNoFree")412 isNoFree = true;413 else if (R->getName() == "IntrWillReturn")414 isWillReturn = !isNoReturn;415 else if (R->getName() == "IntrCold")416 isCold = true;417 else if (R->getName() == "IntrSpeculatable")418 isSpeculatable = true;419 else if (R->getName() == "IntrHasSideEffects")420 hasSideEffects = true;421 else if (R->getName() == "IntrStrictFP")422 isStrictFP = true;423 else if (R->getName() == "IntrNoCreateUndefOrPoison")424 isNoCreateUndefOrPoison = true;425 else if (R->isSubClassOf("NoCapture")) {426 unsigned ArgNo = R->getValueAsInt("ArgNo");427 addArgAttribute(ArgNo, NoCapture);428 } else if (R->isSubClassOf("NoAlias")) {429 unsigned ArgNo = R->getValueAsInt("ArgNo");430 addArgAttribute(ArgNo, NoAlias);431 } else if (R->isSubClassOf("NoUndef")) {432 unsigned ArgNo = R->getValueAsInt("ArgNo");433 addArgAttribute(ArgNo, NoUndef);434 } else if (R->isSubClassOf("NonNull")) {435 unsigned ArgNo = R->getValueAsInt("ArgNo");436 addArgAttribute(ArgNo, NonNull);437 } else if (R->isSubClassOf("Returned")) {438 unsigned ArgNo = R->getValueAsInt("ArgNo");439 addArgAttribute(ArgNo, Returned);440 } else if (R->isSubClassOf("ReadOnly")) {441 unsigned ArgNo = R->getValueAsInt("ArgNo");442 addArgAttribute(ArgNo, ReadOnly);443 } else if (R->isSubClassOf("WriteOnly")) {444 unsigned ArgNo = R->getValueAsInt("ArgNo");445 addArgAttribute(ArgNo, WriteOnly);446 } else if (R->isSubClassOf("ReadNone")) {447 unsigned ArgNo = R->getValueAsInt("ArgNo");448 addArgAttribute(ArgNo, ReadNone);449 } else if (R->isSubClassOf("ImmArg")) {450 unsigned ArgNo = R->getValueAsInt("ArgNo");451 addArgAttribute(ArgNo, ImmArg);452 } else if (R->isSubClassOf("Align")) {453 unsigned ArgNo = R->getValueAsInt("ArgNo");454 uint64_t Align = R->getValueAsInt("Align");455 addArgAttribute(ArgNo, Alignment, Align);456 } else if (R->isSubClassOf("Dereferenceable")) {457 unsigned ArgNo = R->getValueAsInt("ArgNo");458 uint64_t Bytes = R->getValueAsInt("Bytes");459 addArgAttribute(ArgNo, Dereferenceable, Bytes);460 } else if (R->isSubClassOf("Range")) {461 unsigned ArgNo = R->getValueAsInt("ArgNo");462 int64_t Lower = R->getValueAsInt("Lower");463 int64_t Upper = R->getValueAsInt("Upper");464 addArgAttribute(ArgNo, Range, Lower, Upper);465 } else if (R->isSubClassOf("ArgInfo")) {466 unsigned ArgNo = R->getValueAsInt("ArgNo");467 if (ArgNo < 1)468 PrintFatalError(R->getLoc(),469 "ArgInfo requires ArgNo >= 1 (0 is return value)");470 const ListInit *Properties = R->getValueAsListInit("Properties");471 StringRef ArgName;472 StringRef FuncName;473 474 for (const Init *PropInit : Properties->getElements()) {475 if (const auto *PropDef = dyn_cast<DefInit>(PropInit)) {476 const Record *PropRec = PropDef->getDef();477 478 if (PropRec->isSubClassOf("ArgName"))479 ArgName = PropRec->getValueAsString("Name");480 else if (PropRec->isSubClassOf("ImmArgPrinter"))481 FuncName = PropRec->getValueAsString("FuncName");482 else483 PrintFatalError(PropRec->getLoc(),484 "Unknown ArgProperty type: " + PropRec->getName());485 }486 }487 addPrettyPrintFunction(ArgNo - 1, ArgName, FuncName);488 } else {489 llvm_unreachable("Unknown property!");490 }491}492 493llvm::IRMemLocation494CodeGenIntrinsic::getValueAsIRMemLocation(const Record *R) const {495 StringRef Name = R->getName();496 IRMemLocation Loc =497 StringSwitch<IRMemLocation>(Name)498 .Case("TargetMem0", IRMemLocation::TargetMem0)499 .Case("TargetMem1", IRMemLocation::TargetMem1)500 .Case("InaccessibleMem", IRMemLocation::InaccessibleMem)501 .Default(IRMemLocation::Other); // fallback enum502 503 if (Loc == IRMemLocation::Other)504 PrintFatalError(R->getLoc(), "unknown IRMemLocation: " + Name);505 506 return Loc;507}508 509bool CodeGenIntrinsic::isParamAPointer(unsigned ParamIdx) const {510 if (ParamIdx >= IS.ParamTys.size())511 return false;512 return IS.ParamTys[ParamIdx]->isSubClassOf("LLVMQualPointerType") ||513 IS.ParamTys[ParamIdx]->isSubClassOf("LLVMAnyPointerType");514}515 516bool CodeGenIntrinsic::isParamImmArg(unsigned ParamIdx) const {517 // Convert argument index to attribute index starting from `FirstArgIndex`.518 ++ParamIdx;519 if (ParamIdx >= ArgumentAttributes.size())520 return false;521 ArgAttribute Val{ImmArg, 0, 0};522 return llvm::binary_search(ArgumentAttributes[ParamIdx], Val);523}524 525void CodeGenIntrinsic::addArgAttribute(unsigned Idx, ArgAttrKind AK, uint64_t V,526 uint64_t V2) {527 if (Idx >= ArgumentAttributes.size())528 ArgumentAttributes.resize(Idx + 1);529 ArgumentAttributes[Idx].emplace_back(AK, V, V2);530}531 532void CodeGenIntrinsic::addPrettyPrintFunction(unsigned ArgIdx,533 StringRef ArgName,534 StringRef FuncName) {535 auto It = llvm::find_if(PrettyPrintFunctions, [ArgIdx](const auto &Info) {536 return Info.ArgIdx == ArgIdx;537 });538 if (It != PrettyPrintFunctions.end())539 PrintFatalError(TheDef->getLoc(), "ArgInfo for argument " + Twine(ArgIdx) +540 " is already defined as '" +541 It->FuncName + "'");542 PrettyPrintFunctions.emplace_back(ArgIdx, ArgName, FuncName);543}544