2109 lines · cpp
1//===-- SveEmitter.cpp - Generate arm_sve.h for use with clang ------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This tablegen backend is responsible for emitting arm_sve.h, which includes10// a declaration and definition of each function specified by the ARM C/C++11// Language Extensions (ACLE).12//13// For details, visit:14// https://developer.arm.com/architectures/system-architectures/software-standards/acle15//16// Each SVE instruction is implemented in terms of 1 or more functions which17// are suffixed with the element type of the input vectors. Functions may be18// implemented in terms of generic vector operations such as +, *, -, etc. or19// by calling a __builtin_-prefixed function which will be handled by clang's20// CodeGen library.21//22// See also the documentation in include/clang/Basic/arm_sve.td.23//24//===----------------------------------------------------------------------===//25 26#include "llvm/ADT/ArrayRef.h"27#include "llvm/ADT/STLExtras.h"28#include "llvm/ADT/StringExtras.h"29#include "llvm/ADT/StringMap.h"30#include "llvm/Support/raw_ostream.h"31#include "llvm/TableGen/AArch64ImmCheck.h"32#include "llvm/TableGen/Error.h"33#include "llvm/TableGen/Record.h"34#include "llvm/TableGen/StringToOffsetTable.h"35#include <array>36#include <cctype>37#include <set>38#include <string>39#include <tuple>40 41using namespace llvm;42 43enum ClassKind {44 ClassNone,45 ClassS, // signed/unsigned, e.g., "_s8", "_u8" suffix46 ClassG, // Overloaded name without type suffix47};48 49enum class ACLEKind { SVE, SME };50 51using TypeSpec = std::string;52 53namespace {54class SVEType {55 56 enum TypeKind {57 Invalid,58 Void,59 Float,60 SInt,61 UInt,62 BFloat16,63 MFloat8,64 Svcount,65 PrefetchOp,66 PredicatePattern,67 Predicate,68 Fpm69 };70 TypeKind Kind;71 bool Immediate, Constant, Pointer, DefaultType, IsScalable;72 unsigned Bitwidth, ElementBitwidth, NumVectors;73 74public:75 SVEType() : SVEType("", 'v') {}76 77 SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)78 : Kind(Invalid), Immediate(false), Constant(false), Pointer(false),79 DefaultType(false), IsScalable(true), Bitwidth(128),80 ElementBitwidth(~0U), NumVectors(NumVectors) {81 if (!TS.empty())82 applyTypespec(TS);83 applyModifier(CharMod);84 }85 86 SVEType(const SVEType &Base, unsigned NumV) : SVEType(Base) {87 NumVectors = NumV;88 }89 90 bool isPointer() const { return Pointer; }91 bool isConstant() const { return Constant; }92 bool isImmediate() const { return Immediate; }93 bool isScalar() const { return NumVectors == 0; }94 bool isVector() const { return NumVectors > 0; }95 bool isScalableVector() const { return isVector() && IsScalable; }96 bool isFixedLengthVector() const { return isVector() && !IsScalable; }97 bool isChar() const { return ElementBitwidth == 8 && isInteger(); }98 bool isVoid() const { return Kind == Void; }99 bool isDefault() const { return DefaultType; }100 bool isFloat() const { return Kind == Float; }101 bool isBFloat() const { return Kind == BFloat16; }102 bool isMFloat() const { return Kind == MFloat8; }103 bool isFloatingPoint() const {104 return Kind == Float || Kind == BFloat16 || Kind == MFloat8;105 }106 bool isInteger() const { return Kind == SInt || Kind == UInt; }107 bool isSignedInteger() const { return Kind == SInt; }108 bool isUnsignedInteger() const { return Kind == UInt; }109 bool isScalarPredicate() const {110 return Kind == Predicate && NumVectors == 0;111 }112 bool isPredicate() const { return Kind == Predicate; }113 bool isPredicatePattern() const { return Kind == PredicatePattern; }114 bool isPrefetchOp() const { return Kind == PrefetchOp; }115 bool isSvcount() const { return Kind == Svcount; }116 bool isFpm() const { return Kind == Fpm; }117 bool isInvalid() const { return Kind == Invalid; }118 unsigned getElementSizeInBits() const { return ElementBitwidth; }119 unsigned getNumVectors() const { return NumVectors; }120 121 unsigned getNumElements() const {122 assert(ElementBitwidth != ~0U);123 return isPredicate() ? 16 : (Bitwidth / ElementBitwidth);124 }125 unsigned getSizeInBits() const {126 return Bitwidth;127 }128 129 /// Return the string representation of a type, which is an encoded130 /// string for passing to the BUILTIN() macro in Builtins.def.131 std::string builtin_str() const;132 133 /// Return the C/C++ string representation of a type for use in the134 /// arm_sve.h header file.135 std::string str() const;136 137private:138 /// Creates the type based on the typespec string in TS.139 void applyTypespec(StringRef TS);140 141 /// Applies a prototype modifier to the type.142 void applyModifier(char Mod);143 144 /// Get the builtin base for this SVEType, e.g. 'Wi' for svint64_t.145 std::string builtinBaseType() const;146};147 148class SVEEmitter;149 150/// The main grunt class. This represents an instantiation of an intrinsic with151/// a particular typespec and prototype.152class Intrinsic {153 /// The unmangled name.154 std::string Name;155 156 /// The name of the corresponding LLVM IR intrinsic.157 std::string LLVMName;158 159 /// Intrinsic prototype.160 std::string Proto;161 162 /// The base type spec for this intrinsic.163 TypeSpec BaseTypeSpec;164 165 /// The base class kind. Most intrinsics use ClassS, which has full type166 /// info for integers (_s32/_u32), or ClassG which is used for overloaded167 /// intrinsics.168 ClassKind Class;169 170 /// The architectural #ifdef guard.171 std::string SVEGuard, SMEGuard;172 173 // The merge suffix such as _m, _x or _z.174 std::string MergeSuffix;175 176 /// The types of return value [0] and parameters [1..].177 std::vector<SVEType> Types;178 179 /// The "base type", which is VarType('d', BaseTypeSpec).180 SVEType BaseType;181 182 uint64_t Flags;183 184 SmallVector<ImmCheck, 2> ImmChecks;185 186 bool SetsFPMR;187 188public:189 Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,190 StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,191 uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT,192 ClassKind Class, SVEEmitter &Emitter, StringRef SVEGuard,193 StringRef SMEGuard);194 195 ~Intrinsic()=default;196 197 std::string getName() const { return Name; }198 std::string getLLVMName() const { return LLVMName; }199 std::string getProto() const { return Proto; }200 TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; }201 SVEType getBaseType() const { return BaseType; }202 203 StringRef getSVEGuard() const { return SVEGuard; }204 StringRef getSMEGuard() const { return SMEGuard; }205 std::string getGuard() const {206 std::string Guard;207 llvm::raw_string_ostream OS(Guard);208 if (!SVEGuard.empty() && SMEGuard.empty())209 OS << SVEGuard;210 else if (SVEGuard.empty() && !SMEGuard.empty())211 OS << SMEGuard;212 else {213 if (SVEGuard.find(",") != std::string::npos ||214 SVEGuard.find("|") != std::string::npos)215 OS << "(" << SVEGuard << ")";216 else217 OS << SVEGuard;218 OS << "|";219 if (SMEGuard.find(",") != std::string::npos ||220 SMEGuard.find("|") != std::string::npos)221 OS << "(" << SMEGuard << ")";222 else223 OS << SMEGuard;224 }225 return Guard;226 }227 ClassKind getClassKind() const { return Class; }228 229 SVEType getReturnType() const { return Types[0]; }230 ArrayRef<SVEType> getTypes() const { return Types; }231 SVEType getParamType(unsigned I) const { return Types[I + 1]; }232 unsigned getNumParams() const {233 return Proto.size() - (2 * count(Proto, '.')) - 1;234 }235 236 uint64_t getFlags() const { return Flags; }237 bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}238 239 ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; }240 241 /// Return the type string for a BUILTIN() macro in Builtins.def.242 std::string getBuiltinTypeStr();243 244 /// Return the name, mangled with type information. The name is mangled for245 /// ClassS, so will add type suffixes such as _u32/_s32.246 std::string getMangledName(ClassKind CK = ClassS) const {247 return mangleName(CK);248 }249 250 /// As above, but mangles the LLVM name instead.251 std::string getMangledLLVMName() const { return mangleLLVMName(); }252 253 /// Returns true if the intrinsic is overloaded, in that it should also generate254 /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of255 /// 'svld1_u32(..)'.256 static bool isOverloadedIntrinsic(StringRef Name) {257 return Name.contains('[') && Name.contains(']');258 }259 260 /// Return true if the intrinsic takes a splat operand.261 bool hasSplat() const {262 // These prototype modifiers are described in arm_sve.td.263 return Proto.find_first_of("ajfrKLR@!") != std::string::npos;264 }265 266 /// Return the parameter index of the splat operand.267 unsigned getSplatIdx() const {268 unsigned I = 1, Param = 0;269 for (; I < Proto.size(); ++I, ++Param) {270 if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' ||271 Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' ||272 Proto[I] == 'R' || Proto[I] == '@' || Proto[I] == '!')273 break;274 275 // Multivector modifier can be skipped276 if (Proto[I] == '.')277 I += 2;278 }279 assert(I != Proto.size() && "Prototype has no splat operand");280 return Param;281 }282 283 /// Emits the intrinsic declaration to the ostream.284 void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter, ACLEKind Kind) const;285 286private:287 std::string getMergeSuffix() const { return MergeSuffix; }288 StringRef getFPMSuffix() const { return SetsFPMR ? "_fpm" : ""; }289 std::string mangleName(ClassKind LocalCK) const;290 std::string mangleLLVMName() const;291 std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,292 std::string Proto) const;293};294 295class SVEEmitter {296private:297 // The reinterpret builtins are generated separately because they298 // need the cross product of all types (121 functions in total),299 // which is inconvenient to specify in the arm_sve.td file or300 // generate in CGBuiltin.cpp.301 struct ReinterpretTypeInfo {302 SVEType BaseType;303 const char *Suffix;304 };305 306 static const std::array<ReinterpretTypeInfo, 13> Reinterprets;307 308 const RecordKeeper &Records;309 StringMap<uint64_t> EltTypes;310 StringMap<uint64_t> MemEltTypes;311 StringMap<uint64_t> FlagTypes;312 StringMap<uint64_t> MergeTypes;313 StringMap<uint64_t> ImmCheckTypes;314 std::vector<llvm::StringRef> ImmCheckTypeNames;315 316public:317 SVEEmitter(const RecordKeeper &R) : Records(R) {318 for (auto *RV : Records.getAllDerivedDefinitions("EltType"))319 EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");320 for (auto *RV : Records.getAllDerivedDefinitions("MemEltType"))321 MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");322 for (auto *RV : Records.getAllDerivedDefinitions("FlagType"))323 FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");324 for (auto *RV : Records.getAllDerivedDefinitions("MergeType"))325 MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");326 for (auto *RV : Records.getAllDerivedDefinitions("ImmCheckType")) {327 auto [it, inserted] = ImmCheckTypes.try_emplace(328 RV->getNameInitAsString(), RV->getValueAsInt("Value"));329 if (!inserted)330 llvm_unreachable("Duplicate imm check");331 if ((size_t)it->second >= ImmCheckTypeNames.size())332 ImmCheckTypeNames.resize((size_t)it->second + 1);333 ImmCheckTypeNames[it->second] = it->first();334 }335 }336 337 /// Returns the enum value for the immcheck type338 unsigned getEnumValueForImmCheck(StringRef C) const {339 auto It = ImmCheckTypes.find(C);340 if (It != ImmCheckTypes.end())341 return It->getValue();342 llvm_unreachable("Unsupported imm check");343 }344 345 /// Returns the enum value for the flag type346 uint64_t getEnumValueForFlag(StringRef C) const {347 auto Res = FlagTypes.find(C);348 if (Res != FlagTypes.end())349 return Res->getValue();350 llvm_unreachable("Unsupported flag");351 }352 353 /// Returns the name for the immcheck type354 StringRef getImmCheckForEnumValue(unsigned Id) {355 if ((size_t)Id < ImmCheckTypeNames.size())356 return ImmCheckTypeNames[Id];357 llvm_unreachable("Unsupported imm check");358 }359 360 // Returns the SVETypeFlags for a given value and mask.361 uint64_t encodeFlag(uint64_t V, StringRef MaskName) const {362 auto It = FlagTypes.find(MaskName);363 if (It != FlagTypes.end()) {364 uint64_t Mask = It->getValue();365 unsigned Shift = countr_zero(Mask);366 assert(Shift < 64 && "Mask value produced an invalid shift value");367 return (V << Shift) & Mask;368 }369 llvm_unreachable("Unsupported flag");370 }371 372 // Returns the SVETypeFlags for the given element type.373 uint64_t encodeEltType(StringRef EltName) {374 auto It = EltTypes.find(EltName);375 if (It != EltTypes.end())376 return encodeFlag(It->getValue(), "EltTypeMask");377 llvm_unreachable("Unsupported EltType");378 }379 380 // Returns the SVETypeFlags for the given memory element type.381 uint64_t encodeMemoryElementType(uint64_t MT) {382 return encodeFlag(MT, "MemEltTypeMask");383 }384 385 // Returns the SVETypeFlags for the given merge type.386 uint64_t encodeMergeType(uint64_t MT) {387 return encodeFlag(MT, "MergeTypeMask");388 }389 390 // Returns the SVETypeFlags for the given splat operand.391 unsigned encodeSplatOperand(unsigned SplatIdx) {392 assert(SplatIdx < 7 && "SplatIdx out of encodable range");393 return encodeFlag(SplatIdx + 1, "SplatOperandMask");394 }395 396 // Returns the SVETypeFlags value for the given SVEType.397 uint64_t encodeTypeFlags(const SVEType &T);398 399 /// Emit arm_sve.h.400 void createHeader(raw_ostream &o);401 402 // Emits core intrinsics in both arm_sme.h and arm_sve.h403 void createCoreHeaderIntrinsics(raw_ostream &o, SVEEmitter &Emitter,404 ACLEKind Kind);405 406 /// Emit all the __builtin prototypes and code needed by Sema.407 void createBuiltins(raw_ostream &o);408 409 /// Emit all the __builtin prototypes in JSON format.410 void createBuiltinsJSON(raw_ostream &o);411 412 /// Emit all the information needed to map builtin -> LLVM IR intrinsic.413 void createCodeGenMap(raw_ostream &o);414 415 /// Emit all the range checks for the immediates.416 void createRangeChecks(raw_ostream &o);417 418 // Emit all the ImmCheckTypes to arm_immcheck_types.inc419 void createImmCheckTypes(raw_ostream &OS);420 421 /// Create the SVETypeFlags used in CGBuiltins422 void createTypeFlags(raw_ostream &o);423 424 /// Emit arm_sme.h.425 void createSMEHeader(raw_ostream &o);426 427 /// Emit all the SME __builtin prototypes and code needed by Sema.428 void createSMEBuiltins(raw_ostream &o);429 430 /// Emit all the information needed to map builtin -> LLVM IR intrinsic.431 void createSMECodeGenMap(raw_ostream &o);432 433 /// Create a table for a builtin's requirement for PSTATE.SM.434 void createStreamingAttrs(raw_ostream &o, ACLEKind Kind);435 436 /// Emit all the range checks for the immediates.437 void createSMERangeChecks(raw_ostream &o);438 439 /// Create a table for a builtin's requirement for PSTATE.ZA.440 void createBuiltinZAState(raw_ostream &OS);441 442 /// Create intrinsic and add it to \p Out443 void createIntrinsic(const Record *R,444 SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out);445};446 447const std::array<SVEEmitter::ReinterpretTypeInfo, 13> SVEEmitter::Reinterprets =448 {{{SVEType("c", 'd'), "s8"},449 {SVEType("Uc", 'd'), "u8"},450 {SVEType("m", 'd'), "mf8"},451 {SVEType("s", 'd'), "s16"},452 {SVEType("Us", 'd'), "u16"},453 {SVEType("i", 'd'), "s32"},454 {SVEType("Ui", 'd'), "u32"},455 {SVEType("l", 'd'), "s64"},456 {SVEType("Ul", 'd'), "u64"},457 {SVEType("h", 'd'), "f16"},458 {SVEType("b", 'd'), "bf16"},459 {SVEType("f", 'd'), "f32"},460 {SVEType("d", 'd'), "f64"}}};461 462} // end anonymous namespace463 464//===----------------------------------------------------------------------===//465// Type implementation466//===----------------------------------------------------------------------===//467 468std::string SVEType::builtinBaseType() const {469 switch (Kind) {470 case TypeKind::Void:471 return "v";472 case TypeKind::Svcount:473 return "Qa";474 case TypeKind::PrefetchOp:475 case TypeKind::PredicatePattern:476 return "i";477 case TypeKind::Fpm:478 return "UWi";479 case TypeKind::Predicate:480 return "b";481 case TypeKind::BFloat16:482 assert(ElementBitwidth == 16 && "Invalid BFloat16!");483 return "y";484 case TypeKind::MFloat8:485 assert(ElementBitwidth == 8 && "Invalid MFloat8!");486 return "m";487 case TypeKind::Float:488 switch (ElementBitwidth) {489 case 16:490 return "h";491 case 32:492 return "f";493 case 64:494 return "d";495 default:496 llvm_unreachable("Unhandled float width!");497 }498 case TypeKind::SInt:499 case TypeKind::UInt:500 switch (ElementBitwidth) {501 case 1:502 return "b";503 case 8:504 return "c";505 case 16:506 return "s";507 case 32:508 return "i";509 case 64:510 return "Wi";511 case 128:512 return "LLLi";513 default:514 llvm_unreachable("Unhandled bitwidth!");515 }516 case TypeKind::Invalid:517 llvm_unreachable("Attempting to resolve builtin string from Invalid type!");518 }519 llvm_unreachable("Unhandled TypeKind!");520}521 522std::string SVEType::builtin_str() const {523 std::string Prefix;524 525 if (isScalableVector())526 Prefix = "q" + llvm::utostr(getNumElements() * NumVectors);527 else if (isFixedLengthVector())528 Prefix = "V" + llvm::utostr(getNumElements() * NumVectors);529 else if (isImmediate()) {530 assert(!isFloatingPoint() && "fp immediates are not supported");531 Prefix = "I";532 }533 534 // Make chars and integer pointers explicitly signed.535 if ((ElementBitwidth == 8 || isPointer()) && isSignedInteger())536 Prefix += "S";537 else if (isUnsignedInteger())538 Prefix += "U";539 540 std::string BuiltinStr = Prefix + builtinBaseType();541 if (isConstant())542 BuiltinStr += "C";543 if (isPointer())544 BuiltinStr += "*";545 546 return BuiltinStr;547}548 549std::string SVEType::str() const {550 std::string TypeStr;551 552 switch (Kind) {553 case TypeKind::PrefetchOp:554 return "enum svprfop";555 case TypeKind::PredicatePattern:556 return "enum svpattern";557 case TypeKind::Fpm:558 TypeStr += "fpm";559 break;560 case TypeKind::Void:561 TypeStr += "void";562 break;563 case TypeKind::Float:564 TypeStr += "float" + llvm::utostr(ElementBitwidth);565 break;566 case TypeKind::Svcount:567 TypeStr += "svcount";568 break;569 case TypeKind::Predicate:570 TypeStr += "bool";571 break;572 case TypeKind::BFloat16:573 TypeStr += "bfloat16";574 break;575 case TypeKind::MFloat8:576 TypeStr += "mfloat8";577 break;578 case TypeKind::SInt:579 TypeStr += "int" + llvm::utostr(ElementBitwidth);580 break;581 case TypeKind::UInt:582 TypeStr += "uint" + llvm::utostr(ElementBitwidth);583 break;584 case TypeKind::Invalid:585 llvm_unreachable("Attempting to resolve type name from Invalid type!");586 }587 588 if (isFixedLengthVector())589 TypeStr += "x" + llvm::utostr(getNumElements());590 else if (isScalableVector())591 TypeStr = "sv" + TypeStr;592 593 if (NumVectors > 1)594 TypeStr += "x" + llvm::utostr(NumVectors);595 if (!isScalarPredicate() && !isVoid())596 TypeStr += "_t";597 if (isConstant())598 TypeStr += " const";599 if (isPointer())600 TypeStr += " *";601 602 return TypeStr;603}604 605void SVEType::applyTypespec(StringRef TS) {606 for (char I : TS) {607 switch (I) {608 case 'Q':609 assert(isInvalid() && "Unexpected use of typespec modifier");610 Kind = Svcount;611 break;612 case 'P':613 assert(isInvalid() && "Unexpected use of typespec modifier");614 Kind = Predicate;615 break;616 case 'U':617 assert(isInvalid() && "Unexpected use of typespec modifier");618 Kind = UInt;619 break;620 case 'c':621 Kind = isInvalid() ? SInt : Kind;622 ElementBitwidth = 8;623 break;624 case 's':625 Kind = isInvalid() ? SInt : Kind;626 ElementBitwidth = 16;627 break;628 case 'i':629 Kind = isInvalid() ? SInt : Kind;630 ElementBitwidth = 32;631 break;632 case 'l':633 Kind = isInvalid() ? SInt : Kind;634 ElementBitwidth = 64;635 break;636 case 'q':637 Kind = isInvalid() ? SInt : Kind;638 ElementBitwidth = 128;639 break;640 case 'h':641 assert(isInvalid() && "Unexpected use of typespec modifier");642 Kind = Float;643 ElementBitwidth = 16;644 break;645 case 'f':646 assert(isInvalid() && "Unexpected use of typespec modifier");647 Kind = Float;648 ElementBitwidth = 32;649 break;650 case 'd':651 assert(isInvalid() && "Unexpected use of typespec modifier");652 Kind = Float;653 ElementBitwidth = 64;654 break;655 case 'b':656 assert(isInvalid() && "Unexpected use of typespec modifier");657 Kind = BFloat16;658 ElementBitwidth = 16;659 break;660 case 'm':661 assert(isInvalid() && "Unexpected use of typespec modifier");662 Kind = MFloat8;663 ElementBitwidth = 8;664 break;665 default:666 llvm_unreachable("Unhandled type code!");667 }668 }669 assert(ElementBitwidth != ~0U && "Bad element bitwidth!");670}671 672void SVEType::applyModifier(char Mod) {673 switch (Mod) {674 case 'v':675 Kind = Void;676 NumVectors = 0;677 break;678 case 'd':679 DefaultType = true;680 break;681 case 'c':682 Constant = true;683 [[fallthrough]];684 case 'p':685 Pointer = true;686 Bitwidth = ElementBitwidth;687 NumVectors = 0;688 break;689 case 'e':690 Kind = UInt;691 ElementBitwidth /= 2;692 break;693 case 'h':694 ElementBitwidth /= 2;695 break;696 case 'q':697 ElementBitwidth /= 4;698 break;699 case 'b':700 Kind = UInt;701 ElementBitwidth /= 4;702 break;703 case 'o':704 ElementBitwidth *= 4;705 break;706 case 'P':707 Kind = Predicate;708 Bitwidth = 16;709 ElementBitwidth = 1;710 break;711 case '{':712 IsScalable = false;713 Bitwidth = 128;714 NumVectors = 1;715 break;716 case 's':717 case 'a':718 Bitwidth = ElementBitwidth;719 NumVectors = 0;720 break;721 case 'R':722 ElementBitwidth /= 2;723 NumVectors = 0;724 break;725 case 'r':726 ElementBitwidth /= 4;727 NumVectors = 0;728 break;729 case '@':730 Kind = UInt;731 ElementBitwidth /= 4;732 NumVectors = 0;733 break;734 case 'K':735 Kind = SInt;736 Bitwidth = ElementBitwidth;737 NumVectors = 0;738 break;739 case 'L':740 Kind = UInt;741 Bitwidth = ElementBitwidth;742 NumVectors = 0;743 break;744 case 'u':745 Kind = UInt;746 break;747 case 'x':748 Kind = SInt;749 break;750 case 'i':751 Kind = UInt;752 ElementBitwidth = Bitwidth = 64;753 NumVectors = 0;754 Immediate = true;755 break;756 case 'I':757 Kind = PredicatePattern;758 ElementBitwidth = Bitwidth = 32;759 NumVectors = 0;760 Immediate = true;761 break;762 case 'J':763 Kind = PrefetchOp;764 ElementBitwidth = Bitwidth = 32;765 NumVectors = 0;766 Immediate = true;767 break;768 case 'k':769 Kind = SInt;770 ElementBitwidth = Bitwidth = 32;771 NumVectors = 0;772 break;773 case 'l':774 Kind = SInt;775 ElementBitwidth = Bitwidth = 64;776 NumVectors = 0;777 break;778 case 'm':779 Kind = UInt;780 ElementBitwidth = Bitwidth = 32;781 NumVectors = 0;782 break;783 case '>':784 Kind = Fpm;785 ElementBitwidth = Bitwidth = 64;786 NumVectors = 0;787 break;788 case 'n':789 Kind = UInt;790 ElementBitwidth = Bitwidth = 64;791 NumVectors = 0;792 break;793 case 'w':794 ElementBitwidth = 64;795 break;796 case 'j':797 ElementBitwidth = Bitwidth = 64;798 NumVectors = 0;799 break;800 case 'f':801 Kind = UInt;802 ElementBitwidth = Bitwidth = 64;803 NumVectors = 0;804 break;805 case 'g':806 Kind = UInt;807 ElementBitwidth = 64;808 break;809 case '#':810 Kind = SInt;811 ElementBitwidth = 64;812 break;813 case '[':814 Kind = UInt;815 ElementBitwidth = 8;816 break;817 case 't':818 Kind = SInt;819 ElementBitwidth = 32;820 break;821 case 'z':822 Kind = UInt;823 ElementBitwidth = 32;824 break;825 case 'O':826 Kind = Float;827 ElementBitwidth = 16;828 break;829 case 'M':830 Kind = Float;831 ElementBitwidth = 32;832 break;833 case 'N':834 Kind = Float;835 ElementBitwidth = 64;836 break;837 case 'Q':838 Kind = Void;839 Constant = true;840 Pointer = true;841 NumVectors = 0;842 break;843 case 'S':844 Kind = SInt;845 Constant = true;846 Pointer = true;847 ElementBitwidth = Bitwidth = 8;848 NumVectors = 0;849 break;850 case 'W':851 Kind = UInt;852 Constant = true;853 Pointer = true;854 ElementBitwidth = Bitwidth = 8;855 NumVectors = 0;856 break;857 case 'T':858 Kind = SInt;859 Constant = true;860 Pointer = true;861 ElementBitwidth = Bitwidth = 16;862 NumVectors = 0;863 break;864 case 'X':865 Kind = UInt;866 Constant = true;867 Pointer = true;868 ElementBitwidth = Bitwidth = 16;869 NumVectors = 0;870 break;871 case 'Y':872 Kind = UInt;873 Constant = true;874 Pointer = true;875 ElementBitwidth = Bitwidth = 32;876 NumVectors = 0;877 break;878 case 'U':879 Kind = SInt;880 Constant = true;881 Pointer = true;882 ElementBitwidth = Bitwidth = 32;883 NumVectors = 0;884 break;885 case '%':886 Kind = Void;887 Pointer = true;888 NumVectors = 0;889 break;890 case 'A':891 Kind = SInt;892 Pointer = true;893 ElementBitwidth = Bitwidth = 8;894 NumVectors = 0;895 break;896 case 'B':897 Kind = SInt;898 Pointer = true;899 ElementBitwidth = Bitwidth = 16;900 NumVectors = 0;901 break;902 case 'C':903 Kind = SInt;904 Pointer = true;905 ElementBitwidth = Bitwidth = 32;906 NumVectors = 0;907 break;908 case 'D':909 Kind = SInt;910 Pointer = true;911 ElementBitwidth = Bitwidth = 64;912 NumVectors = 0;913 break;914 case 'E':915 Kind = UInt;916 Pointer = true;917 ElementBitwidth = Bitwidth = 8;918 NumVectors = 0;919 break;920 case 'F':921 Kind = UInt;922 Pointer = true;923 ElementBitwidth = Bitwidth = 16;924 NumVectors = 0;925 break;926 case 'G':927 Kind = UInt;928 Pointer = true;929 ElementBitwidth = Bitwidth = 32;930 NumVectors = 0;931 break;932 case '$':933 Kind = BFloat16;934 ElementBitwidth = 16;935 break;936 case '}':937 Kind = Svcount;938 NumVectors = 0;939 break;940 case '~':941 Kind = MFloat8;942 ElementBitwidth = 8;943 break;944 case '!':945 Kind = MFloat8;946 Bitwidth = ElementBitwidth = 8;947 NumVectors = 0;948 break;949 case '.':950 llvm_unreachable(". is never a type in itself");951 break;952 default:953 llvm_unreachable("Unhandled character!");954 }955}956 957/// Returns the modifier and number of vectors for the given operand \p Op.958std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) {959 for (unsigned P = 0; !Proto.empty(); ++P) {960 unsigned NumVectors = 1;961 unsigned CharsToSkip = 1;962 char Mod = Proto[0];963 if (Mod == '2' || Mod == '3' || Mod == '4') {964 NumVectors = Mod - '0';965 Mod = 'd';966 if (Proto.size() > 1 && Proto[1] == '.') {967 Mod = Proto[2];968 CharsToSkip = 3;969 }970 }971 972 if (P == Op)973 return {Mod, NumVectors};974 975 Proto = Proto.drop_front(CharsToSkip);976 }977 llvm_unreachable("Unexpected Op");978}979 980//===----------------------------------------------------------------------===//981// Intrinsic implementation982//===----------------------------------------------------------------------===//983 984Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,985 StringRef MergeSuffix, uint64_t MemoryElementTy,986 StringRef LLVMName, uint64_t Flags,987 ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class,988 SVEEmitter &Emitter, StringRef SVEGuard,989 StringRef SMEGuard)990 : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()),991 BaseTypeSpec(BT), Class(Class), MergeSuffix(MergeSuffix.str()),992 BaseType(BT, 'd'), Flags(Flags), ImmChecks(Checks) {993 994 auto FormatGuard = [](StringRef Guard, StringRef Base) -> std::string {995 if (Guard.empty() || Guard == Base)996 return Guard.str();997 998 unsigned Depth = 0;999 for (auto &C : Guard) {1000 switch (C) {1001 default:1002 break;1003 case '|':1004 if (Depth == 0)1005 // Group top-level ORs before ANDing with the base feature.1006 return Base.str() + ",(" + Guard.str() + ")";1007 break;1008 case '(':1009 ++Depth;1010 break;1011 case ')':1012 if (Depth == 0)1013 llvm_unreachable("Mismatched parentheses!");1014 1015 --Depth;1016 break;1017 }1018 }1019 1020 return Base.str() + "," + Guard.str();1021 };1022 1023 this->SVEGuard = FormatGuard(SVEGuard, "sve");1024 this->SMEGuard = FormatGuard(SMEGuard, "sme");1025 1026 // Types[0] is the return value.1027 for (unsigned I = 0; I < (getNumParams() + 1); ++I) {1028 char Mod;1029 unsigned NumVectors;1030 std::tie(Mod, NumVectors) = getProtoModifier(Proto, I);1031 SVEType T(BaseTypeSpec, Mod, NumVectors);1032 Types.push_back(T);1033 SetsFPMR = T.isFpm();1034 1035 // Add range checks for immediates1036 if (I > 0) {1037 if (T.isPredicatePattern())1038 ImmChecks.emplace_back(1039 I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_31"));1040 else if (T.isPrefetchOp())1041 ImmChecks.emplace_back(1042 I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_13"));1043 }1044 }1045 1046 // Set flags based on properties1047 this->Flags |= Emitter.encodeTypeFlags(BaseType);1048 this->Flags |= Emitter.encodeMemoryElementType(MemoryElementTy);1049 this->Flags |= Emitter.encodeMergeType(MergeTy);1050 if (hasSplat())1051 this->Flags |= Emitter.encodeSplatOperand(getSplatIdx());1052 if (SetsFPMR)1053 this->Flags |= Emitter.getEnumValueForFlag("SetsFPMR");1054}1055 1056std::string Intrinsic::getBuiltinTypeStr() {1057 std::string S = getReturnType().builtin_str();1058 for (unsigned I = 0; I < getNumParams(); ++I)1059 S += getParamType(I).builtin_str();1060 1061 return S;1062}1063 1064std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,1065 std::string Proto) const {1066 std::string Ret = Name;1067 while (Ret.find('{') != std::string::npos) {1068 size_t Pos = Ret.find('{');1069 size_t End = Ret.find('}');1070 unsigned NumChars = End - Pos + 1;1071 assert(NumChars == 3 && "Unexpected template argument");1072 1073 SVEType T;1074 char C = Ret[Pos+1];1075 switch(C) {1076 default:1077 llvm_unreachable("Unknown predication specifier");1078 case 'd':1079 T = SVEType(TS, 'd');1080 break;1081 case '0':1082 case '1':1083 case '2':1084 case '3':1085 // Extract the modifier before passing to SVEType to handle numeric1086 // modifiers1087 auto [Mod, NumVectors] = getProtoModifier(Proto, (C - '0'));1088 T = SVEType(TS, Mod);1089 break;1090 }1091 1092 // Replace templated arg with the right suffix (e.g. u32)1093 std::string TypeCode;1094 1095 if (T.isSignedInteger())1096 TypeCode = 's';1097 else if (T.isUnsignedInteger())1098 TypeCode = 'u';1099 else if (T.isSvcount())1100 TypeCode = 'c';1101 else if (T.isPredicate())1102 TypeCode = 'b';1103 else if (T.isBFloat())1104 TypeCode = "bf";1105 else if (T.isMFloat())1106 TypeCode = "mf";1107 else1108 TypeCode = 'f';1109 Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits()));1110 }1111 1112 return Ret;1113}1114 1115std::string Intrinsic::mangleLLVMName() const {1116 std::string S = getLLVMName();1117 1118 // Replace all {d} like expressions with e.g. 'u32'1119 return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto());1120}1121 1122std::string Intrinsic::mangleName(ClassKind LocalCK) const {1123 std::string S = getName();1124 1125 if (LocalCK == ClassG) {1126 // Remove the square brackets and everything in between.1127 while (S.find('[') != std::string::npos) {1128 auto Start = S.find('[');1129 auto End = S.find(']');1130 S.erase(Start, (End-Start)+1);1131 }1132 } else {1133 // Remove the square brackets.1134 while (S.find('[') != std::string::npos) {1135 auto BrPos = S.find('[');1136 if (BrPos != std::string::npos)1137 S.erase(BrPos, 1);1138 BrPos = S.find(']');1139 if (BrPos != std::string::npos)1140 S.erase(BrPos, 1);1141 }1142 }1143 1144 // Replace all {d} like expressions with e.g. 'u32'1145 return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto())1146 .append(getMergeSuffix())1147 .append(getFPMSuffix());1148}1149 1150void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter,1151 ACLEKind Kind) const {1152 bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1;1153 1154 std::string FullName = mangleName(ClassS);1155 std::string ProtoName = mangleName(getClassKind());1156 OS << (IsOverloaded ? "__aio " : "__ai ")1157 << "__attribute__((__clang_arm_builtin_alias(";1158 1159 switch (Kind) {1160 case ACLEKind::SME:1161 OS << "__builtin_sme_" << FullName << ")";1162 break;1163 case ACLEKind::SVE:1164 OS << "__builtin_sve_" << FullName << ")";1165 break;1166 }1167 1168 OS << "))\n";1169 1170 OS << getTypes()[0].str() << " " << ProtoName << "(";1171 for (unsigned I = 0; I < getTypes().size() - 1; ++I) {1172 if (I != 0)1173 OS << ", ";1174 OS << getTypes()[I + 1].str();1175 }1176 OS << ");\n";1177}1178 1179//===----------------------------------------------------------------------===//1180// SVEEmitter implementation1181//===----------------------------------------------------------------------===//1182uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {1183 if (T.isFloat()) {1184 switch (T.getElementSizeInBits()) {1185 case 16:1186 return encodeEltType("EltTyFloat16");1187 case 32:1188 return encodeEltType("EltTyFloat32");1189 case 64:1190 return encodeEltType("EltTyFloat64");1191 default:1192 llvm_unreachable("Unhandled float element bitwidth!");1193 }1194 }1195 1196 if (T.isBFloat()) {1197 assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");1198 return encodeEltType("EltTyBFloat16");1199 }1200 1201 if (T.isMFloat()) {1202 assert(T.getElementSizeInBits() == 8 && "Not a valid MFloat.");1203 return encodeEltType("EltTyMFloat8");1204 }1205 1206 if (T.isPredicate() || T.isSvcount()) {1207 switch (T.getElementSizeInBits()) {1208 case 8:1209 return encodeEltType("EltTyBool8");1210 case 16:1211 return encodeEltType("EltTyBool16");1212 case 32:1213 return encodeEltType("EltTyBool32");1214 case 64:1215 return encodeEltType("EltTyBool64");1216 default:1217 llvm_unreachable("Unhandled predicate element bitwidth!");1218 }1219 }1220 1221 switch (T.getElementSizeInBits()) {1222 case 8:1223 return encodeEltType("EltTyInt8");1224 case 16:1225 return encodeEltType("EltTyInt16");1226 case 32:1227 return encodeEltType("EltTyInt32");1228 case 64:1229 return encodeEltType("EltTyInt64");1230 case 128:1231 return encodeEltType("EltTyInt128");1232 default:1233 llvm_unreachable("Unhandled integer element bitwidth!");1234 }1235}1236 1237void SVEEmitter::createIntrinsic(1238 const Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) {1239 StringRef Name = R->getValueAsString("Name");1240 StringRef Proto = R->getValueAsString("Prototype");1241 StringRef Types = R->getValueAsString("Types");1242 StringRef SVEGuard = R->getValueAsString("SVETargetGuard");1243 StringRef SMEGuard = R->getValueAsString("SMETargetGuard");1244 StringRef LLVMName = R->getValueAsString("LLVMIntrinsic");1245 uint64_t Merge = R->getValueAsInt("Merge");1246 StringRef MergeSuffix = R->getValueAsString("MergeSuffix");1247 uint64_t MemEltType = R->getValueAsInt("MemEltType");1248 1249 int64_t Flags = 0;1250 for (const Record *FlagRec : R->getValueAsListOfDefs("Flags"))1251 Flags |= FlagRec->getValueAsInt("Value");1252 1253 // Create a dummy TypeSpec for non-overloaded builtins.1254 if (Types.empty()) {1255 assert((Flags & getEnumValueForFlag("IsOverloadNone")) &&1256 "Expect TypeSpec for overloaded builtin!");1257 Types = "i";1258 }1259 1260 // Extract type specs from string1261 SmallVector<TypeSpec, 8> TypeSpecs;1262 TypeSpec Acc;1263 for (char I : Types) {1264 Acc.push_back(I);1265 if (islower(I)) {1266 TypeSpecs.push_back(TypeSpec(Acc));1267 Acc.clear();1268 }1269 }1270 1271 // Remove duplicate type specs.1272 sort(TypeSpecs);1273 TypeSpecs.erase(llvm::unique(TypeSpecs), TypeSpecs.end());1274 1275 // Create an Intrinsic for each type spec.1276 for (auto TS : TypeSpecs) {1277 // Collate a list of range/option checks for the immediates.1278 SmallVector<ImmCheck, 2> ImmChecks;1279 for (const Record *ImmR : R->getValueAsListOfDefs("ImmChecks")) {1280 int64_t ArgIdx = ImmR->getValueAsInt("ImmArgIdx");1281 int64_t EltSizeArgIdx = ImmR->getValueAsInt("TypeContextArgIdx");1282 int64_t Kind = ImmR->getValueAsDef("Kind")->getValueAsInt("Value");1283 assert(ArgIdx >= 0 && Kind >= 0 &&1284 "ImmArgIdx and Kind must be nonnegative");1285 1286 unsigned ElementSizeInBits = 0;1287 auto [Mod, NumVectors] = getProtoModifier(Proto, EltSizeArgIdx + 1);1288 if (EltSizeArgIdx >= 0)1289 ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits();1290 ImmChecks.push_back(ImmCheck(ArgIdx, Kind, ElementSizeInBits));1291 }1292 1293 Out.push_back(std::make_unique<Intrinsic>(1294 Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, ImmChecks,1295 TS, ClassS, *this, SVEGuard, SMEGuard));1296 1297 // Also generate the short-form (e.g. svadd_m) for the given type-spec.1298 if (Intrinsic::isOverloadedIntrinsic(Name))1299 Out.push_back(std::make_unique<Intrinsic>(1300 Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags,1301 ImmChecks, TS, ClassG, *this, SVEGuard, SMEGuard));1302 }1303}1304 1305void SVEEmitter::createCoreHeaderIntrinsics(raw_ostream &OS,1306 SVEEmitter &Emitter,1307 ACLEKind Kind) {1308 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;1309 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");1310 for (auto *R : RV)1311 createIntrinsic(R, Defs);1312 1313 // Sort intrinsics in header file by following order/priority:1314 // - Architectural guard (i.e. does it require SVE2 or SVE2_AES)1315 // - Class (is intrinsic overloaded or not)1316 // - Intrinsic name1317 llvm::stable_sort(Defs, [](const std::unique_ptr<Intrinsic> &A,1318 const std::unique_ptr<Intrinsic> &B) {1319 auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {1320 return std::make_tuple(I->getSVEGuard().str() + I->getSMEGuard().str(),1321 (unsigned)I->getClassKind(), I->getName());1322 };1323 return ToTuple(A) < ToTuple(B);1324 });1325 1326 // Actually emit the intrinsic declarations.1327 for (auto &I : Defs)1328 I->emitIntrinsic(OS, Emitter, Kind);1329}1330 1331void SVEEmitter::createHeader(raw_ostream &OS) {1332 OS << "/*===---- arm_sve.h - ARM SVE intrinsics "1333 "-----------------------------------===\n"1334 " *\n"1335 " *\n"1336 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "1337 "Exceptions.\n"1338 " * See https://llvm.org/LICENSE.txt for license information.\n"1339 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"1340 " *\n"1341 " *===-----------------------------------------------------------------"1342 "------===\n"1343 " */\n\n";1344 1345 OS << "#ifndef __ARM_SVE_H\n";1346 OS << "#define __ARM_SVE_H\n\n";1347 1348 OS << "#if !defined(__LITTLE_ENDIAN__)\n";1349 OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n";1350 OS << "#endif\n";1351 1352 OS << "#include <stdint.h>\n\n";1353 OS << "#ifdef __cplusplus\n";1354 OS << "extern \"C\" {\n";1355 OS << "#else\n";1356 OS << "#include <stdbool.h>\n";1357 OS << "#endif\n\n";1358 1359 OS << "typedef __fp16 float16_t;\n";1360 OS << "typedef float float32_t;\n";1361 OS << "typedef double float64_t;\n";1362 1363 OS << "typedef __SVInt8_t svint8_t;\n";1364 OS << "typedef __SVInt16_t svint16_t;\n";1365 OS << "typedef __SVInt32_t svint32_t;\n";1366 OS << "typedef __SVInt64_t svint64_t;\n";1367 OS << "typedef __SVUint8_t svuint8_t;\n";1368 OS << "typedef __SVUint16_t svuint16_t;\n";1369 OS << "typedef __SVUint32_t svuint32_t;\n";1370 OS << "typedef __SVUint64_t svuint64_t;\n";1371 OS << "typedef __SVFloat16_t svfloat16_t;\n\n";1372 1373 OS << "typedef __SVBfloat16_t svbfloat16_t;\n";1374 1375 OS << "#include <arm_bf16.h>\n";1376 OS << "#include <arm_vector_types.h>\n";1377 1378 OS << "typedef __SVMfloat8_t svmfloat8_t;\n\n";1379 1380 OS << "typedef __SVFloat32_t svfloat32_t;\n";1381 OS << "typedef __SVFloat64_t svfloat64_t;\n";1382 OS << "typedef __clang_svint8x2_t svint8x2_t;\n";1383 OS << "typedef __clang_svint16x2_t svint16x2_t;\n";1384 OS << "typedef __clang_svint32x2_t svint32x2_t;\n";1385 OS << "typedef __clang_svint64x2_t svint64x2_t;\n";1386 OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n";1387 OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n";1388 OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n";1389 OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n";1390 OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n";1391 OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n";1392 OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n";1393 OS << "typedef __clang_svint8x3_t svint8x3_t;\n";1394 OS << "typedef __clang_svint16x3_t svint16x3_t;\n";1395 OS << "typedef __clang_svint32x3_t svint32x3_t;\n";1396 OS << "typedef __clang_svint64x3_t svint64x3_t;\n";1397 OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n";1398 OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n";1399 OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n";1400 OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n";1401 OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n";1402 OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n";1403 OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n";1404 OS << "typedef __clang_svint8x4_t svint8x4_t;\n";1405 OS << "typedef __clang_svint16x4_t svint16x4_t;\n";1406 OS << "typedef __clang_svint32x4_t svint32x4_t;\n";1407 OS << "typedef __clang_svint64x4_t svint64x4_t;\n";1408 OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n";1409 OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n";1410 OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n";1411 OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n";1412 OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n";1413 OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n";1414 OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";1415 OS << "typedef __SVBool_t svbool_t;\n";1416 OS << "typedef __clang_svboolx2_t svboolx2_t;\n";1417 OS << "typedef __clang_svboolx4_t svboolx4_t;\n\n";1418 1419 OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";1420 OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";1421 OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";1422 1423 OS << "typedef __clang_svmfloat8x2_t svmfloat8x2_t;\n";1424 OS << "typedef __clang_svmfloat8x3_t svmfloat8x3_t;\n";1425 OS << "typedef __clang_svmfloat8x4_t svmfloat8x4_t;\n";1426 1427 OS << "typedef __SVCount_t svcount_t;\n\n";1428 1429 OS << "enum svpattern\n";1430 OS << "{\n";1431 OS << " SV_POW2 = 0,\n";1432 OS << " SV_VL1 = 1,\n";1433 OS << " SV_VL2 = 2,\n";1434 OS << " SV_VL3 = 3,\n";1435 OS << " SV_VL4 = 4,\n";1436 OS << " SV_VL5 = 5,\n";1437 OS << " SV_VL6 = 6,\n";1438 OS << " SV_VL7 = 7,\n";1439 OS << " SV_VL8 = 8,\n";1440 OS << " SV_VL16 = 9,\n";1441 OS << " SV_VL32 = 10,\n";1442 OS << " SV_VL64 = 11,\n";1443 OS << " SV_VL128 = 12,\n";1444 OS << " SV_VL256 = 13,\n";1445 OS << " SV_MUL4 = 29,\n";1446 OS << " SV_MUL3 = 30,\n";1447 OS << " SV_ALL = 31\n";1448 OS << "};\n\n";1449 1450 OS << "enum svprfop\n";1451 OS << "{\n";1452 OS << " SV_PLDL1KEEP = 0,\n";1453 OS << " SV_PLDL1STRM = 1,\n";1454 OS << " SV_PLDL2KEEP = 2,\n";1455 OS << " SV_PLDL2STRM = 3,\n";1456 OS << " SV_PLDL3KEEP = 4,\n";1457 OS << " SV_PLDL3STRM = 5,\n";1458 OS << " SV_PSTL1KEEP = 8,\n";1459 OS << " SV_PSTL1STRM = 9,\n";1460 OS << " SV_PSTL2KEEP = 10,\n";1461 OS << " SV_PSTL2STRM = 11,\n";1462 OS << " SV_PSTL3KEEP = 12,\n";1463 OS << " SV_PSTL3STRM = 13\n";1464 OS << "};\n\n";1465 1466 OS << "/* Function attributes */\n";1467 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "1468 "__nodebug__))\n\n";1469 OS << "#define __aio static __inline__ __attribute__((__always_inline__, "1470 "__nodebug__, __overloadable__))\n\n";1471 1472 // Add reinterpret functions.1473 for (auto [N, Suffix] :1474 std::initializer_list<std::pair<unsigned, const char *>>{1475 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {1476 for (auto ShortForm : {false, true})1477 for (const ReinterpretTypeInfo &To : Reinterprets) {1478 SVEType ToV(To.BaseType, N);1479 for (const ReinterpretTypeInfo &From : Reinterprets) {1480 SVEType FromV(From.BaseType, N);1481 OS << "__aio "1482 "__attribute__((__clang_arm_builtin_alias(__builtin_sve_"1483 "reinterpret_"1484 << To.Suffix << "_" << From.Suffix << Suffix << ")))\n"1485 << ToV.str() << " svreinterpret_" << To.Suffix;1486 if (!ShortForm)1487 OS << "_" << From.Suffix << Suffix;1488 OS << "(" << FromV.str() << " op);\n";1489 }1490 }1491 }1492 1493 createCoreHeaderIntrinsics(OS, *this, ACLEKind::SVE);1494 1495 OS << "#define svcvtnt_bf16_x svcvtnt_bf16_m\n";1496 OS << "#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m\n";1497 1498 OS << "#define svcvtnt_f16_x svcvtnt_f16_m\n";1499 OS << "#define svcvtnt_f16_f32_x svcvtnt_f16_f32_m\n";1500 OS << "#define svcvtnt_f32_x svcvtnt_f32_m\n";1501 OS << "#define svcvtnt_f32_f64_x svcvtnt_f32_f64_m\n\n";1502 1503 OS << "#define svcvtxnt_f32_x svcvtxnt_f32_m\n";1504 OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n";1505 1506 OS << "#ifdef __cplusplus\n";1507 OS << "} // extern \"C\"\n";1508 OS << "#endif\n\n";1509 OS << "#undef __ai\n\n";1510 OS << "#undef __aio\n\n";1511 OS << "#endif /* __ARM_SVE_H */\n";1512}1513 1514void SVEEmitter::createBuiltins(raw_ostream &OS) {1515 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");1516 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;1517 for (auto *R : RV)1518 createIntrinsic(R, Defs);1519 1520 // The mappings must be sorted based on BuiltinID.1521 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,1522 const std::unique_ptr<Intrinsic> &B) {1523 return A->getMangledName() < B->getMangledName();1524 });1525 1526 llvm::StringToOffsetTable Table;1527 Table.GetOrAddStringOffset("");1528 Table.GetOrAddStringOffset("n");1529 1530 for (const auto &Def : Defs)1531 if (Def->getClassKind() != ClassG) {1532 Table.GetOrAddStringOffset(Def->getMangledName());1533 Table.GetOrAddStringOffset(Def->getBuiltinTypeStr());1534 Table.GetOrAddStringOffset(Def->getGuard());1535 }1536 1537 Table.GetOrAddStringOffset("sme|sve");1538 SmallVector<std::pair<std::string, std::string>> ReinterpretBuiltins;1539 for (auto [N, Suffix] :1540 std::initializer_list<std::pair<unsigned, const char *>>{1541 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {1542 for (const ReinterpretTypeInfo &To : Reinterprets) {1543 SVEType ToV(To.BaseType, N);1544 for (const ReinterpretTypeInfo &From : Reinterprets) {1545 SVEType FromV(From.BaseType, N);1546 std::string Name =1547 (Twine("reinterpret_") + To.Suffix + "_" + From.Suffix + Suffix)1548 .str();1549 std::string Type = ToV.builtin_str() + FromV.builtin_str();1550 Table.GetOrAddStringOffset(Name);1551 Table.GetOrAddStringOffset(Type);1552 ReinterpretBuiltins.push_back({Name, Type});1553 }1554 }1555 }1556 1557 OS << "#ifdef GET_SVE_BUILTIN_ENUMERATORS\n";1558 for (const auto &Def : Defs)1559 if (Def->getClassKind() != ClassG)1560 OS << " BI__builtin_sve_" << Def->getMangledName() << ",\n";1561 for (const auto &[Name, _] : ReinterpretBuiltins)1562 OS << " BI__builtin_sve_" << Name << ",\n";1563 OS << "#endif // GET_SVE_BUILTIN_ENUMERATORS\n\n";1564 1565 OS << "#ifdef GET_SVE_BUILTIN_STR_TABLE\n";1566 Table.EmitStringTableDef(OS, "BuiltinStrings");1567 OS << "#endif // GET_SVE_BUILTIN_STR_TABLE\n\n";1568 1569 OS << "#ifdef GET_SVE_BUILTIN_INFOS\n";1570 for (const auto &Def : Defs) {1571 // Only create BUILTINs for non-overloaded intrinsics, as overloaded1572 // declarations only live in the header file.1573 if (Def->getClassKind() != ClassG) {1574 OS << " Builtin::Info{Builtin::Info::StrOffsets{"1575 << Table.GetStringOffset(Def->getMangledName()) << " /* "1576 << Def->getMangledName() << " */, ";1577 OS << Table.GetStringOffset(Def->getBuiltinTypeStr()) << " /* "1578 << Def->getBuiltinTypeStr() << " */, ";1579 OS << Table.GetStringOffset("n") << " /* n */, ";1580 OS << Table.GetStringOffset(Def->getGuard()) << " /* " << Def->getGuard()1581 << " */}, ";1582 OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";1583 }1584 }1585 for (const auto &[Name, Type] : ReinterpretBuiltins) {1586 OS << " Builtin::Info{Builtin::Info::StrOffsets{"1587 << Table.GetStringOffset(Name) << " /* " << Name << " */, ";1588 OS << Table.GetStringOffset(Type) << " /* " << Type << " */, ";1589 OS << Table.GetStringOffset("n") << " /* n */, ";1590 OS << Table.GetStringOffset("sme|sve") << " /* sme|sve */}, ";1591 OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";1592 }1593 OS << "#endif // GET_SVE_BUILTIN_INFOS\n\n";1594}1595 1596void SVEEmitter::createBuiltinsJSON(raw_ostream &OS) {1597 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;1598 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");1599 for (auto *R : RV)1600 createIntrinsic(R, Defs);1601 1602 OS << "[\n";1603 bool FirstDef = true;1604 1605 for (auto &Def : Defs) {1606 std::vector<std::string> Flags;1607 1608 if (Def->isFlagSet(getEnumValueForFlag("IsStreaming")))1609 Flags.push_back("streaming-only");1610 else if (Def->isFlagSet(getEnumValueForFlag("IsStreamingCompatible")))1611 Flags.push_back("streaming-compatible");1612 else if (Def->isFlagSet(getEnumValueForFlag("VerifyRuntimeMode")))1613 Flags.push_back("feature-dependent");1614 1615 if (Def->isFlagSet(getEnumValueForFlag("IsInZA")) ||1616 Def->isFlagSet(getEnumValueForFlag("IsOutZA")) ||1617 Def->isFlagSet(getEnumValueForFlag("IsInOutZA")))1618 Flags.push_back("requires-za");1619 1620 if (Def->isFlagSet(getEnumValueForFlag("IsInZT0")) ||1621 Def->isFlagSet(getEnumValueForFlag("IsOutZT0")) ||1622 Def->isFlagSet(getEnumValueForFlag("IsInOutZT0")))1623 Flags.push_back("requires-zt");1624 1625 if (!FirstDef)1626 OS << ",\n";1627 1628 OS << "{ ";1629 OS << "\"guard\": \"" << Def->getSVEGuard() << "\",";1630 OS << "\"streaming_guard\": \"" << Def->getSMEGuard() << "\",";1631 OS << "\"flags\": \"";1632 1633 for (size_t I = 0; I < Flags.size(); ++I) {1634 if (I != 0)1635 OS << ',';1636 OS << Flags[I];1637 }1638 1639 OS << "\",\"builtin\": \"";1640 1641 std::string BuiltinName = Def->getMangledName(Def->getClassKind());1642 1643 OS << Def->getReturnType().str() << " " << BuiltinName << "(";1644 for (unsigned I = 0; I < Def->getTypes().size() - 1; ++I) {1645 if (I != 0)1646 OS << ", ";1647 1648 SVEType ParamType = Def->getParamType(I);1649 1650 // These are ImmCheck'd but their type names are sufficiently clear.1651 if (ParamType.isPredicatePattern() || ParamType.isPrefetchOp()) {1652 OS << ParamType.str();1653 continue;1654 }1655 1656 // Pass ImmCheck information by pretending it's a type.1657 auto Iter = llvm::find_if(Def->getImmChecks(), [I](const auto &Chk) {1658 return (unsigned)Chk.getImmArgIdx() == I;1659 });1660 if (Iter != Def->getImmChecks().end())1661 OS << getImmCheckForEnumValue(Iter->getKind());1662 else1663 OS << ParamType.str();1664 }1665 OS << ");\" }";1666 FirstDef = false;1667 }1668 1669 OS << "\n]\n";1670}1671 1672void SVEEmitter::createCodeGenMap(raw_ostream &OS) {1673 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");1674 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;1675 for (auto *R : RV)1676 createIntrinsic(R, Defs);1677 1678 // The mappings must be sorted based on BuiltinID.1679 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,1680 const std::unique_ptr<Intrinsic> &B) {1681 return A->getMangledName() < B->getMangledName();1682 });1683 1684 OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n";1685 for (auto &Def : Defs) {1686 // Builtins only exist for non-overloaded intrinsics, overloaded1687 // declarations only live in the header file.1688 if (Def->getClassKind() == ClassG)1689 continue;1690 1691 uint64_t Flags = Def->getFlags();1692 auto FlagString = std::to_string(Flags);1693 1694 std::string LLVMName = Def->getMangledLLVMName();1695 std::string Builtin = Def->getMangledName();1696 if (!LLVMName.empty())1697 OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString1698 << "),\n";1699 else1700 OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";1701 }1702 OS << "#endif\n\n";1703}1704 1705void SVEEmitter::createRangeChecks(raw_ostream &OS) {1706 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");1707 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;1708 for (auto *R : RV)1709 createIntrinsic(R, Defs);1710 1711 // The mappings must be sorted based on BuiltinID.1712 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,1713 const std::unique_ptr<Intrinsic> &B) {1714 return A->getMangledName() < B->getMangledName();1715 });1716 1717 OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n";1718 1719 // Ensure these are only emitted once.1720 std::set<std::string> Emitted;1721 1722 for (auto &Def : Defs) {1723 if (Emitted.find(Def->getMangledName()) != Emitted.end() ||1724 Def->getImmChecks().empty())1725 continue;1726 1727 OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n";1728 for (auto &Check : Def->getImmChecks())1729 OS << "ImmChecks.emplace_back(" << Check.getImmArgIdx() << ", "1730 << Check.getKind() << ", " << Check.getElementSizeInBits() << ");\n";1731 OS << " break;\n";1732 1733 Emitted.insert(Def->getMangledName());1734 }1735 1736 OS << "#endif\n\n";1737}1738 1739/// Create the SVETypeFlags used in CGBuiltins1740void SVEEmitter::createTypeFlags(raw_ostream &OS) {1741 OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n";1742 for (auto &KV : FlagTypes)1743 OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n";1744 OS << "#endif\n\n";1745 1746 OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n";1747 for (auto &KV : EltTypes)1748 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";1749 OS << "#endif\n\n";1750 1751 OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n";1752 for (auto &KV : MemEltTypes)1753 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";1754 OS << "#endif\n\n";1755 1756 OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n";1757 for (auto &KV : MergeTypes)1758 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";1759 OS << "#endif\n\n";1760}1761 1762void SVEEmitter::createImmCheckTypes(raw_ostream &OS) {1763 OS << "#ifdef LLVM_GET_ARM_INTRIN_IMMCHECKTYPES\n";1764 for (auto &KV : ImmCheckTypes)1765 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";1766 OS << "#endif\n\n";1767}1768 1769void SVEEmitter::createSMEHeader(raw_ostream &OS) {1770 OS << "/*===---- arm_sme.h - ARM SME intrinsics "1771 "------===\n"1772 " *\n"1773 " *\n"1774 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "1775 "Exceptions.\n"1776 " * See https://llvm.org/LICENSE.txt for license information.\n"1777 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"1778 " *\n"1779 " *===-----------------------------------------------------------------"1780 "------===\n"1781 " */\n\n";1782 1783 OS << "#ifndef __ARM_SME_H\n";1784 OS << "#define __ARM_SME_H\n\n";1785 1786 OS << "#if !defined(__LITTLE_ENDIAN__)\n";1787 OS << "#error \"Big endian is currently not supported for arm_sme.h\"\n";1788 OS << "#endif\n";1789 1790 OS << "#include <arm_sve.h>\n\n";1791 OS << "#include <stddef.h>\n\n";1792 1793 OS << "/* Function attributes */\n";1794 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "1795 "__nodebug__))\n\n";1796 OS << "#define __aio static __inline__ __attribute__((__always_inline__, "1797 "__nodebug__, __overloadable__))\n\n";1798 1799 OS << "#ifdef __cplusplus\n";1800 OS << "extern \"C\" {\n";1801 OS << "#endif\n\n";1802 1803 OS << "void __arm_za_disable(void) __arm_streaming_compatible;\n\n";1804 1805 OS << "__ai bool __arm_has_sme(void) __arm_streaming_compatible {\n";1806 OS << " uint64_t x0, x1;\n";1807 OS << " __builtin_arm_get_sme_state(&x0, &x1);\n";1808 OS << " return x0 & (1ULL << 63);\n";1809 OS << "}\n\n";1810 1811 OS << "void *__arm_sc_memcpy(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";1812 OS << "void *__arm_sc_memmove(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";1813 OS << "void *__arm_sc_memset(void *s, int c, size_t n) __arm_streaming_compatible;\n";1814 OS << "void *__arm_sc_memchr(void *s, int c, size_t n) __arm_streaming_compatible;\n\n";1815 1816 OS << "__ai __attribute__((target(\"sme\"))) void svundef_za(void) "1817 "__arm_streaming_compatible __arm_out(\"za\") "1818 "{ }\n\n";1819 1820 createCoreHeaderIntrinsics(OS, *this, ACLEKind::SME);1821 1822 OS << "#ifdef __cplusplus\n";1823 OS << "} // extern \"C\"\n";1824 OS << "#endif\n\n";1825 OS << "#undef __ai\n\n";1826 OS << "#endif /* __ARM_SME_H */\n";1827}1828 1829void SVEEmitter::createSMEBuiltins(raw_ostream &OS) {1830 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");1831 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;1832 for (auto *R : RV) {1833 createIntrinsic(R, Defs);1834 }1835 1836 // The mappings must be sorted based on BuiltinID.1837 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,1838 const std::unique_ptr<Intrinsic> &B) {1839 return A->getMangledName() < B->getMangledName();1840 });1841 1842 llvm::StringToOffsetTable Table;1843 Table.GetOrAddStringOffset("");1844 Table.GetOrAddStringOffset("n");1845 1846 for (const auto &Def : Defs)1847 if (Def->getClassKind() != ClassG) {1848 Table.GetOrAddStringOffset(Def->getMangledName());1849 Table.GetOrAddStringOffset(Def->getBuiltinTypeStr());1850 Table.GetOrAddStringOffset(Def->getGuard());1851 }1852 1853 OS << "#ifdef GET_SME_BUILTIN_ENUMERATORS\n";1854 for (const auto &Def : Defs)1855 if (Def->getClassKind() != ClassG)1856 OS << " BI__builtin_sme_" << Def->getMangledName() << ",\n";1857 OS << "#endif // GET_SME_BUILTIN_ENUMERATORS\n\n";1858 1859 OS << "#ifdef GET_SME_BUILTIN_STR_TABLE\n";1860 Table.EmitStringTableDef(OS, "BuiltinStrings");1861 OS << "#endif // GET_SME_BUILTIN_STR_TABLE\n\n";1862 1863 OS << "#ifdef GET_SME_BUILTIN_INFOS\n";1864 for (const auto &Def : Defs) {1865 // Only create BUILTINs for non-overloaded intrinsics, as overloaded1866 // declarations only live in the header file.1867 if (Def->getClassKind() != ClassG) {1868 OS << " Builtin::Info{Builtin::Info::StrOffsets{"1869 << Table.GetStringOffset(Def->getMangledName()) << " /* "1870 << Def->getMangledName() << " */, ";1871 OS << Table.GetStringOffset(Def->getBuiltinTypeStr()) << " /* "1872 << Def->getBuiltinTypeStr() << " */, ";1873 OS << Table.GetStringOffset("n") << " /* n */, ";1874 OS << Table.GetStringOffset(Def->getGuard()) << " /* " << Def->getGuard()1875 << " */}, ";1876 OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";1877 }1878 }1879 OS << "#endif // GET_SME_BUILTIN_INFOS\n\n";1880}1881 1882void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) {1883 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");1884 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;1885 for (auto *R : RV) {1886 createIntrinsic(R, Defs);1887 }1888 1889 // The mappings must be sorted based on BuiltinID.1890 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,1891 const std::unique_ptr<Intrinsic> &B) {1892 return A->getMangledName() < B->getMangledName();1893 });1894 1895 OS << "#ifdef GET_SME_LLVM_INTRINSIC_MAP\n";1896 for (auto &Def : Defs) {1897 // Builtins only exist for non-overloaded intrinsics, overloaded1898 // declarations only live in the header file.1899 if (Def->getClassKind() == ClassG)1900 continue;1901 1902 uint64_t Flags = Def->getFlags();1903 auto FlagString = std::to_string(Flags);1904 1905 std::string LLVMName = Def->getLLVMName();1906 std::string Builtin = Def->getMangledName();1907 if (!LLVMName.empty())1908 OS << "SMEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString1909 << "),\n";1910 else1911 OS << "SMEMAP2(" << Builtin << ", " << FlagString << "),\n";1912 }1913 OS << "#endif\n\n";1914}1915 1916void SVEEmitter::createSMERangeChecks(raw_ostream &OS) {1917 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");1918 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;1919 for (auto *R : RV) {1920 createIntrinsic(R, Defs);1921 }1922 1923 // The mappings must be sorted based on BuiltinID.1924 sort(Defs, [](const std::unique_ptr<Intrinsic> &A,1925 const std::unique_ptr<Intrinsic> &B) {1926 return A->getMangledName() < B->getMangledName();1927 });1928 1929 OS << "#ifdef GET_SME_IMMEDIATE_CHECK\n";1930 1931 // Ensure these are only emitted once.1932 std::set<std::string> Emitted;1933 1934 for (auto &Def : Defs) {1935 if (Emitted.find(Def->getMangledName()) != Emitted.end() ||1936 Def->getImmChecks().empty())1937 continue;1938 1939 OS << "case SME::BI__builtin_sme_" << Def->getMangledName() << ":\n";1940 for (auto &Check : Def->getImmChecks())1941 OS << "ImmChecks.push_back(std::make_tuple(" << Check.getImmArgIdx()1942 << ", " << Check.getKind() << ", " << Check.getElementSizeInBits()1943 << "));\n";1944 OS << " break;\n";1945 1946 Emitted.insert(Def->getMangledName());1947 }1948 1949 OS << "#endif\n\n";1950}1951 1952void SVEEmitter::createBuiltinZAState(raw_ostream &OS) {1953 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");1954 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;1955 for (auto *R : RV)1956 createIntrinsic(R, Defs);1957 1958 std::map<std::string, std::set<std::string>> IntrinsicsPerState;1959 for (auto &Def : Defs) {1960 std::string Key;1961 auto AddToKey = [&Key](const std::string &S) -> void {1962 Key = Key.empty() ? S : (Key + " | " + S);1963 };1964 1965 if (Def->isFlagSet(getEnumValueForFlag("IsInZA")))1966 AddToKey("ArmInZA");1967 else if (Def->isFlagSet(getEnumValueForFlag("IsOutZA")))1968 AddToKey("ArmOutZA");1969 else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZA")))1970 AddToKey("ArmInOutZA");1971 1972 if (Def->isFlagSet(getEnumValueForFlag("IsInZT0")))1973 AddToKey("ArmInZT0");1974 else if (Def->isFlagSet(getEnumValueForFlag("IsOutZT0")))1975 AddToKey("ArmOutZT0");1976 else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZT0")))1977 AddToKey("ArmInOutZT0");1978 1979 if (!Key.empty())1980 IntrinsicsPerState[Key].insert(Def->getMangledName());1981 }1982 1983 OS << "#ifdef GET_SME_BUILTIN_GET_STATE\n";1984 for (auto &KV : IntrinsicsPerState) {1985 for (StringRef Name : KV.second)1986 OS << "case SME::BI__builtin_sme_" << Name << ":\n";1987 OS << " return " << KV.first << ";\n";1988 }1989 OS << "#endif\n\n";1990}1991 1992void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) {1993 std::vector<const Record *> RV = Records.getAllDerivedDefinitions("Inst");1994 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;1995 for (auto *R : RV)1996 createIntrinsic(R, Defs);1997 1998 StringRef ExtensionKind;1999 switch (Kind) {2000 case ACLEKind::SME:2001 ExtensionKind = "SME";2002 break;2003 case ACLEKind::SVE:2004 ExtensionKind = "SVE";2005 break;2006 }2007 2008 OS << "#ifdef GET_" << ExtensionKind << "_STREAMING_ATTRS\n";2009 2010 StringMap<std::set<std::string>> StreamingMap;2011 2012 uint64_t IsStreamingFlag = getEnumValueForFlag("IsStreaming");2013 uint64_t VerifyRuntimeMode = getEnumValueForFlag("VerifyRuntimeMode");2014 uint64_t IsStreamingCompatibleFlag =2015 getEnumValueForFlag("IsStreamingCompatible");2016 2017 for (auto &Def : Defs) {2018 if (!Def->isFlagSet(VerifyRuntimeMode) && !Def->getSVEGuard().empty() &&2019 !Def->getSMEGuard().empty())2020 report_fatal_error("Missing VerifyRuntimeMode flag");2021 if (Def->isFlagSet(VerifyRuntimeMode) &&2022 (Def->getSVEGuard().empty() || Def->getSMEGuard().empty()))2023 report_fatal_error("VerifyRuntimeMode requires SVE and SME guards");2024 2025 if (Def->isFlagSet(IsStreamingFlag))2026 StreamingMap["ArmStreaming"].insert(Def->getMangledName());2027 else if (Def->isFlagSet(VerifyRuntimeMode))2028 StreamingMap["VerifyRuntimeMode"].insert(Def->getMangledName());2029 else if (Def->isFlagSet(IsStreamingCompatibleFlag))2030 StreamingMap["ArmStreamingCompatible"].insert(Def->getMangledName());2031 else2032 StreamingMap["ArmNonStreaming"].insert(Def->getMangledName());2033 }2034 2035 for (auto BuiltinType : StreamingMap.keys()) {2036 for (auto Name : StreamingMap[BuiltinType]) {2037 OS << "case " << ExtensionKind << "::BI__builtin_"2038 << ExtensionKind.lower() << "_";2039 OS << Name << ":\n";2040 }2041 OS << " BuiltinType = " << BuiltinType << ";\n";2042 OS << " break;\n";2043 }2044 2045 OS << "#endif\n\n";2046}2047 2048namespace clang {2049void EmitSveHeader(const RecordKeeper &Records, raw_ostream &OS) {2050 SVEEmitter(Records).createHeader(OS);2051}2052 2053void EmitSveBuiltins(const RecordKeeper &Records, raw_ostream &OS) {2054 SVEEmitter(Records).createBuiltins(OS);2055}2056 2057void EmitSveBuiltinsJSON(const RecordKeeper &Records, raw_ostream &OS) {2058 SVEEmitter(Records).createBuiltinsJSON(OS);2059}2060 2061void EmitSveBuiltinCG(const RecordKeeper &Records, raw_ostream &OS) {2062 SVEEmitter(Records).createCodeGenMap(OS);2063}2064 2065void EmitSveRangeChecks(const RecordKeeper &Records, raw_ostream &OS) {2066 SVEEmitter(Records).createRangeChecks(OS);2067}2068 2069void EmitSveTypeFlags(const RecordKeeper &Records, raw_ostream &OS) {2070 SVEEmitter(Records).createTypeFlags(OS);2071}2072 2073void EmitImmCheckTypes(const RecordKeeper &Records, raw_ostream &OS) {2074 SVEEmitter(Records).createImmCheckTypes(OS);2075}2076 2077void EmitSveStreamingAttrs(const RecordKeeper &Records, raw_ostream &OS) {2078 SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SVE);2079}2080 2081void EmitSmeHeader(const RecordKeeper &Records, raw_ostream &OS) {2082 SVEEmitter(Records).createSMEHeader(OS);2083}2084 2085void EmitSmeBuiltins(const RecordKeeper &Records, raw_ostream &OS) {2086 SVEEmitter(Records).createSMEBuiltins(OS);2087}2088 2089void EmitSmeBuiltinsJSON(const RecordKeeper &Records, raw_ostream &OS) {2090 SVEEmitter(Records).createBuiltinsJSON(OS);2091}2092 2093void EmitSmeBuiltinCG(const RecordKeeper &Records, raw_ostream &OS) {2094 SVEEmitter(Records).createSMECodeGenMap(OS);2095}2096 2097void EmitSmeRangeChecks(const RecordKeeper &Records, raw_ostream &OS) {2098 SVEEmitter(Records).createSMERangeChecks(OS);2099}2100 2101void EmitSmeStreamingAttrs(const RecordKeeper &Records, raw_ostream &OS) {2102 SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SME);2103}2104 2105void EmitSmeBuiltinZAState(const RecordKeeper &Records, raw_ostream &OS) {2106 SVEEmitter(Records).createBuiltinZAState(OS);2107}2108} // End namespace clang2109