1253 lines · cpp
1//===- RISCVVIntrinsicUtils.cpp - RISC-V Vector Intrinsic Utils -*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "clang/Support/RISCVVIntrinsicUtils.h"10#include "llvm/ADT/ArrayRef.h"11#include "llvm/ADT/StringExtras.h"12#include "llvm/ADT/Twine.h"13#include "llvm/Support/ErrorHandling.h"14#include "llvm/Support/raw_ostream.h"15#include <optional>16 17using namespace llvm;18 19namespace clang {20namespace RISCV {21 22const PrototypeDescriptor PrototypeDescriptor::Mask = PrototypeDescriptor(23 BaseTypeModifier::Vector, VectorTypeModifier::MaskVector);24const PrototypeDescriptor PrototypeDescriptor::VL =25 PrototypeDescriptor(BaseTypeModifier::SizeT);26const PrototypeDescriptor PrototypeDescriptor::Vector =27 PrototypeDescriptor(BaseTypeModifier::Vector);28 29//===----------------------------------------------------------------------===//30// Type implementation31//===----------------------------------------------------------------------===//32 33LMULType::LMULType(int NewLog2LMUL) {34 // Check Log2LMUL is -3, -2, -1, 0, 1, 2, 335 assert(NewLog2LMUL <= 3 && NewLog2LMUL >= -3 && "Bad LMUL number!");36 Log2LMUL = NewLog2LMUL;37}38 39std::string LMULType::str() const {40 if (Log2LMUL < 0)41 return "mf" + utostr(1ULL << (-Log2LMUL));42 return "m" + utostr(1ULL << Log2LMUL);43}44 45VScaleVal LMULType::getScale(unsigned ElementBitwidth) const {46 int Log2ScaleResult = 0;47 switch (ElementBitwidth) {48 default:49 break;50 case 8:51 Log2ScaleResult = Log2LMUL + 3;52 break;53 case 16:54 Log2ScaleResult = Log2LMUL + 2;55 break;56 case 32:57 Log2ScaleResult = Log2LMUL + 1;58 break;59 case 64:60 Log2ScaleResult = Log2LMUL;61 break;62 }63 // Illegal vscale result would be less than 164 if (Log2ScaleResult < 0)65 return std::nullopt;66 return 1 << Log2ScaleResult;67}68 69void LMULType::MulLog2LMUL(int log2LMUL) { Log2LMUL += log2LMUL; }70 71RVVType::RVVType(BasicType BT, int Log2LMUL,72 const PrototypeDescriptor &prototype)73 : BT(BT), LMUL(LMULType(Log2LMUL)) {74 applyBasicType();75 applyModifier(prototype);76 Valid = verifyType();77 if (Valid) {78 initBuiltinStr();79 initTypeStr();80 if (isVector()) {81 initClangBuiltinStr();82 }83 }84}85 86// clang-format off87// boolean type are encoded the ratio of n (SEW/LMUL)88// SEW/LMUL | 1 | 2 | 4 | 8 | 16 | 32 | 6489// c type | vbool64_t | vbool32_t | vbool16_t | vbool8_t | vbool4_t | vbool2_t | vbool1_t90// IR type | nxv1i1 | nxv2i1 | nxv4i1 | nxv8i1 | nxv16i1 | nxv32i1 | nxv64i191 92// type\lmul | 1/8 | 1/4 | 1/2 | 1 | 2 | 4 | 893// -------- |------ | -------- | ------- | ------- | -------- | -------- | --------94// i64 | N/A | N/A | N/A | nxv1i64 | nxv2i64 | nxv4i64 | nxv8i6495// i32 | N/A | N/A | nxv1i32 | nxv2i32 | nxv4i32 | nxv8i32 | nxv16i3296// i16 | N/A | nxv1i16 | nxv2i16 | nxv4i16 | nxv8i16 | nxv16i16 | nxv32i1697// i8 | nxv1i8 | nxv2i8 | nxv4i8 | nxv8i8 | nxv16i8 | nxv32i8 | nxv64i898// double | N/A | N/A | N/A | nxv1f64 | nxv2f64 | nxv4f64 | nxv8f6499// float | N/A | N/A | nxv1f32 | nxv2f32 | nxv4f32 | nxv8f32 | nxv16f32100// half | N/A | nxv1f16 | nxv2f16 | nxv4f16 | nxv8f16 | nxv16f16 | nxv32f16101// bfloat16 | N/A | nxv1bf16 | nxv2bf16| nxv4bf16| nxv8bf16 | nxv16bf16| nxv32bf16102// clang-format on103 104bool RVVType::verifyType() const {105 if (ScalarType == Invalid)106 return false;107 if (isScalar())108 return true;109 if (!Scale)110 return false;111 if (isFloat() && ElementBitwidth == 8)112 return false;113 if (isBFloat() && ElementBitwidth != 16)114 return false;115 if (IsTuple && (NF == 1 || NF > 8))116 return false;117 if (IsTuple && (1 << std::max(0, LMUL.Log2LMUL)) * NF > 8)118 return false;119 unsigned V = *Scale;120 switch (ElementBitwidth) {121 case 1:122 case 8:123 // Check Scale is 1,2,4,8,16,32,64124 return (V <= 64 && isPowerOf2_32(V));125 case 16:126 // Check Scale is 1,2,4,8,16,32127 return (V <= 32 && isPowerOf2_32(V));128 case 32:129 // Check Scale is 1,2,4,8,16130 return (V <= 16 && isPowerOf2_32(V));131 case 64:132 // Check Scale is 1,2,4,8133 return (V <= 8 && isPowerOf2_32(V));134 }135 return false;136}137 138void RVVType::initBuiltinStr() {139 assert(isValid() && "RVVType is invalid");140 switch (ScalarType) {141 case ScalarTypeKind::Void:142 BuiltinStr = "v";143 return;144 case ScalarTypeKind::Size_t:145 BuiltinStr = "z";146 if (IsImmediate)147 BuiltinStr = "I" + BuiltinStr;148 if (IsPointer)149 BuiltinStr += "*";150 return;151 case ScalarTypeKind::Ptrdiff_t:152 BuiltinStr = "Y";153 return;154 case ScalarTypeKind::UnsignedLong:155 BuiltinStr = "ULi";156 return;157 case ScalarTypeKind::SignedLong:158 BuiltinStr = "Li";159 return;160 case ScalarTypeKind::Boolean:161 assert(ElementBitwidth == 1);162 BuiltinStr += "b";163 break;164 case ScalarTypeKind::SignedInteger:165 case ScalarTypeKind::UnsignedInteger:166 switch (ElementBitwidth) {167 case 8:168 BuiltinStr += "c";169 break;170 case 16:171 BuiltinStr += "s";172 break;173 case 32:174 BuiltinStr += "i";175 break;176 case 64:177 BuiltinStr += "Wi";178 break;179 default:180 llvm_unreachable("Unhandled ElementBitwidth!");181 }182 if (isSignedInteger())183 BuiltinStr = "S" + BuiltinStr;184 else185 BuiltinStr = "U" + BuiltinStr;186 break;187 case ScalarTypeKind::Float:188 switch (ElementBitwidth) {189 case 16:190 BuiltinStr += "x";191 break;192 case 32:193 BuiltinStr += "f";194 break;195 case 64:196 BuiltinStr += "d";197 break;198 default:199 llvm_unreachable("Unhandled ElementBitwidth!");200 }201 break;202 case ScalarTypeKind::BFloat:203 BuiltinStr += "y";204 break;205 default:206 llvm_unreachable("ScalarType is invalid!");207 }208 if (IsImmediate)209 BuiltinStr = "I" + BuiltinStr;210 if (isScalar()) {211 if (IsConstant)212 BuiltinStr += "C";213 if (IsPointer)214 BuiltinStr += "*";215 return;216 }217 BuiltinStr = "q" + utostr(*Scale) + BuiltinStr;218 // Pointer to vector types. Defined for segment load intrinsics.219 // segment load intrinsics have pointer type arguments to store the loaded220 // vector values.221 if (IsPointer)222 BuiltinStr += "*";223 224 if (IsTuple)225 BuiltinStr = "T" + utostr(NF) + BuiltinStr;226}227 228void RVVType::initClangBuiltinStr() {229 assert(isValid() && "RVVType is invalid");230 assert(isVector() && "Handle Vector type only");231 232 ClangBuiltinStr = "__rvv_";233 switch (ScalarType) {234 case ScalarTypeKind::Boolean:235 ClangBuiltinStr += "bool" + utostr(64 / *Scale) + "_t";236 return;237 case ScalarTypeKind::Float:238 ClangBuiltinStr += "float";239 break;240 case ScalarTypeKind::BFloat:241 ClangBuiltinStr += "bfloat";242 break;243 case ScalarTypeKind::SignedInteger:244 ClangBuiltinStr += "int";245 break;246 case ScalarTypeKind::UnsignedInteger:247 ClangBuiltinStr += "uint";248 break;249 default:250 llvm_unreachable("ScalarTypeKind is invalid");251 }252 ClangBuiltinStr += utostr(ElementBitwidth) + LMUL.str() +253 (IsTuple ? "x" + utostr(NF) : "") + "_t";254}255 256void RVVType::initTypeStr() {257 assert(isValid() && "RVVType is invalid");258 259 if (IsConstant)260 Str += "const ";261 262 auto getTypeString = [&](StringRef TypeStr) {263 if (isScalar())264 return Twine(TypeStr + Twine(ElementBitwidth) + "_t").str();265 return Twine("v" + TypeStr + Twine(ElementBitwidth) + LMUL.str() +266 (IsTuple ? "x" + utostr(NF) : "") + "_t")267 .str();268 };269 270 switch (ScalarType) {271 case ScalarTypeKind::Void:272 Str = "void";273 return;274 case ScalarTypeKind::Size_t:275 Str = "size_t";276 if (IsPointer)277 Str += " *";278 return;279 case ScalarTypeKind::Ptrdiff_t:280 Str = "ptrdiff_t";281 return;282 case ScalarTypeKind::UnsignedLong:283 Str = "unsigned long";284 return;285 case ScalarTypeKind::SignedLong:286 Str = "long";287 return;288 case ScalarTypeKind::Boolean:289 if (isScalar())290 Str += "bool";291 else292 // Vector bool is special case, the formulate is293 // `vbool<N>_t = MVT::nxv<64/N>i1` ex. vbool16_t = MVT::4i1294 Str += "vbool" + utostr(64 / *Scale) + "_t";295 break;296 case ScalarTypeKind::Float:297 if (isScalar()) {298 if (ElementBitwidth == 64)299 Str += "double";300 else if (ElementBitwidth == 32)301 Str += "float";302 else if (ElementBitwidth == 16)303 Str += "_Float16";304 else305 llvm_unreachable("Unhandled floating type.");306 } else307 Str += getTypeString("float");308 break;309 case ScalarTypeKind::BFloat:310 if (isScalar()) {311 if (ElementBitwidth == 16)312 Str += "__bf16";313 else314 llvm_unreachable("Unhandled floating type.");315 } else316 Str += getTypeString("bfloat");317 break;318 case ScalarTypeKind::SignedInteger:319 Str += getTypeString("int");320 break;321 case ScalarTypeKind::UnsignedInteger:322 Str += getTypeString("uint");323 break;324 default:325 llvm_unreachable("ScalarType is invalid!");326 }327 if (IsPointer)328 Str += " *";329}330 331void RVVType::initShortStr() {332 switch (ScalarType) {333 case ScalarTypeKind::Boolean:334 assert(isVector());335 ShortStr = "b" + utostr(64 / *Scale);336 return;337 case ScalarTypeKind::Float:338 ShortStr = "f" + utostr(ElementBitwidth);339 break;340 case ScalarTypeKind::BFloat:341 ShortStr = "bf" + utostr(ElementBitwidth);342 break;343 case ScalarTypeKind::SignedInteger:344 ShortStr = "i" + utostr(ElementBitwidth);345 break;346 case ScalarTypeKind::UnsignedInteger:347 ShortStr = "u" + utostr(ElementBitwidth);348 break;349 default:350 llvm_unreachable("Unhandled case!");351 }352 if (isVector())353 ShortStr += LMUL.str();354 if (isTuple())355 ShortStr += "x" + utostr(NF);356}357 358static VectorTypeModifier getTupleVTM(unsigned NF) {359 assert(2 <= NF && NF <= 8 && "2 <= NF <= 8");360 return static_cast<VectorTypeModifier>(361 static_cast<uint8_t>(VectorTypeModifier::Tuple2) + (NF - 2));362}363 364void RVVType::applyBasicType() {365 switch (BT) {366 case BasicType::Int8:367 ElementBitwidth = 8;368 ScalarType = ScalarTypeKind::SignedInteger;369 break;370 case BasicType::Int16:371 ElementBitwidth = 16;372 ScalarType = ScalarTypeKind::SignedInteger;373 break;374 case BasicType::Int32:375 ElementBitwidth = 32;376 ScalarType = ScalarTypeKind::SignedInteger;377 break;378 case BasicType::Int64:379 ElementBitwidth = 64;380 ScalarType = ScalarTypeKind::SignedInteger;381 break;382 case BasicType::Float16:383 ElementBitwidth = 16;384 ScalarType = ScalarTypeKind::Float;385 break;386 case BasicType::Float32:387 ElementBitwidth = 32;388 ScalarType = ScalarTypeKind::Float;389 break;390 case BasicType::Float64:391 ElementBitwidth = 64;392 ScalarType = ScalarTypeKind::Float;393 break;394 case BasicType::BFloat16:395 ElementBitwidth = 16;396 ScalarType = ScalarTypeKind::BFloat;397 break;398 default:399 llvm_unreachable("Unhandled type code!");400 }401 assert(ElementBitwidth != 0 && "Bad element bitwidth!");402}403 404std::optional<PrototypeDescriptor>405PrototypeDescriptor::parsePrototypeDescriptor(406 llvm::StringRef PrototypeDescriptorStr) {407 PrototypeDescriptor PD;408 BaseTypeModifier PT = BaseTypeModifier::Invalid;409 VectorTypeModifier VTM = VectorTypeModifier::NoModifier;410 411 if (PrototypeDescriptorStr.empty())412 return PD;413 414 // Handle base type modifier415 auto PType = PrototypeDescriptorStr.back();416 switch (PType) {417 case 'e':418 PT = BaseTypeModifier::Scalar;419 break;420 case 'v':421 PT = BaseTypeModifier::Vector;422 break;423 case 'w':424 PT = BaseTypeModifier::Vector;425 VTM = VectorTypeModifier::Widening2XVector;426 break;427 case 'q':428 PT = BaseTypeModifier::Vector;429 VTM = VectorTypeModifier::Widening4XVector;430 break;431 case 'o':432 PT = BaseTypeModifier::Vector;433 VTM = VectorTypeModifier::Widening8XVector;434 break;435 case 'm':436 PT = BaseTypeModifier::Vector;437 VTM = VectorTypeModifier::MaskVector;438 break;439 case '0':440 PT = BaseTypeModifier::Void;441 break;442 case 'z':443 PT = BaseTypeModifier::SizeT;444 break;445 case 't':446 PT = BaseTypeModifier::Ptrdiff;447 break;448 case 'u':449 PT = BaseTypeModifier::UnsignedLong;450 break;451 case 'l':452 PT = BaseTypeModifier::SignedLong;453 break;454 case 'f':455 PT = BaseTypeModifier::Float32;456 break;457 default:458 llvm_unreachable("Illegal primitive type transformers!");459 }460 PD.PT = static_cast<uint8_t>(PT);461 PrototypeDescriptorStr = PrototypeDescriptorStr.drop_back();462 463 // Compute the vector type transformers, it can only appear one time.464 if (PrototypeDescriptorStr.starts_with("(")) {465 assert(VTM == VectorTypeModifier::NoModifier &&466 "VectorTypeModifier should only have one modifier");467 size_t Idx = PrototypeDescriptorStr.find(')');468 assert(Idx != StringRef::npos);469 StringRef ComplexType = PrototypeDescriptorStr.slice(1, Idx);470 PrototypeDescriptorStr = PrototypeDescriptorStr.drop_front(Idx + 1);471 assert(!PrototypeDescriptorStr.contains('(') &&472 "Only allow one vector type modifier");473 474 auto ComplexTT = ComplexType.split(":");475 if (ComplexTT.first == "Log2EEW") {476 uint32_t Log2EEW;477 if (ComplexTT.second.getAsInteger(10, Log2EEW)) {478 llvm_unreachable("Invalid Log2EEW value!");479 return std::nullopt;480 }481 switch (Log2EEW) {482 case 3:483 VTM = VectorTypeModifier::Log2EEW3;484 break;485 case 4:486 VTM = VectorTypeModifier::Log2EEW4;487 break;488 case 5:489 VTM = VectorTypeModifier::Log2EEW5;490 break;491 case 6:492 VTM = VectorTypeModifier::Log2EEW6;493 break;494 default:495 llvm_unreachable("Invalid Log2EEW value, should be [3-6]");496 return std::nullopt;497 }498 } else if (ComplexTT.first == "FixedSEW") {499 uint32_t NewSEW;500 if (ComplexTT.second.getAsInteger(10, NewSEW)) {501 llvm_unreachable("Invalid FixedSEW value!");502 return std::nullopt;503 }504 switch (NewSEW) {505 case 8:506 VTM = VectorTypeModifier::FixedSEW8;507 break;508 case 16:509 VTM = VectorTypeModifier::FixedSEW16;510 break;511 case 32:512 VTM = VectorTypeModifier::FixedSEW32;513 break;514 case 64:515 VTM = VectorTypeModifier::FixedSEW64;516 break;517 default:518 llvm_unreachable("Invalid FixedSEW value, should be 8, 16, 32 or 64");519 return std::nullopt;520 }521 } else if (ComplexTT.first == "LFixedLog2LMUL") {522 int32_t Log2LMUL;523 if (ComplexTT.second.getAsInteger(10, Log2LMUL)) {524 llvm_unreachable("Invalid LFixedLog2LMUL value!");525 return std::nullopt;526 }527 switch (Log2LMUL) {528 case -3:529 VTM = VectorTypeModifier::LFixedLog2LMULN3;530 break;531 case -2:532 VTM = VectorTypeModifier::LFixedLog2LMULN2;533 break;534 case -1:535 VTM = VectorTypeModifier::LFixedLog2LMULN1;536 break;537 case 0:538 VTM = VectorTypeModifier::LFixedLog2LMUL0;539 break;540 case 1:541 VTM = VectorTypeModifier::LFixedLog2LMUL1;542 break;543 case 2:544 VTM = VectorTypeModifier::LFixedLog2LMUL2;545 break;546 case 3:547 VTM = VectorTypeModifier::LFixedLog2LMUL3;548 break;549 default:550 llvm_unreachable("Invalid LFixedLog2LMUL value, should be [-3, 3]");551 return std::nullopt;552 }553 } else if (ComplexTT.first == "SFixedLog2LMUL") {554 int32_t Log2LMUL;555 if (ComplexTT.second.getAsInteger(10, Log2LMUL)) {556 llvm_unreachable("Invalid SFixedLog2LMUL value!");557 return std::nullopt;558 }559 switch (Log2LMUL) {560 case -3:561 VTM = VectorTypeModifier::SFixedLog2LMULN3;562 break;563 case -2:564 VTM = VectorTypeModifier::SFixedLog2LMULN2;565 break;566 case -1:567 VTM = VectorTypeModifier::SFixedLog2LMULN1;568 break;569 case 0:570 VTM = VectorTypeModifier::SFixedLog2LMUL0;571 break;572 case 1:573 VTM = VectorTypeModifier::SFixedLog2LMUL1;574 break;575 case 2:576 VTM = VectorTypeModifier::SFixedLog2LMUL2;577 break;578 case 3:579 VTM = VectorTypeModifier::SFixedLog2LMUL3;580 break;581 default:582 llvm_unreachable("Invalid LFixedLog2LMUL value, should be [-3, 3]");583 return std::nullopt;584 }585 586 } else if (ComplexTT.first == "SEFixedLog2LMUL") {587 int32_t Log2LMUL;588 if (ComplexTT.second.getAsInteger(10, Log2LMUL)) {589 llvm_unreachable("Invalid SEFixedLog2LMUL value!");590 return std::nullopt;591 }592 switch (Log2LMUL) {593 case -3:594 VTM = VectorTypeModifier::SEFixedLog2LMULN3;595 break;596 case -2:597 VTM = VectorTypeModifier::SEFixedLog2LMULN2;598 break;599 case -1:600 VTM = VectorTypeModifier::SEFixedLog2LMULN1;601 break;602 case 0:603 VTM = VectorTypeModifier::SEFixedLog2LMUL0;604 break;605 case 1:606 VTM = VectorTypeModifier::SEFixedLog2LMUL1;607 break;608 case 2:609 VTM = VectorTypeModifier::SEFixedLog2LMUL2;610 break;611 case 3:612 VTM = VectorTypeModifier::SEFixedLog2LMUL3;613 break;614 default:615 llvm_unreachable("Invalid LFixedLog2LMUL value, should be [-3, 3]");616 return std::nullopt;617 }618 } else if (ComplexTT.first == "Tuple") {619 unsigned NF = 0;620 if (ComplexTT.second.getAsInteger(10, NF)) {621 llvm_unreachable("Invalid NF value!");622 return std::nullopt;623 }624 VTM = getTupleVTM(NF);625 } else {626 llvm_unreachable("Illegal complex type transformers!");627 }628 }629 PD.VTM = static_cast<uint8_t>(VTM);630 631 // Compute the remain type transformers632 TypeModifier TM = TypeModifier::NoModifier;633 for (char I : PrototypeDescriptorStr) {634 switch (I) {635 case 'P':636 if ((TM & TypeModifier::Const) == TypeModifier::Const)637 llvm_unreachable("'P' transformer cannot be used after 'C'");638 if ((TM & TypeModifier::Pointer) == TypeModifier::Pointer)639 llvm_unreachable("'P' transformer cannot be used twice");640 TM |= TypeModifier::Pointer;641 break;642 case 'C':643 TM |= TypeModifier::Const;644 break;645 case 'K':646 TM |= TypeModifier::Immediate;647 break;648 case 'U':649 TM |= TypeModifier::UnsignedInteger;650 break;651 case 'I':652 TM |= TypeModifier::SignedInteger;653 break;654 case 'F':655 TM |= TypeModifier::Float;656 break;657 case 'Y':658 TM |= TypeModifier::BFloat;659 break;660 case 'S':661 TM |= TypeModifier::LMUL1;662 break;663 default:664 llvm_unreachable("Illegal non-primitive type transformer!");665 }666 }667 PD.TM = static_cast<uint8_t>(TM);668 669 return PD;670}671 672void RVVType::applyModifier(const PrototypeDescriptor &Transformer) {673 // Handle primitive type transformer674 switch (static_cast<BaseTypeModifier>(Transformer.PT)) {675 case BaseTypeModifier::Scalar:676 Scale = 0;677 break;678 case BaseTypeModifier::Vector:679 Scale = LMUL.getScale(ElementBitwidth);680 break;681 case BaseTypeModifier::Void:682 ScalarType = ScalarTypeKind::Void;683 break;684 case BaseTypeModifier::SizeT:685 ScalarType = ScalarTypeKind::Size_t;686 break;687 case BaseTypeModifier::Ptrdiff:688 ScalarType = ScalarTypeKind::Ptrdiff_t;689 break;690 case BaseTypeModifier::UnsignedLong:691 ScalarType = ScalarTypeKind::UnsignedLong;692 break;693 case BaseTypeModifier::SignedLong:694 ScalarType = ScalarTypeKind::SignedLong;695 break;696 case BaseTypeModifier::Float32:697 ElementBitwidth = 32;698 ScalarType = ScalarTypeKind::Float;699 break;700 case BaseTypeModifier::Invalid:701 ScalarType = ScalarTypeKind::Invalid;702 return;703 }704 705 switch (static_cast<VectorTypeModifier>(Transformer.VTM)) {706 case VectorTypeModifier::Widening2XVector:707 ElementBitwidth *= 2;708 LMUL.MulLog2LMUL(1);709 Scale = LMUL.getScale(ElementBitwidth);710 if (ScalarType == ScalarTypeKind::BFloat)711 ScalarType = ScalarTypeKind::Float;712 break;713 case VectorTypeModifier::Widening4XVector:714 ElementBitwidth *= 4;715 LMUL.MulLog2LMUL(2);716 Scale = LMUL.getScale(ElementBitwidth);717 break;718 case VectorTypeModifier::Widening8XVector:719 ElementBitwidth *= 8;720 LMUL.MulLog2LMUL(3);721 Scale = LMUL.getScale(ElementBitwidth);722 break;723 case VectorTypeModifier::MaskVector:724 ScalarType = ScalarTypeKind::Boolean;725 Scale = LMUL.getScale(ElementBitwidth);726 ElementBitwidth = 1;727 break;728 case VectorTypeModifier::Log2EEW3:729 applyLog2EEW(3);730 break;731 case VectorTypeModifier::Log2EEW4:732 applyLog2EEW(4);733 break;734 case VectorTypeModifier::Log2EEW5:735 applyLog2EEW(5);736 break;737 case VectorTypeModifier::Log2EEW6:738 applyLog2EEW(6);739 break;740 case VectorTypeModifier::FixedSEW8:741 applyFixedSEW(8);742 break;743 case VectorTypeModifier::FixedSEW16:744 applyFixedSEW(16);745 break;746 case VectorTypeModifier::FixedSEW32:747 applyFixedSEW(32);748 break;749 case VectorTypeModifier::FixedSEW64:750 applyFixedSEW(64);751 break;752 case VectorTypeModifier::LFixedLog2LMULN3:753 applyFixedLog2LMUL(-3, FixedLMULType::LargerThan);754 break;755 case VectorTypeModifier::LFixedLog2LMULN2:756 applyFixedLog2LMUL(-2, FixedLMULType::LargerThan);757 break;758 case VectorTypeModifier::LFixedLog2LMULN1:759 applyFixedLog2LMUL(-1, FixedLMULType::LargerThan);760 break;761 case VectorTypeModifier::LFixedLog2LMUL0:762 applyFixedLog2LMUL(0, FixedLMULType::LargerThan);763 break;764 case VectorTypeModifier::LFixedLog2LMUL1:765 applyFixedLog2LMUL(1, FixedLMULType::LargerThan);766 break;767 case VectorTypeModifier::LFixedLog2LMUL2:768 applyFixedLog2LMUL(2, FixedLMULType::LargerThan);769 break;770 case VectorTypeModifier::LFixedLog2LMUL3:771 applyFixedLog2LMUL(3, FixedLMULType::LargerThan);772 break;773 case VectorTypeModifier::SFixedLog2LMULN3:774 applyFixedLog2LMUL(-3, FixedLMULType::SmallerThan);775 break;776 case VectorTypeModifier::SFixedLog2LMULN2:777 applyFixedLog2LMUL(-2, FixedLMULType::SmallerThan);778 break;779 case VectorTypeModifier::SFixedLog2LMULN1:780 applyFixedLog2LMUL(-1, FixedLMULType::SmallerThan);781 break;782 case VectorTypeModifier::SFixedLog2LMUL0:783 applyFixedLog2LMUL(0, FixedLMULType::SmallerThan);784 break;785 case VectorTypeModifier::SFixedLog2LMUL1:786 applyFixedLog2LMUL(1, FixedLMULType::SmallerThan);787 break;788 case VectorTypeModifier::SFixedLog2LMUL2:789 applyFixedLog2LMUL(2, FixedLMULType::SmallerThan);790 break;791 case VectorTypeModifier::SFixedLog2LMUL3:792 applyFixedLog2LMUL(3, FixedLMULType::SmallerThan);793 break;794 case VectorTypeModifier::SEFixedLog2LMULN3:795 applyFixedLog2LMUL(-3, FixedLMULType::SmallerOrEqual);796 break;797 case VectorTypeModifier::SEFixedLog2LMULN2:798 applyFixedLog2LMUL(-2, FixedLMULType::SmallerOrEqual);799 break;800 case VectorTypeModifier::SEFixedLog2LMULN1:801 applyFixedLog2LMUL(-1, FixedLMULType::SmallerOrEqual);802 break;803 case VectorTypeModifier::SEFixedLog2LMUL0:804 applyFixedLog2LMUL(0, FixedLMULType::SmallerOrEqual);805 break;806 case VectorTypeModifier::SEFixedLog2LMUL1:807 applyFixedLog2LMUL(1, FixedLMULType::SmallerOrEqual);808 break;809 case VectorTypeModifier::SEFixedLog2LMUL2:810 applyFixedLog2LMUL(2, FixedLMULType::SmallerOrEqual);811 break;812 case VectorTypeModifier::SEFixedLog2LMUL3:813 applyFixedLog2LMUL(3, FixedLMULType::SmallerOrEqual);814 break;815 case VectorTypeModifier::Tuple2:816 case VectorTypeModifier::Tuple3:817 case VectorTypeModifier::Tuple4:818 case VectorTypeModifier::Tuple5:819 case VectorTypeModifier::Tuple6:820 case VectorTypeModifier::Tuple7:821 case VectorTypeModifier::Tuple8: {822 IsTuple = true;823 NF = 2 + static_cast<uint8_t>(Transformer.VTM) -824 static_cast<uint8_t>(VectorTypeModifier::Tuple2);825 break;826 }827 case VectorTypeModifier::NoModifier:828 break;829 }830 831 // Early return if the current type modifier is already invalid.832 if (ScalarType == Invalid)833 return;834 835 for (unsigned TypeModifierMaskShift = 0;836 TypeModifierMaskShift <= static_cast<unsigned>(TypeModifier::MaxOffset);837 ++TypeModifierMaskShift) {838 unsigned TypeModifierMask = 1 << TypeModifierMaskShift;839 if ((static_cast<unsigned>(Transformer.TM) & TypeModifierMask) !=840 TypeModifierMask)841 continue;842 switch (static_cast<TypeModifier>(TypeModifierMask)) {843 case TypeModifier::Pointer:844 IsPointer = true;845 break;846 case TypeModifier::Const:847 IsConstant = true;848 break;849 case TypeModifier::Immediate:850 IsImmediate = true;851 IsConstant = true;852 break;853 case TypeModifier::UnsignedInteger:854 ScalarType = ScalarTypeKind::UnsignedInteger;855 break;856 case TypeModifier::SignedInteger:857 ScalarType = ScalarTypeKind::SignedInteger;858 break;859 case TypeModifier::Float:860 ScalarType = ScalarTypeKind::Float;861 break;862 case TypeModifier::BFloat:863 ScalarType = ScalarTypeKind::BFloat;864 break;865 case TypeModifier::LMUL1:866 LMUL = LMULType(0);867 // Update ElementBitwidth need to update Scale too.868 Scale = LMUL.getScale(ElementBitwidth);869 break;870 default:871 llvm_unreachable("Unknown type modifier mask!");872 }873 }874}875 876void RVVType::applyLog2EEW(unsigned Log2EEW) {877 // update new elmul = (eew/sew) * lmul878 LMUL.MulLog2LMUL(Log2EEW - Log2_32(ElementBitwidth));879 // update new eew880 ElementBitwidth = 1 << Log2EEW;881 ScalarType = ScalarTypeKind::SignedInteger;882 Scale = LMUL.getScale(ElementBitwidth);883}884 885void RVVType::applyFixedSEW(unsigned NewSEW) {886 // Set invalid type if src and dst SEW are same.887 if (ElementBitwidth == NewSEW) {888 ScalarType = ScalarTypeKind::Invalid;889 return;890 }891 // Update new SEW892 ElementBitwidth = NewSEW;893 Scale = LMUL.getScale(ElementBitwidth);894}895 896void RVVType::applyFixedLog2LMUL(int Log2LMUL, enum FixedLMULType Type) {897 switch (Type) {898 case FixedLMULType::LargerThan:899 if (Log2LMUL <= LMUL.Log2LMUL) {900 ScalarType = ScalarTypeKind::Invalid;901 return;902 }903 break;904 case FixedLMULType::SmallerThan:905 if (Log2LMUL >= LMUL.Log2LMUL) {906 ScalarType = ScalarTypeKind::Invalid;907 return;908 }909 break;910 case FixedLMULType::SmallerOrEqual:911 if (Log2LMUL > LMUL.Log2LMUL) {912 ScalarType = ScalarTypeKind::Invalid;913 return;914 }915 break;916 }917 918 // Update new LMUL919 LMUL = LMULType(Log2LMUL);920 Scale = LMUL.getScale(ElementBitwidth);921}922 923std::optional<RVVTypes>924RVVTypeCache::computeTypes(BasicType BT, int Log2LMUL, unsigned NF,925 ArrayRef<PrototypeDescriptor> Prototype) {926 RVVTypes Types;927 for (const PrototypeDescriptor &Proto : Prototype) {928 auto T = computeType(BT, Log2LMUL, Proto);929 if (!T)930 return std::nullopt;931 // Record legal type index932 Types.push_back(*T);933 }934 return Types;935}936 937// Compute the hash value of RVVType, used for cache the result of computeType.938static uint64_t computeRVVTypeHashValue(BasicType BT, int Log2LMUL,939 PrototypeDescriptor Proto) {940 // Layout of hash value:941 // 0 8 16 24 32 40942 // | Log2LMUL + 3 | BT | Proto.PT | Proto.TM | Proto.VTM |943 assert(Log2LMUL >= -3 && Log2LMUL <= 3);944 return (Log2LMUL + 3) | (static_cast<uint64_t>(BT) & 0xff) << 8 |945 ((uint64_t)(Proto.PT & 0xff) << 16) |946 ((uint64_t)(Proto.TM & 0xff) << 24) |947 ((uint64_t)(Proto.VTM & 0xff) << 32);948}949 950std::optional<RVVTypePtr> RVVTypeCache::computeType(BasicType BT, int Log2LMUL,951 PrototypeDescriptor Proto) {952 uint64_t Idx = computeRVVTypeHashValue(BT, Log2LMUL, Proto);953 // Search first954 auto It = LegalTypes.find(Idx);955 if (It != LegalTypes.end())956 return &(It->second);957 958 if (IllegalTypes.count(Idx))959 return std::nullopt;960 961 // Compute type and record the result.962 RVVType T(BT, Log2LMUL, Proto);963 if (T.isValid()) {964 // Record legal type index and value.965 std::pair<std::unordered_map<uint64_t, RVVType>::iterator, bool>966 InsertResult = LegalTypes.insert({Idx, T});967 return &(InsertResult.first->second);968 }969 // Record illegal type index.970 IllegalTypes.insert(Idx);971 return std::nullopt;972}973 974//===----------------------------------------------------------------------===//975// RVVIntrinsic implementation976//===----------------------------------------------------------------------===//977RVVIntrinsic::RVVIntrinsic(978 StringRef NewName, StringRef Suffix, StringRef NewOverloadedName,979 StringRef OverloadedSuffix, StringRef IRName, bool IsMasked,980 bool HasMaskedOffOperand, bool HasVL, PolicyScheme Scheme,981 bool SupportOverloading, bool HasBuiltinAlias, StringRef ManualCodegen,982 const RVVTypes &OutInTypes, const std::vector<int64_t> &NewIntrinsicTypes,983 unsigned NF, Policy NewPolicyAttrs, bool HasFRMRoundModeOp, unsigned TWiden)984 : IRName(IRName), IsMasked(IsMasked),985 HasMaskedOffOperand(HasMaskedOffOperand), HasVL(HasVL), Scheme(Scheme),986 SupportOverloading(SupportOverloading), HasBuiltinAlias(HasBuiltinAlias),987 ManualCodegen(ManualCodegen.str()), NF(NF), PolicyAttrs(NewPolicyAttrs),988 TWiden(TWiden) {989 990 // Init BuiltinName, Name and OverloadedName991 BuiltinName = NewName.str();992 Name = BuiltinName;993 if (NewOverloadedName.empty())994 OverloadedName = NewName.split("_").first.str();995 else996 OverloadedName = NewOverloadedName.str();997 if (!Suffix.empty())998 Name += "_" + Suffix.str();999 if (!OverloadedSuffix.empty())1000 OverloadedName += "_" + OverloadedSuffix.str();1001 1002 updateNamesAndPolicy(IsMasked, hasPolicy(), Name, BuiltinName, OverloadedName,1003 PolicyAttrs, HasFRMRoundModeOp);1004 1005 // Init OutputType and InputTypes1006 OutputType = OutInTypes[0];1007 InputTypes.assign(OutInTypes.begin() + 1, OutInTypes.end());1008 1009 // IntrinsicTypes is unmasked TA version index. Need to update it1010 // if there is merge operand (It is always in first operand).1011 IntrinsicTypes = NewIntrinsicTypes;1012 if ((IsMasked && hasMaskedOffOperand()) ||1013 (!IsMasked && hasPassthruOperand())) {1014 for (auto &I : IntrinsicTypes) {1015 if (I >= 0)1016 I += 1;1017 }1018 }1019}1020 1021std::string RVVIntrinsic::getBuiltinTypeStr() const {1022 std::string S;1023 S += OutputType->getBuiltinStr();1024 for (const auto &T : InputTypes) {1025 S += T->getBuiltinStr();1026 }1027 return S;1028}1029 1030std::string RVVIntrinsic::getSuffixStr(1031 RVVTypeCache &TypeCache, BasicType Type, int Log2LMUL,1032 llvm::ArrayRef<PrototypeDescriptor> PrototypeDescriptors) {1033 SmallVector<std::string> SuffixStrs;1034 for (auto PD : PrototypeDescriptors) {1035 auto T = TypeCache.computeType(Type, Log2LMUL, PD);1036 SuffixStrs.push_back((*T)->getShortStr());1037 }1038 return join(SuffixStrs, "_");1039}1040 1041llvm::SmallVector<PrototypeDescriptor> RVVIntrinsic::computeBuiltinTypes(1042 llvm::ArrayRef<PrototypeDescriptor> Prototype, bool IsMasked,1043 bool HasMaskedOffOperand, bool HasVL, unsigned NF,1044 PolicyScheme DefaultScheme, Policy PolicyAttrs, bool IsTuple) {1045 SmallVector<PrototypeDescriptor> NewPrototype(Prototype);1046 bool HasPassthruOp = DefaultScheme == PolicyScheme::HasPassthruOperand;1047 if (IsMasked) {1048 // If HasMaskedOffOperand, insert result type as first input operand if1049 // need.1050 if (HasMaskedOffOperand && !PolicyAttrs.isTAMAPolicy()) {1051 if (NF == 1) {1052 NewPrototype.insert(NewPrototype.begin() + 1, NewPrototype[0]);1053 } else if (NF > 1) {1054 if (IsTuple) {1055 PrototypeDescriptor BasePtrOperand = Prototype[1];1056 PrototypeDescriptor MaskoffType = PrototypeDescriptor(1057 static_cast<uint8_t>(BaseTypeModifier::Vector),1058 static_cast<uint8_t>(getTupleVTM(NF)),1059 BasePtrOperand.TM & ~static_cast<uint8_t>(TypeModifier::Pointer));1060 NewPrototype.insert(NewPrototype.begin() + 1, MaskoffType);1061 } else {1062 // Convert1063 // (void, op0 address, op1 address, ...)1064 // to1065 // (void, op0 address, op1 address, ..., maskedoff0, maskedoff1, ...)1066 PrototypeDescriptor MaskoffType = NewPrototype[1];1067 MaskoffType.TM &= ~static_cast<uint8_t>(TypeModifier::Pointer);1068 NewPrototype.insert(NewPrototype.begin() + NF + 1, NF, MaskoffType);1069 }1070 }1071 }1072 if (HasMaskedOffOperand && NF > 1) {1073 // Convert1074 // (void, op0 address, op1 address, ..., maskedoff0, maskedoff1, ...)1075 // to1076 // (void, op0 address, op1 address, ..., mask, maskedoff0, maskedoff1,1077 // ...)1078 if (IsTuple)1079 NewPrototype.insert(NewPrototype.begin() + 1,1080 PrototypeDescriptor::Mask);1081 else1082 NewPrototype.insert(NewPrototype.begin() + NF + 1,1083 PrototypeDescriptor::Mask);1084 } else {1085 // If IsMasked, insert PrototypeDescriptor:Mask as first input operand.1086 NewPrototype.insert(NewPrototype.begin() + 1, PrototypeDescriptor::Mask);1087 }1088 } else {1089 if (NF == 1) {1090 if (PolicyAttrs.isTUPolicy() && HasPassthruOp)1091 NewPrototype.insert(NewPrototype.begin(), NewPrototype[0]);1092 } else if (PolicyAttrs.isTUPolicy() && HasPassthruOp) {1093 if (IsTuple) {1094 PrototypeDescriptor BasePtrOperand = Prototype[0];1095 PrototypeDescriptor MaskoffType = PrototypeDescriptor(1096 static_cast<uint8_t>(BaseTypeModifier::Vector),1097 static_cast<uint8_t>(getTupleVTM(NF)),1098 BasePtrOperand.TM & ~static_cast<uint8_t>(TypeModifier::Pointer));1099 NewPrototype.insert(NewPrototype.begin(), MaskoffType);1100 } else {1101 // NF > 1 cases for segment load operations.1102 // Convert1103 // (void, op0 address, op1 address, ...)1104 // to1105 // (void, op0 address, op1 address, maskedoff0, maskedoff1, ...)1106 PrototypeDescriptor MaskoffType = Prototype[1];1107 MaskoffType.TM &= ~static_cast<uint8_t>(TypeModifier::Pointer);1108 NewPrototype.insert(NewPrototype.begin() + NF + 1, NF, MaskoffType);1109 }1110 }1111 }1112 1113 // If HasVL, append PrototypeDescriptor:VL to last operand1114 if (HasVL)1115 NewPrototype.push_back(PrototypeDescriptor::VL);1116 1117 return NewPrototype;1118}1119 1120llvm::SmallVector<Policy> RVVIntrinsic::getSupportedUnMaskedPolicies() {1121 return {Policy(Policy::PolicyType::Undisturbed)}; // TU1122}1123 1124llvm::SmallVector<Policy>1125RVVIntrinsic::getSupportedMaskedPolicies(bool HasTailPolicy,1126 bool HasMaskPolicy) {1127 if (HasTailPolicy && HasMaskPolicy)1128 return {Policy(Policy::PolicyType::Undisturbed,1129 Policy::PolicyType::Agnostic), // TUM1130 Policy(Policy::PolicyType::Undisturbed,1131 Policy::PolicyType::Undisturbed), // TUMU1132 Policy(Policy::PolicyType::Agnostic,1133 Policy::PolicyType::Undisturbed)}; // MU1134 if (HasTailPolicy && !HasMaskPolicy)1135 return {Policy(Policy::PolicyType::Undisturbed,1136 Policy::PolicyType::Agnostic)}; // TU1137 if (!HasTailPolicy && HasMaskPolicy)1138 return {Policy(Policy::PolicyType::Agnostic,1139 Policy::PolicyType::Undisturbed)}; // MU1140 llvm_unreachable("An RVV instruction should not be without both tail policy "1141 "and mask policy");1142}1143 1144void RVVIntrinsic::updateNamesAndPolicy(1145 bool IsMasked, bool HasPolicy, std::string &Name, std::string &BuiltinName,1146 std::string &OverloadedName, Policy &PolicyAttrs, bool HasFRMRoundModeOp) {1147 1148 auto appendPolicySuffix = [&](const std::string &suffix) {1149 Name += suffix;1150 BuiltinName += suffix;1151 OverloadedName += suffix;1152 };1153 1154 if (HasFRMRoundModeOp) {1155 Name += "_rm";1156 BuiltinName += "_rm";1157 }1158 1159 if (IsMasked) {1160 if (PolicyAttrs.isTUMUPolicy())1161 appendPolicySuffix("_tumu");1162 else if (PolicyAttrs.isTUMAPolicy())1163 appendPolicySuffix("_tum");1164 else if (PolicyAttrs.isTAMUPolicy())1165 appendPolicySuffix("_mu");1166 else if (PolicyAttrs.isTAMAPolicy()) {1167 Name += "_m";1168 BuiltinName += "_m";1169 } else1170 llvm_unreachable("Unhandled policy condition");1171 } else {1172 if (PolicyAttrs.isTUPolicy())1173 appendPolicySuffix("_tu");1174 else if (PolicyAttrs.isTAPolicy()) // no suffix needed1175 return;1176 else1177 llvm_unreachable("Unhandled policy condition");1178 }1179}1180 1181SmallVector<PrototypeDescriptor> parsePrototypes(StringRef Prototypes) {1182 SmallVector<PrototypeDescriptor> PrototypeDescriptors;1183 const StringRef Primaries("evwqom0ztulf");1184 while (!Prototypes.empty()) {1185 size_t Idx = 0;1186 // Skip over complex prototype because it could contain primitive type1187 // character.1188 if (Prototypes[0] == '(')1189 Idx = Prototypes.find_first_of(')');1190 Idx = Prototypes.find_first_of(Primaries, Idx);1191 assert(Idx != StringRef::npos);1192 auto PD = PrototypeDescriptor::parsePrototypeDescriptor(1193 Prototypes.slice(0, Idx + 1));1194 if (!PD)1195 llvm_unreachable("Error during parsing prototype.");1196 PrototypeDescriptors.push_back(*PD);1197 Prototypes = Prototypes.drop_front(Idx + 1);1198 }1199 return PrototypeDescriptors;1200}1201 1202#define STRINGIFY(NAME) \1203 case NAME: \1204 OS << #NAME; \1205 break;1206 1207llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, enum PolicyScheme PS) {1208 switch (PS) {1209 STRINGIFY(SchemeNone)1210 STRINGIFY(HasPassthruOperand)1211 STRINGIFY(HasPolicyOperand)1212 }1213 return OS;1214}1215 1216#undef STRINGIFY1217 1218raw_ostream &operator<<(raw_ostream &OS, const RVVIntrinsicRecord &Record) {1219 OS << "{";1220 OS << "/*Name=*/\"" << Record.Name << "\", ";1221 if (Record.OverloadedName == nullptr ||1222 StringRef(Record.OverloadedName).empty())1223 OS << "/*OverloadedName=*/nullptr, ";1224 else1225 OS << "/*OverloadedName=*/\"" << Record.OverloadedName << "\", ";1226 OS << "/*RequiredExtensions=*/\"" << Record.RequiredExtensions << "\", ";1227 OS << "/*PrototypeIndex=*/" << Record.PrototypeIndex << ", ";1228 OS << "/*SuffixIndex=*/" << Record.SuffixIndex << ", ";1229 OS << "/*OverloadedSuffixIndex=*/" << Record.OverloadedSuffixIndex << ", ";1230 OS << "/*PrototypeLength=*/" << (int)Record.PrototypeLength << ", ";1231 OS << "/*SuffixLength=*/" << (int)Record.SuffixLength << ", ";1232 OS << "/*OverloadedSuffixSize=*/" << (int)Record.OverloadedSuffixSize << ", ";1233 OS << "/*TypeRangeMask=*/" << (int)Record.TypeRangeMask << ", ";1234 OS << "/*Log2LMULMask=*/" << (int)Record.Log2LMULMask << ", ";1235 OS << "/*NF=*/" << (int)Record.NF << ", ";1236 OS << "/*HasMasked=*/" << (int)Record.HasMasked << ", ";1237 OS << "/*HasVL=*/" << (int)Record.HasVL << ", ";1238 OS << "/*HasMaskedOffOperand=*/" << (int)Record.HasMaskedOffOperand << ", ";1239 OS << "/*HasTailPolicy=*/" << (int)Record.HasTailPolicy << ", ";1240 OS << "/*HasMaskPolicy=*/" << (int)Record.HasMaskPolicy << ", ";1241 OS << "/*HasFRMRoundModeOp=*/" << (int)Record.HasFRMRoundModeOp << ", ";1242 OS << "/*IsTuple=*/" << (int)Record.IsTuple << ", ";1243 OS << "/*UnMaskedPolicyScheme=*/" << (PolicyScheme)Record.UnMaskedPolicyScheme1244 << ", ";1245 OS << "/*MaskedPolicyScheme=*/" << (PolicyScheme)Record.MaskedPolicyScheme1246 << ", ";1247 OS << "},\n";1248 return OS;1249}1250 1251} // end namespace RISCV1252} // end namespace clang1253