1551 lines · cpp
1//===-- RISCVLegalizerInfo.cpp ----------------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8/// \file9/// This file implements the targeting of the Machinelegalizer class for RISC-V.10/// \todo This should be generated by TableGen.11//===----------------------------------------------------------------------===//12 13#include "RISCVLegalizerInfo.h"14#include "MCTargetDesc/RISCVMatInt.h"15#include "RISCVMachineFunctionInfo.h"16#include "RISCVSubtarget.h"17#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"18#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"19#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"20#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"21#include "llvm/CodeGen/MachineConstantPool.h"22#include "llvm/CodeGen/MachineJumpTableInfo.h"23#include "llvm/CodeGen/MachineMemOperand.h"24#include "llvm/CodeGen/MachineOperand.h"25#include "llvm/CodeGen/MachineRegisterInfo.h"26#include "llvm/CodeGen/TargetOpcodes.h"27#include "llvm/CodeGen/ValueTypes.h"28#include "llvm/IR/DerivedTypes.h"29#include "llvm/IR/Intrinsics.h"30#include "llvm/IR/IntrinsicsRISCV.h"31#include "llvm/IR/Type.h"32 33using namespace llvm;34using namespace LegalityPredicates;35using namespace LegalizeMutations;36 37static LegalityPredicate38typeIsLegalIntOrFPVec(unsigned TypeIdx,39 std::initializer_list<LLT> IntOrFPVecTys,40 const RISCVSubtarget &ST) {41 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {42 return ST.hasVInstructions() &&43 (Query.Types[TypeIdx].getScalarSizeInBits() != 64 ||44 ST.hasVInstructionsI64()) &&45 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||46 ST.getELen() == 64);47 };48 49 return all(typeInSet(TypeIdx, IntOrFPVecTys), P);50}51 52static LegalityPredicate53typeIsLegalBoolVec(unsigned TypeIdx, std::initializer_list<LLT> BoolVecTys,54 const RISCVSubtarget &ST) {55 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {56 return ST.hasVInstructions() &&57 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||58 ST.getELen() == 64);59 };60 return all(typeInSet(TypeIdx, BoolVecTys), P);61}62 63static LegalityPredicate typeIsLegalPtrVec(unsigned TypeIdx,64 std::initializer_list<LLT> PtrVecTys,65 const RISCVSubtarget &ST) {66 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {67 return ST.hasVInstructions() &&68 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||69 ST.getELen() == 64) &&70 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 16 ||71 Query.Types[TypeIdx].getScalarSizeInBits() == 32);72 };73 return all(typeInSet(TypeIdx, PtrVecTys), P);74}75 76RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)77 : STI(ST), XLen(STI.getXLen()), sXLen(LLT::scalar(XLen)) {78 const LLT sDoubleXLen = LLT::scalar(2 * XLen);79 const LLT p0 = LLT::pointer(0, XLen);80 const LLT s1 = LLT::scalar(1);81 const LLT s8 = LLT::scalar(8);82 const LLT s16 = LLT::scalar(16);83 const LLT s32 = LLT::scalar(32);84 const LLT s64 = LLT::scalar(64);85 const LLT s128 = LLT::scalar(128);86 87 const LLT nxv1s1 = LLT::scalable_vector(1, s1);88 const LLT nxv2s1 = LLT::scalable_vector(2, s1);89 const LLT nxv4s1 = LLT::scalable_vector(4, s1);90 const LLT nxv8s1 = LLT::scalable_vector(8, s1);91 const LLT nxv16s1 = LLT::scalable_vector(16, s1);92 const LLT nxv32s1 = LLT::scalable_vector(32, s1);93 const LLT nxv64s1 = LLT::scalable_vector(64, s1);94 95 const LLT nxv1s8 = LLT::scalable_vector(1, s8);96 const LLT nxv2s8 = LLT::scalable_vector(2, s8);97 const LLT nxv4s8 = LLT::scalable_vector(4, s8);98 const LLT nxv8s8 = LLT::scalable_vector(8, s8);99 const LLT nxv16s8 = LLT::scalable_vector(16, s8);100 const LLT nxv32s8 = LLT::scalable_vector(32, s8);101 const LLT nxv64s8 = LLT::scalable_vector(64, s8);102 103 const LLT nxv1s16 = LLT::scalable_vector(1, s16);104 const LLT nxv2s16 = LLT::scalable_vector(2, s16);105 const LLT nxv4s16 = LLT::scalable_vector(4, s16);106 const LLT nxv8s16 = LLT::scalable_vector(8, s16);107 const LLT nxv16s16 = LLT::scalable_vector(16, s16);108 const LLT nxv32s16 = LLT::scalable_vector(32, s16);109 110 const LLT nxv1s32 = LLT::scalable_vector(1, s32);111 const LLT nxv2s32 = LLT::scalable_vector(2, s32);112 const LLT nxv4s32 = LLT::scalable_vector(4, s32);113 const LLT nxv8s32 = LLT::scalable_vector(8, s32);114 const LLT nxv16s32 = LLT::scalable_vector(16, s32);115 116 const LLT nxv1s64 = LLT::scalable_vector(1, s64);117 const LLT nxv2s64 = LLT::scalable_vector(2, s64);118 const LLT nxv4s64 = LLT::scalable_vector(4, s64);119 const LLT nxv8s64 = LLT::scalable_vector(8, s64);120 121 const LLT nxv1p0 = LLT::scalable_vector(1, p0);122 const LLT nxv2p0 = LLT::scalable_vector(2, p0);123 const LLT nxv4p0 = LLT::scalable_vector(4, p0);124 const LLT nxv8p0 = LLT::scalable_vector(8, p0);125 const LLT nxv16p0 = LLT::scalable_vector(16, p0);126 127 using namespace TargetOpcode;128 129 auto BoolVecTys = {nxv1s1, nxv2s1, nxv4s1, nxv8s1, nxv16s1, nxv32s1, nxv64s1};130 131 auto IntOrFPVecTys = {nxv1s8, nxv2s8, nxv4s8, nxv8s8, nxv16s8, nxv32s8,132 nxv64s8, nxv1s16, nxv2s16, nxv4s16, nxv8s16, nxv16s16,133 nxv32s16, nxv1s32, nxv2s32, nxv4s32, nxv8s32, nxv16s32,134 nxv1s64, nxv2s64, nxv4s64, nxv8s64};135 136 auto PtrVecTys = {nxv1p0, nxv2p0, nxv4p0, nxv8p0, nxv16p0};137 138 getActionDefinitionsBuilder({G_ADD, G_SUB})139 .legalFor({sXLen})140 .legalIf(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST))141 .customFor(ST.is64Bit(), {s32})142 .widenScalarToNextPow2(0)143 .clampScalar(0, sXLen, sXLen);144 145 getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})146 .legalFor({sXLen})147 .legalIf(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST))148 .widenScalarToNextPow2(0)149 .clampScalar(0, sXLen, sXLen);150 151 getActionDefinitionsBuilder(152 {G_UADDE, G_UADDO, G_USUBE, G_USUBO}).lower();153 154 getActionDefinitionsBuilder({G_SADDE, G_SADDO, G_SSUBE, G_SSUBO})155 .minScalar(0, sXLen)156 .lower();157 158 // TODO: Use Vector Single-Width Saturating Instructions for vector types.159 getActionDefinitionsBuilder(160 {G_UADDSAT, G_SADDSAT, G_USUBSAT, G_SSUBSAT, G_SSHLSAT, G_USHLSAT})161 .lower();162 163 getActionDefinitionsBuilder({G_SHL, G_ASHR, G_LSHR})164 .legalFor({{sXLen, sXLen}})165 .customFor(ST.is64Bit(), {{s32, s32}})166 .widenScalarToNextPow2(0)167 .clampScalar(1, sXLen, sXLen)168 .clampScalar(0, sXLen, sXLen);169 170 getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT})171 .legalFor({{s32, s16}})172 .legalFor(ST.is64Bit(), {{s64, s16}, {s64, s32}})173 .legalIf(all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),174 typeIsLegalIntOrFPVec(1, IntOrFPVecTys, ST)))175 .customIf(typeIsLegalBoolVec(1, BoolVecTys, ST))176 .maxScalar(0, sXLen);177 178 getActionDefinitionsBuilder(G_SEXT_INREG)179 .customFor({sXLen})180 .clampScalar(0, sXLen, sXLen)181 .lower();182 183 // Merge/Unmerge184 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {185 auto &MergeUnmergeActions = getActionDefinitionsBuilder(Op);186 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;187 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;188 if (XLen == 32 && ST.hasStdExtD()) {189 MergeUnmergeActions.legalIf(190 all(typeIs(BigTyIdx, s64), typeIs(LitTyIdx, s32)));191 }192 MergeUnmergeActions.widenScalarToNextPow2(LitTyIdx, XLen)193 .widenScalarToNextPow2(BigTyIdx, XLen)194 .clampScalar(LitTyIdx, sXLen, sXLen)195 .clampScalar(BigTyIdx, sXLen, sXLen);196 }197 198 getActionDefinitionsBuilder({G_FSHL, G_FSHR}).lower();199 200 getActionDefinitionsBuilder({G_ROTR, G_ROTL})201 .legalFor(ST.hasStdExtZbb() || ST.hasStdExtZbkb(), {{sXLen, sXLen}})202 .customFor(ST.is64Bit() && (ST.hasStdExtZbb() || ST.hasStdExtZbkb()),203 {{s32, s32}})204 .lower();205 206 getActionDefinitionsBuilder(G_BITREVERSE).maxScalar(0, sXLen).lower();207 208 getActionDefinitionsBuilder(G_BITCAST).legalIf(209 all(LegalityPredicates::any(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),210 typeIsLegalBoolVec(0, BoolVecTys, ST)),211 LegalityPredicates::any(typeIsLegalIntOrFPVec(1, IntOrFPVecTys, ST),212 typeIsLegalBoolVec(1, BoolVecTys, ST))));213 214 auto &BSWAPActions = getActionDefinitionsBuilder(G_BSWAP);215 if (ST.hasStdExtZbb() || ST.hasStdExtZbkb())216 BSWAPActions.legalFor({sXLen}).clampScalar(0, sXLen, sXLen);217 else218 BSWAPActions.maxScalar(0, sXLen).lower();219 220 auto &CountZerosActions = getActionDefinitionsBuilder({G_CTLZ, G_CTTZ});221 auto &CountZerosUndefActions =222 getActionDefinitionsBuilder({G_CTLZ_ZERO_UNDEF, G_CTTZ_ZERO_UNDEF});223 if (ST.hasStdExtZbb()) {224 CountZerosActions.legalFor({{sXLen, sXLen}})225 .customFor({{s32, s32}})226 .clampScalar(0, s32, sXLen)227 .widenScalarToNextPow2(0)228 .scalarSameSizeAs(1, 0);229 } else {230 CountZerosActions.maxScalar(0, sXLen).scalarSameSizeAs(1, 0).lower();231 CountZerosUndefActions.maxScalar(0, sXLen).scalarSameSizeAs(1, 0);232 }233 CountZerosUndefActions.lower();234 235 auto &CTPOPActions = getActionDefinitionsBuilder(G_CTPOP);236 if (ST.hasStdExtZbb()) {237 CTPOPActions.legalFor({{sXLen, sXLen}})238 .clampScalar(0, sXLen, sXLen)239 .scalarSameSizeAs(1, 0);240 } else {241 CTPOPActions.widenScalarToNextPow2(0, /*Min*/ 8)242 .clampScalar(0, s8, sXLen)243 .scalarSameSizeAs(1, 0)244 .lower();245 }246 247 getActionDefinitionsBuilder(G_CONSTANT)248 .legalFor({p0})249 .legalFor(!ST.is64Bit(), {s32})250 .customFor(ST.is64Bit(), {s64})251 .widenScalarToNextPow2(0)252 .clampScalar(0, sXLen, sXLen);253 254 // TODO: transform illegal vector types into legal vector type255 getActionDefinitionsBuilder(G_FREEZE)256 .legalFor({s16, s32, p0})257 .legalFor(ST.is64Bit(), {s64})258 .legalIf(typeIsLegalBoolVec(0, BoolVecTys, ST))259 .legalIf(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST))260 .widenScalarToNextPow2(0)261 .clampScalar(0, s16, sXLen);262 263 // TODO: transform illegal vector types into legal vector type264 // TODO: Merge with G_FREEZE?265 getActionDefinitionsBuilder(266 {G_IMPLICIT_DEF, G_CONSTANT_FOLD_BARRIER})267 .legalFor({s32, sXLen, p0})268 .legalIf(typeIsLegalBoolVec(0, BoolVecTys, ST))269 .legalIf(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST))270 .widenScalarToNextPow2(0)271 .clampScalar(0, s32, sXLen);272 273 getActionDefinitionsBuilder(G_ICMP)274 .legalFor({{sXLen, sXLen}, {sXLen, p0}})275 .legalIf(all(typeIsLegalBoolVec(0, BoolVecTys, ST),276 typeIsLegalIntOrFPVec(1, IntOrFPVecTys, ST)))277 .widenScalarOrEltToNextPow2OrMinSize(1, 8)278 .clampScalar(1, sXLen, sXLen)279 .clampScalar(0, sXLen, sXLen);280 281 getActionDefinitionsBuilder(G_SELECT)282 .legalFor({{s32, sXLen}, {p0, sXLen}})283 .legalIf(all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),284 typeIsLegalBoolVec(1, BoolVecTys, ST)))285 .legalFor(XLen == 64 || ST.hasStdExtD(), {{s64, sXLen}})286 .widenScalarToNextPow2(0)287 .clampScalar(0, s32, (XLen == 64 || ST.hasStdExtD()) ? s64 : s32)288 .clampScalar(1, sXLen, sXLen);289 290 auto &LoadActions = getActionDefinitionsBuilder(G_LOAD);291 auto &StoreActions = getActionDefinitionsBuilder(G_STORE);292 auto &ExtLoadActions = getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD});293 294 // Return the alignment needed for scalar memory ops. If unaligned scalar mem295 // is supported, we only require byte alignment. Otherwise, we need the memory296 // op to be natively aligned.297 auto getScalarMemAlign = [&ST](unsigned Size) {298 return ST.enableUnalignedScalarMem() ? 8 : Size;299 };300 301 LoadActions.legalForTypesWithMemDesc(302 {{s16, p0, s8, getScalarMemAlign(8)},303 {s32, p0, s8, getScalarMemAlign(8)},304 {s16, p0, s16, getScalarMemAlign(16)},305 {s32, p0, s16, getScalarMemAlign(16)},306 {s32, p0, s32, getScalarMemAlign(32)},307 {p0, p0, sXLen, getScalarMemAlign(XLen)}});308 StoreActions.legalForTypesWithMemDesc(309 {{s16, p0, s8, getScalarMemAlign(8)},310 {s32, p0, s8, getScalarMemAlign(8)},311 {s16, p0, s16, getScalarMemAlign(16)},312 {s32, p0, s16, getScalarMemAlign(16)},313 {s32, p0, s32, getScalarMemAlign(32)},314 {p0, p0, sXLen, getScalarMemAlign(XLen)}});315 ExtLoadActions.legalForTypesWithMemDesc(316 {{sXLen, p0, s8, getScalarMemAlign(8)},317 {sXLen, p0, s16, getScalarMemAlign(16)}});318 if (XLen == 64) {319 LoadActions.legalForTypesWithMemDesc(320 {{s64, p0, s8, getScalarMemAlign(8)},321 {s64, p0, s16, getScalarMemAlign(16)},322 {s64, p0, s32, getScalarMemAlign(32)},323 {s64, p0, s64, getScalarMemAlign(64)}});324 StoreActions.legalForTypesWithMemDesc(325 {{s64, p0, s8, getScalarMemAlign(8)},326 {s64, p0, s16, getScalarMemAlign(16)},327 {s64, p0, s32, getScalarMemAlign(32)},328 {s64, p0, s64, getScalarMemAlign(64)}});329 ExtLoadActions.legalForTypesWithMemDesc(330 {{s64, p0, s32, getScalarMemAlign(32)}});331 } else if (ST.hasStdExtD()) {332 LoadActions.legalForTypesWithMemDesc(333 {{s64, p0, s64, getScalarMemAlign(64)}});334 StoreActions.legalForTypesWithMemDesc(335 {{s64, p0, s64, getScalarMemAlign(64)}});336 }337 338 // Vector loads/stores.339 if (ST.hasVInstructions()) {340 LoadActions.legalForTypesWithMemDesc({{nxv2s8, p0, nxv2s8, 8},341 {nxv4s8, p0, nxv4s8, 8},342 {nxv8s8, p0, nxv8s8, 8},343 {nxv16s8, p0, nxv16s8, 8},344 {nxv32s8, p0, nxv32s8, 8},345 {nxv64s8, p0, nxv64s8, 8},346 {nxv2s16, p0, nxv2s16, 16},347 {nxv4s16, p0, nxv4s16, 16},348 {nxv8s16, p0, nxv8s16, 16},349 {nxv16s16, p0, nxv16s16, 16},350 {nxv32s16, p0, nxv32s16, 16},351 {nxv2s32, p0, nxv2s32, 32},352 {nxv4s32, p0, nxv4s32, 32},353 {nxv8s32, p0, nxv8s32, 32},354 {nxv16s32, p0, nxv16s32, 32}});355 StoreActions.legalForTypesWithMemDesc({{nxv2s8, p0, nxv2s8, 8},356 {nxv4s8, p0, nxv4s8, 8},357 {nxv8s8, p0, nxv8s8, 8},358 {nxv16s8, p0, nxv16s8, 8},359 {nxv32s8, p0, nxv32s8, 8},360 {nxv64s8, p0, nxv64s8, 8},361 {nxv2s16, p0, nxv2s16, 16},362 {nxv4s16, p0, nxv4s16, 16},363 {nxv8s16, p0, nxv8s16, 16},364 {nxv16s16, p0, nxv16s16, 16},365 {nxv32s16, p0, nxv32s16, 16},366 {nxv2s32, p0, nxv2s32, 32},367 {nxv4s32, p0, nxv4s32, 32},368 {nxv8s32, p0, nxv8s32, 32},369 {nxv16s32, p0, nxv16s32, 32}});370 371 if (ST.getELen() == 64) {372 LoadActions.legalForTypesWithMemDesc({{nxv1s8, p0, nxv1s8, 8},373 {nxv1s16, p0, nxv1s16, 16},374 {nxv1s32, p0, nxv1s32, 32}});375 StoreActions.legalForTypesWithMemDesc({{nxv1s8, p0, nxv1s8, 8},376 {nxv1s16, p0, nxv1s16, 16},377 {nxv1s32, p0, nxv1s32, 32}});378 }379 380 if (ST.hasVInstructionsI64()) {381 LoadActions.legalForTypesWithMemDesc({{nxv1s64, p0, nxv1s64, 64},382 {nxv2s64, p0, nxv2s64, 64},383 {nxv4s64, p0, nxv4s64, 64},384 {nxv8s64, p0, nxv8s64, 64}});385 StoreActions.legalForTypesWithMemDesc({{nxv1s64, p0, nxv1s64, 64},386 {nxv2s64, p0, nxv2s64, 64},387 {nxv4s64, p0, nxv4s64, 64},388 {nxv8s64, p0, nxv8s64, 64}});389 }390 391 // we will take the custom lowering logic if we have scalable vector types392 // with non-standard alignments393 LoadActions.customIf(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST));394 StoreActions.customIf(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST));395 396 // Pointers require that XLen sized elements are legal.397 if (XLen <= ST.getELen()) {398 LoadActions.customIf(typeIsLegalPtrVec(0, PtrVecTys, ST));399 StoreActions.customIf(typeIsLegalPtrVec(0, PtrVecTys, ST));400 }401 }402 403 LoadActions.widenScalarToNextPow2(0, /* MinSize = */ 8)404 .lowerIfMemSizeNotByteSizePow2()405 .clampScalar(0, s16, sXLen)406 .lower();407 StoreActions408 .clampScalar(0, s16, sXLen)409 .lowerIfMemSizeNotByteSizePow2()410 .lower();411 412 ExtLoadActions.widenScalarToNextPow2(0).clampScalar(0, sXLen, sXLen).lower();413 414 getActionDefinitionsBuilder({G_PTR_ADD, G_PTRMASK}).legalFor({{p0, sXLen}});415 416 getActionDefinitionsBuilder(G_PTRTOINT)417 .legalFor({{sXLen, p0}})418 .clampScalar(0, sXLen, sXLen);419 420 getActionDefinitionsBuilder(G_INTTOPTR)421 .legalFor({{p0, sXLen}})422 .clampScalar(1, sXLen, sXLen);423 424 getActionDefinitionsBuilder(G_BRCOND).legalFor({sXLen}).minScalar(0, sXLen);425 426 getActionDefinitionsBuilder(G_BRJT).customFor({{p0, sXLen}});427 428 getActionDefinitionsBuilder(G_BRINDIRECT).legalFor({p0});429 430 getActionDefinitionsBuilder(G_PHI)431 .legalFor({p0, s32, sXLen})432 .widenScalarToNextPow2(0)433 .clampScalar(0, s32, sXLen);434 435 getActionDefinitionsBuilder({G_GLOBAL_VALUE, G_JUMP_TABLE, G_CONSTANT_POOL})436 .legalFor({p0});437 438 if (ST.hasStdExtZmmul()) {439 getActionDefinitionsBuilder(G_MUL)440 .legalFor({sXLen})441 .widenScalarToNextPow2(0)442 .clampScalar(0, sXLen, sXLen);443 444 // clang-format off445 getActionDefinitionsBuilder({G_SMULH, G_UMULH})446 .legalFor({sXLen})447 .lower();448 // clang-format on449 450 getActionDefinitionsBuilder({G_SMULO, G_UMULO}).minScalar(0, sXLen).lower();451 } else {452 getActionDefinitionsBuilder(G_MUL)453 .libcallFor({sXLen, sDoubleXLen})454 .widenScalarToNextPow2(0)455 .clampScalar(0, sXLen, sDoubleXLen);456 457 getActionDefinitionsBuilder({G_SMULH, G_UMULH}).lowerFor({sXLen});458 459 getActionDefinitionsBuilder({G_SMULO, G_UMULO})460 .minScalar(0, sXLen)461 // Widen sXLen to sDoubleXLen so we can use a single libcall to get462 // the low bits for the mul result and high bits to do the overflow463 // check.464 .widenScalarIf(typeIs(0, sXLen),465 LegalizeMutations::changeTo(0, sDoubleXLen))466 .lower();467 }468 469 if (ST.hasStdExtM()) {470 getActionDefinitionsBuilder({G_SDIV, G_UDIV, G_UREM})471 .legalFor({sXLen})472 .customFor({s32})473 .libcallFor({sDoubleXLen})474 .clampScalar(0, s32, sDoubleXLen)475 .widenScalarToNextPow2(0);476 getActionDefinitionsBuilder(G_SREM)477 .legalFor({sXLen})478 .libcallFor({sDoubleXLen})479 .clampScalar(0, sXLen, sDoubleXLen)480 .widenScalarToNextPow2(0);481 } else {482 getActionDefinitionsBuilder({G_UDIV, G_SDIV, G_UREM, G_SREM})483 .libcallFor({sXLen, sDoubleXLen})484 .clampScalar(0, sXLen, sDoubleXLen)485 .widenScalarToNextPow2(0);486 }487 488 // TODO: Use libcall for sDoubleXLen.489 getActionDefinitionsBuilder({G_SDIVREM, G_UDIVREM}).lower();490 491 getActionDefinitionsBuilder(G_ABS)492 .customFor(ST.hasStdExtZbb(), {sXLen})493 .minScalar(ST.hasStdExtZbb(), 0, sXLen)494 .lower();495 496 getActionDefinitionsBuilder({G_ABDS, G_ABDU})497 .minScalar(ST.hasStdExtZbb(), 0, sXLen)498 .lower();499 500 getActionDefinitionsBuilder({G_UMAX, G_UMIN, G_SMAX, G_SMIN})501 .legalFor(ST.hasStdExtZbb(), {sXLen})502 .minScalar(ST.hasStdExtZbb(), 0, sXLen)503 .lower();504 505 getActionDefinitionsBuilder({G_SCMP, G_UCMP}).lower();506 507 getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0});508 509 getActionDefinitionsBuilder({G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();510 511 getActionDefinitionsBuilder({G_DYN_STACKALLOC, G_STACKSAVE, G_STACKRESTORE})512 .lower();513 514 // FP Operations515 516 // FIXME: Support s128 for rv32 when libcall handling is able to use sret.517 getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT,518 G_FMAXNUM, G_FMINNUM, G_FMAXIMUMNUM,519 G_FMINIMUMNUM})520 .legalFor(ST.hasStdExtF(), {s32})521 .legalFor(ST.hasStdExtD(), {s64})522 .legalFor(ST.hasStdExtZfh(), {s16})523 .libcallFor({s32, s64})524 .libcallFor(ST.is64Bit(), {s128});525 526 getActionDefinitionsBuilder({G_FNEG, G_FABS})527 .legalFor(ST.hasStdExtF(), {s32})528 .legalFor(ST.hasStdExtD(), {s64})529 .legalFor(ST.hasStdExtZfh(), {s16})530 .lowerFor({s32, s64, s128});531 532 getActionDefinitionsBuilder(G_FREM)533 .libcallFor({s32, s64})534 .libcallFor(ST.is64Bit(), {s128})535 .minScalar(0, s32)536 .scalarize(0);537 538 getActionDefinitionsBuilder(G_FCOPYSIGN)539 .legalFor(ST.hasStdExtF(), {{s32, s32}})540 .legalFor(ST.hasStdExtD(), {{s64, s64}, {s32, s64}, {s64, s32}})541 .legalFor(ST.hasStdExtZfh(), {{s16, s16}, {s16, s32}, {s32, s16}})542 .legalFor(ST.hasStdExtZfh() && ST.hasStdExtD(), {{s16, s64}, {s64, s16}})543 .lower();544 545 // FIXME: Use Zfhmin.546 getActionDefinitionsBuilder(G_FPTRUNC)547 .legalFor(ST.hasStdExtD(), {{s32, s64}})548 .legalFor(ST.hasStdExtZfh(), {{s16, s32}})549 .legalFor(ST.hasStdExtZfh() && ST.hasStdExtD(), {{s16, s64}})550 .libcallFor({{s32, s64}})551 .libcallFor(ST.is64Bit(), {{s32, s128}, {s64, s128}});552 getActionDefinitionsBuilder(G_FPEXT)553 .legalFor(ST.hasStdExtD(), {{s64, s32}})554 .legalFor(ST.hasStdExtZfh(), {{s32, s16}})555 .legalFor(ST.hasStdExtZfh() && ST.hasStdExtD(), {{s64, s16}})556 .libcallFor({{s64, s32}})557 .libcallFor(ST.is64Bit(), {{s128, s32}, {s128, s64}});558 559 getActionDefinitionsBuilder(G_FCMP)560 .legalFor(ST.hasStdExtF(), {{sXLen, s32}})561 .legalFor(ST.hasStdExtD(), {{sXLen, s64}})562 .legalFor(ST.hasStdExtZfh(), {{sXLen, s16}})563 .clampScalar(0, sXLen, sXLen)564 .libcallFor({{sXLen, s32}, {sXLen, s64}})565 .libcallFor(ST.is64Bit(), {{sXLen, s128}});566 567 // TODO: Support vector version of G_IS_FPCLASS.568 getActionDefinitionsBuilder(G_IS_FPCLASS)569 .customFor(ST.hasStdExtF(), {{s1, s32}})570 .customFor(ST.hasStdExtD(), {{s1, s64}})571 .customFor(ST.hasStdExtZfh(), {{s1, s16}})572 .lowerFor({{s1, s32}, {s1, s64}});573 574 getActionDefinitionsBuilder(G_FCONSTANT)575 .legalFor(ST.hasStdExtF(), {s32})576 .legalFor(ST.hasStdExtD(), {s64})577 .legalFor(ST.hasStdExtZfh(), {s16})578 .customFor(!ST.is64Bit(), {s32})579 .customFor(ST.is64Bit(), {s32, s64})580 .lowerFor({s64, s128});581 582 getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})583 .legalFor(ST.hasStdExtF(), {{sXLen, s32}})584 .legalFor(ST.hasStdExtD(), {{sXLen, s64}})585 .legalFor(ST.hasStdExtZfh(), {{sXLen, s16}})586 .customFor(ST.is64Bit() && ST.hasStdExtF(), {{s32, s32}})587 .customFor(ST.is64Bit() && ST.hasStdExtD(), {{s32, s64}})588 .customFor(ST.is64Bit() && ST.hasStdExtZfh(), {{s32, s16}})589 .widenScalarToNextPow2(0)590 .minScalar(0, s32)591 .libcallFor({{s32, s32}, {s64, s32}, {s32, s64}, {s64, s64}})592 .libcallFor(ST.is64Bit(), {{s32, s128}, {s64, s128}}) // FIXME RV32.593 .libcallFor(ST.is64Bit(), {{s128, s32}, {s128, s64}, {s128, s128}});594 595 getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})596 .legalFor(ST.hasStdExtF(), {{s32, sXLen}})597 .legalFor(ST.hasStdExtD(), {{s64, sXLen}})598 .legalFor(ST.hasStdExtZfh(), {{s16, sXLen}})599 .widenScalarToNextPow2(1)600 // Promote to XLen if the operation is legal.601 .widenScalarIf(602 [=, &ST](const LegalityQuery &Query) {603 return Query.Types[0].isScalar() && Query.Types[1].isScalar() &&604 (Query.Types[1].getSizeInBits() < ST.getXLen()) &&605 ((ST.hasStdExtF() && Query.Types[0].getSizeInBits() == 32) ||606 (ST.hasStdExtD() && Query.Types[0].getSizeInBits() == 64) ||607 (ST.hasStdExtZfh() &&608 Query.Types[0].getSizeInBits() == 16));609 },610 LegalizeMutations::changeTo(1, sXLen))611 // Otherwise only promote to s32 since we have si libcalls.612 .minScalar(1, s32)613 .libcallFor({{s32, s32}, {s64, s32}, {s32, s64}, {s64, s64}})614 .libcallFor(ST.is64Bit(), {{s128, s32}, {s128, s64}}) // FIXME RV32.615 .libcallFor(ST.is64Bit(), {{s32, s128}, {s64, s128}, {s128, s128}});616 617 // FIXME: We can do custom inline expansion like SelectionDAG.618 getActionDefinitionsBuilder({G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,619 G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND,620 G_INTRINSIC_ROUNDEVEN})621 .legalFor(ST.hasStdExtZfa(), {s32})622 .legalFor(ST.hasStdExtZfa() && ST.hasStdExtD(), {s64})623 .legalFor(ST.hasStdExtZfa() && ST.hasStdExtZfh(), {s16})624 .libcallFor({s32, s64})625 .libcallFor(ST.is64Bit(), {s128});626 627 getActionDefinitionsBuilder({G_FMAXIMUM, G_FMINIMUM})628 .legalFor(ST.hasStdExtZfa(), {s32})629 .legalFor(ST.hasStdExtZfa() && ST.hasStdExtD(), {s64})630 .legalFor(ST.hasStdExtZfa() && ST.hasStdExtZfh(), {s16});631 632 getActionDefinitionsBuilder({G_FCOS, G_FSIN, G_FTAN, G_FPOW, G_FLOG, G_FLOG2,633 G_FLOG10, G_FEXP, G_FEXP2, G_FEXP10, G_FACOS,634 G_FASIN, G_FATAN, G_FATAN2, G_FCOSH, G_FSINH,635 G_FTANH, G_FMODF})636 .libcallFor({s32, s64})637 .libcallFor(ST.is64Bit(), {s128});638 getActionDefinitionsBuilder({G_FPOWI, G_FLDEXP})639 .libcallFor({{s32, s32}, {s64, s32}})640 .libcallFor(ST.is64Bit(), {s128, s32});641 642 getActionDefinitionsBuilder(G_VASTART).customFor({p0});643 644 // va_list must be a pointer, but most sized types are pretty easy to handle645 // as the destination.646 getActionDefinitionsBuilder(G_VAARG)647 // TODO: Implement narrowScalar and widenScalar for G_VAARG for types648 // other than sXLen.649 .clampScalar(0, sXLen, sXLen)650 .lowerForCartesianProduct({sXLen, p0}, {p0});651 652 getActionDefinitionsBuilder(G_VSCALE)653 .clampScalar(0, sXLen, sXLen)654 .customFor({sXLen});655 656 auto &SplatActions =657 getActionDefinitionsBuilder(G_SPLAT_VECTOR)658 .legalIf(all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),659 typeIs(1, sXLen)))660 .customIf(all(typeIsLegalBoolVec(0, BoolVecTys, ST), typeIs(1, s1)));661 // Handle case of s64 element vectors on RV32. If the subtarget does not have662 // f64, then try to lower it to G_SPLAT_VECTOR_SPLIT_64_VL. If the subtarget663 // does have f64, then we don't know whether the type is an f64 or an i64,664 // so mark the G_SPLAT_VECTOR as legal and decide later what to do with it,665 // depending on how the instructions it consumes are legalized. They are not666 // legalized yet since legalization is in reverse postorder, so we cannot667 // make the decision at this moment.668 if (XLen == 32) {669 if (ST.hasVInstructionsF64() && ST.hasStdExtD())670 SplatActions.legalIf(all(671 typeInSet(0, {nxv1s64, nxv2s64, nxv4s64, nxv8s64}), typeIs(1, s64)));672 else if (ST.hasVInstructionsI64())673 SplatActions.customIf(all(674 typeInSet(0, {nxv1s64, nxv2s64, nxv4s64, nxv8s64}), typeIs(1, s64)));675 }676 677 SplatActions.clampScalar(1, sXLen, sXLen);678 679 LegalityPredicate ExtractSubvecBitcastPred = [=](const LegalityQuery &Query) {680 LLT DstTy = Query.Types[0];681 LLT SrcTy = Query.Types[1];682 return DstTy.getElementType() == LLT::scalar(1) &&683 DstTy.getElementCount().getKnownMinValue() >= 8 &&684 SrcTy.getElementCount().getKnownMinValue() >= 8;685 };686 getActionDefinitionsBuilder(G_EXTRACT_SUBVECTOR)687 // We don't have the ability to slide mask vectors down indexed by their688 // i1 elements; the smallest we can do is i8. Often we are able to bitcast689 // to equivalent i8 vectors.690 .bitcastIf(691 all(typeIsLegalBoolVec(0, BoolVecTys, ST),692 typeIsLegalBoolVec(1, BoolVecTys, ST), ExtractSubvecBitcastPred),693 [=](const LegalityQuery &Query) {694 LLT CastTy = LLT::vector(695 Query.Types[0].getElementCount().divideCoefficientBy(8), 8);696 return std::pair(0, CastTy);697 })698 .customIf(LegalityPredicates::any(699 all(typeIsLegalBoolVec(0, BoolVecTys, ST),700 typeIsLegalBoolVec(1, BoolVecTys, ST)),701 all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),702 typeIsLegalIntOrFPVec(1, IntOrFPVecTys, ST))));703 704 getActionDefinitionsBuilder(G_INSERT_SUBVECTOR)705 .customIf(all(typeIsLegalBoolVec(0, BoolVecTys, ST),706 typeIsLegalBoolVec(1, BoolVecTys, ST)))707 .customIf(all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),708 typeIsLegalIntOrFPVec(1, IntOrFPVecTys, ST)));709 710 getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG_WITH_SUCCESS)711 .lowerIf(all(typeInSet(0, {s8, s16, s32, s64}), typeIs(2, p0)));712 713 getActionDefinitionsBuilder({G_ATOMIC_CMPXCHG, G_ATOMICRMW_ADD})714 .legalFor(ST.hasStdExtA(), {{sXLen, p0}})715 .libcallFor(!ST.hasStdExtA(), {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})716 .clampScalar(0, sXLen, sXLen);717 718 getActionDefinitionsBuilder(G_ATOMICRMW_SUB)719 .libcallFor(!ST.hasStdExtA(), {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})720 .clampScalar(0, sXLen, sXLen)721 .lower();722 723 LegalityPredicate InsertVectorEltPred = [=](const LegalityQuery &Query) {724 LLT VecTy = Query.Types[0];725 LLT EltTy = Query.Types[1];726 return VecTy.getElementType() == EltTy;727 };728 729 getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT)730 .legalIf(all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),731 InsertVectorEltPred, typeIs(2, sXLen)))732 .legalIf(all(typeIsLegalBoolVec(0, BoolVecTys, ST), InsertVectorEltPred,733 typeIs(2, sXLen)));734 735 getLegacyLegalizerInfo().computeTables();736 verify(*ST.getInstrInfo());737}738 739bool RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,740 MachineInstr &MI) const {741 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();742 743 if (const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =744 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID)) {745 if (II->hasScalarOperand() && !II->IsFPIntrinsic) {746 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;747 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();748 749 auto OldScalar = MI.getOperand(II->ScalarOperand + 2).getReg();750 // Legalize integer vx form intrinsic.751 if (MRI.getType(OldScalar).isScalar()) {752 if (MRI.getType(OldScalar).getSizeInBits() < sXLen.getSizeInBits()) {753 Helper.Observer.changingInstr(MI);754 Helper.widenScalarSrc(MI, sXLen, II->ScalarOperand + 2,755 TargetOpcode::G_ANYEXT);756 Helper.Observer.changedInstr(MI);757 } else if (MRI.getType(OldScalar).getSizeInBits() >758 sXLen.getSizeInBits()) {759 // TODO: i64 in riscv32.760 return false;761 }762 }763 }764 return true;765 }766 767 switch (IntrinsicID) {768 default:769 return false;770 case Intrinsic::vacopy: {771 // vacopy arguments must be legal because of the intrinsic signature.772 // No need to check here.773 774 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;775 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();776 MachineFunction &MF = *MI.getMF();777 const DataLayout &DL = MIRBuilder.getDataLayout();778 LLVMContext &Ctx = MF.getFunction().getContext();779 780 Register DstLst = MI.getOperand(1).getReg();781 LLT PtrTy = MRI.getType(DstLst);782 783 // Load the source va_list784 Align Alignment = DL.getABITypeAlign(getTypeForLLT(PtrTy, Ctx));785 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(786 MachinePointerInfo(), MachineMemOperand::MOLoad, PtrTy, Alignment);787 auto Tmp = MIRBuilder.buildLoad(PtrTy, MI.getOperand(2), *LoadMMO);788 789 // Store the result in the destination va_list790 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(791 MachinePointerInfo(), MachineMemOperand::MOStore, PtrTy, Alignment);792 MIRBuilder.buildStore(Tmp, DstLst, *StoreMMO);793 794 MI.eraseFromParent();795 return true;796 }797 case Intrinsic::riscv_masked_atomicrmw_add:798 case Intrinsic::riscv_masked_atomicrmw_sub:799 case Intrinsic::riscv_masked_cmpxchg:800 return true;801 }802}803 804bool RISCVLegalizerInfo::legalizeVAStart(MachineInstr &MI,805 MachineIRBuilder &MIRBuilder) const {806 // Stores the address of the VarArgsFrameIndex slot into the memory location807 assert(MI.getOpcode() == TargetOpcode::G_VASTART);808 MachineFunction *MF = MI.getParent()->getParent();809 RISCVMachineFunctionInfo *FuncInfo = MF->getInfo<RISCVMachineFunctionInfo>();810 int FI = FuncInfo->getVarArgsFrameIndex();811 LLT AddrTy = MIRBuilder.getMRI()->getType(MI.getOperand(0).getReg());812 auto FINAddr = MIRBuilder.buildFrameIndex(AddrTy, FI);813 assert(MI.hasOneMemOperand());814 MIRBuilder.buildStore(FINAddr, MI.getOperand(0).getReg(),815 *MI.memoperands()[0]);816 MI.eraseFromParent();817 return true;818}819 820bool RISCVLegalizerInfo::legalizeBRJT(MachineInstr &MI,821 MachineIRBuilder &MIRBuilder) const {822 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();823 auto &MF = *MI.getParent()->getParent();824 const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();825 unsigned EntrySize = MJTI->getEntrySize(MF.getDataLayout());826 827 Register PtrReg = MI.getOperand(0).getReg();828 LLT PtrTy = MRI.getType(PtrReg);829 Register IndexReg = MI.getOperand(2).getReg();830 LLT IndexTy = MRI.getType(IndexReg);831 832 if (!isPowerOf2_32(EntrySize))833 return false;834 835 auto ShiftAmt = MIRBuilder.buildConstant(IndexTy, Log2_32(EntrySize));836 IndexReg = MIRBuilder.buildShl(IndexTy, IndexReg, ShiftAmt).getReg(0);837 838 auto Addr = MIRBuilder.buildPtrAdd(PtrTy, PtrReg, IndexReg);839 840 MachineMemOperand *MMO = MF.getMachineMemOperand(841 MachinePointerInfo::getJumpTable(MF), MachineMemOperand::MOLoad,842 EntrySize, Align(MJTI->getEntryAlignment(MF.getDataLayout())));843 844 Register TargetReg;845 switch (MJTI->getEntryKind()) {846 default:847 return false;848 case MachineJumpTableInfo::EK_LabelDifference32: {849 // For PIC, the sequence is:850 // BRIND(load(Jumptable + index) + RelocBase)851 // RelocBase can be JumpTable, GOT or some sort of global base.852 unsigned LoadOpc =853 STI.is64Bit() ? TargetOpcode::G_SEXTLOAD : TargetOpcode::G_LOAD;854 auto Load = MIRBuilder.buildLoadInstr(LoadOpc, IndexTy, Addr, *MMO);855 TargetReg = MIRBuilder.buildPtrAdd(PtrTy, PtrReg, Load).getReg(0);856 break;857 }858 case MachineJumpTableInfo::EK_Custom32: {859 auto Load = MIRBuilder.buildLoadInstr(TargetOpcode::G_SEXTLOAD, IndexTy,860 Addr, *MMO);861 TargetReg = MIRBuilder.buildIntToPtr(PtrTy, Load).getReg(0);862 break;863 }864 case MachineJumpTableInfo::EK_BlockAddress:865 TargetReg = MIRBuilder.buildLoad(PtrTy, Addr, *MMO).getReg(0);866 break;867 }868 869 MIRBuilder.buildBrIndirect(TargetReg);870 871 MI.eraseFromParent();872 return true;873}874 875bool RISCVLegalizerInfo::shouldBeInConstantPool(const APInt &APImm,876 bool ShouldOptForSize) const {877 assert(APImm.getBitWidth() == 32 || APImm.getBitWidth() == 64);878 int64_t Imm = APImm.getSExtValue();879 // All simm32 constants should be handled by isel.880 // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making881 // this check redundant, but small immediates are common so this check882 // should have better compile time.883 if (isInt<32>(Imm))884 return false;885 886 // We only need to cost the immediate, if constant pool lowering is enabled.887 if (!STI.useConstantPoolForLargeInts())888 return false;889 890 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, STI);891 if (Seq.size() <= STI.getMaxBuildIntsCost())892 return false;893 894 // Optimizations below are disabled for opt size. If we're optimizing for895 // size, use a constant pool.896 if (ShouldOptForSize)897 return true;898 //899 // Special case. See if we can build the constant as (ADD (SLLI X, C), X) do900 // that if it will avoid a constant pool.901 // It will require an extra temporary register though.902 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where903 // low and high 32 bits are the same and bit 31 and 63 are set.904 unsigned ShiftAmt, AddOpc;905 RISCVMatInt::InstSeq SeqLo =906 RISCVMatInt::generateTwoRegInstSeq(Imm, STI, ShiftAmt, AddOpc);907 return !(!SeqLo.empty() && (SeqLo.size() + 2) <= STI.getMaxBuildIntsCost());908}909 910bool RISCVLegalizerInfo::legalizeVScale(MachineInstr &MI,911 MachineIRBuilder &MIB) const {912 const LLT XLenTy(STI.getXLenVT());913 Register Dst = MI.getOperand(0).getReg();914 915 // We define our scalable vector types for lmul=1 to use a 64 bit known916 // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate917 // vscale as VLENB / 8.918 static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");919 if (STI.getRealMinVLen() < RISCV::RVVBitsPerBlock)920 // Support for VLEN==32 is incomplete.921 return false;922 923 // We assume VLENB is a multiple of 8. We manually choose the best shift924 // here because SimplifyDemandedBits isn't always able to simplify it.925 uint64_t Val = MI.getOperand(1).getCImm()->getZExtValue();926 if (isPowerOf2_64(Val)) {927 uint64_t Log2 = Log2_64(Val);928 if (Log2 < 3) {929 auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});930 MIB.buildLShr(Dst, VLENB, MIB.buildConstant(XLenTy, 3 - Log2));931 } else if (Log2 > 3) {932 auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});933 MIB.buildShl(Dst, VLENB, MIB.buildConstant(XLenTy, Log2 - 3));934 } else {935 MIB.buildInstr(RISCV::G_READ_VLENB, {Dst}, {});936 }937 } else if ((Val % 8) == 0) {938 // If the multiplier is a multiple of 8, scale it down to avoid needing939 // to shift the VLENB value.940 auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});941 MIB.buildMul(Dst, VLENB, MIB.buildConstant(XLenTy, Val / 8));942 } else {943 auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});944 auto VScale = MIB.buildLShr(XLenTy, VLENB, MIB.buildConstant(XLenTy, 3));945 MIB.buildMul(Dst, VScale, MIB.buildConstant(XLenTy, Val));946 }947 MI.eraseFromParent();948 return true;949}950 951// Custom-lower extensions from mask vectors by using a vselect either with 1952// for zero/any-extension or -1 for sign-extension:953// (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)954// Note that any-extension is lowered identically to zero-extension.955bool RISCVLegalizerInfo::legalizeExt(MachineInstr &MI,956 MachineIRBuilder &MIB) const {957 958 unsigned Opc = MI.getOpcode();959 assert(Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_SEXT ||960 Opc == TargetOpcode::G_ANYEXT);961 962 MachineRegisterInfo &MRI = *MIB.getMRI();963 Register Dst = MI.getOperand(0).getReg();964 Register Src = MI.getOperand(1).getReg();965 966 LLT DstTy = MRI.getType(Dst);967 int64_t ExtTrueVal = Opc == TargetOpcode::G_SEXT ? -1 : 1;968 LLT DstEltTy = DstTy.getElementType();969 auto SplatZero = MIB.buildSplatVector(DstTy, MIB.buildConstant(DstEltTy, 0));970 auto SplatTrue =971 MIB.buildSplatVector(DstTy, MIB.buildConstant(DstEltTy, ExtTrueVal));972 MIB.buildSelect(Dst, Src, SplatTrue, SplatZero);973 974 MI.eraseFromParent();975 return true;976}977 978bool RISCVLegalizerInfo::legalizeLoadStore(MachineInstr &MI,979 LegalizerHelper &Helper,980 MachineIRBuilder &MIB) const {981 assert((isa<GLoad>(MI) || isa<GStore>(MI)) &&982 "Machine instructions must be Load/Store.");983 MachineRegisterInfo &MRI = *MIB.getMRI();984 MachineFunction *MF = MI.getMF();985 const DataLayout &DL = MIB.getDataLayout();986 LLVMContext &Ctx = MF->getFunction().getContext();987 988 Register DstReg = MI.getOperand(0).getReg();989 LLT DataTy = MRI.getType(DstReg);990 if (!DataTy.isVector())991 return false;992 993 if (!MI.hasOneMemOperand())994 return false;995 996 MachineMemOperand *MMO = *MI.memoperands_begin();997 998 const auto *TLI = STI.getTargetLowering();999 EVT VT = EVT::getEVT(getTypeForLLT(DataTy, Ctx));1000 1001 if (TLI->allowsMemoryAccessForAlignment(Ctx, DL, VT, *MMO))1002 return true;1003 1004 unsigned EltSizeBits = DataTy.getScalarSizeInBits();1005 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&1006 "Unexpected unaligned RVV load type");1007 1008 // Calculate the new vector type with i8 elements1009 unsigned NumElements =1010 DataTy.getElementCount().getKnownMinValue() * (EltSizeBits / 8);1011 LLT NewDataTy = LLT::scalable_vector(NumElements, 8);1012 1013 Helper.bitcast(MI, 0, NewDataTy);1014 1015 return true;1016}1017 1018/// Return the type of the mask type suitable for masking the provided1019/// vector type. This is simply an i1 element type vector of the same1020/// (possibly scalable) length.1021static LLT getMaskTypeFor(LLT VecTy) {1022 assert(VecTy.isVector());1023 ElementCount EC = VecTy.getElementCount();1024 return LLT::vector(EC, LLT::scalar(1));1025}1026 1027/// Creates an all ones mask suitable for masking a vector of type VecTy with1028/// vector length VL.1029static MachineInstrBuilder buildAllOnesMask(LLT VecTy, const SrcOp &VL,1030 MachineIRBuilder &MIB,1031 MachineRegisterInfo &MRI) {1032 LLT MaskTy = getMaskTypeFor(VecTy);1033 return MIB.buildInstr(RISCV::G_VMSET_VL, {MaskTy}, {VL});1034}1035 1036/// Gets the two common "VL" operands: an all-ones mask and the vector length.1037/// VecTy is a scalable vector type.1038static std::pair<MachineInstrBuilder, MachineInstrBuilder>1039buildDefaultVLOps(LLT VecTy, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) {1040 assert(VecTy.isScalableVector() && "Expecting scalable container type");1041 const RISCVSubtarget &STI = MIB.getMF().getSubtarget<RISCVSubtarget>();1042 LLT XLenTy(STI.getXLenVT());1043 auto VL = MIB.buildConstant(XLenTy, -1);1044 auto Mask = buildAllOnesMask(VecTy, VL, MIB, MRI);1045 return {Mask, VL};1046}1047 1048static MachineInstrBuilder1049buildSplatPartsS64WithVL(const DstOp &Dst, const SrcOp &Passthru, Register Lo,1050 Register Hi, const SrcOp &VL, MachineIRBuilder &MIB,1051 MachineRegisterInfo &MRI) {1052 // TODO: If the Hi bits of the splat are undefined, then it's fine to just1053 // splat Lo even if it might be sign extended. I don't think we have1054 // introduced a case where we're build a s64 where the upper bits are undef1055 // yet.1056 1057 // Fall back to a stack store and stride x0 vector load.1058 // TODO: need to lower G_SPLAT_VECTOR_SPLIT_I64. This is done in1059 // preprocessDAG in SDAG.1060 return MIB.buildInstr(RISCV::G_SPLAT_VECTOR_SPLIT_I64_VL, {Dst},1061 {Passthru, Lo, Hi, VL});1062}1063 1064static MachineInstrBuilder1065buildSplatSplitS64WithVL(const DstOp &Dst, const SrcOp &Passthru,1066 const SrcOp &Scalar, const SrcOp &VL,1067 MachineIRBuilder &MIB, MachineRegisterInfo &MRI) {1068 assert(Scalar.getLLTTy(MRI) == LLT::scalar(64) && "Unexpected VecTy!");1069 auto Unmerge = MIB.buildUnmerge(LLT::scalar(32), Scalar);1070 return buildSplatPartsS64WithVL(Dst, Passthru, Unmerge.getReg(0),1071 Unmerge.getReg(1), VL, MIB, MRI);1072}1073 1074// Lower splats of s1 types to G_ICMP. For each mask vector type, we have a1075// legal equivalently-sized i8 type, so we can use that as a go-between.1076// Splats of s1 types that have constant value can be legalized as VMSET_VL or1077// VMCLR_VL.1078bool RISCVLegalizerInfo::legalizeSplatVector(MachineInstr &MI,1079 MachineIRBuilder &MIB) const {1080 assert(MI.getOpcode() == TargetOpcode::G_SPLAT_VECTOR);1081 1082 MachineRegisterInfo &MRI = *MIB.getMRI();1083 1084 Register Dst = MI.getOperand(0).getReg();1085 Register SplatVal = MI.getOperand(1).getReg();1086 1087 LLT VecTy = MRI.getType(Dst);1088 LLT XLenTy(STI.getXLenVT());1089 1090 // Handle case of s64 element vectors on rv321091 if (XLenTy.getSizeInBits() == 32 &&1092 VecTy.getElementType().getSizeInBits() == 64) {1093 auto [_, VL] = buildDefaultVLOps(MRI.getType(Dst), MIB, MRI);1094 buildSplatSplitS64WithVL(Dst, MIB.buildUndef(VecTy), SplatVal, VL, MIB,1095 MRI);1096 MI.eraseFromParent();1097 return true;1098 }1099 1100 // All-zeros or all-ones splats are handled specially.1101 MachineInstr &SplatValMI = *MRI.getVRegDef(SplatVal);1102 if (isAllOnesOrAllOnesSplat(SplatValMI, MRI)) {1103 auto VL = buildDefaultVLOps(VecTy, MIB, MRI).second;1104 MIB.buildInstr(RISCV::G_VMSET_VL, {Dst}, {VL});1105 MI.eraseFromParent();1106 return true;1107 }1108 if (isNullOrNullSplat(SplatValMI, MRI)) {1109 auto VL = buildDefaultVLOps(VecTy, MIB, MRI).second;1110 MIB.buildInstr(RISCV::G_VMCLR_VL, {Dst}, {VL});1111 MI.eraseFromParent();1112 return true;1113 }1114 1115 // Handle non-constant mask splat (i.e. not sure if it's all zeros or all1116 // ones) by promoting it to an s8 splat.1117 LLT InterEltTy = LLT::scalar(8);1118 LLT InterTy = VecTy.changeElementType(InterEltTy);1119 auto ZExtSplatVal = MIB.buildZExt(InterEltTy, SplatVal);1120 auto And =1121 MIB.buildAnd(InterEltTy, ZExtSplatVal, MIB.buildConstant(InterEltTy, 1));1122 auto LHS = MIB.buildSplatVector(InterTy, And);1123 auto ZeroSplat =1124 MIB.buildSplatVector(InterTy, MIB.buildConstant(InterEltTy, 0));1125 MIB.buildICmp(CmpInst::Predicate::ICMP_NE, Dst, LHS, ZeroSplat);1126 MI.eraseFromParent();1127 return true;1128}1129 1130static LLT getLMUL1Ty(LLT VecTy) {1131 assert(VecTy.getElementType().getSizeInBits() <= 64 &&1132 "Unexpected vector LLT");1133 return LLT::scalable_vector(RISCV::RVVBitsPerBlock /1134 VecTy.getElementType().getSizeInBits(),1135 VecTy.getElementType());1136}1137 1138bool RISCVLegalizerInfo::legalizeExtractSubvector(MachineInstr &MI,1139 MachineIRBuilder &MIB) const {1140 GExtractSubvector &ES = cast<GExtractSubvector>(MI);1141 1142 MachineRegisterInfo &MRI = *MIB.getMRI();1143 1144 Register Dst = ES.getReg(0);1145 Register Src = ES.getSrcVec();1146 uint64_t Idx = ES.getIndexImm();1147 1148 // With an index of 0 this is a cast-like subvector, which can be performed1149 // with subregister operations.1150 if (Idx == 0)1151 return true;1152 1153 LLT LitTy = MRI.getType(Dst);1154 LLT BigTy = MRI.getType(Src);1155 1156 if (LitTy.getElementType() == LLT::scalar(1)) {1157 // We can't slide this mask vector up indexed by its i1 elements.1158 // This poses a problem when we wish to insert a scalable vector which1159 // can't be re-expressed as a larger type. Just choose the slow path and1160 // extend to a larger type, then truncate back down.1161 LLT ExtBigTy = BigTy.changeElementType(LLT::scalar(8));1162 LLT ExtLitTy = LitTy.changeElementType(LLT::scalar(8));1163 auto BigZExt = MIB.buildZExt(ExtBigTy, Src);1164 auto ExtractZExt = MIB.buildExtractSubvector(ExtLitTy, BigZExt, Idx);1165 auto SplatZero = MIB.buildSplatVector(1166 ExtLitTy, MIB.buildConstant(ExtLitTy.getElementType(), 0));1167 MIB.buildICmp(CmpInst::Predicate::ICMP_NE, Dst, ExtractZExt, SplatZero);1168 MI.eraseFromParent();1169 return true;1170 }1171 1172 // extract_subvector scales the index by vscale if the subvector is scalable,1173 // and decomposeSubvectorInsertExtractToSubRegs takes this into account.1174 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();1175 MVT LitTyMVT = getMVTForLLT(LitTy);1176 auto Decompose =1177 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(1178 getMVTForLLT(BigTy), LitTyMVT, Idx, TRI);1179 unsigned RemIdx = Decompose.second;1180 1181 // If the Idx has been completely eliminated then this is a subvector extract1182 // which naturally aligns to a vector register. These can easily be handled1183 // using subregister manipulation.1184 if (RemIdx == 0)1185 return true;1186 1187 // Else LitTy is M1 or smaller and may need to be slid down: if LitTy1188 // was > M1 then the index would need to be a multiple of VLMAX, and so would1189 // divide exactly.1190 assert(1191 RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(LitTyMVT)).second ||1192 RISCVTargetLowering::getLMUL(LitTyMVT) == RISCVVType::LMUL_1);1193 1194 // If the vector type is an LMUL-group type, extract a subvector equal to the1195 // nearest full vector register type.1196 LLT InterLitTy = BigTy;1197 Register Vec = Src;1198 if (TypeSize::isKnownGT(BigTy.getSizeInBits(),1199 getLMUL1Ty(BigTy).getSizeInBits())) {1200 // If BigTy has an LMUL > 1, then LitTy should have a smaller LMUL, and1201 // we should have successfully decomposed the extract into a subregister.1202 assert(Decompose.first != RISCV::NoSubRegister);1203 InterLitTy = getLMUL1Ty(BigTy);1204 // SDAG builds a TargetExtractSubreg. We cannot create a a Copy with SubReg1205 // specified on the source Register (the equivalent) since generic virtual1206 // register does not allow subregister index.1207 Vec = MIB.buildExtractSubvector(InterLitTy, Src, Idx - RemIdx).getReg(0);1208 }1209 1210 // Slide this vector register down by the desired number of elements in order1211 // to place the desired subvector starting at element 0.1212 const LLT XLenTy(STI.getXLenVT());1213 auto SlidedownAmt = MIB.buildVScale(XLenTy, RemIdx);1214 auto [Mask, VL] = buildDefaultVLOps(LitTy, MIB, MRI);1215 uint64_t Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;1216 auto Slidedown = MIB.buildInstr(1217 RISCV::G_VSLIDEDOWN_VL, {InterLitTy},1218 {MIB.buildUndef(InterLitTy), Vec, SlidedownAmt, Mask, VL, Policy});1219 1220 // Now the vector is in the right position, extract our final subvector. This1221 // should resolve to a COPY.1222 MIB.buildExtractSubvector(Dst, Slidedown, 0);1223 1224 MI.eraseFromParent();1225 return true;1226}1227 1228bool RISCVLegalizerInfo::legalizeInsertSubvector(MachineInstr &MI,1229 LegalizerHelper &Helper,1230 MachineIRBuilder &MIB) const {1231 GInsertSubvector &IS = cast<GInsertSubvector>(MI);1232 1233 MachineRegisterInfo &MRI = *MIB.getMRI();1234 1235 Register Dst = IS.getReg(0);1236 Register BigVec = IS.getBigVec();1237 Register LitVec = IS.getSubVec();1238 uint64_t Idx = IS.getIndexImm();1239 1240 LLT BigTy = MRI.getType(BigVec);1241 LLT LitTy = MRI.getType(LitVec);1242 1243 if (Idx == 0 ||1244 MRI.getVRegDef(BigVec)->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)1245 return true;1246 1247 // We don't have the ability to slide mask vectors up indexed by their i11248 // elements; the smallest we can do is i8. Often we are able to bitcast to1249 // equivalent i8 vectors. Otherwise, we can must zeroextend to equivalent i81250 // vectors and truncate down after the insert.1251 if (LitTy.getElementType() == LLT::scalar(1)) {1252 auto BigTyMinElts = BigTy.getElementCount().getKnownMinValue();1253 auto LitTyMinElts = LitTy.getElementCount().getKnownMinValue();1254 if (BigTyMinElts >= 8 && LitTyMinElts >= 8)1255 return Helper.bitcast(1256 IS, 0,1257 LLT::vector(BigTy.getElementCount().divideCoefficientBy(8), 8));1258 1259 // We can't slide this mask vector up indexed by its i1 elements.1260 // This poses a problem when we wish to insert a scalable vector which1261 // can't be re-expressed as a larger type. Just choose the slow path and1262 // extend to a larger type, then truncate back down.1263 LLT ExtBigTy = BigTy.changeElementType(LLT::scalar(8));1264 return Helper.widenScalar(IS, 0, ExtBigTy);1265 }1266 1267 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();1268 unsigned SubRegIdx, RemIdx;1269 std::tie(SubRegIdx, RemIdx) =1270 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(1271 getMVTForLLT(BigTy), getMVTForLLT(LitTy), Idx, TRI);1272 1273 TypeSize VecRegSize = TypeSize::getScalable(RISCV::RVVBitsPerBlock);1274 assert(isPowerOf2_64(1275 STI.expandVScale(LitTy.getSizeInBits()).getKnownMinValue()));1276 bool ExactlyVecRegSized =1277 STI.expandVScale(LitTy.getSizeInBits())1278 .isKnownMultipleOf(STI.expandVScale(VecRegSize));1279 1280 // If the Idx has been completely eliminated and this subvector's size is a1281 // vector register or a multiple thereof, or the surrounding elements are1282 // undef, then this is a subvector insert which naturally aligns to a vector1283 // register. These can easily be handled using subregister manipulation.1284 if (RemIdx == 0 && ExactlyVecRegSized)1285 return true;1286 1287 // If the subvector is smaller than a vector register, then the insertion1288 // must preserve the undisturbed elements of the register. We do this by1289 // lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1 vector type1290 // (which resolves to a subregister copy), performing a VSLIDEUP to place the1291 // subvector within the vector register, and an INSERT_SUBVECTOR of that1292 // LMUL=1 type back into the larger vector (resolving to another subregister1293 // operation). See below for how our VSLIDEUP works. We go via a LMUL=1 type1294 // to avoid allocating a large register group to hold our subvector.1295 1296 // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements1297 // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy1298 // (in our case undisturbed). This means we can set up a subvector insertion1299 // where OFFSET is the insertion offset, and the VL is the OFFSET plus the1300 // size of the subvector.1301 const LLT XLenTy(STI.getXLenVT());1302 LLT InterLitTy = BigTy;1303 Register AlignedExtract = BigVec;1304 unsigned AlignedIdx = Idx - RemIdx;1305 if (TypeSize::isKnownGT(BigTy.getSizeInBits(),1306 getLMUL1Ty(BigTy).getSizeInBits())) {1307 InterLitTy = getLMUL1Ty(BigTy);1308 // Extract a subvector equal to the nearest full vector register type. This1309 // should resolve to a G_EXTRACT on a subreg.1310 AlignedExtract =1311 MIB.buildExtractSubvector(InterLitTy, BigVec, AlignedIdx).getReg(0);1312 }1313 1314 auto Insert = MIB.buildInsertSubvector(InterLitTy, MIB.buildUndef(InterLitTy),1315 LitVec, 0);1316 1317 auto [Mask, _] = buildDefaultVLOps(BigTy, MIB, MRI);1318 auto VL = MIB.buildVScale(XLenTy, LitTy.getElementCount().getKnownMinValue());1319 1320 // If we're inserting into the lowest elements, use a tail undisturbed1321 // vmv.v.v.1322 MachineInstrBuilder Inserted;1323 bool NeedInsertSubvec =1324 TypeSize::isKnownGT(BigTy.getSizeInBits(), InterLitTy.getSizeInBits());1325 Register InsertedDst =1326 NeedInsertSubvec ? MRI.createGenericVirtualRegister(InterLitTy) : Dst;1327 if (RemIdx == 0) {1328 Inserted = MIB.buildInstr(RISCV::G_VMV_V_V_VL, {InsertedDst},1329 {AlignedExtract, Insert, VL});1330 } else {1331 auto SlideupAmt = MIB.buildVScale(XLenTy, RemIdx);1332 // Construct the vector length corresponding to RemIdx + length(LitTy).1333 VL = MIB.buildAdd(XLenTy, SlideupAmt, VL);1334 // Use tail agnostic policy if we're inserting over InterLitTy's tail.1335 ElementCount EndIndex =1336 ElementCount::getScalable(RemIdx) + LitTy.getElementCount();1337 uint64_t Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;1338 if (STI.expandVScale(EndIndex) ==1339 STI.expandVScale(InterLitTy.getElementCount()))1340 Policy = RISCVVType::TAIL_AGNOSTIC;1341 1342 Inserted =1343 MIB.buildInstr(RISCV::G_VSLIDEUP_VL, {InsertedDst},1344 {AlignedExtract, Insert, SlideupAmt, Mask, VL, Policy});1345 }1346 1347 // If required, insert this subvector back into the correct vector register.1348 // This should resolve to an INSERT_SUBREG instruction.1349 if (NeedInsertSubvec)1350 MIB.buildInsertSubvector(Dst, BigVec, Inserted, AlignedIdx);1351 1352 MI.eraseFromParent();1353 return true;1354}1355 1356static unsigned getRISCVWOpcode(unsigned Opcode) {1357 switch (Opcode) {1358 default:1359 llvm_unreachable("Unexpected opcode");1360 case TargetOpcode::G_ASHR:1361 return RISCV::G_SRAW;1362 case TargetOpcode::G_LSHR:1363 return RISCV::G_SRLW;1364 case TargetOpcode::G_SHL:1365 return RISCV::G_SLLW;1366 case TargetOpcode::G_SDIV:1367 return RISCV::G_DIVW;1368 case TargetOpcode::G_UDIV:1369 return RISCV::G_DIVUW;1370 case TargetOpcode::G_UREM:1371 return RISCV::G_REMUW;1372 case TargetOpcode::G_ROTL:1373 return RISCV::G_ROLW;1374 case TargetOpcode::G_ROTR:1375 return RISCV::G_RORW;1376 case TargetOpcode::G_CTLZ:1377 return RISCV::G_CLZW;1378 case TargetOpcode::G_CTTZ:1379 return RISCV::G_CTZW;1380 case TargetOpcode::G_FPTOSI:1381 return RISCV::G_FCVT_W_RV64;1382 case TargetOpcode::G_FPTOUI:1383 return RISCV::G_FCVT_WU_RV64;1384 }1385}1386 1387bool RISCVLegalizerInfo::legalizeCustom(1388 LegalizerHelper &Helper, MachineInstr &MI,1389 LostDebugLocObserver &LocObserver) const {1390 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;1391 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();1392 MachineFunction &MF = *MI.getParent()->getParent();1393 switch (MI.getOpcode()) {1394 default:1395 // No idea what to do.1396 return false;1397 case TargetOpcode::G_ABS:1398 return Helper.lowerAbsToMaxNeg(MI);1399 case TargetOpcode::G_FCONSTANT: {1400 const APFloat &FVal = MI.getOperand(1).getFPImm()->getValueAPF();1401 1402 // Convert G_FCONSTANT to G_CONSTANT.1403 Register DstReg = MI.getOperand(0).getReg();1404 MIRBuilder.buildConstant(DstReg, FVal.bitcastToAPInt());1405 1406 MI.eraseFromParent();1407 return true;1408 }1409 case TargetOpcode::G_CONSTANT: {1410 const Function &F = MF.getFunction();1411 // TODO: if PSI and BFI are present, add " ||1412 // llvm::shouldOptForSize(*CurMBB, PSI, BFI)".1413 bool ShouldOptForSize = F.hasOptSize();1414 const ConstantInt *ConstVal = MI.getOperand(1).getCImm();1415 if (!shouldBeInConstantPool(ConstVal->getValue(), ShouldOptForSize))1416 return true;1417 return Helper.lowerConstant(MI);1418 }1419 case TargetOpcode::G_SUB:1420 case TargetOpcode::G_ADD: {1421 Helper.Observer.changingInstr(MI);1422 Helper.widenScalarSrc(MI, sXLen, 1, TargetOpcode::G_ANYEXT);1423 Helper.widenScalarSrc(MI, sXLen, 2, TargetOpcode::G_ANYEXT);1424 1425 Register DstALU = MRI.createGenericVirtualRegister(sXLen);1426 1427 MachineOperand &MO = MI.getOperand(0);1428 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());1429 auto DstSext = MIRBuilder.buildSExtInReg(sXLen, DstALU, 32);1430 1431 MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, {MO}, {DstSext});1432 MO.setReg(DstALU);1433 1434 Helper.Observer.changedInstr(MI);1435 return true;1436 }1437 case TargetOpcode::G_SEXT_INREG: {1438 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());1439 int64_t SizeInBits = MI.getOperand(2).getImm();1440 // Source size of 32 is sext.w.1441 if (DstTy.getSizeInBits() == 64 && SizeInBits == 32)1442 return true;1443 1444 if (STI.hasStdExtZbb() && (SizeInBits == 8 || SizeInBits == 16))1445 return true;1446 1447 return Helper.lower(MI, 0, /* Unused hint type */ LLT()) ==1448 LegalizerHelper::Legalized;1449 }1450 case TargetOpcode::G_ASHR:1451 case TargetOpcode::G_LSHR:1452 case TargetOpcode::G_SHL: {1453 if (getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI)) {1454 // We don't need a custom node for shift by constant. Just widen the1455 // source and the shift amount.1456 unsigned ExtOpc = TargetOpcode::G_ANYEXT;1457 if (MI.getOpcode() == TargetOpcode::G_ASHR)1458 ExtOpc = TargetOpcode::G_SEXT;1459 else if (MI.getOpcode() == TargetOpcode::G_LSHR)1460 ExtOpc = TargetOpcode::G_ZEXT;1461 1462 Helper.Observer.changingInstr(MI);1463 Helper.widenScalarSrc(MI, sXLen, 1, ExtOpc);1464 Helper.widenScalarSrc(MI, sXLen, 2, TargetOpcode::G_ZEXT);1465 Helper.widenScalarDst(MI, sXLen);1466 Helper.Observer.changedInstr(MI);1467 return true;1468 }1469 1470 Helper.Observer.changingInstr(MI);1471 Helper.widenScalarSrc(MI, sXLen, 1, TargetOpcode::G_ANYEXT);1472 Helper.widenScalarSrc(MI, sXLen, 2, TargetOpcode::G_ANYEXT);1473 Helper.widenScalarDst(MI, sXLen);1474 MI.setDesc(MIRBuilder.getTII().get(getRISCVWOpcode(MI.getOpcode())));1475 Helper.Observer.changedInstr(MI);1476 return true;1477 }1478 case TargetOpcode::G_SDIV:1479 case TargetOpcode::G_UDIV:1480 case TargetOpcode::G_UREM:1481 case TargetOpcode::G_ROTL:1482 case TargetOpcode::G_ROTR: {1483 Helper.Observer.changingInstr(MI);1484 Helper.widenScalarSrc(MI, sXLen, 1, TargetOpcode::G_ANYEXT);1485 Helper.widenScalarSrc(MI, sXLen, 2, TargetOpcode::G_ANYEXT);1486 Helper.widenScalarDst(MI, sXLen);1487 MI.setDesc(MIRBuilder.getTII().get(getRISCVWOpcode(MI.getOpcode())));1488 Helper.Observer.changedInstr(MI);1489 return true;1490 }1491 case TargetOpcode::G_CTLZ:1492 case TargetOpcode::G_CTTZ: {1493 Helper.Observer.changingInstr(MI);1494 Helper.widenScalarSrc(MI, sXLen, 1, TargetOpcode::G_ANYEXT);1495 Helper.widenScalarDst(MI, sXLen);1496 MI.setDesc(MIRBuilder.getTII().get(getRISCVWOpcode(MI.getOpcode())));1497 Helper.Observer.changedInstr(MI);1498 return true;1499 }1500 case TargetOpcode::G_FPTOSI:1501 case TargetOpcode::G_FPTOUI: {1502 Helper.Observer.changingInstr(MI);1503 Helper.widenScalarDst(MI, sXLen);1504 MI.setDesc(MIRBuilder.getTII().get(getRISCVWOpcode(MI.getOpcode())));1505 MI.addOperand(MachineOperand::CreateImm(RISCVFPRndMode::RTZ));1506 Helper.Observer.changedInstr(MI);1507 return true;1508 }1509 case TargetOpcode::G_IS_FPCLASS: {1510 Register GISFPCLASS = MI.getOperand(0).getReg();1511 Register Src = MI.getOperand(1).getReg();1512 const MachineOperand &ImmOp = MI.getOperand(2);1513 MachineIRBuilder MIB(MI);1514 1515 // Turn LLVM IR's floating point classes to that in RISC-V,1516 // by simply rotating the 10-bit immediate right by two bits.1517 APInt GFpClassImm(10, static_cast<uint64_t>(ImmOp.getImm()));1518 auto FClassMask = MIB.buildConstant(sXLen, GFpClassImm.rotr(2).zext(XLen));1519 auto ConstZero = MIB.buildConstant(sXLen, 0);1520 1521 auto GFClass = MIB.buildInstr(RISCV::G_FCLASS, {sXLen}, {Src});1522 auto And = MIB.buildAnd(sXLen, GFClass, FClassMask);1523 MIB.buildICmp(CmpInst::ICMP_NE, GISFPCLASS, And, ConstZero);1524 1525 MI.eraseFromParent();1526 return true;1527 }1528 case TargetOpcode::G_BRJT:1529 return legalizeBRJT(MI, MIRBuilder);1530 case TargetOpcode::G_VASTART:1531 return legalizeVAStart(MI, MIRBuilder);1532 case TargetOpcode::G_VSCALE:1533 return legalizeVScale(MI, MIRBuilder);1534 case TargetOpcode::G_ZEXT:1535 case TargetOpcode::G_SEXT:1536 case TargetOpcode::G_ANYEXT:1537 return legalizeExt(MI, MIRBuilder);1538 case TargetOpcode::G_SPLAT_VECTOR:1539 return legalizeSplatVector(MI, MIRBuilder);1540 case TargetOpcode::G_EXTRACT_SUBVECTOR:1541 return legalizeExtractSubvector(MI, MIRBuilder);1542 case TargetOpcode::G_INSERT_SUBVECTOR:1543 return legalizeInsertSubvector(MI, Helper, MIRBuilder);1544 case TargetOpcode::G_LOAD:1545 case TargetOpcode::G_STORE:1546 return legalizeLoadStore(MI, Helper, MIRBuilder);1547 }1548 1549 llvm_unreachable("expected switch to return");1550}1551