3704 lines · cpp
1//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This file implements the WebAssemblyTargetLowering class.11///12//===----------------------------------------------------------------------===//13 14#include "WebAssemblyISelLowering.h"15#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"16#include "Utils/WebAssemblyTypeUtilities.h"17#include "WebAssemblyMachineFunctionInfo.h"18#include "WebAssemblySubtarget.h"19#include "WebAssemblyTargetMachine.h"20#include "WebAssemblyUtilities.h"21#include "llvm/CodeGen/CallingConvLower.h"22#include "llvm/CodeGen/MachineFrameInfo.h"23#include "llvm/CodeGen/MachineInstrBuilder.h"24#include "llvm/CodeGen/MachineJumpTableInfo.h"25#include "llvm/CodeGen/MachineModuleInfo.h"26#include "llvm/CodeGen/MachineRegisterInfo.h"27#include "llvm/CodeGen/SDPatternMatch.h"28#include "llvm/CodeGen/SelectionDAG.h"29#include "llvm/CodeGen/SelectionDAGNodes.h"30#include "llvm/IR/DiagnosticInfo.h"31#include "llvm/IR/DiagnosticPrinter.h"32#include "llvm/IR/Function.h"33#include "llvm/IR/IntrinsicInst.h"34#include "llvm/IR/Intrinsics.h"35#include "llvm/IR/IntrinsicsWebAssembly.h"36#include "llvm/Support/ErrorHandling.h"37#include "llvm/Support/KnownBits.h"38#include "llvm/Support/MathExtras.h"39#include "llvm/Target/TargetOptions.h"40using namespace llvm;41 42#define DEBUG_TYPE "wasm-lower"43 44WebAssemblyTargetLowering::WebAssemblyTargetLowering(45 const TargetMachine &TM, const WebAssemblySubtarget &STI)46 : TargetLowering(TM, STI), Subtarget(&STI) {47 auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32;48 49 // Set the load count for memcmp expand optimization50 MaxLoadsPerMemcmp = 8;51 MaxLoadsPerMemcmpOptSize = 4;52 53 // Booleans always contain 0 or 1.54 setBooleanContents(ZeroOrOneBooleanContent);55 // Except in SIMD vectors56 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);57 // We don't know the microarchitecture here, so just reduce register pressure.58 setSchedulingPreference(Sched::RegPressure);59 // Tell ISel that we have a stack pointer.60 setStackPointerRegisterToSaveRestore(61 Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);62 // Set up the register classes.63 addRegisterClass(MVT::i32, &WebAssembly::I32RegClass);64 addRegisterClass(MVT::i64, &WebAssembly::I64RegClass);65 addRegisterClass(MVT::f32, &WebAssembly::F32RegClass);66 addRegisterClass(MVT::f64, &WebAssembly::F64RegClass);67 if (Subtarget->hasSIMD128()) {68 addRegisterClass(MVT::v16i8, &WebAssembly::V128RegClass);69 addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass);70 addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass);71 addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass);72 addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass);73 addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass);74 }75 if (Subtarget->hasFP16()) {76 addRegisterClass(MVT::v8f16, &WebAssembly::V128RegClass);77 }78 if (Subtarget->hasReferenceTypes()) {79 addRegisterClass(MVT::externref, &WebAssembly::EXTERNREFRegClass);80 addRegisterClass(MVT::funcref, &WebAssembly::FUNCREFRegClass);81 if (Subtarget->hasExceptionHandling()) {82 addRegisterClass(MVT::exnref, &WebAssembly::EXNREFRegClass);83 }84 }85 // Compute derived properties from the register classes.86 computeRegisterProperties(Subtarget->getRegisterInfo());87 88 // Transform loads and stores to pointers in address space 1 to loads and89 // stores to WebAssembly global variables, outside linear memory.90 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) {91 setOperationAction(ISD::LOAD, T, Custom);92 setOperationAction(ISD::STORE, T, Custom);93 }94 if (Subtarget->hasSIMD128()) {95 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,96 MVT::v2f64}) {97 setOperationAction(ISD::LOAD, T, Custom);98 setOperationAction(ISD::STORE, T, Custom);99 }100 }101 if (Subtarget->hasFP16()) {102 setOperationAction(ISD::LOAD, MVT::v8f16, Custom);103 setOperationAction(ISD::STORE, MVT::v8f16, Custom);104 }105 if (Subtarget->hasReferenceTypes()) {106 // We need custom load and store lowering for both externref, funcref and107 // Other. The MVT::Other here represents tables of reference types.108 for (auto T : {MVT::externref, MVT::funcref, MVT::Other}) {109 setOperationAction(ISD::LOAD, T, Custom);110 setOperationAction(ISD::STORE, T, Custom);111 }112 }113 114 setOperationAction(ISD::GlobalAddress, MVTPtr, Custom);115 setOperationAction(ISD::GlobalTLSAddress, MVTPtr, Custom);116 setOperationAction(ISD::ExternalSymbol, MVTPtr, Custom);117 setOperationAction(ISD::JumpTable, MVTPtr, Custom);118 setOperationAction(ISD::BlockAddress, MVTPtr, Custom);119 setOperationAction(ISD::BRIND, MVT::Other, Custom);120 setOperationAction(ISD::CLEAR_CACHE, MVT::Other, Custom);121 122 // Take the default expansion for va_arg, va_copy, and va_end. There is no123 // default action for va_start, so we do that custom.124 setOperationAction(ISD::VASTART, MVT::Other, Custom);125 setOperationAction(ISD::VAARG, MVT::Other, Expand);126 setOperationAction(ISD::VACOPY, MVT::Other, Expand);127 setOperationAction(ISD::VAEND, MVT::Other, Expand);128 129 for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64, MVT::v8f16}) {130 if (!Subtarget->hasFP16() && T == MVT::v8f16) {131 continue;132 }133 // Don't expand the floating-point types to constant pools.134 setOperationAction(ISD::ConstantFP, T, Legal);135 // Expand floating-point comparisons.136 for (auto CC : {ISD::SETO, ISD::SETUO, ISD::SETUEQ, ISD::SETONE,137 ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE})138 setCondCodeAction(CC, T, Expand);139 // Expand floating-point library function operators.140 for (auto Op :141 {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FMA})142 setOperationAction(Op, T, Expand);143 // Note supported floating-point library function operators that otherwise144 // default to expand.145 for (auto Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT,146 ISD::FRINT, ISD::FROUNDEVEN})147 setOperationAction(Op, T, Legal);148 // Support minimum and maximum, which otherwise default to expand.149 setOperationAction(ISD::FMINIMUM, T, Legal);150 setOperationAction(ISD::FMAXIMUM, T, Legal);151 // When experimental v8f16 support is enabled these instructions don't need152 // to be expanded.153 if (T != MVT::v8f16) {154 setOperationAction(ISD::FP16_TO_FP, T, Expand);155 setOperationAction(ISD::FP_TO_FP16, T, Expand);156 }157 setLoadExtAction(ISD::EXTLOAD, T, MVT::f16, Expand);158 setTruncStoreAction(T, MVT::f16, Expand);159 }160 161 // Expand unavailable integer operations.162 for (auto Op :163 {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU,164 ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS,165 ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) {166 for (auto T : {MVT::i32, MVT::i64})167 setOperationAction(Op, T, Expand);168 if (Subtarget->hasSIMD128())169 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})170 setOperationAction(Op, T, Expand);171 }172 173 if (Subtarget->hasWideArithmetic()) {174 setOperationAction(ISD::ADD, MVT::i128, Custom);175 setOperationAction(ISD::SUB, MVT::i128, Custom);176 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Custom);177 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Custom);178 setOperationAction(ISD::UADDO, MVT::i64, Custom);179 }180 181 if (Subtarget->hasNontrappingFPToInt())182 for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT})183 for (auto T : {MVT::i32, MVT::i64})184 setOperationAction(Op, T, Custom);185 186 if (Subtarget->hasRelaxedSIMD()) {187 setOperationAction(188 {ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM},189 {MVT::v4f32, MVT::v2f64}, Legal);190 }191 // SIMD-specific configuration192 if (Subtarget->hasSIMD128()) {193 194 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);195 196 // Combine wide-vector muls, with extend inputs, to extmul_half.197 setTargetDAGCombine(ISD::MUL);198 199 // Combine vector mask reductions into alltrue/anytrue200 setTargetDAGCombine(ISD::SETCC);201 202 // Convert vector to integer bitcasts to bitmask203 setTargetDAGCombine(ISD::BITCAST);204 205 // Hoist bitcasts out of shuffles206 setTargetDAGCombine(ISD::VECTOR_SHUFFLE);207 208 // Combine extends of extract_subvectors into widening ops209 setTargetDAGCombine({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND});210 211 // Combine int_to_fp or fp_extend of extract_vectors and vice versa into212 // conversions ops213 setTargetDAGCombine({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_EXTEND,214 ISD::EXTRACT_SUBVECTOR});215 216 // Combine fp_to_{s,u}int_sat or fp_round of concat_vectors or vice versa217 // into conversion ops218 setTargetDAGCombine({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT,219 ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_ROUND,220 ISD::CONCAT_VECTORS});221 222 setTargetDAGCombine(ISD::TRUNCATE);223 224 // Support saturating add/sub for i8x16 and i16x8225 for (auto Op : {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT})226 for (auto T : {MVT::v16i8, MVT::v8i16})227 setOperationAction(Op, T, Legal);228 229 // Support integer abs230 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})231 setOperationAction(ISD::ABS, T, Legal);232 233 // Custom lower BUILD_VECTORs to minimize number of replace_lanes234 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,235 MVT::v2f64})236 setOperationAction(ISD::BUILD_VECTOR, T, Custom);237 238 if (Subtarget->hasFP16())239 setOperationAction(ISD::BUILD_VECTOR, MVT::f16, Custom);240 241 // We have custom shuffle lowering to expose the shuffle mask242 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,243 MVT::v2f64})244 setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom);245 246 if (Subtarget->hasFP16())247 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8f16, Custom);248 249 // Support splatting250 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,251 MVT::v2f64})252 setOperationAction(ISD::SPLAT_VECTOR, T, Legal);253 254 setOperationAction(ISD::AVGCEILU, {MVT::v8i16, MVT::v16i8}, Legal);255 256 // Custom lowering since wasm shifts must have a scalar shift amount257 for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})258 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})259 setOperationAction(Op, T, Custom);260 261 // Custom lower lane accesses to expand out variable indices262 for (auto Op : {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT})263 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,264 MVT::v2f64})265 setOperationAction(Op, T, Custom);266 267 // There is no i8x16.mul instruction268 setOperationAction(ISD::MUL, MVT::v16i8, Expand);269 270 // There is no vector conditional select instruction271 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,272 MVT::v2f64})273 setOperationAction(ISD::SELECT_CC, T, Expand);274 275 // Expand integer operations supported for scalars but not SIMD276 for (auto Op :277 {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM, ISD::ROTL, ISD::ROTR})278 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})279 setOperationAction(Op, T, Expand);280 281 // But we do have integer min and max operations282 for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})283 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})284 setOperationAction(Op, T, Legal);285 286 // And we have popcnt for i8x16. It can be used to expand ctlz/cttz.287 setOperationAction(ISD::CTPOP, MVT::v16i8, Legal);288 setOperationAction(ISD::CTLZ, MVT::v16i8, Expand);289 setOperationAction(ISD::CTTZ, MVT::v16i8, Expand);290 291 // Custom lower bit counting operations for other types to scalarize them.292 for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP})293 for (auto T : {MVT::v8i16, MVT::v4i32, MVT::v2i64})294 setOperationAction(Op, T, Custom);295 296 // Expand float operations supported for scalars but not SIMD297 for (auto Op : {ISD::FCOPYSIGN, ISD::FLOG, ISD::FLOG2, ISD::FLOG10,298 ISD::FEXP, ISD::FEXP2, ISD::FEXP10})299 for (auto T : {MVT::v4f32, MVT::v2f64})300 setOperationAction(Op, T, Expand);301 302 // Unsigned comparison operations are unavailable for i64x2 vectors.303 for (auto CC : {ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE})304 setCondCodeAction(CC, MVT::v2i64, Custom);305 306 // 64x2 conversions are not in the spec307 for (auto Op :308 {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT})309 for (auto T : {MVT::v2i64, MVT::v2f64})310 setOperationAction(Op, T, Expand);311 312 // But saturating fp_to_int converstions are313 for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}) {314 setOperationAction(Op, MVT::v4i32, Custom);315 if (Subtarget->hasFP16()) {316 setOperationAction(Op, MVT::v8i16, Custom);317 }318 }319 320 // Support vector extending321 for (auto T : MVT::integer_fixedlen_vector_valuetypes()) {322 setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom);323 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Custom);324 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Custom);325 }326 327 if (Subtarget->hasFP16()) {328 setOperationAction(ISD::FMA, MVT::v8f16, Legal);329 }330 331 if (Subtarget->hasRelaxedSIMD()) {332 setOperationAction(ISD::FMULADD, MVT::v4f32, Legal);333 setOperationAction(ISD::FMULADD, MVT::v2f64, Legal);334 }335 336 // Partial MLA reductions.337 for (auto Op : {ISD::PARTIAL_REDUCE_SMLA, ISD::PARTIAL_REDUCE_UMLA}) {338 setPartialReduceMLAAction(Op, MVT::v4i32, MVT::v16i8, Legal);339 setPartialReduceMLAAction(Op, MVT::v4i32, MVT::v8i16, Legal);340 }341 }342 343 // As a special case, these operators use the type to mean the type to344 // sign-extend from.345 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);346 if (!Subtarget->hasSignExt()) {347 // Sign extends are legal only when extending a vector extract348 auto Action = Subtarget->hasSIMD128() ? Custom : Expand;349 for (auto T : {MVT::i8, MVT::i16, MVT::i32})350 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Action);351 }352 for (auto T : MVT::integer_fixedlen_vector_valuetypes())353 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand);354 355 // Dynamic stack allocation: use the default expansion.356 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);357 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);358 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVTPtr, Expand);359 360 setOperationAction(ISD::FrameIndex, MVT::i32, Custom);361 setOperationAction(ISD::FrameIndex, MVT::i64, Custom);362 setOperationAction(ISD::CopyToReg, MVT::Other, Custom);363 364 // Expand these forms; we pattern-match the forms that we can handle in isel.365 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64})366 for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})367 setOperationAction(Op, T, Expand);368 369 // We have custom switch handling.370 setOperationAction(ISD::BR_JT, MVT::Other, Custom);371 372 // WebAssembly doesn't have:373 // - Floating-point extending loads.374 // - Floating-point truncating stores.375 // - i1 extending loads.376 // - truncating SIMD stores and most extending loads377 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);378 setTruncStoreAction(MVT::f64, MVT::f32, Expand);379 for (auto T : MVT::integer_valuetypes())380 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})381 setLoadExtAction(Ext, T, MVT::i1, Promote);382 if (Subtarget->hasSIMD128()) {383 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,384 MVT::v2f64}) {385 for (auto MemT : MVT::fixedlen_vector_valuetypes()) {386 if (MVT(T) != MemT) {387 setTruncStoreAction(T, MemT, Expand);388 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})389 setLoadExtAction(Ext, T, MemT, Expand);390 }391 }392 }393 // But some vector extending loads are legal394 for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) {395 setLoadExtAction(Ext, MVT::v8i16, MVT::v8i8, Legal);396 setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal);397 setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal);398 }399 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Legal);400 }401 402 // Don't do anything clever with build_pairs403 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);404 405 // Trap lowers to wasm unreachable406 setOperationAction(ISD::TRAP, MVT::Other, Legal);407 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);408 409 // Exception handling intrinsics410 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);411 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);412 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);413 414 setMaxAtomicSizeInBitsSupported(64);415 416 // Always convert switches to br_tables unless there is only one case, which417 // is equivalent to a simple branch. This reduces code size for wasm, and we418 // defer possible jump table optimizations to the VM.419 setMinimumJumpTableEntries(2);420}421 422MVT WebAssemblyTargetLowering::getPointerTy(const DataLayout &DL,423 uint32_t AS) const {424 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_EXTERNREF)425 return MVT::externref;426 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF)427 return MVT::funcref;428 return TargetLowering::getPointerTy(DL, AS);429}430 431MVT WebAssemblyTargetLowering::getPointerMemTy(const DataLayout &DL,432 uint32_t AS) const {433 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_EXTERNREF)434 return MVT::externref;435 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF)436 return MVT::funcref;437 return TargetLowering::getPointerMemTy(DL, AS);438}439 440TargetLowering::AtomicExpansionKind441WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {442 // We have wasm instructions for these443 switch (AI->getOperation()) {444 case AtomicRMWInst::Add:445 case AtomicRMWInst::Sub:446 case AtomicRMWInst::And:447 case AtomicRMWInst::Or:448 case AtomicRMWInst::Xor:449 case AtomicRMWInst::Xchg:450 return AtomicExpansionKind::None;451 default:452 break;453 }454 return AtomicExpansionKind::CmpXChg;455}456 457bool WebAssemblyTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {458 // Implementation copied from X86TargetLowering.459 unsigned Opc = VecOp.getOpcode();460 461 // Assume target opcodes can't be scalarized.462 // TODO - do we have any exceptions?463 if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc))464 return false;465 466 // If the vector op is not supported, try to convert to scalar.467 EVT VecVT = VecOp.getValueType();468 if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))469 return true;470 471 // If the vector op is supported, but the scalar op is not, the transform may472 // not be worthwhile.473 EVT ScalarVT = VecVT.getScalarType();474 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);475}476 477FastISel *WebAssemblyTargetLowering::createFastISel(478 FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const {479 return WebAssembly::createFastISel(FuncInfo, LibInfo);480}481 482MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,483 EVT VT) const {484 unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1);485 if (BitWidth > 1 && BitWidth < 8)486 BitWidth = 8;487 488 if (BitWidth > 64) {489 // The shift will be lowered to a libcall, and compiler-rt libcalls expect490 // the count to be an i32.491 BitWidth = 32;492 assert(BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) &&493 "32-bit shift counts ought to be enough for anyone");494 }495 496 MVT Result = MVT::getIntegerVT(BitWidth);497 assert(Result != MVT::INVALID_SIMPLE_VALUE_TYPE &&498 "Unable to represent scalar shift amount type");499 return Result;500}501 502// Lower an fp-to-int conversion operator from the LLVM opcode, which has an503// undefined result on invalid/overflow, to the WebAssembly opcode, which504// traps on invalid/overflow.505static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL,506 MachineBasicBlock *BB,507 const TargetInstrInfo &TII,508 bool IsUnsigned, bool Int64,509 bool Float64, unsigned LoweredOpcode) {510 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();511 512 Register OutReg = MI.getOperand(0).getReg();513 Register InReg = MI.getOperand(1).getReg();514 515 unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32;516 unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32;517 unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32;518 unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32;519 unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;520 unsigned Eqz = WebAssembly::EQZ_I32;521 unsigned And = WebAssembly::AND_I32;522 int64_t Limit = Int64 ? INT64_MIN : INT32_MIN;523 int64_t Substitute = IsUnsigned ? 0 : Limit;524 double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit;525 auto &Context = BB->getParent()->getFunction().getContext();526 Type *Ty = Float64 ? Type::getDoubleTy(Context) : Type::getFloatTy(Context);527 528 const BasicBlock *LLVMBB = BB->getBasicBlock();529 MachineFunction *F = BB->getParent();530 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);531 MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB);532 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);533 534 MachineFunction::iterator It = ++BB->getIterator();535 F->insert(It, FalseMBB);536 F->insert(It, TrueMBB);537 F->insert(It, DoneMBB);538 539 // Transfer the remainder of BB and its successor edges to DoneMBB.540 DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());541 DoneMBB->transferSuccessorsAndUpdatePHIs(BB);542 543 BB->addSuccessor(TrueMBB);544 BB->addSuccessor(FalseMBB);545 TrueMBB->addSuccessor(DoneMBB);546 FalseMBB->addSuccessor(DoneMBB);547 548 unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg;549 Tmp0 = MRI.createVirtualRegister(MRI.getRegClass(InReg));550 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));551 CmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);552 EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);553 FalseReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));554 TrueReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));555 556 MI.eraseFromParent();557 // For signed numbers, we can do a single comparison to determine whether558 // fabs(x) is within range.559 if (IsUnsigned) {560 Tmp0 = InReg;561 } else {562 BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg);563 }564 BuildMI(BB, DL, TII.get(FConst), Tmp1)565 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal)));566 BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1);567 568 // For unsigned numbers, we have to do a separate comparison with zero.569 if (IsUnsigned) {570 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));571 Register SecondCmpReg =572 MRI.createVirtualRegister(&WebAssembly::I32RegClass);573 Register AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);574 BuildMI(BB, DL, TII.get(FConst), Tmp1)575 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0)));576 BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1);577 BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg);578 CmpReg = AndReg;579 }580 581 BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg);582 583 // Create the CFG diamond to select between doing the conversion or using584 // the substitute value.585 BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg);586 BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg);587 BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);588 BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute);589 BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg)590 .addReg(FalseReg)591 .addMBB(FalseMBB)592 .addReg(TrueReg)593 .addMBB(TrueMBB);594 595 return DoneMBB;596}597 598// Lower a `MEMCPY` instruction into a CFG triangle around a `MEMORY_COPY`599// instuction to handle the zero-length case.600static MachineBasicBlock *LowerMemcpy(MachineInstr &MI, DebugLoc DL,601 MachineBasicBlock *BB,602 const TargetInstrInfo &TII, bool Int64) {603 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();604 605 MachineOperand DstMem = MI.getOperand(0);606 MachineOperand SrcMem = MI.getOperand(1);607 MachineOperand Dst = MI.getOperand(2);608 MachineOperand Src = MI.getOperand(3);609 MachineOperand Len = MI.getOperand(4);610 611 // If the length is a constant, we don't actually need the check.612 if (MachineInstr *Def = MRI.getVRegDef(Len.getReg())) {613 if (Def->getOpcode() == WebAssembly::CONST_I32 ||614 Def->getOpcode() == WebAssembly::CONST_I64) {615 if (Def->getOperand(1).getImm() == 0) {616 // A zero-length memcpy is a no-op.617 MI.eraseFromParent();618 return BB;619 }620 // A non-zero-length memcpy doesn't need a zero check.621 unsigned MemoryCopy =622 Int64 ? WebAssembly::MEMORY_COPY_A64 : WebAssembly::MEMORY_COPY_A32;623 BuildMI(*BB, MI, DL, TII.get(MemoryCopy))624 .add(DstMem)625 .add(SrcMem)626 .add(Dst)627 .add(Src)628 .add(Len);629 MI.eraseFromParent();630 return BB;631 }632 }633 634 // We're going to add an extra use to `Len` to test if it's zero; that635 // use shouldn't be a kill, even if the original use is.636 MachineOperand NoKillLen = Len;637 NoKillLen.setIsKill(false);638 639 // Decide on which `MachineInstr` opcode we're going to use.640 unsigned Eqz = Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32;641 unsigned MemoryCopy =642 Int64 ? WebAssembly::MEMORY_COPY_A64 : WebAssembly::MEMORY_COPY_A32;643 644 // Create two new basic blocks; one for the new `memory.fill` that we can645 // branch over, and one for the rest of the instructions after the original646 // `memory.fill`.647 const BasicBlock *LLVMBB = BB->getBasicBlock();648 MachineFunction *F = BB->getParent();649 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);650 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);651 652 MachineFunction::iterator It = ++BB->getIterator();653 F->insert(It, TrueMBB);654 F->insert(It, DoneMBB);655 656 // Transfer the remainder of BB and its successor edges to DoneMBB.657 DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());658 DoneMBB->transferSuccessorsAndUpdatePHIs(BB);659 660 // Connect the CFG edges.661 BB->addSuccessor(TrueMBB);662 BB->addSuccessor(DoneMBB);663 TrueMBB->addSuccessor(DoneMBB);664 665 // Create a virtual register for the `Eqz` result.666 unsigned EqzReg;667 EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);668 669 // Erase the original `memory.copy`.670 MI.eraseFromParent();671 672 // Test if `Len` is zero.673 BuildMI(BB, DL, TII.get(Eqz), EqzReg).add(NoKillLen);674 675 // Insert a new `memory.copy`.676 BuildMI(TrueMBB, DL, TII.get(MemoryCopy))677 .add(DstMem)678 .add(SrcMem)679 .add(Dst)680 .add(Src)681 .add(Len);682 683 // Create the CFG triangle.684 BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(DoneMBB).addReg(EqzReg);685 BuildMI(TrueMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);686 687 return DoneMBB;688}689 690// Lower a `MEMSET` instruction into a CFG triangle around a `MEMORY_FILL`691// instuction to handle the zero-length case.692static MachineBasicBlock *LowerMemset(MachineInstr &MI, DebugLoc DL,693 MachineBasicBlock *BB,694 const TargetInstrInfo &TII, bool Int64) {695 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();696 697 MachineOperand Mem = MI.getOperand(0);698 MachineOperand Dst = MI.getOperand(1);699 MachineOperand Val = MI.getOperand(2);700 MachineOperand Len = MI.getOperand(3);701 702 // If the length is a constant, we don't actually need the check.703 if (MachineInstr *Def = MRI.getVRegDef(Len.getReg())) {704 if (Def->getOpcode() == WebAssembly::CONST_I32 ||705 Def->getOpcode() == WebAssembly::CONST_I64) {706 if (Def->getOperand(1).getImm() == 0) {707 // A zero-length memset is a no-op.708 MI.eraseFromParent();709 return BB;710 }711 // A non-zero-length memset doesn't need a zero check.712 unsigned MemoryFill =713 Int64 ? WebAssembly::MEMORY_FILL_A64 : WebAssembly::MEMORY_FILL_A32;714 BuildMI(*BB, MI, DL, TII.get(MemoryFill))715 .add(Mem)716 .add(Dst)717 .add(Val)718 .add(Len);719 MI.eraseFromParent();720 return BB;721 }722 }723 724 // We're going to add an extra use to `Len` to test if it's zero; that725 // use shouldn't be a kill, even if the original use is.726 MachineOperand NoKillLen = Len;727 NoKillLen.setIsKill(false);728 729 // Decide on which `MachineInstr` opcode we're going to use.730 unsigned Eqz = Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32;731 unsigned MemoryFill =732 Int64 ? WebAssembly::MEMORY_FILL_A64 : WebAssembly::MEMORY_FILL_A32;733 734 // Create two new basic blocks; one for the new `memory.fill` that we can735 // branch over, and one for the rest of the instructions after the original736 // `memory.fill`.737 const BasicBlock *LLVMBB = BB->getBasicBlock();738 MachineFunction *F = BB->getParent();739 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);740 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);741 742 MachineFunction::iterator It = ++BB->getIterator();743 F->insert(It, TrueMBB);744 F->insert(It, DoneMBB);745 746 // Transfer the remainder of BB and its successor edges to DoneMBB.747 DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());748 DoneMBB->transferSuccessorsAndUpdatePHIs(BB);749 750 // Connect the CFG edges.751 BB->addSuccessor(TrueMBB);752 BB->addSuccessor(DoneMBB);753 TrueMBB->addSuccessor(DoneMBB);754 755 // Create a virtual register for the `Eqz` result.756 unsigned EqzReg;757 EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);758 759 // Erase the original `memory.fill`.760 MI.eraseFromParent();761 762 // Test if `Len` is zero.763 BuildMI(BB, DL, TII.get(Eqz), EqzReg).add(NoKillLen);764 765 // Insert a new `memory.copy`.766 BuildMI(TrueMBB, DL, TII.get(MemoryFill)).add(Mem).add(Dst).add(Val).add(Len);767 768 // Create the CFG triangle.769 BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(DoneMBB).addReg(EqzReg);770 BuildMI(TrueMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);771 772 return DoneMBB;773}774 775static MachineBasicBlock *776LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB,777 const WebAssemblySubtarget *Subtarget,778 const TargetInstrInfo &TII) {779 MachineInstr &CallParams = *CallResults.getPrevNode();780 assert(CallParams.getOpcode() == WebAssembly::CALL_PARAMS);781 assert(CallResults.getOpcode() == WebAssembly::CALL_RESULTS ||782 CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS);783 784 bool IsIndirect =785 CallParams.getOperand(0).isReg() || CallParams.getOperand(0).isFI();786 bool IsRetCall = CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS;787 788 bool IsFuncrefCall = false;789 if (IsIndirect && CallParams.getOperand(0).isReg()) {790 Register Reg = CallParams.getOperand(0).getReg();791 const MachineFunction *MF = BB->getParent();792 const MachineRegisterInfo &MRI = MF->getRegInfo();793 const TargetRegisterClass *TRC = MRI.getRegClass(Reg);794 IsFuncrefCall = (TRC == &WebAssembly::FUNCREFRegClass);795 assert(!IsFuncrefCall || Subtarget->hasReferenceTypes());796 }797 798 unsigned CallOp;799 if (IsIndirect && IsRetCall) {800 CallOp = WebAssembly::RET_CALL_INDIRECT;801 } else if (IsIndirect) {802 CallOp = WebAssembly::CALL_INDIRECT;803 } else if (IsRetCall) {804 CallOp = WebAssembly::RET_CALL;805 } else {806 CallOp = WebAssembly::CALL;807 }808 809 MachineFunction &MF = *BB->getParent();810 const MCInstrDesc &MCID = TII.get(CallOp);811 MachineInstrBuilder MIB(MF, MF.CreateMachineInstr(MCID, DL));812 813 // Move the function pointer to the end of the arguments for indirect calls814 if (IsIndirect) {815 auto FnPtr = CallParams.getOperand(0);816 CallParams.removeOperand(0);817 818 // For funcrefs, call_indirect is done through __funcref_call_table and the819 // funcref is always installed in slot 0 of the table, therefore instead of820 // having the function pointer added at the end of the params list, a zero821 // (the index in822 // __funcref_call_table is added).823 if (IsFuncrefCall) {824 Register RegZero =825 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);826 MachineInstrBuilder MIBC0 =827 BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);828 829 BB->insert(CallResults.getIterator(), MIBC0);830 MachineInstrBuilder(MF, CallParams).addReg(RegZero);831 } else832 CallParams.addOperand(FnPtr);833 }834 835 for (auto Def : CallResults.defs())836 MIB.add(Def);837 838 if (IsIndirect) {839 // Placeholder for the type index.840 // This gets replaced with the correct value in WebAssemblyMCInstLower.cpp841 MIB.addImm(0);842 // The table into which this call_indirect indexes.843 MCSymbolWasm *Table = IsFuncrefCall844 ? WebAssembly::getOrCreateFuncrefCallTableSymbol(845 MF.getContext(), Subtarget)846 : WebAssembly::getOrCreateFunctionTableSymbol(847 MF.getContext(), Subtarget);848 if (Subtarget->hasCallIndirectOverlong()) {849 MIB.addSym(Table);850 } else {851 // For the MVP there is at most one table whose number is 0, but we can't852 // write a table symbol or issue relocations. Instead we just ensure the853 // table is live and write a zero.854 Table->setNoStrip();855 MIB.addImm(0);856 }857 }858 859 for (auto Use : CallParams.uses())860 MIB.add(Use);861 862 BB->insert(CallResults.getIterator(), MIB);863 CallParams.eraseFromParent();864 CallResults.eraseFromParent();865 866 // If this is a funcref call, to avoid hidden GC roots, we need to clear the867 // table slot with ref.null upon call_indirect return.868 //869 // This generates the following code, which comes right after a call_indirect870 // of a funcref:871 //872 // i32.const 0873 // ref.null func874 // table.set __funcref_call_table875 if (IsIndirect && IsFuncrefCall) {876 MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(877 MF.getContext(), Subtarget);878 Register RegZero =879 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);880 MachineInstr *Const0 =881 BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);882 BB->insertAfter(MIB.getInstr()->getIterator(), Const0);883 884 Register RegFuncref =885 MF.getRegInfo().createVirtualRegister(&WebAssembly::FUNCREFRegClass);886 MachineInstr *RefNull =887 BuildMI(MF, DL, TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref);888 BB->insertAfter(Const0->getIterator(), RefNull);889 890 MachineInstr *TableSet =891 BuildMI(MF, DL, TII.get(WebAssembly::TABLE_SET_FUNCREF))892 .addSym(Table)893 .addReg(RegZero)894 .addReg(RegFuncref);895 BB->insertAfter(RefNull->getIterator(), TableSet);896 }897 898 return BB;899}900 901MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(902 MachineInstr &MI, MachineBasicBlock *BB) const {903 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();904 DebugLoc DL = MI.getDebugLoc();905 906 switch (MI.getOpcode()) {907 default:908 llvm_unreachable("Unexpected instr type to insert");909 case WebAssembly::FP_TO_SINT_I32_F32:910 return LowerFPToInt(MI, DL, BB, TII, false, false, false,911 WebAssembly::I32_TRUNC_S_F32);912 case WebAssembly::FP_TO_UINT_I32_F32:913 return LowerFPToInt(MI, DL, BB, TII, true, false, false,914 WebAssembly::I32_TRUNC_U_F32);915 case WebAssembly::FP_TO_SINT_I64_F32:916 return LowerFPToInt(MI, DL, BB, TII, false, true, false,917 WebAssembly::I64_TRUNC_S_F32);918 case WebAssembly::FP_TO_UINT_I64_F32:919 return LowerFPToInt(MI, DL, BB, TII, true, true, false,920 WebAssembly::I64_TRUNC_U_F32);921 case WebAssembly::FP_TO_SINT_I32_F64:922 return LowerFPToInt(MI, DL, BB, TII, false, false, true,923 WebAssembly::I32_TRUNC_S_F64);924 case WebAssembly::FP_TO_UINT_I32_F64:925 return LowerFPToInt(MI, DL, BB, TII, true, false, true,926 WebAssembly::I32_TRUNC_U_F64);927 case WebAssembly::FP_TO_SINT_I64_F64:928 return LowerFPToInt(MI, DL, BB, TII, false, true, true,929 WebAssembly::I64_TRUNC_S_F64);930 case WebAssembly::FP_TO_UINT_I64_F64:931 return LowerFPToInt(MI, DL, BB, TII, true, true, true,932 WebAssembly::I64_TRUNC_U_F64);933 case WebAssembly::MEMCPY_A32:934 return LowerMemcpy(MI, DL, BB, TII, false);935 case WebAssembly::MEMCPY_A64:936 return LowerMemcpy(MI, DL, BB, TII, true);937 case WebAssembly::MEMSET_A32:938 return LowerMemset(MI, DL, BB, TII, false);939 case WebAssembly::MEMSET_A64:940 return LowerMemset(MI, DL, BB, TII, true);941 case WebAssembly::CALL_RESULTS:942 case WebAssembly::RET_CALL_RESULTS:943 return LowerCallResults(MI, DL, BB, Subtarget, TII);944 }945}946 947std::pair<unsigned, const TargetRegisterClass *>948WebAssemblyTargetLowering::getRegForInlineAsmConstraint(949 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {950 // First, see if this is a constraint that directly corresponds to a951 // WebAssembly register class.952 if (Constraint.size() == 1) {953 switch (Constraint[0]) {954 case 'r':955 assert(VT != MVT::iPTR && "Pointer MVT not expected here");956 if (Subtarget->hasSIMD128() && VT.isVector()) {957 if (VT.getSizeInBits() == 128)958 return std::make_pair(0U, &WebAssembly::V128RegClass);959 }960 if (VT.isInteger() && !VT.isVector()) {961 if (VT.getSizeInBits() <= 32)962 return std::make_pair(0U, &WebAssembly::I32RegClass);963 if (VT.getSizeInBits() <= 64)964 return std::make_pair(0U, &WebAssembly::I64RegClass);965 }966 if (VT.isFloatingPoint() && !VT.isVector()) {967 switch (VT.getSizeInBits()) {968 case 32:969 return std::make_pair(0U, &WebAssembly::F32RegClass);970 case 64:971 return std::make_pair(0U, &WebAssembly::F64RegClass);972 default:973 break;974 }975 }976 break;977 default:978 break;979 }980 }981 982 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);983}984 985bool WebAssemblyTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {986 // Assume ctz is a relatively cheap operation.987 return true;988}989 990bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {991 // Assume clz is a relatively cheap operation.992 return true;993}994 995bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,996 const AddrMode &AM,997 Type *Ty, unsigned AS,998 Instruction *I) const {999 // WebAssembly offsets are added as unsigned without wrapping. The1000 // isLegalAddressingMode gives us no way to determine if wrapping could be1001 // happening, so we approximate this by accepting only non-negative offsets.1002 if (AM.BaseOffs < 0)1003 return false;1004 1005 // WebAssembly has no scale register operands.1006 if (AM.Scale != 0)1007 return false;1008 1009 // Everything else is legal.1010 return true;1011}1012 1013bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(1014 EVT /*VT*/, unsigned /*AddrSpace*/, Align /*Align*/,1015 MachineMemOperand::Flags /*Flags*/, unsigned *Fast) const {1016 // WebAssembly supports unaligned accesses, though it should be declared1017 // with the p2align attribute on loads and stores which do so, and there1018 // may be a performance impact. We tell LLVM they're "fast" because1019 // for the kinds of things that LLVM uses this for (merging adjacent stores1020 // of constants, etc.), WebAssembly implementations will either want the1021 // unaligned access or they'll split anyway.1022 if (Fast)1023 *Fast = 1;1024 return true;1025}1026 1027bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT,1028 AttributeList Attr) const {1029 // The current thinking is that wasm engines will perform this optimization,1030 // so we can save on code size.1031 return true;1032}1033 1034bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {1035 EVT ExtT = ExtVal.getValueType();1036 EVT MemT = cast<LoadSDNode>(ExtVal->getOperand(0))->getValueType(0);1037 return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||1038 (ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||1039 (ExtT == MVT::v2i64 && MemT == MVT::v2i32);1040}1041 1042bool WebAssemblyTargetLowering::isOffsetFoldingLegal(1043 const GlobalAddressSDNode *GA) const {1044 // Wasm doesn't support function addresses with offsets1045 const GlobalValue *GV = GA->getGlobal();1046 return isa<Function>(GV) ? false : TargetLowering::isOffsetFoldingLegal(GA);1047}1048 1049EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,1050 LLVMContext &C,1051 EVT VT) const {1052 if (VT.isVector())1053 return VT.changeVectorElementTypeToInteger();1054 1055 // So far, all branch instructions in Wasm take an I32 condition.1056 // The default TargetLowering::getSetCCResultType returns the pointer size,1057 // which would be useful to reduce instruction counts when testing1058 // against 64-bit pointers/values if at some point Wasm supports that.1059 return EVT::getIntegerVT(C, 32);1060}1061 1062bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,1063 const CallInst &I,1064 MachineFunction &MF,1065 unsigned Intrinsic) const {1066 switch (Intrinsic) {1067 case Intrinsic::wasm_memory_atomic_notify:1068 Info.opc = ISD::INTRINSIC_W_CHAIN;1069 Info.memVT = MVT::i32;1070 Info.ptrVal = I.getArgOperand(0);1071 Info.offset = 0;1072 Info.align = Align(4);1073 // atomic.notify instruction does not really load the memory specified with1074 // this argument, but MachineMemOperand should either be load or store, so1075 // we set this to a load.1076 // FIXME Volatile isn't really correct, but currently all LLVM atomic1077 // instructions are treated as volatiles in the backend, so we should be1078 // consistent. The same applies for wasm_atomic_wait intrinsics too.1079 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;1080 return true;1081 case Intrinsic::wasm_memory_atomic_wait32:1082 Info.opc = ISD::INTRINSIC_W_CHAIN;1083 Info.memVT = MVT::i32;1084 Info.ptrVal = I.getArgOperand(0);1085 Info.offset = 0;1086 Info.align = Align(4);1087 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;1088 return true;1089 case Intrinsic::wasm_memory_atomic_wait64:1090 Info.opc = ISD::INTRINSIC_W_CHAIN;1091 Info.memVT = MVT::i64;1092 Info.ptrVal = I.getArgOperand(0);1093 Info.offset = 0;1094 Info.align = Align(8);1095 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;1096 return true;1097 case Intrinsic::wasm_loadf16_f32:1098 Info.opc = ISD::INTRINSIC_W_CHAIN;1099 Info.memVT = MVT::f16;1100 Info.ptrVal = I.getArgOperand(0);1101 Info.offset = 0;1102 Info.align = Align(2);1103 Info.flags = MachineMemOperand::MOLoad;1104 return true;1105 case Intrinsic::wasm_storef16_f32:1106 Info.opc = ISD::INTRINSIC_VOID;1107 Info.memVT = MVT::f16;1108 Info.ptrVal = I.getArgOperand(1);1109 Info.offset = 0;1110 Info.align = Align(2);1111 Info.flags = MachineMemOperand::MOStore;1112 return true;1113 default:1114 return false;1115 }1116}1117 1118void WebAssemblyTargetLowering::computeKnownBitsForTargetNode(1119 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,1120 const SelectionDAG &DAG, unsigned Depth) const {1121 switch (Op.getOpcode()) {1122 default:1123 break;1124 case ISD::INTRINSIC_WO_CHAIN: {1125 unsigned IntNo = Op.getConstantOperandVal(0);1126 switch (IntNo) {1127 default:1128 break;1129 case Intrinsic::wasm_bitmask: {1130 unsigned BitWidth = Known.getBitWidth();1131 EVT VT = Op.getOperand(1).getSimpleValueType();1132 unsigned PossibleBits = VT.getVectorNumElements();1133 APInt ZeroMask = APInt::getHighBitsSet(BitWidth, BitWidth - PossibleBits);1134 Known.Zero |= ZeroMask;1135 break;1136 }1137 }1138 break;1139 }1140 case WebAssemblyISD::EXTEND_LOW_U:1141 case WebAssemblyISD::EXTEND_HIGH_U: {1142 // We know the high half, of each destination vector element, will be zero.1143 SDValue SrcOp = Op.getOperand(0);1144 EVT VT = SrcOp.getSimpleValueType();1145 unsigned BitWidth = Known.getBitWidth();1146 if (VT == MVT::v8i8 || VT == MVT::v16i8) {1147 assert(BitWidth >= 8 && "Unexpected width!");1148 APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 8);1149 Known.Zero |= Mask;1150 } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {1151 assert(BitWidth >= 16 && "Unexpected width!");1152 APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 16);1153 Known.Zero |= Mask;1154 } else if (VT == MVT::v2i32 || VT == MVT::v4i32) {1155 assert(BitWidth >= 32 && "Unexpected width!");1156 APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 32);1157 Known.Zero |= Mask;1158 }1159 break;1160 }1161 // For 128-bit addition if the upper bits are all zero then it's known that1162 // the upper bits of the result will have all bits guaranteed zero except the1163 // first.1164 case WebAssemblyISD::I64_ADD128:1165 if (Op.getResNo() == 1) {1166 SDValue LHS_HI = Op.getOperand(1);1167 SDValue RHS_HI = Op.getOperand(3);1168 if (isNullConstant(LHS_HI) && isNullConstant(RHS_HI))1169 Known.Zero.setBitsFrom(1);1170 }1171 break;1172 }1173}1174 1175TargetLoweringBase::LegalizeTypeAction1176WebAssemblyTargetLowering::getPreferredVectorAction(MVT VT) const {1177 if (VT.isFixedLengthVector()) {1178 MVT EltVT = VT.getVectorElementType();1179 // We have legal vector types with these lane types, so widening the1180 // vector would let us use some of the lanes directly without having to1181 // extend or truncate values.1182 if (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||1183 EltVT == MVT::i64 || EltVT == MVT::f32 || EltVT == MVT::f64)1184 return TypeWidenVector;1185 }1186 1187 return TargetLoweringBase::getPreferredVectorAction(VT);1188}1189 1190bool WebAssemblyTargetLowering::isFMAFasterThanFMulAndFAdd(1191 const MachineFunction &MF, EVT VT) const {1192 if (!Subtarget->hasFP16() || !VT.isVector())1193 return false;1194 1195 EVT ScalarVT = VT.getScalarType();1196 if (!ScalarVT.isSimple())1197 return false;1198 1199 return ScalarVT.getSimpleVT().SimpleTy == MVT::f16;1200}1201 1202bool WebAssemblyTargetLowering::shouldSimplifyDemandedVectorElts(1203 SDValue Op, const TargetLoweringOpt &TLO) const {1204 // ISel process runs DAGCombiner after legalization; this step is called1205 // SelectionDAG optimization phase. This post-legalization combining process1206 // runs DAGCombiner on each node, and if there was a change to be made,1207 // re-runs legalization again on it and its user nodes to make sure1208 // everythiing is in a legalized state.1209 //1210 // The legalization calls lowering routines, and we do our custom lowering for1211 // build_vectors (LowerBUILD_VECTOR), which converts undef vector elements1212 // into zeros. But there is a set of routines in DAGCombiner that turns unused1213 // (= not demanded) nodes into undef, among which SimplifyDemandedVectorElts1214 // turns unused vector elements into undefs. But this routine does not work1215 // with our custom LowerBUILD_VECTOR, which turns undefs into zeros. This1216 // combination can result in a infinite loop, in which undefs are converted to1217 // zeros in legalization and back to undefs in combining.1218 //1219 // So after DAG is legalized, we prevent SimplifyDemandedVectorElts from1220 // running for build_vectors.1221 if (Op.getOpcode() == ISD::BUILD_VECTOR && TLO.LegalOps && TLO.LegalTys)1222 return false;1223 return true;1224}1225 1226//===----------------------------------------------------------------------===//1227// WebAssembly Lowering private implementation.1228//===----------------------------------------------------------------------===//1229 1230//===----------------------------------------------------------------------===//1231// Lowering Code1232//===----------------------------------------------------------------------===//1233 1234static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) {1235 MachineFunction &MF = DAG.getMachineFunction();1236 DAG.getContext()->diagnose(1237 DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));1238}1239 1240// Test whether the given calling convention is supported.1241static bool callingConvSupported(CallingConv::ID CallConv) {1242 // We currently support the language-independent target-independent1243 // conventions. We don't yet have a way to annotate calls with properties like1244 // "cold", and we don't have any call-clobbered registers, so these are mostly1245 // all handled the same.1246 return CallConv == CallingConv::C || CallConv == CallingConv::Fast ||1247 CallConv == CallingConv::Cold ||1248 CallConv == CallingConv::PreserveMost ||1249 CallConv == CallingConv::PreserveAll ||1250 CallConv == CallingConv::CXX_FAST_TLS ||1251 CallConv == CallingConv::WASM_EmscriptenInvoke ||1252 CallConv == CallingConv::Swift;1253}1254 1255SDValue1256WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,1257 SmallVectorImpl<SDValue> &InVals) const {1258 SelectionDAG &DAG = CLI.DAG;1259 SDLoc DL = CLI.DL;1260 SDValue Chain = CLI.Chain;1261 SDValue Callee = CLI.Callee;1262 MachineFunction &MF = DAG.getMachineFunction();1263 auto Layout = MF.getDataLayout();1264 1265 CallingConv::ID CallConv = CLI.CallConv;1266 if (!callingConvSupported(CallConv))1267 fail(DL, DAG,1268 "WebAssembly doesn't support language-specific or target-specific "1269 "calling conventions yet");1270 if (CLI.IsPatchPoint)1271 fail(DL, DAG, "WebAssembly doesn't support patch point yet");1272 1273 if (CLI.IsTailCall) {1274 auto NoTail = [&](const char *Msg) {1275 if (CLI.CB && CLI.CB->isMustTailCall())1276 fail(DL, DAG, Msg);1277 CLI.IsTailCall = false;1278 };1279 1280 if (!Subtarget->hasTailCall())1281 NoTail("WebAssembly 'tail-call' feature not enabled");1282 1283 // Varargs calls cannot be tail calls because the buffer is on the stack1284 if (CLI.IsVarArg)1285 NoTail("WebAssembly does not support varargs tail calls");1286 1287 // Do not tail call unless caller and callee return types match1288 const Function &F = MF.getFunction();1289 const TargetMachine &TM = getTargetMachine();1290 Type *RetTy = F.getReturnType();1291 SmallVector<MVT, 4> CallerRetTys;1292 SmallVector<MVT, 4> CalleeRetTys;1293 computeLegalValueVTs(F, TM, RetTy, CallerRetTys);1294 computeLegalValueVTs(F, TM, CLI.RetTy, CalleeRetTys);1295 bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() &&1296 std::equal(CallerRetTys.begin(), CallerRetTys.end(),1297 CalleeRetTys.begin());1298 if (!TypesMatch)1299 NoTail("WebAssembly tail call requires caller and callee return types to "1300 "match");1301 1302 // If pointers to local stack values are passed, we cannot tail call1303 if (CLI.CB) {1304 for (auto &Arg : CLI.CB->args()) {1305 Value *Val = Arg.get();1306 // Trace the value back through pointer operations1307 while (true) {1308 Value *Src = Val->stripPointerCastsAndAliases();1309 if (auto *GEP = dyn_cast<GetElementPtrInst>(Src))1310 Src = GEP->getPointerOperand();1311 if (Val == Src)1312 break;1313 Val = Src;1314 }1315 if (isa<AllocaInst>(Val)) {1316 NoTail(1317 "WebAssembly does not support tail calling with stack arguments");1318 break;1319 }1320 }1321 }1322 }1323 1324 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;1325 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;1326 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;1327 1328 // The generic code may have added an sret argument. If we're lowering an1329 // invoke function, the ABI requires that the function pointer be the first1330 // argument, so we may have to swap the arguments.1331 if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 &&1332 Outs[0].Flags.isSRet()) {1333 std::swap(Outs[0], Outs[1]);1334 std::swap(OutVals[0], OutVals[1]);1335 }1336 1337 bool HasSwiftSelfArg = false;1338 bool HasSwiftErrorArg = false;1339 unsigned NumFixedArgs = 0;1340 for (unsigned I = 0; I < Outs.size(); ++I) {1341 const ISD::OutputArg &Out = Outs[I];1342 SDValue &OutVal = OutVals[I];1343 HasSwiftSelfArg |= Out.Flags.isSwiftSelf();1344 HasSwiftErrorArg |= Out.Flags.isSwiftError();1345 if (Out.Flags.isNest())1346 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");1347 if (Out.Flags.isInAlloca())1348 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");1349 if (Out.Flags.isInConsecutiveRegs())1350 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");1351 if (Out.Flags.isInConsecutiveRegsLast())1352 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");1353 if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) {1354 auto &MFI = MF.getFrameInfo();1355 int FI = MFI.CreateStackObject(Out.Flags.getByValSize(),1356 Out.Flags.getNonZeroByValAlign(),1357 /*isSS=*/false);1358 SDValue SizeNode =1359 DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32);1360 SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));1361 Chain = DAG.getMemcpy(Chain, DL, FINode, OutVal, SizeNode,1362 Out.Flags.getNonZeroByValAlign(),1363 /*isVolatile*/ false, /*AlwaysInline=*/false,1364 /*CI=*/nullptr, std::nullopt, MachinePointerInfo(),1365 MachinePointerInfo());1366 OutVal = FINode;1367 }1368 // Count the number of fixed args *after* legalization.1369 NumFixedArgs += !Out.Flags.isVarArg();1370 }1371 1372 bool IsVarArg = CLI.IsVarArg;1373 auto PtrVT = getPointerTy(Layout);1374 1375 // For swiftcc, emit additional swiftself and swifterror arguments1376 // if there aren't. These additional arguments are also added for callee1377 // signature They are necessary to match callee and caller signature for1378 // indirect call.1379 if (CallConv == CallingConv::Swift) {1380 Type *PtrTy = PointerType::getUnqual(*DAG.getContext());1381 if (!HasSwiftSelfArg) {1382 NumFixedArgs++;1383 ISD::ArgFlagsTy Flags;1384 Flags.setSwiftSelf();1385 ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);1386 CLI.Outs.push_back(Arg);1387 SDValue ArgVal = DAG.getUNDEF(PtrVT);1388 CLI.OutVals.push_back(ArgVal);1389 }1390 if (!HasSwiftErrorArg) {1391 NumFixedArgs++;1392 ISD::ArgFlagsTy Flags;1393 Flags.setSwiftError();1394 ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);1395 CLI.Outs.push_back(Arg);1396 SDValue ArgVal = DAG.getUNDEF(PtrVT);1397 CLI.OutVals.push_back(ArgVal);1398 }1399 }1400 1401 // Analyze operands of the call, assigning locations to each operand.1402 SmallVector<CCValAssign, 16> ArgLocs;1403 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());1404 1405 if (IsVarArg) {1406 // Outgoing non-fixed arguments are placed in a buffer. First1407 // compute their offsets and the total amount of buffer space needed.1408 for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) {1409 const ISD::OutputArg &Out = Outs[I];1410 SDValue &Arg = OutVals[I];1411 EVT VT = Arg.getValueType();1412 assert(VT != MVT::iPTR && "Legalized args should be concrete");1413 Type *Ty = VT.getTypeForEVT(*DAG.getContext());1414 Align Alignment =1415 std::max(Out.Flags.getNonZeroOrigAlign(), Layout.getABITypeAlign(Ty));1416 unsigned Offset =1417 CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty), Alignment);1418 CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(),1419 Offset, VT.getSimpleVT(),1420 CCValAssign::Full));1421 }1422 }1423 1424 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();1425 1426 SDValue FINode;1427 if (IsVarArg && NumBytes) {1428 // For non-fixed arguments, next emit stores to store the argument values1429 // to the stack buffer at the offsets computed above.1430 MaybeAlign StackAlign = Layout.getStackAlignment();1431 assert(StackAlign && "data layout string is missing stack alignment");1432 int FI = MF.getFrameInfo().CreateStackObject(NumBytes, *StackAlign,1433 /*isSS=*/false);1434 unsigned ValNo = 0;1435 SmallVector<SDValue, 8> Chains;1436 for (SDValue Arg : drop_begin(OutVals, NumFixedArgs)) {1437 assert(ArgLocs[ValNo].getValNo() == ValNo &&1438 "ArgLocs should remain in order and only hold varargs args");1439 unsigned Offset = ArgLocs[ValNo++].getLocMemOffset();1440 FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));1441 SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode,1442 DAG.getConstant(Offset, DL, PtrVT));1443 Chains.push_back(1444 DAG.getStore(Chain, DL, Arg, Add,1445 MachinePointerInfo::getFixedStack(MF, FI, Offset)));1446 }1447 if (!Chains.empty())1448 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);1449 } else if (IsVarArg) {1450 FINode = DAG.getIntPtrConstant(0, DL);1451 }1452 1453 if (Callee->getOpcode() == ISD::GlobalAddress) {1454 // If the callee is a GlobalAddress node (quite common, every direct call1455 // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress1456 // doesn't at MO_GOT which is not needed for direct calls.1457 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Callee);1458 Callee = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,1459 getPointerTy(DAG.getDataLayout()),1460 GA->getOffset());1461 Callee = DAG.getNode(WebAssemblyISD::Wrapper, DL,1462 getPointerTy(DAG.getDataLayout()), Callee);1463 }1464 1465 // Compute the operands for the CALLn node.1466 SmallVector<SDValue, 16> Ops;1467 Ops.push_back(Chain);1468 Ops.push_back(Callee);1469 1470 // Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs1471 // isn't reliable.1472 Ops.append(OutVals.begin(),1473 IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end());1474 // Add a pointer to the vararg buffer.1475 if (IsVarArg)1476 Ops.push_back(FINode);1477 1478 SmallVector<EVT, 8> InTys;1479 for (const auto &In : Ins) {1480 assert(!In.Flags.isByVal() && "byval is not valid for return values");1481 assert(!In.Flags.isNest() && "nest is not valid for return values");1482 if (In.Flags.isInAlloca())1483 fail(DL, DAG, "WebAssembly hasn't implemented inalloca return values");1484 if (In.Flags.isInConsecutiveRegs())1485 fail(DL, DAG, "WebAssembly hasn't implemented cons regs return values");1486 if (In.Flags.isInConsecutiveRegsLast())1487 fail(DL, DAG,1488 "WebAssembly hasn't implemented cons regs last return values");1489 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in1490 // registers.1491 InTys.push_back(In.VT);1492 }1493 1494 // Lastly, if this is a call to a funcref we need to add an instruction1495 // table.set to the chain and transform the call.1496 if (CLI.CB && WebAssembly::isWebAssemblyFuncrefType(1497 CLI.CB->getCalledOperand()->getType())) {1498 // In the absence of function references proposal where a funcref call is1499 // lowered to call_ref, using reference types we generate a table.set to set1500 // the funcref to a special table used solely for this purpose, followed by1501 // a call_indirect. Here we just generate the table set, and return the1502 // SDValue of the table.set so that LowerCall can finalize the lowering by1503 // generating the call_indirect.1504 SDValue Chain = Ops[0];1505 1506 MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(1507 MF.getContext(), Subtarget);1508 SDValue Sym = DAG.getMCSymbol(Table, PtrVT);1509 SDValue TableSlot = DAG.getConstant(0, DL, MVT::i32);1510 SDValue TableSetOps[] = {Chain, Sym, TableSlot, Callee};1511 SDValue TableSet = DAG.getMemIntrinsicNode(1512 WebAssemblyISD::TABLE_SET, DL, DAG.getVTList(MVT::Other), TableSetOps,1513 MVT::funcref,1514 // Machine Mem Operand args1515 MachinePointerInfo(1516 WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF),1517 CLI.CB->getCalledOperand()->getPointerAlignment(DAG.getDataLayout()),1518 MachineMemOperand::MOStore);1519 1520 Ops[0] = TableSet; // The new chain is the TableSet itself1521 }1522 1523 if (CLI.IsTailCall) {1524 // ret_calls do not return values to the current frame1525 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);1526 return DAG.getNode(WebAssemblyISD::RET_CALL, DL, NodeTys, Ops);1527 }1528 1529 InTys.push_back(MVT::Other);1530 SDVTList InTyList = DAG.getVTList(InTys);1531 SDValue Res = DAG.getNode(WebAssemblyISD::CALL, DL, InTyList, Ops);1532 1533 for (size_t I = 0; I < Ins.size(); ++I)1534 InVals.push_back(Res.getValue(I));1535 1536 // Return the chain1537 return Res.getValue(Ins.size());1538}1539 1540bool WebAssemblyTargetLowering::CanLowerReturn(1541 CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/,1542 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext & /*Context*/,1543 const Type *RetTy) const {1544 // WebAssembly can only handle returning tuples with multivalue enabled1545 return WebAssembly::canLowerReturn(Outs.size(), Subtarget);1546}1547 1548SDValue WebAssemblyTargetLowering::LowerReturn(1549 SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/,1550 const SmallVectorImpl<ISD::OutputArg> &Outs,1551 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,1552 SelectionDAG &DAG) const {1553 assert(WebAssembly::canLowerReturn(Outs.size(), Subtarget) &&1554 "MVP WebAssembly can only return up to one value");1555 if (!callingConvSupported(CallConv))1556 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");1557 1558 SmallVector<SDValue, 4> RetOps(1, Chain);1559 RetOps.append(OutVals.begin(), OutVals.end());1560 Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, RetOps);1561 1562 // Record the number and types of the return values.1563 for (const ISD::OutputArg &Out : Outs) {1564 assert(!Out.Flags.isByVal() && "byval is not valid for return values");1565 assert(!Out.Flags.isNest() && "nest is not valid for return values");1566 assert(!Out.Flags.isVarArg() && "non-fixed return value is not valid");1567 if (Out.Flags.isInAlloca())1568 fail(DL, DAG, "WebAssembly hasn't implemented inalloca results");1569 if (Out.Flags.isInConsecutiveRegs())1570 fail(DL, DAG, "WebAssembly hasn't implemented cons regs results");1571 if (Out.Flags.isInConsecutiveRegsLast())1572 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last results");1573 }1574 1575 return Chain;1576}1577 1578SDValue WebAssemblyTargetLowering::LowerFormalArguments(1579 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,1580 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,1581 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {1582 if (!callingConvSupported(CallConv))1583 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");1584 1585 MachineFunction &MF = DAG.getMachineFunction();1586 auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>();1587 1588 // Set up the incoming ARGUMENTS value, which serves to represent the liveness1589 // of the incoming values before they're represented by virtual registers.1590 MF.getRegInfo().addLiveIn(WebAssembly::ARGUMENTS);1591 1592 bool HasSwiftErrorArg = false;1593 bool HasSwiftSelfArg = false;1594 for (const ISD::InputArg &In : Ins) {1595 HasSwiftSelfArg |= In.Flags.isSwiftSelf();1596 HasSwiftErrorArg |= In.Flags.isSwiftError();1597 if (In.Flags.isInAlloca())1598 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");1599 if (In.Flags.isNest())1600 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");1601 if (In.Flags.isInConsecutiveRegs())1602 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");1603 if (In.Flags.isInConsecutiveRegsLast())1604 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");1605 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in1606 // registers.1607 InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,1608 DAG.getTargetConstant(InVals.size(),1609 DL, MVT::i32))1610 : DAG.getUNDEF(In.VT));1611 1612 // Record the number and types of arguments.1613 MFI->addParam(In.VT);1614 }1615 1616 // For swiftcc, emit additional swiftself and swifterror arguments1617 // if there aren't. These additional arguments are also added for callee1618 // signature They are necessary to match callee and caller signature for1619 // indirect call.1620 auto PtrVT = getPointerTy(MF.getDataLayout());1621 if (CallConv == CallingConv::Swift) {1622 if (!HasSwiftSelfArg) {1623 MFI->addParam(PtrVT);1624 }1625 if (!HasSwiftErrorArg) {1626 MFI->addParam(PtrVT);1627 }1628 }1629 // Varargs are copied into a buffer allocated by the caller, and a pointer to1630 // the buffer is passed as an argument.1631 if (IsVarArg) {1632 MVT PtrVT = getPointerTy(MF.getDataLayout());1633 Register VarargVreg =1634 MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrVT));1635 MFI->setVarargBufferVreg(VarargVreg);1636 Chain = DAG.getCopyToReg(1637 Chain, DL, VarargVreg,1638 DAG.getNode(WebAssemblyISD::ARGUMENT, DL, PtrVT,1639 DAG.getTargetConstant(Ins.size(), DL, MVT::i32)));1640 MFI->addParam(PtrVT);1641 }1642 1643 // Record the number and types of arguments and results.1644 SmallVector<MVT, 4> Params;1645 SmallVector<MVT, 4> Results;1646 computeSignatureVTs(MF.getFunction().getFunctionType(), &MF.getFunction(),1647 MF.getFunction(), DAG.getTarget(), Params, Results);1648 for (MVT VT : Results)1649 MFI->addResult(VT);1650 // TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify1651 // the param logic here with ComputeSignatureVTs1652 assert(MFI->getParams().size() == Params.size() &&1653 std::equal(MFI->getParams().begin(), MFI->getParams().end(),1654 Params.begin()));1655 1656 return Chain;1657}1658 1659void WebAssemblyTargetLowering::ReplaceNodeResults(1660 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {1661 switch (N->getOpcode()) {1662 case ISD::SIGN_EXTEND_INREG:1663 // Do not add any results, signifying that N should not be custom lowered1664 // after all. This happens because simd128 turns on custom lowering for1665 // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an1666 // illegal type.1667 break;1668 case ISD::SIGN_EXTEND_VECTOR_INREG:1669 case ISD::ZERO_EXTEND_VECTOR_INREG:1670 // Do not add any results, signifying that N should not be custom lowered.1671 // EXTEND_VECTOR_INREG is implemented for some vectors, but not all.1672 break;1673 case ISD::ADD:1674 case ISD::SUB:1675 Results.push_back(Replace128Op(N, DAG));1676 break;1677 default:1678 llvm_unreachable(1679 "ReplaceNodeResults not implemented for this op for WebAssembly!");1680 }1681}1682 1683//===----------------------------------------------------------------------===//1684// Custom lowering hooks.1685//===----------------------------------------------------------------------===//1686 1687SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,1688 SelectionDAG &DAG) const {1689 SDLoc DL(Op);1690 switch (Op.getOpcode()) {1691 default:1692 llvm_unreachable("unimplemented operation lowering");1693 return SDValue();1694 case ISD::FrameIndex:1695 return LowerFrameIndex(Op, DAG);1696 case ISD::GlobalAddress:1697 return LowerGlobalAddress(Op, DAG);1698 case ISD::GlobalTLSAddress:1699 return LowerGlobalTLSAddress(Op, DAG);1700 case ISD::ExternalSymbol:1701 return LowerExternalSymbol(Op, DAG);1702 case ISD::JumpTable:1703 return LowerJumpTable(Op, DAG);1704 case ISD::BR_JT:1705 return LowerBR_JT(Op, DAG);1706 case ISD::VASTART:1707 return LowerVASTART(Op, DAG);1708 case ISD::BlockAddress:1709 case ISD::BRIND:1710 fail(DL, DAG, "WebAssembly hasn't implemented computed gotos");1711 return SDValue();1712 case ISD::RETURNADDR:1713 return LowerRETURNADDR(Op, DAG);1714 case ISD::FRAMEADDR:1715 return LowerFRAMEADDR(Op, DAG);1716 case ISD::CopyToReg:1717 return LowerCopyToReg(Op, DAG);1718 case ISD::EXTRACT_VECTOR_ELT:1719 case ISD::INSERT_VECTOR_ELT:1720 return LowerAccessVectorElement(Op, DAG);1721 case ISD::INTRINSIC_VOID:1722 case ISD::INTRINSIC_WO_CHAIN:1723 case ISD::INTRINSIC_W_CHAIN:1724 return LowerIntrinsic(Op, DAG);1725 case ISD::SIGN_EXTEND_INREG:1726 return LowerSIGN_EXTEND_INREG(Op, DAG);1727 case ISD::ZERO_EXTEND_VECTOR_INREG:1728 case ISD::SIGN_EXTEND_VECTOR_INREG:1729 case ISD::ANY_EXTEND_VECTOR_INREG:1730 return LowerEXTEND_VECTOR_INREG(Op, DAG);1731 case ISD::BUILD_VECTOR:1732 return LowerBUILD_VECTOR(Op, DAG);1733 case ISD::VECTOR_SHUFFLE:1734 return LowerVECTOR_SHUFFLE(Op, DAG);1735 case ISD::SETCC:1736 return LowerSETCC(Op, DAG);1737 case ISD::SHL:1738 case ISD::SRA:1739 case ISD::SRL:1740 return LowerShift(Op, DAG);1741 case ISD::FP_TO_SINT_SAT:1742 case ISD::FP_TO_UINT_SAT:1743 return LowerFP_TO_INT_SAT(Op, DAG);1744 case ISD::LOAD:1745 return LowerLoad(Op, DAG);1746 case ISD::STORE:1747 return LowerStore(Op, DAG);1748 case ISD::CTPOP:1749 case ISD::CTLZ:1750 case ISD::CTTZ:1751 return DAG.UnrollVectorOp(Op.getNode());1752 case ISD::CLEAR_CACHE:1753 report_fatal_error("llvm.clear_cache is not supported on wasm");1754 case ISD::SMUL_LOHI:1755 case ISD::UMUL_LOHI:1756 return LowerMUL_LOHI(Op, DAG);1757 case ISD::UADDO:1758 return LowerUADDO(Op, DAG);1759 }1760}1761 1762static bool IsWebAssemblyGlobal(SDValue Op) {1763 if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op))1764 return WebAssembly::isWasmVarAddressSpace(GA->getAddressSpace());1765 1766 return false;1767}1768 1769static std::optional<unsigned> IsWebAssemblyLocal(SDValue Op,1770 SelectionDAG &DAG) {1771 const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op);1772 if (!FI)1773 return std::nullopt;1774 1775 auto &MF = DAG.getMachineFunction();1776 return WebAssemblyFrameLowering::getLocalForStackObject(MF, FI->getIndex());1777}1778 1779SDValue WebAssemblyTargetLowering::LowerStore(SDValue Op,1780 SelectionDAG &DAG) const {1781 SDLoc DL(Op);1782 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());1783 const SDValue &Value = SN->getValue();1784 const SDValue &Base = SN->getBasePtr();1785 const SDValue &Offset = SN->getOffset();1786 1787 if (IsWebAssemblyGlobal(Base)) {1788 if (!Offset->isUndef())1789 report_fatal_error("unexpected offset when storing to webassembly global",1790 false);1791 1792 SDVTList Tys = DAG.getVTList(MVT::Other);1793 SDValue Ops[] = {SN->getChain(), Value, Base};1794 return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_SET, DL, Tys, Ops,1795 SN->getMemoryVT(), SN->getMemOperand());1796 }1797 1798 if (std::optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {1799 if (!Offset->isUndef())1800 report_fatal_error("unexpected offset when storing to webassembly local",1801 false);1802 1803 SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);1804 SDVTList Tys = DAG.getVTList(MVT::Other); // The chain.1805 SDValue Ops[] = {SN->getChain(), Idx, Value};1806 return DAG.getNode(WebAssemblyISD::LOCAL_SET, DL, Tys, Ops);1807 }1808 1809 if (WebAssembly::isWasmVarAddressSpace(SN->getAddressSpace()))1810 report_fatal_error(1811 "Encountered an unlowerable store to the wasm_var address space",1812 false);1813 1814 return Op;1815}1816 1817SDValue WebAssemblyTargetLowering::LowerLoad(SDValue Op,1818 SelectionDAG &DAG) const {1819 SDLoc DL(Op);1820 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());1821 const SDValue &Base = LN->getBasePtr();1822 const SDValue &Offset = LN->getOffset();1823 1824 if (IsWebAssemblyGlobal(Base)) {1825 if (!Offset->isUndef())1826 report_fatal_error(1827 "unexpected offset when loading from webassembly global", false);1828 1829 SDVTList Tys = DAG.getVTList(LN->getValueType(0), MVT::Other);1830 SDValue Ops[] = {LN->getChain(), Base};1831 return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_GET, DL, Tys, Ops,1832 LN->getMemoryVT(), LN->getMemOperand());1833 }1834 1835 if (std::optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {1836 if (!Offset->isUndef())1837 report_fatal_error(1838 "unexpected offset when loading from webassembly local", false);1839 1840 SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);1841 EVT LocalVT = LN->getValueType(0);1842 return DAG.getNode(WebAssemblyISD::LOCAL_GET, DL, {LocalVT, MVT::Other},1843 {LN->getChain(), Idx});1844 }1845 1846 if (WebAssembly::isWasmVarAddressSpace(LN->getAddressSpace()))1847 report_fatal_error(1848 "Encountered an unlowerable load from the wasm_var address space",1849 false);1850 1851 return Op;1852}1853 1854SDValue WebAssemblyTargetLowering::LowerMUL_LOHI(SDValue Op,1855 SelectionDAG &DAG) const {1856 assert(Subtarget->hasWideArithmetic());1857 assert(Op.getValueType() == MVT::i64);1858 SDLoc DL(Op);1859 unsigned Opcode;1860 switch (Op.getOpcode()) {1861 case ISD::UMUL_LOHI:1862 Opcode = WebAssemblyISD::I64_MUL_WIDE_U;1863 break;1864 case ISD::SMUL_LOHI:1865 Opcode = WebAssemblyISD::I64_MUL_WIDE_S;1866 break;1867 default:1868 llvm_unreachable("unexpected opcode");1869 }1870 SDValue LHS = Op.getOperand(0);1871 SDValue RHS = Op.getOperand(1);1872 SDValue Lo =1873 DAG.getNode(Opcode, DL, DAG.getVTList(MVT::i64, MVT::i64), LHS, RHS);1874 SDValue Hi(Lo.getNode(), 1);1875 SDValue Ops[] = {Lo, Hi};1876 return DAG.getMergeValues(Ops, DL);1877}1878 1879// Lowers `UADDO` intrinsics to an `i64.add128` instruction when it's enabled.1880//1881// This enables generating a single wasm instruction for this operation where1882// the upper half of both operands are constant zeros. The upper half of the1883// result is then whether the overflow happened.1884SDValue WebAssemblyTargetLowering::LowerUADDO(SDValue Op,1885 SelectionDAG &DAG) const {1886 assert(Subtarget->hasWideArithmetic());1887 assert(Op.getValueType() == MVT::i64);1888 assert(Op.getOpcode() == ISD::UADDO);1889 SDLoc DL(Op);1890 SDValue LHS = Op.getOperand(0);1891 SDValue RHS = Op.getOperand(1);1892 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);1893 SDValue Result =1894 DAG.getNode(WebAssemblyISD::I64_ADD128, DL,1895 DAG.getVTList(MVT::i64, MVT::i64), LHS, Zero, RHS, Zero);1896 SDValue CarryI64(Result.getNode(), 1);1897 SDValue CarryI32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, CarryI64);1898 SDValue Ops[] = {Result, CarryI32};1899 return DAG.getMergeValues(Ops, DL);1900}1901 1902SDValue WebAssemblyTargetLowering::Replace128Op(SDNode *N,1903 SelectionDAG &DAG) const {1904 assert(Subtarget->hasWideArithmetic());1905 assert(N->getValueType(0) == MVT::i128);1906 SDLoc DL(N);1907 unsigned Opcode;1908 switch (N->getOpcode()) {1909 case ISD::ADD:1910 Opcode = WebAssemblyISD::I64_ADD128;1911 break;1912 case ISD::SUB:1913 Opcode = WebAssemblyISD::I64_SUB128;1914 break;1915 default:1916 llvm_unreachable("unexpected opcode");1917 }1918 SDValue LHS = N->getOperand(0);1919 SDValue RHS = N->getOperand(1);1920 1921 SDValue C0 = DAG.getConstant(0, DL, MVT::i64);1922 SDValue C1 = DAG.getConstant(1, DL, MVT::i64);1923 SDValue LHS_0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, LHS, C0);1924 SDValue LHS_1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, LHS, C1);1925 SDValue RHS_0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, RHS, C0);1926 SDValue RHS_1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, RHS, C1);1927 SDValue Result_LO = DAG.getNode(Opcode, DL, DAG.getVTList(MVT::i64, MVT::i64),1928 LHS_0, LHS_1, RHS_0, RHS_1);1929 SDValue Result_HI(Result_LO.getNode(), 1);1930 return DAG.getNode(ISD::BUILD_PAIR, DL, N->getVTList(), Result_LO, Result_HI);1931}1932 1933SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,1934 SelectionDAG &DAG) const {1935 SDValue Src = Op.getOperand(2);1936 if (isa<FrameIndexSDNode>(Src.getNode())) {1937 // CopyToReg nodes don't support FrameIndex operands. Other targets select1938 // the FI to some LEA-like instruction, but since we don't have that, we1939 // need to insert some kind of instruction that can take an FI operand and1940 // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy1941 // local.copy between Op and its FI operand.1942 SDValue Chain = Op.getOperand(0);1943 SDLoc DL(Op);1944 Register Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();1945 EVT VT = Src.getValueType();1946 SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I321947 : WebAssembly::COPY_I64,1948 DL, VT, Src),1949 0);1950 return Op.getNode()->getNumValues() == 11951 ? DAG.getCopyToReg(Chain, DL, Reg, Copy)1952 : DAG.getCopyToReg(Chain, DL, Reg, Copy,1953 Op.getNumOperands() == 4 ? Op.getOperand(3)1954 : SDValue());1955 }1956 return SDValue();1957}1958 1959SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,1960 SelectionDAG &DAG) const {1961 int FI = cast<FrameIndexSDNode>(Op)->getIndex();1962 return DAG.getTargetFrameIndex(FI, Op.getValueType());1963}1964 1965SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,1966 SelectionDAG &DAG) const {1967 SDLoc DL(Op);1968 1969 if (!Subtarget->getTargetTriple().isOSEmscripten()) {1970 // A frame's return address lives in the protected call stack, not in1971 // addressable linear memory, so wasm has no sound return-address value to1972 // hand back. Degrade the builtin to a null pointer rather than aborting the1973 // compile: __builtin_return_address is representable and lawfully no-op-able1974 // here, so erroring on it would be a never-silent violation. NULL matches1975 // the musl wasm32 port (its dlsym passes 0 because the ELF return-address1976 // dance has no wasm analog); the only cost is that backtrace / caller1977 // identification degrade to "unknown" on wasm, which is expected. Emscripten1978 // keeps its libcall-based lowering below, untouched.1979 return DAG.getConstant(0, DL, Op.getValueType());1980 }1981 1982 unsigned Depth = Op.getConstantOperandVal(0);1983 MakeLibCallOptions CallOptions;1984 return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(),1985 {DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions, DL)1986 .first;1987}1988 1989SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,1990 SelectionDAG &DAG) const {1991 // Non-zero depths are not supported by WebAssembly currently. Use the1992 // legalizer's default expansion, which is to return 0 (what this function is1993 // documented to do).1994 if (Op.getConstantOperandVal(0) > 0)1995 return SDValue();1996 1997 DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true);1998 EVT VT = Op.getValueType();1999 Register FP =2000 Subtarget->getRegisterInfo()->getFrameRegister(DAG.getMachineFunction());2001 return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), FP, VT);2002}2003 2004SDValue2005WebAssemblyTargetLowering::LowerGlobalTLSAddress(SDValue Op,2006 SelectionDAG &DAG) const {2007 SDLoc DL(Op);2008 const auto *GA = cast<GlobalAddressSDNode>(Op);2009 2010 MachineFunction &MF = DAG.getMachineFunction();2011 if (!MF.getSubtarget<WebAssemblySubtarget>().hasBulkMemory())2012 report_fatal_error("cannot use thread-local storage without bulk memory",2013 false);2014 2015 const GlobalValue *GV = GA->getGlobal();2016 2017 // Currently only Emscripten supports dynamic linking with threads. Therefore,2018 // on other targets, if we have thread-local storage, only the local-exec2019 // model is possible.2020 auto model = Subtarget->getTargetTriple().isOSEmscripten()2021 ? GV->getThreadLocalMode()2022 : GlobalValue::LocalExecTLSModel;2023 2024 // Unsupported TLS modes2025 assert(model != GlobalValue::NotThreadLocal);2026 assert(model != GlobalValue::InitialExecTLSModel);2027 2028 if (model == GlobalValue::LocalExecTLSModel ||2029 model == GlobalValue::LocalDynamicTLSModel ||2030 (model == GlobalValue::GeneralDynamicTLSModel &&2031 getTargetMachine().shouldAssumeDSOLocal(GV))) {2032 // For DSO-local TLS variables we use offset from __tls_base2033 2034 MVT PtrVT = getPointerTy(DAG.getDataLayout());2035 auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I642036 : WebAssembly::GLOBAL_GET_I32;2037 const char *BaseName = MF.createExternalSymbolName("__tls_base");2038 2039 SDValue BaseAddr(2040 DAG.getMachineNode(GlobalGet, DL, PtrVT,2041 DAG.getTargetExternalSymbol(BaseName, PtrVT)),2042 0);2043 2044 SDValue TLSOffset = DAG.getTargetGlobalAddress(2045 GV, DL, PtrVT, GA->getOffset(), WebAssemblyII::MO_TLS_BASE_REL);2046 SDValue SymOffset =2047 DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, TLSOffset);2048 2049 return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymOffset);2050 }2051 2052 assert(model == GlobalValue::GeneralDynamicTLSModel);2053 2054 EVT VT = Op.getValueType();2055 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,2056 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,2057 GA->getOffset(),2058 WebAssemblyII::MO_GOT_TLS));2059}2060 2061SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,2062 SelectionDAG &DAG) const {2063 SDLoc DL(Op);2064 const auto *GA = cast<GlobalAddressSDNode>(Op);2065 EVT VT = Op.getValueType();2066 assert(GA->getTargetFlags() == 0 &&2067 "Unexpected target flags on generic GlobalAddressSDNode");2068 if (!WebAssembly::isValidAddressSpace(GA->getAddressSpace()))2069 fail(DL, DAG, "Invalid address space for WebAssembly target");2070 2071 unsigned OperandFlags = 0;2072 const GlobalValue *GV = GA->getGlobal();2073 // Since WebAssembly tables cannot yet be shared accross modules, we don't2074 // need special treatment for tables in PIC mode.2075 if (isPositionIndependent() &&2076 !WebAssembly::isWebAssemblyTableType(GV->getValueType())) {2077 if (getTargetMachine().shouldAssumeDSOLocal(GV)) {2078 MachineFunction &MF = DAG.getMachineFunction();2079 MVT PtrVT = getPointerTy(MF.getDataLayout());2080 const char *BaseName;2081 if (GV->getValueType()->isFunctionTy()) {2082 BaseName = MF.createExternalSymbolName("__table_base");2083 OperandFlags = WebAssemblyII::MO_TABLE_BASE_REL;2084 } else {2085 BaseName = MF.createExternalSymbolName("__memory_base");2086 OperandFlags = WebAssemblyII::MO_MEMORY_BASE_REL;2087 }2088 SDValue BaseAddr =2089 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,2090 DAG.getTargetExternalSymbol(BaseName, PtrVT));2091 2092 SDValue SymAddr = DAG.getNode(2093 WebAssemblyISD::WrapperREL, DL, VT,2094 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset(),2095 OperandFlags));2096 2097 return DAG.getNode(ISD::ADD, DL, VT, BaseAddr, SymAddr);2098 }2099 OperandFlags = WebAssemblyII::MO_GOT;2100 }2101 2102 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,2103 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,2104 GA->getOffset(), OperandFlags));2105}2106 2107SDValue2108WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,2109 SelectionDAG &DAG) const {2110 SDLoc DL(Op);2111 const auto *ES = cast<ExternalSymbolSDNode>(Op);2112 EVT VT = Op.getValueType();2113 assert(ES->getTargetFlags() == 0 &&2114 "Unexpected target flags on generic ExternalSymbolSDNode");2115 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,2116 DAG.getTargetExternalSymbol(ES->getSymbol(), VT));2117}2118 2119SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,2120 SelectionDAG &DAG) const {2121 // There's no need for a Wrapper node because we always incorporate a jump2122 // table operand into a BR_TABLE instruction, rather than ever2123 // materializing it in a register.2124 const JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);2125 return DAG.getTargetJumpTable(JT->getIndex(), Op.getValueType(),2126 JT->getTargetFlags());2127}2128 2129SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,2130 SelectionDAG &DAG) const {2131 SDLoc DL(Op);2132 SDValue Chain = Op.getOperand(0);2133 const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1));2134 SDValue Index = Op.getOperand(2);2135 assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags");2136 2137 SmallVector<SDValue, 8> Ops;2138 Ops.push_back(Chain);2139 Ops.push_back(Index);2140 2141 MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo();2142 const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs;2143 2144 // Add an operand for each case.2145 for (auto *MBB : MBBs)2146 Ops.push_back(DAG.getBasicBlock(MBB));2147 2148 // Add the first MBB as a dummy default target for now. This will be replaced2149 // with the proper default target (and the preceding range check eliminated)2150 // if possible by WebAssemblyFixBrTableDefaults.2151 Ops.push_back(DAG.getBasicBlock(*MBBs.begin()));2152 return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops);2153}2154 2155SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,2156 SelectionDAG &DAG) const {2157 SDLoc DL(Op);2158 EVT PtrVT = getPointerTy(DAG.getMachineFunction().getDataLayout());2159 2160 auto *MFI = DAG.getMachineFunction().getInfo<WebAssemblyFunctionInfo>();2161 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();2162 2163 SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL,2164 MFI->getVarargBufferVreg(), PtrVT);2165 return DAG.getStore(Op.getOperand(0), DL, ArgN, Op.getOperand(1),2166 MachinePointerInfo(SV));2167}2168 2169SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,2170 SelectionDAG &DAG) const {2171 MachineFunction &MF = DAG.getMachineFunction();2172 unsigned IntNo;2173 switch (Op.getOpcode()) {2174 case ISD::INTRINSIC_VOID:2175 case ISD::INTRINSIC_W_CHAIN:2176 IntNo = Op.getConstantOperandVal(1);2177 break;2178 case ISD::INTRINSIC_WO_CHAIN:2179 IntNo = Op.getConstantOperandVal(0);2180 break;2181 default:2182 llvm_unreachable("Invalid intrinsic");2183 }2184 SDLoc DL(Op);2185 2186 switch (IntNo) {2187 default:2188 return SDValue(); // Don't custom lower most intrinsics.2189 2190 case Intrinsic::wasm_lsda: {2191 auto PtrVT = getPointerTy(MF.getDataLayout());2192 const char *SymName = MF.createExternalSymbolName(2193 "GCC_except_table" + std::to_string(MF.getFunctionNumber()));2194 if (isPositionIndependent()) {2195 SDValue Node = DAG.getTargetExternalSymbol(2196 SymName, PtrVT, WebAssemblyII::MO_MEMORY_BASE_REL);2197 const char *BaseName = MF.createExternalSymbolName("__memory_base");2198 SDValue BaseAddr =2199 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,2200 DAG.getTargetExternalSymbol(BaseName, PtrVT));2201 SDValue SymAddr =2202 DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, Node);2203 return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymAddr);2204 }2205 SDValue Node = DAG.getTargetExternalSymbol(SymName, PtrVT);2206 return DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, Node);2207 }2208 2209 case Intrinsic::wasm_shuffle: {2210 // Drop in-chain and replace undefs, but otherwise pass through unchanged2211 SDValue Ops[18];2212 size_t OpIdx = 0;2213 Ops[OpIdx++] = Op.getOperand(1);2214 Ops[OpIdx++] = Op.getOperand(2);2215 while (OpIdx < 18) {2216 const SDValue &MaskIdx = Op.getOperand(OpIdx + 1);2217 if (MaskIdx.isUndef() || MaskIdx.getNode()->getAsZExtVal() >= 32) {2218 bool isTarget = MaskIdx.getNode()->getOpcode() == ISD::TargetConstant;2219 Ops[OpIdx++] = DAG.getConstant(0, DL, MVT::i32, isTarget);2220 } else {2221 Ops[OpIdx++] = MaskIdx;2222 }2223 }2224 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);2225 }2226 2227 case Intrinsic::thread_pointer: {2228 MVT PtrVT = getPointerTy(DAG.getDataLayout());2229 auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I642230 : WebAssembly::GLOBAL_GET_I32;2231 const char *TlsBase = MF.createExternalSymbolName("__tls_base");2232 return SDValue(2233 DAG.getMachineNode(GlobalGet, DL, PtrVT,2234 DAG.getTargetExternalSymbol(TlsBase, PtrVT)),2235 0);2236 }2237 }2238}2239 2240SDValue2241WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,2242 SelectionDAG &DAG) const {2243 SDLoc DL(Op);2244 // If sign extension operations are disabled, allow sext_inreg only if operand2245 // is a vector extract of an i8 or i16 lane. SIMD does not depend on sign2246 // extension operations, but allowing sext_inreg in this context lets us have2247 // simple patterns to select extract_lane_s instructions. Expanding sext_inreg2248 // everywhere would be simpler in this file, but would necessitate large and2249 // brittle patterns to undo the expansion and select extract_lane_s2250 // instructions.2251 assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128());2252 if (Op.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)2253 return SDValue();2254 2255 const SDValue &Extract = Op.getOperand(0);2256 MVT VecT = Extract.getOperand(0).getSimpleValueType();2257 if (VecT.getVectorElementType().getSizeInBits() > 32)2258 return SDValue();2259 MVT ExtractedLaneT =2260 cast<VTSDNode>(Op.getOperand(1).getNode())->getVT().getSimpleVT();2261 MVT ExtractedVecT =2262 MVT::getVectorVT(ExtractedLaneT, 128 / ExtractedLaneT.getSizeInBits());2263 if (ExtractedVecT == VecT)2264 return Op;2265 2266 // Bitcast vector to appropriate type to ensure ISel pattern coverage2267 const SDNode *Index = Extract.getOperand(1).getNode();2268 if (!isa<ConstantSDNode>(Index))2269 return SDValue();2270 unsigned IndexVal = Index->getAsZExtVal();2271 unsigned Scale =2272 ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements();2273 assert(Scale > 1);2274 SDValue NewIndex =2275 DAG.getConstant(IndexVal * Scale, DL, Index->getValueType(0));2276 SDValue NewExtract = DAG.getNode(2277 ISD::EXTRACT_VECTOR_ELT, DL, Extract.getValueType(),2278 DAG.getBitcast(ExtractedVecT, Extract.getOperand(0)), NewIndex);2279 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), NewExtract,2280 Op.getOperand(1));2281}2282 2283static SDValue GetExtendHigh(SDValue Op, unsigned UserOpc, EVT VT,2284 SelectionDAG &DAG) {2285 if (Op.getOpcode() != ISD::VECTOR_SHUFFLE)2286 return SDValue();2287 2288 assert((UserOpc == WebAssemblyISD::EXTEND_LOW_U ||2289 UserOpc == WebAssemblyISD::EXTEND_LOW_S) &&2290 "expected extend_low");2291 auto *Shuffle = cast<ShuffleVectorSDNode>(Op.getNode());2292 2293 ArrayRef<int> Mask = Shuffle->getMask();2294 // Look for a shuffle which moves from the high half to the low half.2295 size_t FirstIdx = Mask.size() / 2;2296 for (size_t i = 0; i < Mask.size() / 2; ++i) {2297 if (Mask[i] != static_cast<int>(FirstIdx + i)) {2298 return SDValue();2299 }2300 }2301 2302 SDLoc DL(Op);2303 unsigned Opc = UserOpc == WebAssemblyISD::EXTEND_LOW_S2304 ? WebAssemblyISD::EXTEND_HIGH_S2305 : WebAssemblyISD::EXTEND_HIGH_U;2306 return DAG.getNode(Opc, DL, VT, Shuffle->getOperand(0));2307}2308 2309SDValue2310WebAssemblyTargetLowering::LowerEXTEND_VECTOR_INREG(SDValue Op,2311 SelectionDAG &DAG) const {2312 SDLoc DL(Op);2313 EVT VT = Op.getValueType();2314 SDValue Src = Op.getOperand(0);2315 EVT SrcVT = Src.getValueType();2316 2317 if (SrcVT.getVectorElementType() == MVT::i1 ||2318 SrcVT.getVectorElementType() == MVT::i64)2319 return SDValue();2320 2321 assert(VT.getScalarSizeInBits() % SrcVT.getScalarSizeInBits() == 0 &&2322 "Unexpected extension factor.");2323 unsigned Scale = VT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits();2324 2325 if (Scale != 2 && Scale != 4 && Scale != 8)2326 return SDValue();2327 2328 unsigned Ext;2329 switch (Op.getOpcode()) {2330 default:2331 llvm_unreachable("unexpected opcode");2332 case ISD::ANY_EXTEND_VECTOR_INREG:2333 case ISD::ZERO_EXTEND_VECTOR_INREG:2334 Ext = WebAssemblyISD::EXTEND_LOW_U;2335 break;2336 case ISD::SIGN_EXTEND_VECTOR_INREG:2337 Ext = WebAssemblyISD::EXTEND_LOW_S;2338 break;2339 }2340 2341 if (Scale == 2) {2342 // See if we can use EXTEND_HIGH.2343 if (auto ExtendHigh = GetExtendHigh(Op.getOperand(0), Ext, VT, DAG))2344 return ExtendHigh;2345 }2346 2347 SDValue Ret = Src;2348 while (Scale != 1) {2349 Ret = DAG.getNode(Ext, DL,2350 Ret.getValueType()2351 .widenIntegerVectorElementType(*DAG.getContext())2352 .getHalfNumVectorElementsVT(*DAG.getContext()),2353 Ret);2354 Scale /= 2;2355 }2356 assert(Ret.getValueType() == VT);2357 return Ret;2358}2359 2360static SDValue LowerConvertLow(SDValue Op, SelectionDAG &DAG) {2361 SDLoc DL(Op);2362 if (Op.getValueType() != MVT::v2f64)2363 return SDValue();2364 2365 auto GetConvertedLane = [](SDValue Op, unsigned &Opcode, SDValue &SrcVec,2366 unsigned &Index) -> bool {2367 switch (Op.getOpcode()) {2368 case ISD::SINT_TO_FP:2369 Opcode = WebAssemblyISD::CONVERT_LOW_S;2370 break;2371 case ISD::UINT_TO_FP:2372 Opcode = WebAssemblyISD::CONVERT_LOW_U;2373 break;2374 case ISD::FP_EXTEND:2375 Opcode = WebAssemblyISD::PROMOTE_LOW;2376 break;2377 default:2378 return false;2379 }2380 2381 auto ExtractVector = Op.getOperand(0);2382 if (ExtractVector.getOpcode() != ISD::EXTRACT_VECTOR_ELT)2383 return false;2384 2385 if (!isa<ConstantSDNode>(ExtractVector.getOperand(1).getNode()))2386 return false;2387 2388 SrcVec = ExtractVector.getOperand(0);2389 Index = ExtractVector.getConstantOperandVal(1);2390 return true;2391 };2392 2393 unsigned LHSOpcode, RHSOpcode, LHSIndex, RHSIndex;2394 SDValue LHSSrcVec, RHSSrcVec;2395 if (!GetConvertedLane(Op.getOperand(0), LHSOpcode, LHSSrcVec, LHSIndex) ||2396 !GetConvertedLane(Op.getOperand(1), RHSOpcode, RHSSrcVec, RHSIndex))2397 return SDValue();2398 2399 if (LHSOpcode != RHSOpcode)2400 return SDValue();2401 2402 MVT ExpectedSrcVT;2403 switch (LHSOpcode) {2404 case WebAssemblyISD::CONVERT_LOW_S:2405 case WebAssemblyISD::CONVERT_LOW_U:2406 ExpectedSrcVT = MVT::v4i32;2407 break;2408 case WebAssemblyISD::PROMOTE_LOW:2409 ExpectedSrcVT = MVT::v4f32;2410 break;2411 }2412 if (LHSSrcVec.getValueType() != ExpectedSrcVT)2413 return SDValue();2414 2415 auto Src = LHSSrcVec;2416 if (LHSIndex != 0 || RHSIndex != 1 || LHSSrcVec != RHSSrcVec) {2417 // Shuffle the source vector so that the converted lanes are the low lanes.2418 Src = DAG.getVectorShuffle(2419 ExpectedSrcVT, DL, LHSSrcVec, RHSSrcVec,2420 {static_cast<int>(LHSIndex), static_cast<int>(RHSIndex) + 4, -1, -1});2421 }2422 return DAG.getNode(LHSOpcode, DL, MVT::v2f64, Src);2423}2424 2425SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,2426 SelectionDAG &DAG) const {2427 MVT VT = Op.getSimpleValueType();2428 if (VT == MVT::v8f16) {2429 // BUILD_VECTOR can't handle FP16 operands since Wasm doesn't have a scaler2430 // FP16 type, so cast them to I16s.2431 MVT IVT = VT.changeVectorElementType(MVT::i16);2432 SmallVector<SDValue, 8> NewOps;2433 for (unsigned I = 0, E = Op.getNumOperands(); I < E; ++I)2434 NewOps.push_back(DAG.getBitcast(MVT::i16, Op.getOperand(I)));2435 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(), IVT, NewOps);2436 return DAG.getBitcast(VT, Res);2437 }2438 2439 if (auto ConvertLow = LowerConvertLow(Op, DAG))2440 return ConvertLow;2441 2442 SDLoc DL(Op);2443 const EVT VecT = Op.getValueType();2444 const EVT LaneT = Op.getOperand(0).getValueType();2445 const size_t Lanes = Op.getNumOperands();2446 bool CanSwizzle = VecT == MVT::v16i8;2447 2448 // BUILD_VECTORs are lowered to the instruction that initializes the highest2449 // possible number of lanes at once followed by a sequence of replace_lane2450 // instructions to individually initialize any remaining lanes.2451 2452 // TODO: Tune this. For example, lanewise swizzling is very expensive, so2453 // swizzled lanes should be given greater weight.2454 2455 // TODO: Investigate looping rather than always extracting/replacing specific2456 // lanes to fill gaps.2457 2458 auto IsConstant = [](const SDValue &V) {2459 return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP;2460 };2461 2462 // Returns the source vector and index vector pair if they exist. Checks for:2463 // (extract_vector_elt2464 // $src,2465 // (sign_extend_inreg (extract_vector_elt $indices, $i))2466 // )2467 auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) {2468 auto Bail = std::make_pair(SDValue(), SDValue());2469 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)2470 return Bail;2471 const SDValue &SwizzleSrc = Lane->getOperand(0);2472 const SDValue &IndexExt = Lane->getOperand(1);2473 if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG)2474 return Bail;2475 const SDValue &Index = IndexExt->getOperand(0);2476 if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT)2477 return Bail;2478 const SDValue &SwizzleIndices = Index->getOperand(0);2479 if (SwizzleSrc.getValueType() != MVT::v16i8 ||2480 SwizzleIndices.getValueType() != MVT::v16i8 ||2481 Index->getOperand(1)->getOpcode() != ISD::Constant ||2482 Index->getConstantOperandVal(1) != I)2483 return Bail;2484 return std::make_pair(SwizzleSrc, SwizzleIndices);2485 };2486 2487 // If the lane is extracted from another vector at a constant index, return2488 // that vector. The source vector must not have more lanes than the dest2489 // because the shufflevector indices are in terms of the destination lanes and2490 // would not be able to address the smaller individual source lanes.2491 auto GetShuffleSrc = [&](const SDValue &Lane) {2492 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)2493 return SDValue();2494 if (!isa<ConstantSDNode>(Lane->getOperand(1).getNode()))2495 return SDValue();2496 if (Lane->getOperand(0).getValueType().getVectorNumElements() >2497 VecT.getVectorNumElements())2498 return SDValue();2499 return Lane->getOperand(0);2500 };2501 2502 using ValueEntry = std::pair<SDValue, size_t>;2503 SmallVector<ValueEntry, 16> SplatValueCounts;2504 2505 using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;2506 SmallVector<SwizzleEntry, 16> SwizzleCounts;2507 2508 using ShuffleEntry = std::pair<SDValue, size_t>;2509 SmallVector<ShuffleEntry, 16> ShuffleCounts;2510 2511 auto AddCount = [](auto &Counts, const auto &Val) {2512 auto CountIt =2513 llvm::find_if(Counts, [&Val](auto E) { return E.first == Val; });2514 if (CountIt == Counts.end()) {2515 Counts.emplace_back(Val, 1);2516 } else {2517 CountIt->second++;2518 }2519 };2520 2521 auto GetMostCommon = [](auto &Counts) {2522 auto CommonIt = llvm::max_element(Counts, llvm::less_second());2523 assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector");2524 return *CommonIt;2525 };2526 2527 size_t NumConstantLanes = 0;2528 2529 // Count eligible lanes for each type of vector creation op2530 for (size_t I = 0; I < Lanes; ++I) {2531 const SDValue &Lane = Op->getOperand(I);2532 if (Lane.isUndef())2533 continue;2534 2535 AddCount(SplatValueCounts, Lane);2536 2537 if (IsConstant(Lane))2538 NumConstantLanes++;2539 if (auto ShuffleSrc = GetShuffleSrc(Lane))2540 AddCount(ShuffleCounts, ShuffleSrc);2541 if (CanSwizzle) {2542 auto SwizzleSrcs = GetSwizzleSrcs(I, Lane);2543 if (SwizzleSrcs.first)2544 AddCount(SwizzleCounts, SwizzleSrcs);2545 }2546 }2547 2548 SDValue SplatValue;2549 size_t NumSplatLanes;2550 std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts);2551 2552 SDValue SwizzleSrc;2553 SDValue SwizzleIndices;2554 size_t NumSwizzleLanes = 0;2555 if (SwizzleCounts.size())2556 std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices),2557 NumSwizzleLanes) = GetMostCommon(SwizzleCounts);2558 2559 // Shuffles can draw from up to two vectors, so find the two most common2560 // sources.2561 SDValue ShuffleSrc1, ShuffleSrc2;2562 size_t NumShuffleLanes = 0;2563 if (ShuffleCounts.size()) {2564 std::tie(ShuffleSrc1, NumShuffleLanes) = GetMostCommon(ShuffleCounts);2565 llvm::erase_if(ShuffleCounts,2566 [&](const auto &Pair) { return Pair.first == ShuffleSrc1; });2567 }2568 if (ShuffleCounts.size()) {2569 size_t AdditionalShuffleLanes;2570 std::tie(ShuffleSrc2, AdditionalShuffleLanes) =2571 GetMostCommon(ShuffleCounts);2572 NumShuffleLanes += AdditionalShuffleLanes;2573 }2574 2575 // Predicate returning true if the lane is properly initialized by the2576 // original instruction2577 std::function<bool(size_t, const SDValue &)> IsLaneConstructed;2578 SDValue Result;2579 // Prefer swizzles over shuffles over vector consts over splats2580 if (NumSwizzleLanes >= NumShuffleLanes &&2581 NumSwizzleLanes >= NumConstantLanes && NumSwizzleLanes >= NumSplatLanes) {2582 Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc,2583 SwizzleIndices);2584 auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices);2585 IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) {2586 return Swizzled == GetSwizzleSrcs(I, Lane);2587 };2588 } else if (NumShuffleLanes >= NumConstantLanes &&2589 NumShuffleLanes >= NumSplatLanes) {2590 size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits() / 8;2591 size_t DestLaneCount = VecT.getVectorNumElements();2592 size_t Scale1 = 1;2593 size_t Scale2 = 1;2594 SDValue Src1 = ShuffleSrc1;2595 SDValue Src2 = ShuffleSrc2 ? ShuffleSrc2 : DAG.getUNDEF(VecT);2596 if (Src1.getValueType() != VecT) {2597 size_t LaneSize =2598 Src1.getValueType().getVectorElementType().getFixedSizeInBits() / 8;2599 assert(LaneSize > DestLaneSize);2600 Scale1 = LaneSize / DestLaneSize;2601 Src1 = DAG.getBitcast(VecT, Src1);2602 }2603 if (Src2.getValueType() != VecT) {2604 size_t LaneSize =2605 Src2.getValueType().getVectorElementType().getFixedSizeInBits() / 8;2606 assert(LaneSize > DestLaneSize);2607 Scale2 = LaneSize / DestLaneSize;2608 Src2 = DAG.getBitcast(VecT, Src2);2609 }2610 2611 int Mask[16];2612 assert(DestLaneCount <= 16);2613 for (size_t I = 0; I < DestLaneCount; ++I) {2614 const SDValue &Lane = Op->getOperand(I);2615 SDValue Src = GetShuffleSrc(Lane);2616 if (Src == ShuffleSrc1) {2617 Mask[I] = Lane->getConstantOperandVal(1) * Scale1;2618 } else if (Src && Src == ShuffleSrc2) {2619 Mask[I] = DestLaneCount + Lane->getConstantOperandVal(1) * Scale2;2620 } else {2621 Mask[I] = -1;2622 }2623 }2624 ArrayRef<int> MaskRef(Mask, DestLaneCount);2625 Result = DAG.getVectorShuffle(VecT, DL, Src1, Src2, MaskRef);2626 IsLaneConstructed = [&](size_t, const SDValue &Lane) {2627 auto Src = GetShuffleSrc(Lane);2628 return Src == ShuffleSrc1 || (Src && Src == ShuffleSrc2);2629 };2630 } else if (NumConstantLanes >= NumSplatLanes) {2631 SmallVector<SDValue, 16> ConstLanes;2632 for (const SDValue &Lane : Op->op_values()) {2633 if (IsConstant(Lane)) {2634 // Values may need to be fixed so that they will sign extend to be2635 // within the expected range during ISel. Check whether the value is in2636 // bounds based on the lane bit width and if it is out of bounds, lop2637 // off the extra bits.2638 uint64_t LaneBits = 128 / Lanes;2639 if (auto *Const = dyn_cast<ConstantSDNode>(Lane.getNode())) {2640 ConstLanes.push_back(DAG.getConstant(2641 Const->getAPIntValue().trunc(LaneBits).getZExtValue(),2642 SDLoc(Lane), LaneT));2643 } else {2644 ConstLanes.push_back(Lane);2645 }2646 } else if (LaneT.isFloatingPoint()) {2647 ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT));2648 } else {2649 ConstLanes.push_back(DAG.getConstant(0, DL, LaneT));2650 }2651 }2652 Result = DAG.getBuildVector(VecT, DL, ConstLanes);2653 IsLaneConstructed = [&IsConstant](size_t _, const SDValue &Lane) {2654 return IsConstant(Lane);2655 };2656 } else {2657 size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits();2658 if (NumSplatLanes == 1 && Op->getOperand(0) == SplatValue &&2659 (DestLaneSize == 32 || DestLaneSize == 64)) {2660 // Could be selected to load_zero.2661 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecT, SplatValue);2662 } else {2663 // Use a splat (which might be selected as a load splat)2664 Result = DAG.getSplatBuildVector(VecT, DL, SplatValue);2665 }2666 IsLaneConstructed = [&SplatValue](size_t _, const SDValue &Lane) {2667 return Lane == SplatValue;2668 };2669 }2670 2671 assert(Result);2672 assert(IsLaneConstructed);2673 2674 // Add replace_lane instructions for any unhandled values2675 for (size_t I = 0; I < Lanes; ++I) {2676 const SDValue &Lane = Op->getOperand(I);2677 if (!Lane.isUndef() && !IsLaneConstructed(I, Lane))2678 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane,2679 DAG.getConstant(I, DL, MVT::i32));2680 }2681 2682 return Result;2683}2684 2685SDValue2686WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,2687 SelectionDAG &DAG) const {2688 SDLoc DL(Op);2689 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask();2690 MVT VecType = Op.getOperand(0).getSimpleValueType();2691 assert(VecType.is128BitVector() && "Unexpected shuffle vector type");2692 size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8;2693 2694 // Space for two vector args and sixteen mask indices2695 SDValue Ops[18];2696 size_t OpIdx = 0;2697 Ops[OpIdx++] = Op.getOperand(0);2698 Ops[OpIdx++] = Op.getOperand(1);2699 2700 // Expand mask indices to byte indices and materialize them as operands2701 for (int M : Mask) {2702 for (size_t J = 0; J < LaneBytes; ++J) {2703 // Lower undefs (represented by -1 in mask) to {0..J}, which use a2704 // whole lane of vector input, to allow further reduction at VM. E.g.2705 // match an 8x16 byte shuffle to an equivalent cheaper 32x4 shuffle.2706 uint64_t ByteIndex = M == -1 ? J : (uint64_t)M * LaneBytes + J;2707 Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32);2708 }2709 }2710 2711 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);2712}2713 2714SDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op,2715 SelectionDAG &DAG) const {2716 SDLoc DL(Op);2717 // The legalizer does not know how to expand the unsupported comparison modes2718 // of i64x2 vectors, so we manually unroll them here.2719 assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64);2720 SmallVector<SDValue, 2> LHS, RHS;2721 DAG.ExtractVectorElements(Op->getOperand(0), LHS);2722 DAG.ExtractVectorElements(Op->getOperand(1), RHS);2723 const SDValue &CC = Op->getOperand(2);2724 auto MakeLane = [&](unsigned I) {2725 return DAG.getNode(ISD::SELECT_CC, DL, MVT::i64, LHS[I], RHS[I],2726 DAG.getConstant(uint64_t(-1), DL, MVT::i64),2727 DAG.getConstant(uint64_t(0), DL, MVT::i64), CC);2728 };2729 return DAG.getBuildVector(Op->getValueType(0), DL,2730 {MakeLane(0), MakeLane(1)});2731}2732 2733SDValue2734WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,2735 SelectionDAG &DAG) const {2736 // Allow constant lane indices, expand variable lane indices2737 SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode();2738 if (isa<ConstantSDNode>(IdxNode)) {2739 // Ensure the index type is i32 to match the tablegen patterns2740 uint64_t Idx = IdxNode->getAsZExtVal();2741 SmallVector<SDValue, 3> Ops(Op.getNode()->ops());2742 Ops[Op.getNumOperands() - 1] =2743 DAG.getConstant(Idx, SDLoc(IdxNode), MVT::i32);2744 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), Ops);2745 }2746 // Perform default expansion2747 return SDValue();2748}2749 2750static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) {2751 EVT LaneT = Op.getSimpleValueType().getVectorElementType();2752 // 32-bit and 64-bit unrolled shifts will have proper semantics2753 if (LaneT.bitsGE(MVT::i32))2754 return DAG.UnrollVectorOp(Op.getNode());2755 // Otherwise mask the shift value to get proper semantics from 32-bit shift2756 SDLoc DL(Op);2757 size_t NumLanes = Op.getSimpleValueType().getVectorNumElements();2758 SDValue Mask = DAG.getConstant(LaneT.getSizeInBits() - 1, DL, MVT::i32);2759 unsigned ShiftOpcode = Op.getOpcode();2760 SmallVector<SDValue, 16> ShiftedElements;2761 DAG.ExtractVectorElements(Op.getOperand(0), ShiftedElements, 0, 0, MVT::i32);2762 SmallVector<SDValue, 16> ShiftElements;2763 DAG.ExtractVectorElements(Op.getOperand(1), ShiftElements, 0, 0, MVT::i32);2764 SmallVector<SDValue, 16> UnrolledOps;2765 for (size_t i = 0; i < NumLanes; ++i) {2766 SDValue MaskedShiftValue =2767 DAG.getNode(ISD::AND, DL, MVT::i32, ShiftElements[i], Mask);2768 SDValue ShiftedValue = ShiftedElements[i];2769 if (ShiftOpcode == ISD::SRA)2770 ShiftedValue = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32,2771 ShiftedValue, DAG.getValueType(LaneT));2772 UnrolledOps.push_back(2773 DAG.getNode(ShiftOpcode, DL, MVT::i32, ShiftedValue, MaskedShiftValue));2774 }2775 return DAG.getBuildVector(Op.getValueType(), DL, UnrolledOps);2776}2777 2778SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,2779 SelectionDAG &DAG) const {2780 SDLoc DL(Op);2781 2782 // Only manually lower vector shifts2783 assert(Op.getSimpleValueType().isVector());2784 2785 uint64_t LaneBits = Op.getValueType().getScalarSizeInBits();2786 auto ShiftVal = Op.getOperand(1);2787 2788 // Try to skip bitmask operation since it is implied inside shift instruction2789 auto SkipImpliedMask = [](SDValue MaskOp, uint64_t MaskBits) {2790 if (MaskOp.getOpcode() != ISD::AND)2791 return MaskOp;2792 SDValue LHS = MaskOp.getOperand(0);2793 SDValue RHS = MaskOp.getOperand(1);2794 if (MaskOp.getValueType().isVector()) {2795 APInt MaskVal;2796 if (!ISD::isConstantSplatVector(RHS.getNode(), MaskVal))2797 std::swap(LHS, RHS);2798 2799 if (ISD::isConstantSplatVector(RHS.getNode(), MaskVal) &&2800 MaskVal == MaskBits)2801 MaskOp = LHS;2802 } else {2803 if (!isa<ConstantSDNode>(RHS.getNode()))2804 std::swap(LHS, RHS);2805 2806 auto ConstantRHS = dyn_cast<ConstantSDNode>(RHS.getNode());2807 if (ConstantRHS && ConstantRHS->getAPIntValue() == MaskBits)2808 MaskOp = LHS;2809 }2810 2811 return MaskOp;2812 };2813 2814 // Skip vector and operation2815 ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);2816 ShiftVal = DAG.getSplatValue(ShiftVal);2817 if (!ShiftVal)2818 return unrollVectorShift(Op, DAG);2819 2820 // Skip scalar and operation2821 ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);2822 // Use anyext because none of the high bits can affect the shift2823 ShiftVal = DAG.getAnyExtOrTrunc(ShiftVal, DL, MVT::i32);2824 2825 unsigned Opcode;2826 switch (Op.getOpcode()) {2827 case ISD::SHL:2828 Opcode = WebAssemblyISD::VEC_SHL;2829 break;2830 case ISD::SRA:2831 Opcode = WebAssemblyISD::VEC_SHR_S;2832 break;2833 case ISD::SRL:2834 Opcode = WebAssemblyISD::VEC_SHR_U;2835 break;2836 default:2837 llvm_unreachable("unexpected opcode");2838 }2839 2840 return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0), ShiftVal);2841}2842 2843SDValue WebAssemblyTargetLowering::LowerFP_TO_INT_SAT(SDValue Op,2844 SelectionDAG &DAG) const {2845 EVT ResT = Op.getValueType();2846 EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();2847 2848 if ((ResT == MVT::i32 || ResT == MVT::i64) &&2849 (SatVT == MVT::i32 || SatVT == MVT::i64))2850 return Op;2851 2852 if (ResT == MVT::v4i32 && SatVT == MVT::i32)2853 return Op;2854 2855 if (ResT == MVT::v8i16 && SatVT == MVT::i16)2856 return Op;2857 2858 return SDValue();2859}2860 2861//===----------------------------------------------------------------------===//2862// Custom DAG combine hooks2863//===----------------------------------------------------------------------===//2864static SDValue2865performVECTOR_SHUFFLECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {2866 auto &DAG = DCI.DAG;2867 auto Shuffle = cast<ShuffleVectorSDNode>(N);2868 2869 // Hoist vector bitcasts that don't change the number of lanes out of unary2870 // shuffles, where they are less likely to get in the way of other combines.2871 // (shuffle (vNxT1 (bitcast (vNxT0 x))), undef, mask) ->2872 // (vNxT1 (bitcast (vNxT0 (shuffle x, undef, mask))))2873 SDValue Bitcast = N->getOperand(0);2874 if (Bitcast.getOpcode() != ISD::BITCAST)2875 return SDValue();2876 if (!N->getOperand(1).isUndef())2877 return SDValue();2878 SDValue CastOp = Bitcast.getOperand(0);2879 EVT SrcType = CastOp.getValueType();2880 EVT DstType = Bitcast.getValueType();2881 if (!SrcType.is128BitVector() ||2882 SrcType.getVectorNumElements() != DstType.getVectorNumElements())2883 return SDValue();2884 SDValue NewShuffle = DAG.getVectorShuffle(2885 SrcType, SDLoc(N), CastOp, DAG.getUNDEF(SrcType), Shuffle->getMask());2886 return DAG.getBitcast(DstType, NewShuffle);2887}2888 2889/// Convert ({u,s}itofp vec) --> ({u,s}itofp ({s,z}ext vec)) so it doesn't get2890/// split up into scalar instructions during legalization, and the vector2891/// extending instructions are selected in performVectorExtendCombine below.2892static SDValue2893performVectorExtendToFPCombine(SDNode *N,2894 TargetLowering::DAGCombinerInfo &DCI) {2895 auto &DAG = DCI.DAG;2896 assert(N->getOpcode() == ISD::UINT_TO_FP ||2897 N->getOpcode() == ISD::SINT_TO_FP);2898 2899 EVT InVT = N->getOperand(0)->getValueType(0);2900 EVT ResVT = N->getValueType(0);2901 MVT ExtVT;2902 if (ResVT == MVT::v4f32 && (InVT == MVT::v4i16 || InVT == MVT::v4i8))2903 ExtVT = MVT::v4i32;2904 else if (ResVT == MVT::v2f64 && (InVT == MVT::v2i16 || InVT == MVT::v2i8))2905 ExtVT = MVT::v2i32;2906 else2907 return SDValue();2908 2909 unsigned Op =2910 N->getOpcode() == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;2911 SDValue Conv = DAG.getNode(Op, SDLoc(N), ExtVT, N->getOperand(0));2912 return DAG.getNode(N->getOpcode(), SDLoc(N), ResVT, Conv);2913}2914 2915static SDValue2916performVectorNonNegToFPCombine(SDNode *N,2917 TargetLowering::DAGCombinerInfo &DCI) {2918 auto &DAG = DCI.DAG;2919 2920 SDNodeFlags Flags = N->getFlags();2921 SDValue Op0 = N->getOperand(0);2922 EVT VT = N->getValueType(0);2923 2924 // Optimize uitofp to sitofp when the sign bit is known to be zero.2925 // Depending on the target (runtime) backend, this might be performance2926 // neutral (e.g. AArch64) or a significant improvement (e.g. x86_64).2927 if (VT.isVector() && (Flags.hasNonNeg() || DAG.SignBitIsZero(Op0))) {2928 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, Op0);2929 }2930 2931 return SDValue();2932}2933 2934static SDValue2935performVectorExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {2936 auto &DAG = DCI.DAG;2937 assert(N->getOpcode() == ISD::SIGN_EXTEND ||2938 N->getOpcode() == ISD::ZERO_EXTEND);2939 2940 // Combine ({s,z}ext (extract_subvector src, i)) into a widening operation if2941 // possible before the extract_subvector can be expanded.2942 auto Extract = N->getOperand(0);2943 if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR)2944 return SDValue();2945 auto Source = Extract.getOperand(0);2946 auto *IndexNode = dyn_cast<ConstantSDNode>(Extract.getOperand(1));2947 if (IndexNode == nullptr)2948 return SDValue();2949 auto Index = IndexNode->getZExtValue();2950 2951 // Only v8i8, v4i16, and v2i32 extracts can be widened, and only if the2952 // extracted subvector is the low or high half of its source.2953 EVT ResVT = N->getValueType(0);2954 if (ResVT == MVT::v8i16) {2955 if (Extract.getValueType() != MVT::v8i8 ||2956 Source.getValueType() != MVT::v16i8 || (Index != 0 && Index != 8))2957 return SDValue();2958 } else if (ResVT == MVT::v4i32) {2959 if (Extract.getValueType() != MVT::v4i16 ||2960 Source.getValueType() != MVT::v8i16 || (Index != 0 && Index != 4))2961 return SDValue();2962 } else if (ResVT == MVT::v2i64) {2963 if (Extract.getValueType() != MVT::v2i32 ||2964 Source.getValueType() != MVT::v4i32 || (Index != 0 && Index != 2))2965 return SDValue();2966 } else {2967 return SDValue();2968 }2969 2970 bool IsSext = N->getOpcode() == ISD::SIGN_EXTEND;2971 bool IsLow = Index == 0;2972 2973 unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::EXTEND_LOW_S2974 : WebAssemblyISD::EXTEND_HIGH_S)2975 : (IsLow ? WebAssemblyISD::EXTEND_LOW_U2976 : WebAssemblyISD::EXTEND_HIGH_U);2977 2978 return DAG.getNode(Op, SDLoc(N), ResVT, Source);2979}2980 2981static SDValue2982performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {2983 auto &DAG = DCI.DAG;2984 2985 auto GetWasmConversionOp = [](unsigned Op) {2986 switch (Op) {2987 case ISD::FP_TO_SINT_SAT:2988 return WebAssemblyISD::TRUNC_SAT_ZERO_S;2989 case ISD::FP_TO_UINT_SAT:2990 return WebAssemblyISD::TRUNC_SAT_ZERO_U;2991 case ISD::FP_ROUND:2992 return WebAssemblyISD::DEMOTE_ZERO;2993 }2994 llvm_unreachable("unexpected op");2995 };2996 2997 auto IsZeroSplat = [](SDValue SplatVal) {2998 auto *Splat = dyn_cast<BuildVectorSDNode>(SplatVal.getNode());2999 APInt SplatValue, SplatUndef;3000 unsigned SplatBitSize;3001 bool HasAnyUndefs;3002 // Endianness doesn't matter in this context because we are looking for3003 // an all-zero value.3004 return Splat &&3005 Splat->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,3006 HasAnyUndefs) &&3007 SplatValue == 0;3008 };3009 3010 if (N->getOpcode() == ISD::CONCAT_VECTORS) {3011 // Combine this:3012 //3013 // (concat_vectors (v2i32 (fp_to_{s,u}int_sat $x, 32)), (v2i32 (splat 0)))3014 //3015 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).3016 //3017 // Or this:3018 //3019 // (concat_vectors (v2f32 (fp_round (v2f64 $x))), (v2f32 (splat 0)))3020 //3021 // into (f32x4.demote_zero_f64x2 $x).3022 EVT ResVT;3023 EVT ExpectedConversionType;3024 auto Conversion = N->getOperand(0);3025 auto ConversionOp = Conversion.getOpcode();3026 switch (ConversionOp) {3027 case ISD::FP_TO_SINT_SAT:3028 case ISD::FP_TO_UINT_SAT:3029 ResVT = MVT::v4i32;3030 ExpectedConversionType = MVT::v2i32;3031 break;3032 case ISD::FP_ROUND:3033 ResVT = MVT::v4f32;3034 ExpectedConversionType = MVT::v2f32;3035 break;3036 default:3037 return SDValue();3038 }3039 3040 if (N->getValueType(0) != ResVT)3041 return SDValue();3042 3043 if (Conversion.getValueType() != ExpectedConversionType)3044 return SDValue();3045 3046 auto Source = Conversion.getOperand(0);3047 if (Source.getValueType() != MVT::v2f64)3048 return SDValue();3049 3050 if (!IsZeroSplat(N->getOperand(1)) ||3051 N->getOperand(1).getValueType() != ExpectedConversionType)3052 return SDValue();3053 3054 unsigned Op = GetWasmConversionOp(ConversionOp);3055 return DAG.getNode(Op, SDLoc(N), ResVT, Source);3056 }3057 3058 // Combine this:3059 //3060 // (fp_to_{s,u}int_sat (concat_vectors $x, (v2f64 (splat 0))), 32)3061 //3062 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).3063 //3064 // Or this:3065 //3066 // (v4f32 (fp_round (concat_vectors $x, (v2f64 (splat 0)))))3067 //3068 // into (f32x4.demote_zero_f64x2 $x).3069 EVT ResVT;3070 auto ConversionOp = N->getOpcode();3071 switch (ConversionOp) {3072 case ISD::FP_TO_SINT_SAT:3073 case ISD::FP_TO_UINT_SAT:3074 ResVT = MVT::v4i32;3075 break;3076 case ISD::FP_ROUND:3077 ResVT = MVT::v4f32;3078 break;3079 default:3080 llvm_unreachable("unexpected op");3081 }3082 3083 if (N->getValueType(0) != ResVT)3084 return SDValue();3085 3086 auto Concat = N->getOperand(0);3087 if (Concat.getValueType() != MVT::v4f64)3088 return SDValue();3089 3090 auto Source = Concat.getOperand(0);3091 if (Source.getValueType() != MVT::v2f64)3092 return SDValue();3093 3094 if (!IsZeroSplat(Concat.getOperand(1)) ||3095 Concat.getOperand(1).getValueType() != MVT::v2f64)3096 return SDValue();3097 3098 unsigned Op = GetWasmConversionOp(ConversionOp);3099 return DAG.getNode(Op, SDLoc(N), ResVT, Source);3100}3101 3102// Helper to extract VectorWidth bits from Vec, starting from IdxVal.3103static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,3104 const SDLoc &DL, unsigned VectorWidth) {3105 EVT VT = Vec.getValueType();3106 EVT ElVT = VT.getVectorElementType();3107 unsigned Factor = VT.getSizeInBits() / VectorWidth;3108 EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,3109 VT.getVectorNumElements() / Factor);3110 3111 // Extract the relevant VectorWidth bits. Generate an EXTRACT_SUBVECTOR3112 unsigned ElemsPerChunk = VectorWidth / ElVT.getSizeInBits();3113 assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");3114 3115 // This is the index of the first element of the VectorWidth-bit chunk3116 // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.3117 IdxVal &= ~(ElemsPerChunk - 1);3118 3119 // If the input is a buildvector just emit a smaller one.3120 if (Vec.getOpcode() == ISD::BUILD_VECTOR)3121 return DAG.getBuildVector(ResultVT, DL,3122 Vec->ops().slice(IdxVal, ElemsPerChunk));3123 3124 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, DL);3125 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResultVT, Vec, VecIdx);3126}3127 3128// Helper to recursively truncate vector elements in half with NARROW_U. DstVT3129// is the expected destination value type after recursion. In is the initial3130// input. Note that the input should have enough leading zero bits to prevent3131// NARROW_U from saturating results.3132static SDValue truncateVectorWithNARROW(EVT DstVT, SDValue In, const SDLoc &DL,3133 SelectionDAG &DAG) {3134 EVT SrcVT = In.getValueType();3135 3136 // No truncation required, we might get here due to recursive calls.3137 if (SrcVT == DstVT)3138 return In;3139 3140 unsigned SrcSizeInBits = SrcVT.getSizeInBits();3141 unsigned NumElems = SrcVT.getVectorNumElements();3142 if (!isPowerOf2_32(NumElems))3143 return SDValue();3144 assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation");3145 assert(SrcSizeInBits > DstVT.getSizeInBits() && "Illegal truncation");3146 3147 LLVMContext &Ctx = *DAG.getContext();3148 EVT PackedSVT = EVT::getIntegerVT(Ctx, SrcVT.getScalarSizeInBits() / 2);3149 3150 // Narrow to the largest type possible:3151 // vXi64/vXi32 -> i16x8.narrow_i32x4_u and vXi16 -> i8x16.narrow_i16x8_u.3152 EVT InVT = MVT::i16, OutVT = MVT::i8;3153 if (SrcVT.getScalarSizeInBits() > 16) {3154 InVT = MVT::i32;3155 OutVT = MVT::i16;3156 }3157 unsigned SubSizeInBits = SrcSizeInBits / 2;3158 InVT = EVT::getVectorVT(Ctx, InVT, SubSizeInBits / InVT.getSizeInBits());3159 OutVT = EVT::getVectorVT(Ctx, OutVT, SubSizeInBits / OutVT.getSizeInBits());3160 3161 // Split lower/upper subvectors.3162 SDValue Lo = extractSubVector(In, 0, DAG, DL, SubSizeInBits);3163 SDValue Hi = extractSubVector(In, NumElems / 2, DAG, DL, SubSizeInBits);3164 3165 // 256bit -> 128bit truncate - Narrow lower/upper 128-bit subvectors.3166 if (SrcVT.is256BitVector() && DstVT.is128BitVector()) {3167 Lo = DAG.getBitcast(InVT, Lo);3168 Hi = DAG.getBitcast(InVT, Hi);3169 SDValue Res = DAG.getNode(WebAssemblyISD::NARROW_U, DL, OutVT, Lo, Hi);3170 return DAG.getBitcast(DstVT, Res);3171 }3172 3173 // Recursively narrow lower/upper subvectors, concat result and narrow again.3174 EVT PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems / 2);3175 Lo = truncateVectorWithNARROW(PackedVT, Lo, DL, DAG);3176 Hi = truncateVectorWithNARROW(PackedVT, Hi, DL, DAG);3177 3178 PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems);3179 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);3180 return truncateVectorWithNARROW(DstVT, Res, DL, DAG);3181}3182 3183static SDValue performTruncateCombine(SDNode *N,3184 TargetLowering::DAGCombinerInfo &DCI) {3185 auto &DAG = DCI.DAG;3186 3187 SDValue In = N->getOperand(0);3188 EVT InVT = In.getValueType();3189 if (!InVT.isSimple())3190 return SDValue();3191 3192 EVT OutVT = N->getValueType(0);3193 if (!OutVT.isVector())3194 return SDValue();3195 3196 EVT OutSVT = OutVT.getVectorElementType();3197 EVT InSVT = InVT.getVectorElementType();3198 // Currently only cover truncate to v16i8 or v8i16.3199 if (!((InSVT == MVT::i16 || InSVT == MVT::i32 || InSVT == MVT::i64) &&3200 (OutSVT == MVT::i8 || OutSVT == MVT::i16) && OutVT.is128BitVector()))3201 return SDValue();3202 3203 SDLoc DL(N);3204 APInt Mask = APInt::getLowBitsSet(InVT.getScalarSizeInBits(),3205 OutVT.getScalarSizeInBits());3206 In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(Mask, DL, InVT));3207 return truncateVectorWithNARROW(OutVT, In, DL, DAG);3208}3209 3210static SDValue performBitcastCombine(SDNode *N,3211 TargetLowering::DAGCombinerInfo &DCI) {3212 using namespace llvm::SDPatternMatch;3213 auto &DAG = DCI.DAG;3214 SDLoc DL(N);3215 SDValue Src = N->getOperand(0);3216 EVT VT = N->getValueType(0);3217 EVT SrcVT = Src.getValueType();3218 3219 if (!(DCI.isBeforeLegalize() && VT.isScalarInteger() &&3220 SrcVT.isFixedLengthVector() && SrcVT.getScalarType() == MVT::i1))3221 return SDValue();3222 3223 unsigned NumElts = SrcVT.getVectorNumElements();3224 EVT Width = MVT::getIntegerVT(128 / NumElts);3225 3226 // bitcast <N x i1> to iN, where N = 2, 4, 8, 16 (legal)3227 // ==> bitmask3228 if (NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) {3229 return DAG.getZExtOrTrunc(3230 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,3231 {DAG.getConstant(Intrinsic::wasm_bitmask, DL, MVT::i32),3232 DAG.getSExtOrTrunc(N->getOperand(0), DL,3233 SrcVT.changeVectorElementType(Width))}),3234 DL, VT);3235 }3236 3237 // bitcast <N x i1>(setcc ...) to concat iN, where N = 32 and 64 (illegal)3238 if (NumElts == 32 || NumElts == 64) {3239 // Strategy: We will setcc them seperately in v16i8 -> v16i13240 // Bitcast them to i16, extend them to either i32 or i64.3241 // Add them together, shifting left 1 by 1.3242 SDValue Concat, SetCCVector;3243 ISD::CondCode SetCond;3244 3245 if (!sd_match(N, m_BitCast(m_c_SetCC(m_Value(Concat), m_Value(SetCCVector),3246 m_CondCode(SetCond)))))3247 return SDValue();3248 if (Concat.getOpcode() != ISD::CONCAT_VECTORS)3249 return SDValue();3250 3251 uint64_t ElementWidth =3252 SetCCVector.getValueType().getVectorElementType().getFixedSizeInBits();3253 3254 SmallVector<SDValue> VectorsToShuffle;3255 for (size_t I = 0; I < Concat->ops().size(); I++) {3256 VectorsToShuffle.push_back(DAG.getBitcast(3257 MVT::i16,3258 DAG.getSetCC(DL, MVT::v16i1, Concat->ops()[I],3259 extractSubVector(SetCCVector, I * (128 / ElementWidth),3260 DAG, DL, 128),3261 SetCond)));3262 }3263 3264 MVT ReturnType = VectorsToShuffle.size() == 2 ? MVT::i32 : MVT::i64;3265 SDValue ReturningInteger = DAG.getConstant(0, DL, ReturnType);3266 3267 for (SDValue V : VectorsToShuffle) {3268 ReturningInteger = DAG.getNode(3269 ISD::SHL, DL, ReturnType,3270 {DAG.getShiftAmountConstant(16, ReturnType, DL), ReturningInteger});3271 3272 SDValue ExtendedV = DAG.getZExtOrTrunc(V, DL, ReturnType);3273 ReturningInteger =3274 DAG.getNode(ISD::ADD, DL, ReturnType, {ReturningInteger, ExtendedV});3275 }3276 3277 return ReturningInteger;3278 }3279 3280 return SDValue();3281}3282 3283static SDValue performAnyAllCombine(SDNode *N, SelectionDAG &DAG) {3284 // any_true (setcc <X>, 0, eq) => (not (all_true X))3285 // all_true (setcc <X>, 0, eq) => (not (any_true X))3286 // any_true (setcc <X>, 0, ne) => (any_true X)3287 // all_true (setcc <X>, 0, ne) => (all_true X)3288 assert(N->getOpcode() == ISD::INTRINSIC_WO_CHAIN);3289 using namespace llvm::SDPatternMatch;3290 3291 SDValue LHS;3292 if (N->getNumOperands() < 2 ||3293 !sd_match(N->getOperand(1),3294 m_c_SetCC(m_Value(LHS), m_Zero(), m_CondCode())))3295 return SDValue();3296 EVT LT = LHS.getValueType();3297 if (LT.getScalarSizeInBits() > 128 / LT.getVectorNumElements())3298 return SDValue();3299 3300 auto CombineSetCC = [&N, &DAG](Intrinsic::WASMIntrinsics InPre,3301 ISD::CondCode SetType,3302 Intrinsic::WASMIntrinsics InPost) {3303 if (N->getConstantOperandVal(0) != InPre)3304 return SDValue();3305 3306 SDValue LHS;3307 if (!sd_match(N->getOperand(1), m_c_SetCC(m_Value(LHS), m_Zero(),3308 m_SpecificCondCode(SetType))))3309 return SDValue();3310 3311 SDLoc DL(N);3312 SDValue Ret = DAG.getZExtOrTrunc(3313 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,3314 {DAG.getConstant(InPost, DL, MVT::i32), LHS}),3315 DL, MVT::i1);3316 if (SetType == ISD::SETEQ)3317 Ret = DAG.getNOT(DL, Ret, MVT::i1);3318 return DAG.getZExtOrTrunc(Ret, DL, N->getValueType(0));3319 };3320 3321 if (SDValue AnyTrueEQ = CombineSetCC(Intrinsic::wasm_anytrue, ISD::SETEQ,3322 Intrinsic::wasm_alltrue))3323 return AnyTrueEQ;3324 if (SDValue AllTrueEQ = CombineSetCC(Intrinsic::wasm_alltrue, ISD::SETEQ,3325 Intrinsic::wasm_anytrue))3326 return AllTrueEQ;3327 if (SDValue AnyTrueNE = CombineSetCC(Intrinsic::wasm_anytrue, ISD::SETNE,3328 Intrinsic::wasm_anytrue))3329 return AnyTrueNE;3330 if (SDValue AllTrueNE = CombineSetCC(Intrinsic::wasm_alltrue, ISD::SETNE,3331 Intrinsic::wasm_alltrue))3332 return AllTrueNE;3333 3334 return SDValue();3335}3336 3337template <int MatchRHS, ISD::CondCode MatchCond, bool RequiresNegate,3338 Intrinsic::ID Intrin>3339static SDValue TryMatchTrue(SDNode *N, EVT VecVT, SelectionDAG &DAG) {3340 SDValue LHS = N->getOperand(0);3341 SDValue RHS = N->getOperand(1);3342 SDValue Cond = N->getOperand(2);3343 if (MatchCond != cast<CondCodeSDNode>(Cond)->get())3344 return SDValue();3345 3346 if (MatchRHS != cast<ConstantSDNode>(RHS)->getSExtValue())3347 return SDValue();3348 3349 SDLoc DL(N);3350 SDValue Ret = DAG.getZExtOrTrunc(3351 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,3352 {DAG.getConstant(Intrin, DL, MVT::i32),3353 DAG.getSExtOrTrunc(LHS->getOperand(0), DL, VecVT)}),3354 DL, MVT::i1);3355 if (RequiresNegate)3356 Ret = DAG.getNOT(DL, Ret, MVT::i1);3357 return DAG.getZExtOrTrunc(Ret, DL, N->getValueType(0));3358}3359 3360/// Try to convert a i128 comparison to a v16i8 comparison before type3361/// legalization splits it up into chunks3362static SDValue3363combineVectorSizedSetCCEquality(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,3364 const WebAssemblySubtarget *Subtarget) {3365 3366 SDLoc DL(N);3367 SDValue X = N->getOperand(0);3368 SDValue Y = N->getOperand(1);3369 EVT VT = N->getValueType(0);3370 EVT OpVT = X.getValueType();3371 3372 SelectionDAG &DAG = DCI.DAG;3373 if (DCI.DAG.getMachineFunction().getFunction().hasFnAttribute(3374 Attribute::NoImplicitFloat))3375 return SDValue();3376 3377 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();3378 // We're looking for an oversized integer equality comparison with SIMD3379 if (!OpVT.isScalarInteger() || !OpVT.isByteSized() || OpVT != MVT::i128 ||3380 !Subtarget->hasSIMD128() || !isIntEqualitySetCC(CC))3381 return SDValue();3382 3383 // Don't perform this combine if constructing the vector will be expensive.3384 auto IsVectorBitCastCheap = [](SDValue X) {3385 X = peekThroughBitcasts(X);3386 return isa<ConstantSDNode>(X) || X.getOpcode() == ISD::LOAD;3387 };3388 3389 if (!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y))3390 return SDValue();3391 3392 SDValue VecX = DAG.getBitcast(MVT::v16i8, X);3393 SDValue VecY = DAG.getBitcast(MVT::v16i8, Y);3394 SDValue Cmp = DAG.getSetCC(DL, MVT::v16i8, VecX, VecY, CC);3395 3396 SDValue Intr =3397 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,3398 {DAG.getConstant(CC == ISD::SETEQ ? Intrinsic::wasm_alltrue3399 : Intrinsic::wasm_anytrue,3400 DL, MVT::i32),3401 Cmp});3402 3403 return DAG.getSetCC(DL, VT, Intr, DAG.getConstant(0, DL, MVT::i32),3404 ISD::SETNE);3405}3406 3407static SDValue performSETCCCombine(SDNode *N,3408 TargetLowering::DAGCombinerInfo &DCI,3409 const WebAssemblySubtarget *Subtarget) {3410 if (!DCI.isBeforeLegalize())3411 return SDValue();3412 3413 EVT VT = N->getValueType(0);3414 if (!VT.isScalarInteger())3415 return SDValue();3416 3417 if (SDValue V = combineVectorSizedSetCCEquality(N, DCI, Subtarget))3418 return V;3419 3420 SDValue LHS = N->getOperand(0);3421 if (LHS->getOpcode() != ISD::BITCAST)3422 return SDValue();3423 3424 EVT FromVT = LHS->getOperand(0).getValueType();3425 if (!FromVT.isFixedLengthVector() || FromVT.getVectorElementType() != MVT::i1)3426 return SDValue();3427 3428 unsigned NumElts = FromVT.getVectorNumElements();3429 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)3430 return SDValue();3431 3432 if (!cast<ConstantSDNode>(N->getOperand(1)))3433 return SDValue();3434 3435 EVT VecVT = FromVT.changeVectorElementType(MVT::getIntegerVT(128 / NumElts));3436 auto &DAG = DCI.DAG;3437 // setcc (iN (bitcast (vNi1 X))), 0, ne3438 // ==> any_true (vNi1 X)3439 if (auto Match = TryMatchTrue<0, ISD::SETNE, false, Intrinsic::wasm_anytrue>(3440 N, VecVT, DAG)) {3441 return Match;3442 }3443 // setcc (iN (bitcast (vNi1 X))), 0, eq3444 // ==> xor (any_true (vNi1 X)), -13445 if (auto Match = TryMatchTrue<0, ISD::SETEQ, true, Intrinsic::wasm_anytrue>(3446 N, VecVT, DAG)) {3447 return Match;3448 }3449 // setcc (iN (bitcast (vNi1 X))), -1, eq3450 // ==> all_true (vNi1 X)3451 if (auto Match = TryMatchTrue<-1, ISD::SETEQ, false, Intrinsic::wasm_alltrue>(3452 N, VecVT, DAG)) {3453 return Match;3454 }3455 // setcc (iN (bitcast (vNi1 X))), -1, ne3456 // ==> xor (all_true (vNi1 X)), -13457 if (auto Match = TryMatchTrue<-1, ISD::SETNE, true, Intrinsic::wasm_alltrue>(3458 N, VecVT, DAG)) {3459 return Match;3460 }3461 return SDValue();3462}3463 3464static SDValue TryWideExtMulCombine(SDNode *N, SelectionDAG &DAG) {3465 EVT VT = N->getValueType(0);3466 if (VT != MVT::v8i32 && VT != MVT::v16i32)3467 return SDValue();3468 3469 // Mul with extending inputs.3470 SDValue LHS = N->getOperand(0);3471 SDValue RHS = N->getOperand(1);3472 if (LHS.getOpcode() != RHS.getOpcode())3473 return SDValue();3474 3475 if (LHS.getOpcode() != ISD::SIGN_EXTEND &&3476 LHS.getOpcode() != ISD::ZERO_EXTEND)3477 return SDValue();3478 3479 if (LHS->getOperand(0).getValueType() != RHS->getOperand(0).getValueType())3480 return SDValue();3481 3482 EVT FromVT = LHS->getOperand(0).getValueType();3483 EVT EltTy = FromVT.getVectorElementType();3484 if (EltTy != MVT::i8)3485 return SDValue();3486 3487 // For an input DAG that looks like this3488 // %a = input_type3489 // %b = input_type3490 // %lhs = extend %a to output_type3491 // %rhs = extend %b to output_type3492 // %mul = mul %lhs, %rhs3493 3494 // input_type | output_type | instructions3495 // v16i8 | v16i32 | %low = i16x8.extmul_low_i8x16_ %a, %b3496 // | | %high = i16x8.extmul_high_i8x16_, %a, %b3497 // | | %low_low = i32x4.ext_low_i16x8_ %low3498 // | | %low_high = i32x4.ext_high_i16x8_ %low3499 // | | %high_low = i32x4.ext_low_i16x8_ %high3500 // | | %high_high = i32x4.ext_high_i16x8_ %high3501 // | | %res = concat_vector(...)3502 // v8i8 | v8i32 | %low = i16x8.extmul_low_i8x16_ %a, %b3503 // | | %low_low = i32x4.ext_low_i16x8_ %low3504 // | | %low_high = i32x4.ext_high_i16x8_ %low3505 // | | %res = concat_vector(%low_low, %low_high)3506 3507 SDLoc DL(N);3508 unsigned NumElts = VT.getVectorNumElements();3509 SDValue ExtendInLHS = LHS->getOperand(0);3510 SDValue ExtendInRHS = RHS->getOperand(0);3511 bool IsSigned = LHS->getOpcode() == ISD::SIGN_EXTEND;3512 unsigned ExtendLowOpc =3513 IsSigned ? WebAssemblyISD::EXTEND_LOW_S : WebAssemblyISD::EXTEND_LOW_U;3514 unsigned ExtendHighOpc =3515 IsSigned ? WebAssemblyISD::EXTEND_HIGH_S : WebAssemblyISD::EXTEND_HIGH_U;3516 3517 auto GetExtendLow = [&DAG, &DL, &ExtendLowOpc](EVT VT, SDValue Op) {3518 return DAG.getNode(ExtendLowOpc, DL, VT, Op);3519 };3520 auto GetExtendHigh = [&DAG, &DL, &ExtendHighOpc](EVT VT, SDValue Op) {3521 return DAG.getNode(ExtendHighOpc, DL, VT, Op);3522 };3523 3524 if (NumElts == 16) {3525 SDValue LowLHS = GetExtendLow(MVT::v8i16, ExtendInLHS);3526 SDValue LowRHS = GetExtendLow(MVT::v8i16, ExtendInRHS);3527 SDValue MulLow = DAG.getNode(ISD::MUL, DL, MVT::v8i16, LowLHS, LowRHS);3528 SDValue HighLHS = GetExtendHigh(MVT::v8i16, ExtendInLHS);3529 SDValue HighRHS = GetExtendHigh(MVT::v8i16, ExtendInRHS);3530 SDValue MulHigh = DAG.getNode(ISD::MUL, DL, MVT::v8i16, HighLHS, HighRHS);3531 SDValue SubVectors[] = {3532 GetExtendLow(MVT::v4i32, MulLow),3533 GetExtendHigh(MVT::v4i32, MulLow),3534 GetExtendLow(MVT::v4i32, MulHigh),3535 GetExtendHigh(MVT::v4i32, MulHigh),3536 };3537 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SubVectors);3538 } else {3539 assert(NumElts == 8);3540 SDValue LowLHS = DAG.getNode(LHS->getOpcode(), DL, MVT::v8i16, ExtendInLHS);3541 SDValue LowRHS = DAG.getNode(RHS->getOpcode(), DL, MVT::v8i16, ExtendInRHS);3542 SDValue MulLow = DAG.getNode(ISD::MUL, DL, MVT::v8i16, LowLHS, LowRHS);3543 SDValue Lo = GetExtendLow(MVT::v4i32, MulLow);3544 SDValue Hi = GetExtendHigh(MVT::v4i32, MulLow);3545 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);3546 }3547 return SDValue();3548}3549 3550static SDValue performMulCombine(SDNode *N,3551 TargetLowering::DAGCombinerInfo &DCI) {3552 assert(N->getOpcode() == ISD::MUL);3553 EVT VT = N->getValueType(0);3554 if (!VT.isVector())3555 return SDValue();3556 3557 if (auto Res = TryWideExtMulCombine(N, DCI.DAG))3558 return Res;3559 3560 // We don't natively support v16i8 or v8i8 mul, but we do support v8i16. So,3561 // extend them to v8i16. Only do this before legalization in case a narrow3562 // vector is widened and may be simplified later.3563 if (!DCI.isBeforeLegalize() || (VT != MVT::v8i8 && VT != MVT::v16i8))3564 return SDValue();3565 3566 SDLoc DL(N);3567 SelectionDAG &DAG = DCI.DAG;3568 SDValue LHS = N->getOperand(0);3569 SDValue RHS = N->getOperand(1);3570 EVT MulVT = MVT::v8i16;3571 3572 if (VT == MVT::v8i8) {3573 SDValue PromotedLHS = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, LHS,3574 DAG.getUNDEF(MVT::v8i8));3575 SDValue PromotedRHS = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, RHS,3576 DAG.getUNDEF(MVT::v8i8));3577 SDValue LowLHS =3578 DAG.getNode(WebAssemblyISD::EXTEND_LOW_U, DL, MulVT, PromotedLHS);3579 SDValue LowRHS =3580 DAG.getNode(WebAssemblyISD::EXTEND_LOW_U, DL, MulVT, PromotedRHS);3581 SDValue MulLow = DAG.getBitcast(3582 MVT::v16i8, DAG.getNode(ISD::MUL, DL, MulVT, LowLHS, LowRHS));3583 // Take the low byte of each lane.3584 SDValue Shuffle = DAG.getVectorShuffle(3585 MVT::v16i8, DL, MulLow, DAG.getUNDEF(MVT::v16i8),3586 {0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1});3587 return extractSubVector(Shuffle, 0, DAG, DL, 64);3588 } else {3589 assert(VT == MVT::v16i8 && "Expected v16i8");3590 SDValue LowLHS = DAG.getNode(WebAssemblyISD::EXTEND_LOW_U, DL, MulVT, LHS);3591 SDValue LowRHS = DAG.getNode(WebAssemblyISD::EXTEND_LOW_U, DL, MulVT, RHS);3592 SDValue HighLHS =3593 DAG.getNode(WebAssemblyISD::EXTEND_HIGH_U, DL, MulVT, LHS);3594 SDValue HighRHS =3595 DAG.getNode(WebAssemblyISD::EXTEND_HIGH_U, DL, MulVT, RHS);3596 3597 SDValue MulLow =3598 DAG.getBitcast(VT, DAG.getNode(ISD::MUL, DL, MulVT, LowLHS, LowRHS));3599 SDValue MulHigh =3600 DAG.getBitcast(VT, DAG.getNode(ISD::MUL, DL, MulVT, HighLHS, HighRHS));3601 3602 // Take the low byte of each lane.3603 return DAG.getVectorShuffle(3604 VT, DL, MulLow, MulHigh,3605 {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30});3606 }3607}3608 3609SDValue DoubleVectorWidth(SDValue In, unsigned RequiredNumElems,3610 SelectionDAG &DAG) {3611 SDLoc DL(In);3612 LLVMContext &Ctx = *DAG.getContext();3613 EVT InVT = In.getValueType();3614 unsigned NumElems = InVT.getVectorNumElements() * 2;3615 EVT OutVT = EVT::getVectorVT(Ctx, InVT.getVectorElementType(), NumElems);3616 SDValue Concat =3617 DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, In, DAG.getPOISON(InVT));3618 if (NumElems < RequiredNumElems) {3619 return DoubleVectorWidth(Concat, RequiredNumElems, DAG);3620 }3621 return Concat;3622}3623 3624SDValue performConvertFPCombine(SDNode *N, SelectionDAG &DAG) {3625 EVT OutVT = N->getValueType(0);3626 if (!OutVT.isVector())3627 return SDValue();3628 3629 EVT OutElTy = OutVT.getVectorElementType();3630 if (OutElTy != MVT::i8 && OutElTy != MVT::i16)3631 return SDValue();3632 3633 unsigned NumElems = OutVT.getVectorNumElements();3634 if (!isPowerOf2_32(NumElems))3635 return SDValue();3636 3637 EVT FPVT = N->getOperand(0)->getValueType(0);3638 if (FPVT.getVectorElementType() != MVT::f32)3639 return SDValue();3640 3641 SDLoc DL(N);3642 3643 // First, convert to i32.3644 LLVMContext &Ctx = *DAG.getContext();3645 EVT IntVT = EVT::getVectorVT(Ctx, MVT::i32, NumElems);3646 SDValue ToInt = DAG.getNode(N->getOpcode(), DL, IntVT, N->getOperand(0));3647 APInt Mask = APInt::getLowBitsSet(IntVT.getScalarSizeInBits(),3648 OutVT.getScalarSizeInBits());3649 // Mask out the top MSBs.3650 SDValue Masked =3651 DAG.getNode(ISD::AND, DL, IntVT, ToInt, DAG.getConstant(Mask, DL, IntVT));3652 3653 if (OutVT.getSizeInBits() < 128) {3654 // Create a wide enough vector that we can use narrow.3655 EVT NarrowedVT = OutElTy == MVT::i8 ? MVT::v16i8 : MVT::v8i16;3656 unsigned NumRequiredElems = NarrowedVT.getVectorNumElements();3657 SDValue WideVector = DoubleVectorWidth(Masked, NumRequiredElems, DAG);3658 SDValue Trunc = truncateVectorWithNARROW(NarrowedVT, WideVector, DL, DAG);3659 return DAG.getBitcast(3660 OutVT, extractSubVector(Trunc, 0, DAG, DL, OutVT.getSizeInBits()));3661 } else {3662 return truncateVectorWithNARROW(OutVT, Masked, DL, DAG);3663 }3664 return SDValue();3665}3666 3667SDValue3668WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,3669 DAGCombinerInfo &DCI) const {3670 switch (N->getOpcode()) {3671 default:3672 return SDValue();3673 case ISD::BITCAST:3674 return performBitcastCombine(N, DCI);3675 case ISD::SETCC:3676 return performSETCCCombine(N, DCI, Subtarget);3677 case ISD::VECTOR_SHUFFLE:3678 return performVECTOR_SHUFFLECombine(N, DCI);3679 case ISD::SIGN_EXTEND:3680 case ISD::ZERO_EXTEND:3681 return performVectorExtendCombine(N, DCI);3682 case ISD::UINT_TO_FP:3683 if (auto ExtCombine = performVectorExtendToFPCombine(N, DCI))3684 return ExtCombine;3685 return performVectorNonNegToFPCombine(N, DCI);3686 case ISD::SINT_TO_FP:3687 return performVectorExtendToFPCombine(N, DCI);3688 case ISD::FP_TO_SINT_SAT:3689 case ISD::FP_TO_UINT_SAT:3690 case ISD::FP_ROUND:3691 case ISD::CONCAT_VECTORS:3692 return performVectorTruncZeroCombine(N, DCI);3693 case ISD::FP_TO_SINT:3694 case ISD::FP_TO_UINT:3695 return performConvertFPCombine(N, DCI.DAG);3696 case ISD::TRUNCATE:3697 return performTruncateCombine(N, DCI);3698 case ISD::INTRINSIC_WO_CHAIN:3699 return performAnyAllCombine(N, DCI.DAG);3700 case ISD::MUL:3701 return performMulCombine(N, DCI);3702 }3703}3704