brintos / llvm-project-archived public Read only

1//===- TargetLoweringBase.cpp - Implement the TargetLoweringBase class ----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This implements the TargetLoweringBase class.10//11//===----------------------------------------------------------------------===//12 13#include "llvm/ADT/BitVector.h"14#include "llvm/ADT/DenseMap.h"15#include "llvm/ADT/STLExtras.h"16#include "llvm/ADT/SmallVector.h"17#include "llvm/ADT/StringExtras.h"18#include "llvm/ADT/StringRef.h"19#include "llvm/ADT/Twine.h"20#include "llvm/Analysis/Loads.h"21#include "llvm/Analysis/TargetTransformInfo.h"22#include "llvm/CodeGen/Analysis.h"23#include "llvm/CodeGen/ISDOpcodes.h"24#include "llvm/CodeGen/MachineBasicBlock.h"25#include "llvm/CodeGen/MachineFrameInfo.h"26#include "llvm/CodeGen/MachineFunction.h"27#include "llvm/CodeGen/MachineInstr.h"28#include "llvm/CodeGen/MachineInstrBuilder.h"29#include "llvm/CodeGen/MachineMemOperand.h"30#include "llvm/CodeGen/MachineOperand.h"31#include "llvm/CodeGen/MachineRegisterInfo.h"32#include "llvm/CodeGen/RuntimeLibcallUtil.h"33#include "llvm/CodeGen/StackMaps.h"34#include "llvm/CodeGen/TargetLowering.h"35#include "llvm/CodeGen/TargetOpcodes.h"36#include "llvm/CodeGen/TargetRegisterInfo.h"37#include "llvm/CodeGen/ValueTypes.h"38#include "llvm/CodeGenTypes/MachineValueType.h"39#include "llvm/IR/Attributes.h"40#include "llvm/IR/CallingConv.h"41#include "llvm/IR/DataLayout.h"42#include "llvm/IR/DerivedTypes.h"43#include "llvm/IR/Function.h"44#include "llvm/IR/GlobalValue.h"45#include "llvm/IR/GlobalVariable.h"46#include "llvm/IR/IRBuilder.h"47#include "llvm/IR/Module.h"48#include "llvm/IR/Type.h"49#include "llvm/Support/Casting.h"50#include "llvm/Support/CommandLine.h"51#include "llvm/Support/Compiler.h"52#include "llvm/Support/ErrorHandling.h"53#include "llvm/Support/MathExtras.h"54#include "llvm/Target/TargetMachine.h"55#include "llvm/Target/TargetOptions.h"56#include "llvm/TargetParser/Triple.h"57#include "llvm/Transforms/Utils/SizeOpts.h"58#include <algorithm>59#include <cassert>60#include <cstdint>61#include <cstring>62#include <string>63#include <tuple>64#include <utility>65 66using namespace llvm;67 68static cl::opt<bool> JumpIsExpensiveOverride(69    "jump-is-expensive", cl::init(false),70    cl::desc("Do not create extra branches to split comparison logic."),71    cl::Hidden);72 73static cl::opt<unsigned> MinimumJumpTableEntries74  ("min-jump-table-entries", cl::init(4), cl::Hidden,75   cl::desc("Set minimum number of entries to use a jump table."));76 77static cl::opt<unsigned> MaximumJumpTableSize78  ("max-jump-table-size", cl::init(UINT_MAX), cl::Hidden,79   cl::desc("Set maximum size of jump tables."));80 81/// Minimum jump table density for normal functions.82static cl::opt<unsigned>83    JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden,84                     cl::desc("Minimum density for building a jump table in "85                              "a normal function"));86 87/// Minimum jump table density for -Os or -Oz functions.88static cl::opt<unsigned> OptsizeJumpTableDensity(89    "optsize-jump-table-density", cl::init(40), cl::Hidden,90    cl::desc("Minimum density for building a jump table in "91             "an optsize function"));92 93static cl::opt<unsigned> MinimumBitTestCmpsOverride(94    "min-bit-test-cmps", cl::init(2), cl::Hidden,95    cl::desc("Set minimum of largest number of comparisons "96             "to use bit test for switch."));97 98// FIXME: This option is only to test if the strict fp operation processed99// correctly by preventing mutating strict fp operation to normal fp operation100// during development. When the backend supports strict float operation, this101// option will be meaningless.102static cl::opt<bool> DisableStrictNodeMutation("disable-strictnode-mutation",103       cl::desc("Don't mutate strict-float node to a legalize node"),104       cl::init(false), cl::Hidden);105 106/// GetFPLibCall - Helper to return the right libcall for the given floating107/// point type, or UNKNOWN_LIBCALL if there is none.108RTLIB::Libcall RTLIB::getFPLibCall(EVT VT,109                                   RTLIB::Libcall Call_F32,110                                   RTLIB::Libcall Call_F64,111                                   RTLIB::Libcall Call_F80,112                                   RTLIB::Libcall Call_F128,113                                   RTLIB::Libcall Call_PPCF128) {114  return115    VT == MVT::f32 ? Call_F32 :116    VT == MVT::f64 ? Call_F64 :117    VT == MVT::f80 ? Call_F80 :118    VT == MVT::f128 ? Call_F128 :119    VT == MVT::ppcf128 ? Call_PPCF128 :120    RTLIB::UNKNOWN_LIBCALL;121}122 123/// getFPEXT - Return the FPEXT_*_* value for the given types, or124/// UNKNOWN_LIBCALL if there is none.125RTLIB::Libcall RTLIB::getFPEXT(EVT OpVT, EVT RetVT) {126  if (OpVT == MVT::f16) {127    if (RetVT == MVT::f32)128      return FPEXT_F16_F32;129    if (RetVT == MVT::f64)130      return FPEXT_F16_F64;131    if (RetVT == MVT::f80)132      return FPEXT_F16_F80;133    if (RetVT == MVT::f128)134      return FPEXT_F16_F128;135  } else if (OpVT == MVT::f32) {136    if (RetVT == MVT::f64)137      return FPEXT_F32_F64;138    if (RetVT == MVT::f128)139      return FPEXT_F32_F128;140    if (RetVT == MVT::ppcf128)141      return FPEXT_F32_PPCF128;142  } else if (OpVT == MVT::f64) {143    if (RetVT == MVT::f128)144      return FPEXT_F64_F128;145    else if (RetVT == MVT::ppcf128)146      return FPEXT_F64_PPCF128;147  } else if (OpVT == MVT::f80) {148    if (RetVT == MVT::f128)149      return FPEXT_F80_F128;150  } else if (OpVT == MVT::bf16) {151    if (RetVT == MVT::f32)152      return FPEXT_BF16_F32;153  }154 155  return UNKNOWN_LIBCALL;156}157 158/// getFPROUND - Return the FPROUND_*_* value for the given types, or159/// UNKNOWN_LIBCALL if there is none.160RTLIB::Libcall RTLIB::getFPROUND(EVT OpVT, EVT RetVT) {161  if (RetVT == MVT::f16) {162    if (OpVT == MVT::f32)163      return FPROUND_F32_F16;164    if (OpVT == MVT::f64)165      return FPROUND_F64_F16;166    if (OpVT == MVT::f80)167      return FPROUND_F80_F16;168    if (OpVT == MVT::f128)169      return FPROUND_F128_F16;170    if (OpVT == MVT::ppcf128)171      return FPROUND_PPCF128_F16;172  } else if (RetVT == MVT::bf16) {173    if (OpVT == MVT::f32)174      return FPROUND_F32_BF16;175    if (OpVT == MVT::f64)176      return FPROUND_F64_BF16;177    if (OpVT == MVT::f80)178      return FPROUND_F80_BF16;179    if (OpVT == MVT::f128)180      return FPROUND_F128_BF16;181  } else if (RetVT == MVT::f32) {182    if (OpVT == MVT::f64)183      return FPROUND_F64_F32;184    if (OpVT == MVT::f80)185      return FPROUND_F80_F32;186    if (OpVT == MVT::f128)187      return FPROUND_F128_F32;188    if (OpVT == MVT::ppcf128)189      return FPROUND_PPCF128_F32;190  } else if (RetVT == MVT::f64) {191    if (OpVT == MVT::f80)192      return FPROUND_F80_F64;193    if (OpVT == MVT::f128)194      return FPROUND_F128_F64;195    if (OpVT == MVT::ppcf128)196      return FPROUND_PPCF128_F64;197  } else if (RetVT == MVT::f80) {198    if (OpVT == MVT::f128)199      return FPROUND_F128_F80;200  }201 202  return UNKNOWN_LIBCALL;203}204 205/// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or206/// UNKNOWN_LIBCALL if there is none.207RTLIB::Libcall RTLIB::getFPTOSINT(EVT OpVT, EVT RetVT) {208  if (OpVT == MVT::f16) {209    if (RetVT == MVT::i32)210      return FPTOSINT_F16_I32;211    if (RetVT == MVT::i64)212      return FPTOSINT_F16_I64;213    if (RetVT == MVT::i128)214      return FPTOSINT_F16_I128;215  } else if (OpVT == MVT::f32) {216    if (RetVT == MVT::i32)217      return FPTOSINT_F32_I32;218    if (RetVT == MVT::i64)219      return FPTOSINT_F32_I64;220    if (RetVT == MVT::i128)221      return FPTOSINT_F32_I128;222  } else if (OpVT == MVT::f64) {223    if (RetVT == MVT::i32)224      return FPTOSINT_F64_I32;225    if (RetVT == MVT::i64)226      return FPTOSINT_F64_I64;227    if (RetVT == MVT::i128)228      return FPTOSINT_F64_I128;229  } else if (OpVT == MVT::f80) {230    if (RetVT == MVT::i32)231      return FPTOSINT_F80_I32;232    if (RetVT == MVT::i64)233      return FPTOSINT_F80_I64;234    if (RetVT == MVT::i128)235      return FPTOSINT_F80_I128;236  } else if (OpVT == MVT::f128) {237    if (RetVT == MVT::i32)238      return FPTOSINT_F128_I32;239    if (RetVT == MVT::i64)240      return FPTOSINT_F128_I64;241    if (RetVT == MVT::i128)242      return FPTOSINT_F128_I128;243  } else if (OpVT == MVT::ppcf128) {244    if (RetVT == MVT::i32)245      return FPTOSINT_PPCF128_I32;246    if (RetVT == MVT::i64)247      return FPTOSINT_PPCF128_I64;248    if (RetVT == MVT::i128)249      return FPTOSINT_PPCF128_I128;250  }251  return UNKNOWN_LIBCALL;252}253 254/// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or255/// UNKNOWN_LIBCALL if there is none.256RTLIB::Libcall RTLIB::getFPTOUINT(EVT OpVT, EVT RetVT) {257  if (OpVT == MVT::f16) {258    if (RetVT == MVT::i32)259      return FPTOUINT_F16_I32;260    if (RetVT == MVT::i64)261      return FPTOUINT_F16_I64;262    if (RetVT == MVT::i128)263      return FPTOUINT_F16_I128;264  } else if (OpVT == MVT::f32) {265    if (RetVT == MVT::i32)266      return FPTOUINT_F32_I32;267    if (RetVT == MVT::i64)268      return FPTOUINT_F32_I64;269    if (RetVT == MVT::i128)270      return FPTOUINT_F32_I128;271  } else if (OpVT == MVT::f64) {272    if (RetVT == MVT::i32)273      return FPTOUINT_F64_I32;274    if (RetVT == MVT::i64)275      return FPTOUINT_F64_I64;276    if (RetVT == MVT::i128)277      return FPTOUINT_F64_I128;278  } else if (OpVT == MVT::f80) {279    if (RetVT == MVT::i32)280      return FPTOUINT_F80_I32;281    if (RetVT == MVT::i64)282      return FPTOUINT_F80_I64;283    if (RetVT == MVT::i128)284      return FPTOUINT_F80_I128;285  } else if (OpVT == MVT::f128) {286    if (RetVT == MVT::i32)287      return FPTOUINT_F128_I32;288    if (RetVT == MVT::i64)289      return FPTOUINT_F128_I64;290    if (RetVT == MVT::i128)291      return FPTOUINT_F128_I128;292  } else if (OpVT == MVT::ppcf128) {293    if (RetVT == MVT::i32)294      return FPTOUINT_PPCF128_I32;295    if (RetVT == MVT::i64)296      return FPTOUINT_PPCF128_I64;297    if (RetVT == MVT::i128)298      return FPTOUINT_PPCF128_I128;299  }300  return UNKNOWN_LIBCALL;301}302 303/// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or304/// UNKNOWN_LIBCALL if there is none.305RTLIB::Libcall RTLIB::getSINTTOFP(EVT OpVT, EVT RetVT) {306  if (OpVT == MVT::i32) {307    if (RetVT == MVT::f16)308      return SINTTOFP_I32_F16;309    if (RetVT == MVT::f32)310      return SINTTOFP_I32_F32;311    if (RetVT == MVT::f64)312      return SINTTOFP_I32_F64;313    if (RetVT == MVT::f80)314      return SINTTOFP_I32_F80;315    if (RetVT == MVT::f128)316      return SINTTOFP_I32_F128;317    if (RetVT == MVT::ppcf128)318      return SINTTOFP_I32_PPCF128;319  } else if (OpVT == MVT::i64) {320    if (RetVT == MVT::bf16)321      return SINTTOFP_I64_BF16;322    if (RetVT == MVT::f16)323      return SINTTOFP_I64_F16;324    if (RetVT == MVT::f32)325      return SINTTOFP_I64_F32;326    if (RetVT == MVT::f64)327      return SINTTOFP_I64_F64;328    if (RetVT == MVT::f80)329      return SINTTOFP_I64_F80;330    if (RetVT == MVT::f128)331      return SINTTOFP_I64_F128;332    if (RetVT == MVT::ppcf128)333      return SINTTOFP_I64_PPCF128;334  } else if (OpVT == MVT::i128) {335    if (RetVT == MVT::f16)336      return SINTTOFP_I128_F16;337    if (RetVT == MVT::f32)338      return SINTTOFP_I128_F32;339    if (RetVT == MVT::f64)340      return SINTTOFP_I128_F64;341    if (RetVT == MVT::f80)342      return SINTTOFP_I128_F80;343    if (RetVT == MVT::f128)344      return SINTTOFP_I128_F128;345    if (RetVT == MVT::ppcf128)346      return SINTTOFP_I128_PPCF128;347  }348  return UNKNOWN_LIBCALL;349}350 351/// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or352/// UNKNOWN_LIBCALL if there is none.353RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) {354  if (OpVT == MVT::i32) {355    if (RetVT == MVT::f16)356      return UINTTOFP_I32_F16;357    if (RetVT == MVT::f32)358      return UINTTOFP_I32_F32;359    if (RetVT == MVT::f64)360      return UINTTOFP_I32_F64;361    if (RetVT == MVT::f80)362      return UINTTOFP_I32_F80;363    if (RetVT == MVT::f128)364      return UINTTOFP_I32_F128;365    if (RetVT == MVT::ppcf128)366      return UINTTOFP_I32_PPCF128;367  } else if (OpVT == MVT::i64) {368    if (RetVT == MVT::bf16)369      return UINTTOFP_I64_BF16;370    if (RetVT == MVT::f16)371      return UINTTOFP_I64_F16;372    if (RetVT == MVT::f32)373      return UINTTOFP_I64_F32;374    if (RetVT == MVT::f64)375      return UINTTOFP_I64_F64;376    if (RetVT == MVT::f80)377      return UINTTOFP_I64_F80;378    if (RetVT == MVT::f128)379      return UINTTOFP_I64_F128;380    if (RetVT == MVT::ppcf128)381      return UINTTOFP_I64_PPCF128;382  } else if (OpVT == MVT::i128) {383    if (RetVT == MVT::f16)384      return UINTTOFP_I128_F16;385    if (RetVT == MVT::f32)386      return UINTTOFP_I128_F32;387    if (RetVT == MVT::f64)388      return UINTTOFP_I128_F64;389    if (RetVT == MVT::f80)390      return UINTTOFP_I128_F80;391    if (RetVT == MVT::f128)392      return UINTTOFP_I128_F128;393    if (RetVT == MVT::ppcf128)394      return UINTTOFP_I128_PPCF128;395  }396  return UNKNOWN_LIBCALL;397}398 399RTLIB::Libcall RTLIB::getPOWI(EVT RetVT) {400  return getFPLibCall(RetVT, POWI_F32, POWI_F64, POWI_F80, POWI_F128,401                      POWI_PPCF128);402}403 404RTLIB::Libcall RTLIB::getPOW(EVT RetVT) {405  return getFPLibCall(RetVT, POW_F32, POW_F64, POW_F80, POW_F128, POW_PPCF128);406}407 408RTLIB::Libcall RTLIB::getLDEXP(EVT RetVT) {409  return getFPLibCall(RetVT, LDEXP_F32, LDEXP_F64, LDEXP_F80, LDEXP_F128,410                      LDEXP_PPCF128);411}412 413RTLIB::Libcall RTLIB::getFREXP(EVT RetVT) {414  return getFPLibCall(RetVT, FREXP_F32, FREXP_F64, FREXP_F80, FREXP_F128,415                      FREXP_PPCF128);416}417 418RTLIB::Libcall RTLIB::getSIN(EVT RetVT) {419  return getFPLibCall(RetVT, SIN_F32, SIN_F64, SIN_F80, SIN_F128, SIN_PPCF128);420}421 422RTLIB::Libcall RTLIB::getCOS(EVT RetVT) {423  return getFPLibCall(RetVT, COS_F32, COS_F64, COS_F80, COS_F128, COS_PPCF128);424}425 426RTLIB::Libcall RTLIB::getSINCOS(EVT RetVT) {427  // TODO: Tablegen should generate this function428  if (RetVT.isVector()) {429    if (!RetVT.isSimple())430      return RTLIB::UNKNOWN_LIBCALL;431    switch (RetVT.getSimpleVT().SimpleTy) {432    case MVT::v4f32:433      return RTLIB::SINCOS_V4F32;434    case MVT::v2f64:435      return RTLIB::SINCOS_V2F64;436    case MVT::nxv4f32:437      return RTLIB::SINCOS_NXV4F32;438    case MVT::nxv2f64:439      return RTLIB::SINCOS_NXV2F64;440    default:441      return RTLIB::UNKNOWN_LIBCALL;442    }443  }444 445  return getFPLibCall(RetVT, SINCOS_F32, SINCOS_F64, SINCOS_F80, SINCOS_F128,446                      SINCOS_PPCF128);447}448 449RTLIB::Libcall RTLIB::getSINCOSPI(EVT RetVT) {450  // TODO: Tablegen should generate this function451  if (RetVT.isVector()) {452    if (!RetVT.isSimple())453      return RTLIB::UNKNOWN_LIBCALL;454    switch (RetVT.getSimpleVT().SimpleTy) {455    case MVT::v4f32:456      return RTLIB::SINCOSPI_V4F32;457    case MVT::v2f64:458      return RTLIB::SINCOSPI_V2F64;459    case MVT::nxv4f32:460      return RTLIB::SINCOSPI_NXV4F32;461    case MVT::nxv2f64:462      return RTLIB::SINCOSPI_NXV2F64;463    default:464      return RTLIB::UNKNOWN_LIBCALL;465    }466  }467 468  return getFPLibCall(RetVT, SINCOSPI_F32, SINCOSPI_F64, SINCOSPI_F80,469                      SINCOSPI_F128, SINCOSPI_PPCF128);470}471 472RTLIB::Libcall RTLIB::getSINCOS_STRET(EVT RetVT) {473  return getFPLibCall(RetVT, SINCOS_STRET_F32, SINCOS_STRET_F64,474                      UNKNOWN_LIBCALL, UNKNOWN_LIBCALL, UNKNOWN_LIBCALL);475}476 477RTLIB::Libcall RTLIB::getMODF(EVT RetVT) {478  // TODO: Tablegen should generate this function479  if (RetVT.isVector()) {480    if (!RetVT.isSimple())481      return RTLIB::UNKNOWN_LIBCALL;482    switch (RetVT.getSimpleVT().SimpleTy) {483    case MVT::v4f32:484      return RTLIB::MODF_V4F32;485    case MVT::v2f64:486      return RTLIB::MODF_V2F64;487    case MVT::nxv4f32:488      return RTLIB::MODF_NXV4F32;489    case MVT::nxv2f64:490      return RTLIB::MODF_NXV2F64;491    default:492      return RTLIB::UNKNOWN_LIBCALL;493    }494  }495 496  return getFPLibCall(RetVT, MODF_F32, MODF_F64, MODF_F80, MODF_F128,497                      MODF_PPCF128);498}499 500RTLIB::Libcall RTLIB::getOutlineAtomicHelper(const Libcall (&LC)[5][4],501                                             AtomicOrdering Order,502                                             uint64_t MemSize) {503  unsigned ModeN, ModelN;504  switch (MemSize) {505  case 1:506    ModeN = 0;507    break;508  case 2:509    ModeN = 1;510    break;511  case 4:512    ModeN = 2;513    break;514  case 8:515    ModeN = 3;516    break;517  case 16:518    ModeN = 4;519    break;520  default:521    return RTLIB::UNKNOWN_LIBCALL;522  }523 524  switch (Order) {525  case AtomicOrdering::Monotonic:526    ModelN = 0;527    break;528  case AtomicOrdering::Acquire:529    ModelN = 1;530    break;531  case AtomicOrdering::Release:532    ModelN = 2;533    break;534  case AtomicOrdering::AcquireRelease:535  case AtomicOrdering::SequentiallyConsistent:536    ModelN = 3;537    break;538  default:539    return UNKNOWN_LIBCALL;540  }541 542  return LC[ModeN][ModelN];543}544 545RTLIB::Libcall RTLIB::getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order,546                                        MVT VT) {547  if (!VT.isScalarInteger())548    return UNKNOWN_LIBCALL;549  uint64_t MemSize = VT.getScalarSizeInBits() / 8;550 551#define LCALLS(A, B)                                                           \552  { A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL }553#define LCALL5(A)                                                              \554  LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)555  switch (Opc) {556  case ISD::ATOMIC_CMP_SWAP: {557    const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_CAS)};558    return getOutlineAtomicHelper(LC, Order, MemSize);559  }560  case ISD::ATOMIC_SWAP: {561    const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_SWP)};562    return getOutlineAtomicHelper(LC, Order, MemSize);563  }564  case ISD::ATOMIC_LOAD_ADD: {565    const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDADD)};566    return getOutlineAtomicHelper(LC, Order, MemSize);567  }568  case ISD::ATOMIC_LOAD_OR: {569    const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDSET)};570    return getOutlineAtomicHelper(LC, Order, MemSize);571  }572  case ISD::ATOMIC_LOAD_CLR: {573    const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDCLR)};574    return getOutlineAtomicHelper(LC, Order, MemSize);575  }576  case ISD::ATOMIC_LOAD_XOR: {577    const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDEOR)};578    return getOutlineAtomicHelper(LC, Order, MemSize);579  }580  default:581    return UNKNOWN_LIBCALL;582  }583#undef LCALLS584#undef LCALL5585}586 587RTLIB::Libcall RTLIB::getSYNC(unsigned Opc, MVT VT) {588#define OP_TO_LIBCALL(Name, Enum)                                              \589  case Name:                                                                   \590    switch (VT.SimpleTy) {                                                     \591    default:                                                                   \592      return UNKNOWN_LIBCALL;                                                  \593    case MVT::i8:                                                              \594      return Enum##_1;                                                         \595    case MVT::i16:                                                             \596      return Enum##_2;                                                         \597    case MVT::i32:                                                             \598      return Enum##_4;                                                         \599    case MVT::i64:                                                             \600      return Enum##_8;                                                         \601    case MVT::i128:                                                            \602      return Enum##_16;                                                        \603    }604 605  switch (Opc) {606    OP_TO_LIBCALL(ISD::ATOMIC_SWAP, SYNC_LOCK_TEST_AND_SET)607    OP_TO_LIBCALL(ISD::ATOMIC_CMP_SWAP, SYNC_VAL_COMPARE_AND_SWAP)608    OP_TO_LIBCALL(ISD::ATOMIC_LOAD_ADD, SYNC_FETCH_AND_ADD)609    OP_TO_LIBCALL(ISD::ATOMIC_LOAD_SUB, SYNC_FETCH_AND_SUB)610    OP_TO_LIBCALL(ISD::ATOMIC_LOAD_AND, SYNC_FETCH_AND_AND)611    OP_TO_LIBCALL(ISD::ATOMIC_LOAD_OR, SYNC_FETCH_AND_OR)612    OP_TO_LIBCALL(ISD::ATOMIC_LOAD_XOR, SYNC_FETCH_AND_XOR)613    OP_TO_LIBCALL(ISD::ATOMIC_LOAD_NAND, SYNC_FETCH_AND_NAND)614    OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MAX, SYNC_FETCH_AND_MAX)615    OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMAX, SYNC_FETCH_AND_UMAX)616    OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MIN, SYNC_FETCH_AND_MIN)617    OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMIN, SYNC_FETCH_AND_UMIN)618  }619 620#undef OP_TO_LIBCALL621 622  return UNKNOWN_LIBCALL;623}624 625RTLIB::Libcall RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) {626  switch (ElementSize) {627  case 1:628    return MEMCPY_ELEMENT_UNORDERED_ATOMIC_1;629  case 2:630    return MEMCPY_ELEMENT_UNORDERED_ATOMIC_2;631  case 4:632    return MEMCPY_ELEMENT_UNORDERED_ATOMIC_4;633  case 8:634    return MEMCPY_ELEMENT_UNORDERED_ATOMIC_8;635  case 16:636    return MEMCPY_ELEMENT_UNORDERED_ATOMIC_16;637  default:638    return UNKNOWN_LIBCALL;639  }640}641 642RTLIB::Libcall RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) {643  switch (ElementSize) {644  case 1:645    return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1;646  case 2:647    return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2;648  case 4:649    return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4;650  case 8:651    return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8;652  case 16:653    return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16;654  default:655    return UNKNOWN_LIBCALL;656  }657}658 659RTLIB::Libcall RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) {660  switch (ElementSize) {661  case 1:662    return MEMSET_ELEMENT_UNORDERED_ATOMIC_1;663  case 2:664    return MEMSET_ELEMENT_UNORDERED_ATOMIC_2;665  case 4:666    return MEMSET_ELEMENT_UNORDERED_ATOMIC_4;667  case 8:668    return MEMSET_ELEMENT_UNORDERED_ATOMIC_8;669  case 16:670    return MEMSET_ELEMENT_UNORDERED_ATOMIC_16;671  default:672    return UNKNOWN_LIBCALL;673  }674}675 676ISD::CondCode TargetLoweringBase::getSoftFloatCmpLibcallPredicate(677    RTLIB::LibcallImpl Impl) const {678  switch (Impl) {679  case RTLIB::impl___aeabi_dcmpeq__une:680  case RTLIB::impl___aeabi_fcmpeq__une:681    // Usage in the eq case, so we have to invert the comparison.682    return ISD::SETEQ;683  case RTLIB::impl___aeabi_dcmpeq__oeq:684  case RTLIB::impl___aeabi_fcmpeq__oeq:685    // Normal comparison to boolean value.686    return ISD::SETNE;687  case RTLIB::impl___aeabi_dcmplt:688  case RTLIB::impl___aeabi_dcmple:689  case RTLIB::impl___aeabi_dcmpge:690  case RTLIB::impl___aeabi_dcmpgt:691  case RTLIB::impl___aeabi_dcmpun:692  case RTLIB::impl___aeabi_fcmplt:693  case RTLIB::impl___aeabi_fcmple:694  case RTLIB::impl___aeabi_fcmpge:695  case RTLIB::impl___aeabi_fcmpgt:696    /// The AEABI versions return a typical boolean value, so we can compare697    /// against the integer result as simply != 0.698    return ISD::SETNE;699  default:700    break;701  }702 703  // Assume libgcc/compiler-rt behavior. Most of the cases are really aliases of704  // each other, and return a 3-way comparison style result of -1, 0, or 1705  // depending on lt/eq/gt.706  //707  // FIXME: It would be cleaner to directly express this as a 3-way comparison708  // soft FP libcall instead of individual compares.709  RTLIB::Libcall LC = RTLIB::RuntimeLibcallsInfo::getLibcallFromImpl(Impl);710  switch (LC) {711  case RTLIB::OEQ_F32:712  case RTLIB::OEQ_F64:713  case RTLIB::OEQ_F128:714  case RTLIB::OEQ_PPCF128:715    return ISD::SETEQ;716  case RTLIB::UNE_F32:717  case RTLIB::UNE_F64:718  case RTLIB::UNE_F128:719  case RTLIB::UNE_PPCF128:720    return ISD::SETNE;721  case RTLIB::OGE_F32:722  case RTLIB::OGE_F64:723  case RTLIB::OGE_F128:724  case RTLIB::OGE_PPCF128:725    return ISD::SETGE;726  case RTLIB::OLT_F32:727  case RTLIB::OLT_F64:728  case RTLIB::OLT_F128:729  case RTLIB::OLT_PPCF128:730    return ISD::SETLT;731  case RTLIB::OLE_F32:732  case RTLIB::OLE_F64:733  case RTLIB::OLE_F128:734  case RTLIB::OLE_PPCF128:735    return ISD::SETLE;736  case RTLIB::OGT_F32:737  case RTLIB::OGT_F64:738  case RTLIB::OGT_F128:739  case RTLIB::OGT_PPCF128:740    return ISD::SETGT;741  case RTLIB::UO_F32:742  case RTLIB::UO_F64:743  case RTLIB::UO_F128:744  case RTLIB::UO_PPCF128:745    return ISD::SETNE;746  default:747    llvm_unreachable("not a compare libcall");748  }749}750 751/// NOTE: The TargetMachine owns TLOF.752TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm,753                                       const TargetSubtargetInfo &STI)754    : TM(tm),755      RuntimeLibcallInfo(TM.getTargetTriple(), TM.Options.ExceptionModel,756                         TM.Options.FloatABIType, TM.Options.EABIVersion,757                         TM.Options.MCOptions.getABIName(), TM.Options.VecLib),758      Libcalls(RuntimeLibcallInfo, STI) {759  initActions();760 761  // Perform these initializations only once.762  MaxStoresPerMemset = MaxStoresPerMemcpy = MaxStoresPerMemmove =763      MaxLoadsPerMemcmp = 8;764  MaxGluedStoresPerMemcpy = 0;765  MaxStoresPerMemsetOptSize = MaxStoresPerMemcpyOptSize =766      MaxStoresPerMemmoveOptSize = MaxLoadsPerMemcmpOptSize = 4;767  HasExtractBitsInsn = false;768  JumpIsExpensive = JumpIsExpensiveOverride;769  PredictableSelectIsExpensive = false;770  EnableExtLdPromotion = false;771  StackPointerRegisterToSaveRestore = 0;772  BooleanContents = UndefinedBooleanContent;773  BooleanFloatContents = UndefinedBooleanContent;774  BooleanVectorContents = UndefinedBooleanContent;775  SchedPreferenceInfo = Sched::ILP;776  GatherAllAliasesMaxDepth = 18;777  IsStrictFPEnabled = DisableStrictNodeMutation;778  MaxBytesForAlignment = 0;779  MaxAtomicSizeInBitsSupported = 0;780 781  // Assume that even with libcalls, no target supports wider than 128 bit782  // division.783  MaxDivRemBitWidthSupported = 128;784 785  MaxLargeFPConvertBitWidthSupported = llvm::IntegerType::MAX_INT_BITS;786 787  MinCmpXchgSizeInBits = 0;788  SupportsUnalignedAtomics = false;789 790  MinimumBitTestCmps = MinimumBitTestCmpsOverride;791}792 793// Define the virtual destructor out-of-line to act as a key method to anchor794// debug info (see coding standards).795TargetLoweringBase::~TargetLoweringBase() = default;796 797void TargetLoweringBase::initActions() {798  // All operations default to being supported.799  memset(OpActions, 0, sizeof(OpActions));800  memset(LoadExtActions, 0, sizeof(LoadExtActions));801  memset(TruncStoreActions, 0, sizeof(TruncStoreActions));802  memset(IndexedModeActions, 0, sizeof(IndexedModeActions));803  memset(CondCodeActions, 0, sizeof(CondCodeActions));804  llvm::fill(RegClassForVT, nullptr);805  llvm::fill(TargetDAGCombineArray, 0);806 807  // Let extending atomic loads be unsupported by default.808  for (MVT ValVT : MVT::all_valuetypes())809    for (MVT MemVT : MVT::all_valuetypes())810      setAtomicLoadExtAction({ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT, MemVT,811                             Expand);812 813  // We're somewhat special casing MVT::i2 and MVT::i4. Ideally we want to814  // remove this and targets should individually set these types if not legal.815  for (ISD::NodeType NT : enum_seq(ISD::DELETED_NODE, ISD::BUILTIN_OP_END,816                                   force_iteration_on_noniterable_enum)) {817    for (MVT VT : {MVT::i2, MVT::i4})818      OpActions[(unsigned)VT.SimpleTy][NT] = Expand;819  }820  for (MVT AVT : MVT::all_valuetypes()) {821    for (MVT VT : {MVT::i2, MVT::i4, MVT::v128i2, MVT::v64i4}) {822      setTruncStoreAction(AVT, VT, Expand);823      setLoadExtAction(ISD::EXTLOAD, AVT, VT, Expand);824      setLoadExtAction(ISD::ZEXTLOAD, AVT, VT, Expand);825    }826  }827  for (unsigned IM = (unsigned)ISD::PRE_INC;828       IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) {829    for (MVT VT : {MVT::i2, MVT::i4}) {830      setIndexedLoadAction(IM, VT, Expand);831      setIndexedStoreAction(IM, VT, Expand);832      setIndexedMaskedLoadAction(IM, VT, Expand);833      setIndexedMaskedStoreAction(IM, VT, Expand);834    }835  }836 837  for (MVT VT : MVT::fp_valuetypes()) {838    MVT IntVT = MVT::getIntegerVT(VT.getFixedSizeInBits());839    if (IntVT.isValid()) {840      setOperationAction(ISD::ATOMIC_SWAP, VT, Promote);841      AddPromotedToType(ISD::ATOMIC_SWAP, VT, IntVT);842    }843  }844 845  // Set default actions for various operations.846  for (MVT VT : MVT::all_valuetypes()) {847    // Default all indexed load / store to expand.848    for (unsigned IM = (unsigned)ISD::PRE_INC;849         IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) {850      setIndexedLoadAction(IM, VT, Expand);851      setIndexedStoreAction(IM, VT, Expand);852      setIndexedMaskedLoadAction(IM, VT, Expand);853      setIndexedMaskedStoreAction(IM, VT, Expand);854    }855 856    // Most backends expect to see the node which just returns the value loaded.857    setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Expand);858 859    // These operations default to expand.860    setOperationAction({ISD::FGETSIGN,       ISD::CONCAT_VECTORS,861                        ISD::FMINNUM,        ISD::FMAXNUM,862                        ISD::FMINNUM_IEEE,   ISD::FMAXNUM_IEEE,863                        ISD::FMINIMUM,       ISD::FMAXIMUM,864                        ISD::FMINIMUMNUM,    ISD::FMAXIMUMNUM,865                        ISD::FMAD,           ISD::SMIN,866                        ISD::SMAX,           ISD::UMIN,867                        ISD::UMAX,           ISD::ABS,868                        ISD::FSHL,           ISD::FSHR,869                        ISD::SADDSAT,        ISD::UADDSAT,870                        ISD::SSUBSAT,        ISD::USUBSAT,871                        ISD::SSHLSAT,        ISD::USHLSAT,872                        ISD::SMULFIX,        ISD::SMULFIXSAT,873                        ISD::UMULFIX,        ISD::UMULFIXSAT,874                        ISD::SDIVFIX,        ISD::SDIVFIXSAT,875                        ISD::UDIVFIX,        ISD::UDIVFIXSAT,876                        ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT,877                        ISD::IS_FPCLASS,     ISD::FCBRT,878                        ISD::FLOG,           ISD::FLOG2,879                        ISD::FLOG10,         ISD::FEXP,880                        ISD::FEXP2,          ISD::FEXP10,881                        ISD::FFLOOR,         ISD::FNEARBYINT,882                        ISD::FCEIL,          ISD::FRINT,883                        ISD::FTRUNC,         ISD::FROUNDEVEN,884                        ISD::FTAN,           ISD::FACOS,885                        ISD::FASIN,          ISD::FATAN,886                        ISD::FCOSH,          ISD::FSINH,887                        ISD::FTANH,          ISD::FATAN2,888                        ISD::FMULADD},889                       VT, Expand);890 891    // Overflow operations default to expand892    setOperationAction({ISD::SADDO, ISD::SSUBO, ISD::UADDO, ISD::USUBO,893                        ISD::SMULO, ISD::UMULO},894                       VT, Expand);895 896    // Carry-using overflow operations default to expand.897    setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY, ISD::SETCCCARRY,898                        ISD::SADDO_CARRY, ISD::SSUBO_CARRY},899                       VT, Expand);900 901    // ADDC/ADDE/SUBC/SUBE default to expand.902    setOperationAction({ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}, VT,903                       Expand);904 905    // [US]CMP default to expand906    setOperationAction({ISD::UCMP, ISD::SCMP}, VT, Expand);907 908    // Halving adds909    setOperationAction(910        {ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS, ISD::AVGCEILU}, VT,911        Expand);912 913    // Absolute difference914    setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Expand);915 916    // Saturated trunc917    setOperationAction(ISD::TRUNCATE_SSAT_S, VT, Expand);918    setOperationAction(ISD::TRUNCATE_SSAT_U, VT, Expand);919    setOperationAction(ISD::TRUNCATE_USAT_U, VT, Expand);920 921    // These default to Expand so they will be expanded to CTLZ/CTTZ by default.922    setOperationAction({ISD::CTLZ_ZERO_UNDEF, ISD::CTTZ_ZERO_UNDEF}, VT,923                       Expand);924 925    setOperationAction({ISD::BITREVERSE, ISD::PARITY}, VT, Expand);926 927    // These library functions default to expand.928    setOperationAction({ISD::FROUND, ISD::FPOWI, ISD::FLDEXP, ISD::FFREXP,929                        ISD::FSINCOS, ISD::FSINCOSPI, ISD::FMODF},930                       VT, Expand);931 932    // These operations default to expand for vector types.933    if (VT.isVector())934      setOperationAction({ISD::FCOPYSIGN, ISD::SIGN_EXTEND_INREG,935                          ISD::ANY_EXTEND_VECTOR_INREG,936                          ISD::SIGN_EXTEND_VECTOR_INREG,937                          ISD::ZERO_EXTEND_VECTOR_INREG, ISD::SPLAT_VECTOR,938                          ISD::LRINT, ISD::LLRINT, ISD::LROUND, ISD::LLROUND},939                         VT, Expand);940 941      // Constrained floating-point operations default to expand.942#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \943    setOperationAction(ISD::STRICT_##DAGN, VT, Expand);944#include "llvm/IR/ConstrainedOps.def"945 946    // For most targets @llvm.get.dynamic.area.offset just returns 0.947    setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, VT, Expand);948 949    // Vector reduction default to expand.950    setOperationAction(951        {ISD::VECREDUCE_FADD, ISD::VECREDUCE_FMUL, ISD::VECREDUCE_ADD,952         ISD::VECREDUCE_MUL, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,953         ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,954         ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN, ISD::VECREDUCE_FMAX,955         ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAXIMUM, ISD::VECREDUCE_FMINIMUM,956         ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_SEQ_FMUL},957        VT, Expand);958 959    // Named vector shuffles default to expand.960    setOperationAction(ISD::VECTOR_SPLICE, VT, Expand);961 962    // Only some target support this vector operation. Most need to expand it.963    setOperationAction(ISD::VECTOR_COMPRESS, VT, Expand);964 965    // VP operations default to expand.966#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...)                                   \967    setOperationAction(ISD::SDOPC, VT, Expand);968#include "llvm/IR/VPIntrinsics.def"969 970    // Masked vector extracts default to expand.971    setOperationAction(ISD::VECTOR_FIND_LAST_ACTIVE, VT, Expand);972 973    setOperationAction(ISD::LOOP_DEPENDENCE_RAW_MASK, VT, Expand);974    setOperationAction(ISD::LOOP_DEPENDENCE_WAR_MASK, VT, Expand);975 976    // FP environment operations default to expand.977    setOperationAction(ISD::GET_FPENV, VT, Expand);978    setOperationAction(ISD::SET_FPENV, VT, Expand);979    setOperationAction(ISD::RESET_FPENV, VT, Expand);980 981    setOperationAction(ISD::MSTORE, VT, Expand);982  }983 984  // Most targets ignore the @llvm.prefetch intrinsic.985  setOperationAction(ISD::PREFETCH, MVT::Other, Expand);986 987  // Most targets also ignore the @llvm.readcyclecounter intrinsic.988  setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Expand);989 990  // Most targets also ignore the @llvm.readsteadycounter intrinsic.991  setOperationAction(ISD::READSTEADYCOUNTER, MVT::i64, Expand);992 993  // ConstantFP nodes default to expand.  Targets can either change this to994  // Legal, in which case all fp constants are legal, or use isFPImmLegal()995  // to optimize expansions for certain constants.996  setOperationAction(ISD::ConstantFP,997                     {MVT::bf16, MVT::f16, MVT::f32, MVT::f64, MVT::f80, MVT::f128},998                     Expand);999 1000  // Insert custom handling default for llvm.canonicalize.*.1001  setOperationAction(ISD::FCANONICALIZE,1002                     {MVT::f16, MVT::f32, MVT::f64, MVT::f128}, Expand);1003 1004  // FIXME: Query RuntimeLibCalls to make the decision.1005  setOperationAction({ISD::LRINT, ISD::LLRINT, ISD::LROUND, ISD::LLROUND},1006                     {MVT::f32, MVT::f64, MVT::f128}, LibCall);1007 1008  setOperationAction({ISD::FTAN, ISD::FACOS, ISD::FASIN, ISD::FATAN, ISD::FCOSH,1009                      ISD::FSINH, ISD::FTANH, ISD::FATAN2},1010                     MVT::f16, Promote);1011  // Default ISD::TRAP to expand (which turns it into abort).1012  setOperationAction(ISD::TRAP, MVT::Other, Expand);1013 1014  // On most systems, DEBUGTRAP and TRAP have no difference. The "Expand"1015  // here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP.1016  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Expand);1017 1018  setOperationAction(ISD::UBSANTRAP, MVT::Other, Expand);1019 1020  setOperationAction(ISD::GET_FPENV_MEM, MVT::Other, Expand);1021  setOperationAction(ISD::SET_FPENV_MEM, MVT::Other, Expand);1022 1023  for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {1024    setOperationAction(ISD::GET_FPMODE, VT, Expand);1025    setOperationAction(ISD::SET_FPMODE, VT, Expand);1026  }1027  setOperationAction(ISD::RESET_FPMODE, MVT::Other, Expand);1028 1029  // This one by default will call __clear_cache unless the target1030  // wants something different.1031  setOperationAction(ISD::CLEAR_CACHE, MVT::Other, LibCall);1032}1033 1034MVT TargetLoweringBase::getScalarShiftAmountTy(const DataLayout &DL,1035                                               EVT) const {1036  return MVT::getIntegerVT(DL.getPointerSizeInBits(0));1037}1038 1039EVT TargetLoweringBase::getShiftAmountTy(EVT LHSTy,1040                                         const DataLayout &DL) const {1041  assert(LHSTy.isInteger() && "Shift amount is not an integer type!");1042  if (LHSTy.isVector())1043    return LHSTy;1044  MVT ShiftVT = getScalarShiftAmountTy(DL, LHSTy);1045  // If any possible shift value won't fit in the prefered type, just use1046  // something safe. Assume it will be legalized when the shift is expanded.1047  if (ShiftVT.getSizeInBits() < Log2_32_Ceil(LHSTy.getSizeInBits()))1048    ShiftVT = MVT::i32;1049  assert(ShiftVT.getSizeInBits() >= Log2_32_Ceil(LHSTy.getSizeInBits()) &&1050         "ShiftVT is still too small!");1051  return ShiftVT;1052}1053 1054bool TargetLoweringBase::canOpTrap(unsigned Op, EVT VT) const {1055  assert(isTypeLegal(VT));1056  switch (Op) {1057  default:1058    return false;1059  case ISD::SDIV:1060  case ISD::UDIV:1061  case ISD::SREM:1062  case ISD::UREM:1063    return true;1064  }1065}1066 1067bool TargetLoweringBase::isFreeAddrSpaceCast(unsigned SrcAS,1068                                             unsigned DestAS) const {1069  return TM.isNoopAddrSpaceCast(SrcAS, DestAS);1070}1071 1072unsigned TargetLoweringBase::getBitWidthForCttzElements(1073    Type *RetTy, ElementCount EC, bool ZeroIsPoison,1074    const ConstantRange *VScaleRange) const {1075  // Find the smallest "sensible" element type to use for the expansion.1076  ConstantRange CR(APInt(64, EC.getKnownMinValue()));1077  if (EC.isScalable())1078    CR = CR.umul_sat(*VScaleRange);1079 1080  if (ZeroIsPoison)1081    CR = CR.subtract(APInt(64, 1));1082 1083  unsigned EltWidth = RetTy->getScalarSizeInBits();1084  EltWidth = std::min(EltWidth, CR.getActiveBits());1085  EltWidth = std::max(llvm::bit_ceil(EltWidth), (unsigned)8);1086 1087  return EltWidth;1088}1089 1090void TargetLoweringBase::setJumpIsExpensive(bool isExpensive) {1091  // If the command-line option was specified, ignore this request.1092  if (!JumpIsExpensiveOverride.getNumOccurrences())1093    JumpIsExpensive = isExpensive;1094}1095 1096TargetLoweringBase::LegalizeKind1097TargetLoweringBase::getTypeConversion(LLVMContext &Context, EVT VT) const {1098  // If this is a simple type, use the ComputeRegisterProp mechanism.1099  if (VT.isSimple()) {1100    MVT SVT = VT.getSimpleVT();1101    assert((unsigned)SVT.SimpleTy < std::size(TransformToType));1102    MVT NVT = TransformToType[SVT.SimpleTy];1103    LegalizeTypeAction LA = ValueTypeActions.getTypeAction(SVT);1104 1105    assert((LA == TypeLegal || LA == TypeSoftenFloat ||1106            LA == TypeSoftPromoteHalf ||1107            (NVT.isVector() ||1108             ValueTypeActions.getTypeAction(NVT) != TypePromoteInteger)) &&1109           "Promote may not follow Expand or Promote");1110 1111    if (LA == TypeSplitVector)1112      return LegalizeKind(LA, EVT(SVT).getHalfNumVectorElementsVT(Context));1113    if (LA == TypeScalarizeVector)1114      return LegalizeKind(LA, SVT.getVectorElementType());1115    return LegalizeKind(LA, NVT);1116  }1117 1118  // Handle Extended Scalar Types.1119  if (!VT.isVector()) {1120    assert(VT.isInteger() && "Float types must be simple");1121    unsigned BitSize = VT.getSizeInBits();1122    // First promote to a power-of-two size, then expand if necessary.1123    if (BitSize < 8 || !isPowerOf2_32(BitSize)) {1124      EVT NVT = VT.getRoundIntegerType(Context);1125      assert(NVT != VT && "Unable to round integer VT");1126      LegalizeKind NextStep = getTypeConversion(Context, NVT);1127      // Avoid multi-step promotion.1128      if (NextStep.first == TypePromoteInteger)1129        return NextStep;1130      // Return rounded integer type.1131      return LegalizeKind(TypePromoteInteger, NVT);1132    }1133 1134    return LegalizeKind(TypeExpandInteger,1135                        EVT::getIntegerVT(Context, VT.getSizeInBits() / 2));1136  }1137 1138  // Handle vector types.1139  ElementCount NumElts = VT.getVectorElementCount();1140  EVT EltVT = VT.getVectorElementType();1141 1142  // Vectors with only one element are always scalarized.1143  if (NumElts.isScalar())1144    return LegalizeKind(TypeScalarizeVector, EltVT);1145 1146  // Try to widen vector elements until the element type is a power of two and1147  // promote it to a legal type later on, for example:1148  // <3 x i8> -> <4 x i8> -> <4 x i32>1149  if (EltVT.isInteger()) {1150    // Vectors with a number of elements that is not a power of two are always1151    // widened, for example <3 x i8> -> <4 x i8>.1152    if (!VT.isPow2VectorType()) {1153      NumElts = NumElts.coefficientNextPowerOf2();1154      EVT NVT = EVT::getVectorVT(Context, EltVT, NumElts);1155      return LegalizeKind(TypeWidenVector, NVT);1156    }1157 1158    // Examine the element type.1159    LegalizeKind LK = getTypeConversion(Context, EltVT);1160 1161    // If type is to be expanded, split the vector.1162    //  <4 x i140> -> <2 x i140>1163    if (LK.first == TypeExpandInteger) {1164      if (NumElts.isScalable() && NumElts.getKnownMinValue() == 1)1165        return LegalizeKind(TypeScalarizeScalableVector, EltVT);1166      return LegalizeKind(TypeSplitVector,1167                          VT.getHalfNumVectorElementsVT(Context));1168    }1169 1170    // Promote the integer element types until a legal vector type is found1171    // or until the element integer type is too big. If a legal type was not1172    // found, fallback to the usual mechanism of widening/splitting the1173    // vector.1174    EVT OldEltVT = EltVT;1175    while (true) {1176      // Increase the bitwidth of the element to the next pow-of-two1177      // (which is greater than 8 bits).1178      EltVT = EVT::getIntegerVT(Context, 1 + EltVT.getSizeInBits())1179                  .getRoundIntegerType(Context);1180 1181      // Stop trying when getting a non-simple element type.1182      // Note that vector elements may be greater than legal vector element1183      // types. Example: X86 XMM registers hold 64bit element on 32bit1184      // systems.1185      if (!EltVT.isSimple())1186        break;1187 1188      // Build a new vector type and check if it is legal.1189      MVT NVT = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);1190      // Found a legal promoted vector type.1191      if (NVT != MVT() && ValueTypeActions.getTypeAction(NVT) == TypeLegal)1192        return LegalizeKind(TypePromoteInteger,1193                            EVT::getVectorVT(Context, EltVT, NumElts));1194    }1195 1196    // Reset the type to the unexpanded type if we did not find a legal vector1197    // type with a promoted vector element type.1198    EltVT = OldEltVT;1199  }1200 1201  // Try to widen the vector until a legal type is found.1202  // If there is no wider legal type, split the vector.1203  while (true) {1204    // Round up to the next power of 2.1205    NumElts = NumElts.coefficientNextPowerOf2();1206 1207    // If there is no simple vector type with this many elements then there1208    // cannot be a larger legal vector type.  Note that this assumes that1209    // there are no skipped intermediate vector types in the simple types.1210    if (!EltVT.isSimple())1211      break;1212    MVT LargerVector = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);1213    if (LargerVector == MVT())1214      break;1215 1216    // If this type is legal then widen the vector.1217    if (ValueTypeActions.getTypeAction(LargerVector) == TypeLegal)1218      return LegalizeKind(TypeWidenVector, LargerVector);1219  }1220 1221  // Widen odd vectors to next power of two.1222  if (!VT.isPow2VectorType()) {1223    EVT NVT = VT.getPow2VectorType(Context);1224    return LegalizeKind(TypeWidenVector, NVT);1225  }1226 1227  if (VT.getVectorElementCount() == ElementCount::getScalable(1))1228    return LegalizeKind(TypeScalarizeScalableVector, EltVT);1229 1230  // Vectors with illegal element types are expanded.1231  EVT NVT = EVT::getVectorVT(Context, EltVT,1232                             VT.getVectorElementCount().divideCoefficientBy(2));1233  return LegalizeKind(TypeSplitVector, NVT);1234}1235 1236static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,1237                                          unsigned &NumIntermediates,1238                                          MVT &RegisterVT,1239                                          TargetLoweringBase *TLI) {1240  // Figure out the right, legal destination reg to copy into.1241  ElementCount EC = VT.getVectorElementCount();1242  MVT EltTy = VT.getVectorElementType();1243 1244  unsigned NumVectorRegs = 1;1245 1246  // Scalable vectors cannot be scalarized, so splitting or widening is1247  // required.1248  if (VT.isScalableVector() && !isPowerOf2_32(EC.getKnownMinValue()))1249    llvm_unreachable(1250        "Splitting or widening of non-power-of-2 MVTs is not implemented.");1251 1252  // FIXME: We don't support non-power-of-2-sized vectors for now.1253  // Ideally we could break down into LHS/RHS like LegalizeDAG does.1254  if (!isPowerOf2_32(EC.getKnownMinValue())) {1255    // Split EC to unit size (scalable property is preserved).1256    NumVectorRegs = EC.getKnownMinValue();1257    EC = ElementCount::getFixed(1);1258  }1259 1260  // Divide the input until we get to a supported size. This will1261  // always end up with an EC that represent a scalar or a scalable1262  // scalar.1263  while (EC.getKnownMinValue() > 1 &&1264         !TLI->isTypeLegal(MVT::getVectorVT(EltTy, EC))) {1265    EC = EC.divideCoefficientBy(2);1266    NumVectorRegs <<= 1;1267  }1268 1269  NumIntermediates = NumVectorRegs;1270 1271  MVT NewVT = MVT::getVectorVT(EltTy, EC);1272  if (!TLI->isTypeLegal(NewVT))1273    NewVT = EltTy;1274  IntermediateVT = NewVT;1275 1276  unsigned LaneSizeInBits = NewVT.getScalarSizeInBits();1277 1278  // Convert sizes such as i33 to i64.1279  LaneSizeInBits = llvm::bit_ceil(LaneSizeInBits);1280 1281  MVT DestVT = TLI->getRegisterType(NewVT);1282  RegisterVT = DestVT;1283  if (EVT(DestVT).bitsLT(NewVT))    // Value is expanded, e.g. i64 -> i16.1284    return NumVectorRegs * (LaneSizeInBits / DestVT.getScalarSizeInBits());1285 1286  // Otherwise, promotion or legal types use the same number of registers as1287  // the vector decimated to the appropriate level.1288  return NumVectorRegs;1289}1290 1291/// isLegalRC - Return true if the value types that can be represented by the1292/// specified register class are all legal.1293bool TargetLoweringBase::isLegalRC(const TargetRegisterInfo &TRI,1294                                   const TargetRegisterClass &RC) const {1295  for (const auto *I = TRI.legalclasstypes_begin(RC); *I != MVT::Other; ++I)1296    if (isTypeLegal(*I))1297      return true;1298  return false;1299}1300 1301/// Replace/modify any TargetFrameIndex operands with a targte-dependent1302/// sequence of memory operands that is recognized by PrologEpilogInserter.1303MachineBasicBlock *1304TargetLoweringBase::emitPatchPoint(MachineInstr &InitialMI,1305                                   MachineBasicBlock *MBB) const {1306  MachineInstr *MI = &InitialMI;1307  MachineFunction &MF = *MI->getMF();1308  MachineFrameInfo &MFI = MF.getFrameInfo();1309 1310  // We're handling multiple types of operands here:1311  // PATCHPOINT MetaArgs - live-in, read only, direct1312  // STATEPOINT Deopt Spill - live-through, read only, indirect1313  // STATEPOINT Deopt Alloca - live-through, read only, direct1314  // (We're currently conservative and mark the deopt slots read/write in1315  // practice.)1316  // STATEPOINT GC Spill - live-through, read/write, indirect1317  // STATEPOINT GC Alloca - live-through, read/write, direct1318  // The live-in vs live-through is handled already (the live through ones are1319  // all stack slots), but we need to handle the different type of stackmap1320  // operands and memory effects here.1321 1322  if (llvm::none_of(MI->operands(),1323                    [](MachineOperand &Operand) { return Operand.isFI(); }))1324    return MBB;1325 1326  MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), MI->getDesc());1327 1328  // Inherit previous memory operands.1329  MIB.cloneMemRefs(*MI);1330 1331  for (unsigned i = 0; i < MI->getNumOperands(); ++i) {1332    MachineOperand &MO = MI->getOperand(i);1333    if (!MO.isFI()) {1334      // Index of Def operand this Use it tied to.1335      // Since Defs are coming before Uses, if Use is tied, then1336      // index of Def must be smaller that index of that Use.1337      // Also, Defs preserve their position in new MI.1338      unsigned TiedTo = i;1339      if (MO.isReg() && MO.isTied())1340        TiedTo = MI->findTiedOperandIdx(i);1341      MIB.add(MO);1342      if (TiedTo < i)1343        MIB->tieOperands(TiedTo, MIB->getNumOperands() - 1);1344      continue;1345    }1346 1347    // foldMemoryOperand builds a new MI after replacing a single FI operand1348    // with the canonical set of five x86 addressing-mode operands.1349    int FI = MO.getIndex();1350 1351    // Add frame index operands recognized by stackmaps.cpp1352    if (MFI.isStatepointSpillSlotObjectIndex(FI)) {1353      // indirect-mem-ref tag, size, #FI, offset.1354      // Used for spills inserted by StatepointLowering.  This codepath is not1355      // used for patchpoints/stackmaps at all, for these spilling is done via1356      // foldMemoryOperand callback only.1357      assert(MI->getOpcode() == TargetOpcode::STATEPOINT && "sanity");1358      MIB.addImm(StackMaps::IndirectMemRefOp);1359      MIB.addImm(MFI.getObjectSize(FI));1360      MIB.add(MO);1361      MIB.addImm(0);1362    } else {1363      // direct-mem-ref tag, #FI, offset.1364      // Used by patchpoint, and direct alloca arguments to statepoints1365      MIB.addImm(StackMaps::DirectMemRefOp);1366      MIB.add(MO);1367      MIB.addImm(0);1368    }1369 1370    assert(MIB->mayLoad() && "Folded a stackmap use to a non-load!");1371 1372    // Add a new memory operand for this FI.1373    assert(MFI.getObjectOffset(FI) != -1);1374 1375    // Note: STATEPOINT MMOs are added during SelectionDAG.  STACKMAP, and1376    // PATCHPOINT should be updated to do the same. (TODO)1377    if (MI->getOpcode() != TargetOpcode::STATEPOINT) {1378      auto Flags = MachineMemOperand::MOLoad;1379      MachineMemOperand *MMO = MF.getMachineMemOperand(1380          MachinePointerInfo::getFixedStack(MF, FI), Flags,1381          MF.getDataLayout().getPointerSize(), MFI.getObjectAlign(FI));1382      MIB->addMemOperand(MF, MMO);1383    }1384  }1385  MBB->insert(MachineBasicBlock::iterator(MI), MIB);1386  MI->eraseFromParent();1387  return MBB;1388}1389 1390/// findRepresentativeClass - Return the largest legal super-reg register class1391/// of the register class for the specified type and its associated "cost".1392// This function is in TargetLowering because it uses RegClassForVT which would1393// need to be moved to TargetRegisterInfo and would necessitate moving1394// isTypeLegal over as well - a massive change that would just require1395// TargetLowering having a TargetRegisterInfo class member that it would use.1396std::pair<const TargetRegisterClass *, uint8_t>1397TargetLoweringBase::findRepresentativeClass(const TargetRegisterInfo *TRI,1398                                            MVT VT) const {1399  const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy];1400  if (!RC)1401    return std::make_pair(RC, 0);1402 1403  // Compute the set of all super-register classes.1404  BitVector SuperRegRC(TRI->getNumRegClasses());1405  for (SuperRegClassIterator RCI(RC, TRI); RCI.isValid(); ++RCI)1406    SuperRegRC.setBitsInMask(RCI.getMask());1407 1408  // Find the first legal register class with the largest spill size.1409  const TargetRegisterClass *BestRC = RC;1410  for (unsigned i : SuperRegRC.set_bits()) {1411    const TargetRegisterClass *SuperRC = TRI->getRegClass(i);1412    // We want the largest possible spill size.1413    if (TRI->getSpillSize(*SuperRC) <= TRI->getSpillSize(*BestRC))1414      continue;1415    if (!isLegalRC(*TRI, *SuperRC))1416      continue;1417    BestRC = SuperRC;1418  }1419  return std::make_pair(BestRC, 1);1420}1421 1422/// computeRegisterProperties - Once all of the register classes are added,1423/// this allows us to compute derived properties we expose.1424void TargetLoweringBase::computeRegisterProperties(1425    const TargetRegisterInfo *TRI) {1426  // Everything defaults to needing one register.1427  for (unsigned i = 0; i != MVT::VALUETYPE_SIZE; ++i) {1428    NumRegistersForVT[i] = 1;1429    RegisterTypeForVT[i] = TransformToType[i] = (MVT::SimpleValueType)i;1430  }1431  // ...except isVoid, which doesn't need any registers.1432  NumRegistersForVT[MVT::isVoid] = 0;1433 1434  // Find the largest integer register class.1435  unsigned LargestIntReg = MVT::LAST_INTEGER_VALUETYPE;1436  for (; RegClassForVT[LargestIntReg] == nullptr; --LargestIntReg)1437    assert(LargestIntReg != MVT::i1 && "No integer registers defined!");1438 1439  // Every integer value type larger than this largest register takes twice as1440  // many registers to represent as the previous ValueType.1441  for (unsigned ExpandedReg = LargestIntReg + 1;1442       ExpandedReg <= MVT::LAST_INTEGER_VALUETYPE; ++ExpandedReg) {1443    NumRegistersForVT[ExpandedReg] = 2*NumRegistersForVT[ExpandedReg-1];1444    RegisterTypeForVT[ExpandedReg] = (MVT::SimpleValueType)LargestIntReg;1445    TransformToType[ExpandedReg] = (MVT::SimpleValueType)(ExpandedReg - 1);1446    ValueTypeActions.setTypeAction((MVT::SimpleValueType)ExpandedReg,1447                                   TypeExpandInteger);1448  }1449 1450  // Inspect all of the ValueType's smaller than the largest integer1451  // register to see which ones need promotion.1452  unsigned LegalIntReg = LargestIntReg;1453  for (unsigned IntReg = LargestIntReg - 1;1454       IntReg >= (unsigned)MVT::i1; --IntReg) {1455    MVT IVT = (MVT::SimpleValueType)IntReg;1456    if (isTypeLegal(IVT)) {1457      LegalIntReg = IntReg;1458    } else {1459      RegisterTypeForVT[IntReg] = TransformToType[IntReg] =1460        (MVT::SimpleValueType)LegalIntReg;1461      ValueTypeActions.setTypeAction(IVT, TypePromoteInteger);1462    }1463  }1464 1465  // ppcf128 type is really two f64's.1466  if (!isTypeLegal(MVT::ppcf128)) {1467    if (isTypeLegal(MVT::f64)) {1468      NumRegistersForVT[MVT::ppcf128] = 2*NumRegistersForVT[MVT::f64];1469      RegisterTypeForVT[MVT::ppcf128] = MVT::f64;1470      TransformToType[MVT::ppcf128] = MVT::f64;1471      ValueTypeActions.setTypeAction(MVT::ppcf128, TypeExpandFloat);1472    } else {1473      NumRegistersForVT[MVT::ppcf128] = NumRegistersForVT[MVT::i128];1474      RegisterTypeForVT[MVT::ppcf128] = RegisterTypeForVT[MVT::i128];1475      TransformToType[MVT::ppcf128] = MVT::i128;1476      ValueTypeActions.setTypeAction(MVT::ppcf128, TypeSoftenFloat);1477    }1478  }1479 1480  // Decide how to handle f128. If the target does not have native f128 support,1481  // expand it to i128 and we will be generating soft float library calls.1482  if (!isTypeLegal(MVT::f128)) {1483    NumRegistersForVT[MVT::f128] = NumRegistersForVT[MVT::i128];1484    RegisterTypeForVT[MVT::f128] = RegisterTypeForVT[MVT::i128];1485    TransformToType[MVT::f128] = MVT::i128;1486    ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat);1487  }1488 1489  // Decide how to handle f80. If the target does not have native f80 support,1490  // expand it to i96 and we will be generating soft float library calls.1491  if (!isTypeLegal(MVT::f80)) {1492    NumRegistersForVT[MVT::f80] = 3*NumRegistersForVT[MVT::i32];1493    RegisterTypeForVT[MVT::f80] = RegisterTypeForVT[MVT::i32];1494    TransformToType[MVT::f80] = MVT::i32;1495    ValueTypeActions.setTypeAction(MVT::f80, TypeSoftenFloat);1496  }1497 1498  // Decide how to handle f64. If the target does not have native f64 support,1499  // expand it to i64 and we will be generating soft float library calls.1500  if (!isTypeLegal(MVT::f64)) {1501    NumRegistersForVT[MVT::f64] = NumRegistersForVT[MVT::i64];1502    RegisterTypeForVT[MVT::f64] = RegisterTypeForVT[MVT::i64];1503    TransformToType[MVT::f64] = MVT::i64;1504    ValueTypeActions.setTypeAction(MVT::f64, TypeSoftenFloat);1505  }1506 1507  // Decide how to handle f32. If the target does not have native f32 support,1508  // expand it to i32 and we will be generating soft float library calls.1509  if (!isTypeLegal(MVT::f32)) {1510    NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32];1511    RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32];1512    TransformToType[MVT::f32] = MVT::i32;1513    ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat);1514  }1515 1516  // Decide how to handle f16. If the target does not have native f16 support,1517  // promote it to f32, because there are no f16 library calls (except for1518  // conversions).1519  if (!isTypeLegal(MVT::f16)) {1520    // Allow targets to control how we legalize half.1521    bool SoftPromoteHalfType = softPromoteHalfType();1522    bool UseFPRegsForHalfType = !SoftPromoteHalfType || useFPRegsForHalfType();1523 1524    if (!UseFPRegsForHalfType) {1525      NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::i16];1526      RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::i16];1527    } else {1528      NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::f32];1529      RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::f32];1530    }1531    TransformToType[MVT::f16] = MVT::f32;1532    if (SoftPromoteHalfType) {1533      ValueTypeActions.setTypeAction(MVT::f16, TypeSoftPromoteHalf);1534    } else {1535      ValueTypeActions.setTypeAction(MVT::f16, TypePromoteFloat);1536    }1537  }1538 1539  // Decide how to handle bf16. If the target does not have native bf16 support,1540  // promote it to f32, because there are no bf16 library calls (except for1541  // converting from f32 to bf16).1542  if (!isTypeLegal(MVT::bf16)) {1543    NumRegistersForVT[MVT::bf16] = NumRegistersForVT[MVT::f32];1544    RegisterTypeForVT[MVT::bf16] = RegisterTypeForVT[MVT::f32];1545    TransformToType[MVT::bf16] = MVT::f32;1546    ValueTypeActions.setTypeAction(MVT::bf16, TypeSoftPromoteHalf);1547  }1548 1549  // Loop over all of the vector value types to see which need transformations.1550  for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE;1551       i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) {1552    MVT VT = (MVT::SimpleValueType) i;1553    if (isTypeLegal(VT))1554      continue;1555 1556    MVT EltVT = VT.getVectorElementType();1557    ElementCount EC = VT.getVectorElementCount();1558    bool IsLegalWiderType = false;1559    bool IsScalable = VT.isScalableVector();1560    LegalizeTypeAction PreferredAction = getPreferredVectorAction(VT);1561    switch (PreferredAction) {1562    case TypePromoteInteger: {1563      MVT::SimpleValueType EndVT = IsScalable ?1564                                   MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE :1565                                   MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE;1566      // Try to promote the elements of integer vectors. If no legal1567      // promotion was found, fall through to the widen-vector method.1568      for (unsigned nVT = i + 1;1569           (MVT::SimpleValueType)nVT <= EndVT; ++nVT) {1570        MVT SVT = (MVT::SimpleValueType) nVT;1571        // Promote vectors of integers to vectors with the same number1572        // of elements, with a wider element type.1573        if (SVT.getScalarSizeInBits() > EltVT.getFixedSizeInBits() &&1574            SVT.getVectorElementCount() == EC && isTypeLegal(SVT)) {1575          TransformToType[i] = SVT;1576          RegisterTypeForVT[i] = SVT;1577          NumRegistersForVT[i] = 1;1578          ValueTypeActions.setTypeAction(VT, TypePromoteInteger);1579          IsLegalWiderType = true;1580          break;1581        }1582      }1583      if (IsLegalWiderType)1584        break;1585      [[fallthrough]];1586    }1587 1588    case TypeWidenVector:1589      if (isPowerOf2_32(EC.getKnownMinValue())) {1590        // Try to widen the vector.1591        for (unsigned nVT = i + 1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) {1592          MVT SVT = (MVT::SimpleValueType) nVT;1593          if (SVT.getVectorElementType() == EltVT &&1594              SVT.isScalableVector() == IsScalable &&1595              SVT.getVectorElementCount().getKnownMinValue() >1596                  EC.getKnownMinValue() &&1597              isTypeLegal(SVT)) {1598            TransformToType[i] = SVT;1599            RegisterTypeForVT[i] = SVT;1600            NumRegistersForVT[i] = 1;1601            ValueTypeActions.setTypeAction(VT, TypeWidenVector);1602            IsLegalWiderType = true;1603            break;1604          }1605        }1606        if (IsLegalWiderType)1607          break;1608      } else {1609        // Only widen to the next power of 2 to keep consistency with EVT.1610        MVT NVT = VT.getPow2VectorType();1611        if (isTypeLegal(NVT)) {1612          TransformToType[i] = NVT;1613          ValueTypeActions.setTypeAction(VT, TypeWidenVector);1614          RegisterTypeForVT[i] = NVT;1615          NumRegistersForVT[i] = 1;1616          break;1617        }1618      }1619      [[fallthrough]];1620 1621    case TypeSplitVector:1622    case TypeScalarizeVector: {1623      MVT IntermediateVT;1624      MVT RegisterVT;1625      unsigned NumIntermediates;1626      unsigned NumRegisters = getVectorTypeBreakdownMVT(VT, IntermediateVT,1627          NumIntermediates, RegisterVT, this);1628      NumRegistersForVT[i] = NumRegisters;1629      assert(NumRegistersForVT[i] == NumRegisters &&1630             "NumRegistersForVT size cannot represent NumRegisters!");1631      RegisterTypeForVT[i] = RegisterVT;1632 1633      MVT NVT = VT.getPow2VectorType();1634      if (NVT == VT) {1635        // Type is already a power of 2.  The default action is to split.1636        TransformToType[i] = MVT::Other;1637        if (PreferredAction == TypeScalarizeVector)1638          ValueTypeActions.setTypeAction(VT, TypeScalarizeVector);1639        else if (PreferredAction == TypeSplitVector)1640          ValueTypeActions.setTypeAction(VT, TypeSplitVector);1641        else if (EC.getKnownMinValue() > 1)1642          ValueTypeActions.setTypeAction(VT, TypeSplitVector);1643        else1644          ValueTypeActions.setTypeAction(VT, EC.isScalable()1645                                                 ? TypeScalarizeScalableVector1646                                                 : TypeScalarizeVector);1647      } else {1648        TransformToType[i] = NVT;1649        ValueTypeActions.setTypeAction(VT, TypeWidenVector);1650      }1651      break;1652    }1653    default:1654      llvm_unreachable("Unknown vector legalization action!");1655    }1656  }1657 1658  // Determine the 'representative' register class for each value type.1659  // An representative register class is the largest (meaning one which is1660  // not a sub-register class / subreg register class) legal register class for1661  // a group of value types. For example, on i386, i8, i16, and i321662  // representative would be GR32; while on x86_64 it's GR64.1663  for (unsigned i = 0; i != MVT::VALUETYPE_SIZE; ++i) {1664    const TargetRegisterClass* RRC;1665    uint8_t Cost;1666    std::tie(RRC, Cost) = findRepresentativeClass(TRI, (MVT::SimpleValueType)i);1667    RepRegClassForVT[i] = RRC;1668    RepRegClassCostForVT[i] = Cost;1669  }1670}1671 1672EVT TargetLoweringBase::getSetCCResultType(const DataLayout &DL, LLVMContext &,1673                                           EVT VT) const {1674  assert(!VT.isVector() && "No default SetCC type for vectors!");1675  return getPointerTy(DL).SimpleTy;1676}1677 1678MVT::SimpleValueType TargetLoweringBase::getCmpLibcallReturnType() const {1679  return MVT::i32; // return the default value1680}1681 1682/// getVectorTypeBreakdown - Vector types are broken down into some number of1683/// legal first class types.  For example, MVT::v8f32 maps to 2 MVT::v4f321684/// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack.1685/// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86.1686///1687/// This method returns the number of registers needed, and the VT for each1688/// register.  It also returns the VT and quantity of the intermediate values1689/// before they are promoted/expanded.1690unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context,1691                                                    EVT VT, EVT &IntermediateVT,1692                                                    unsigned &NumIntermediates,1693                                                    MVT &RegisterVT) const {1694  ElementCount EltCnt = VT.getVectorElementCount();1695 1696  // If there is a wider vector type with the same element type as this one,1697  // or a promoted vector type that has the same number of elements which1698  // are wider, then we should convert to that legal vector type.1699  // This handles things like <2 x float> -> <4 x float> and1700  // <4 x i1> -> <4 x i32>.1701  LegalizeTypeAction TA = getTypeAction(Context, VT);1702  if (!EltCnt.isScalar() &&1703      (TA == TypeWidenVector || TA == TypePromoteInteger)) {1704    EVT RegisterEVT = getTypeToTransformTo(Context, VT);1705    if (isTypeLegal(RegisterEVT)) {1706      IntermediateVT = RegisterEVT;1707      RegisterVT = RegisterEVT.getSimpleVT();1708      NumIntermediates = 1;1709      return 1;1710    }1711  }1712 1713  // Figure out the right, legal destination reg to copy into.1714  EVT EltTy = VT.getVectorElementType();1715 1716  unsigned NumVectorRegs = 1;1717 1718  // Scalable vectors cannot be scalarized, so handle the legalisation of the1719  // types like done elsewhere in SelectionDAG.1720  if (EltCnt.isScalable()) {1721    LegalizeKind LK;1722    EVT PartVT = VT;1723    do {1724      // Iterate until we've found a legal (part) type to hold VT.1725      LK = getTypeConversion(Context, PartVT);1726      PartVT = LK.second;1727    } while (LK.first != TypeLegal);1728 1729    if (!PartVT.isVector()) {1730      report_fatal_error(1731          "Don't know how to legalize this scalable vector type");1732    }1733 1734    NumIntermediates =1735        divideCeil(VT.getVectorElementCount().getKnownMinValue(),1736                   PartVT.getVectorElementCount().getKnownMinValue());1737    IntermediateVT = PartVT;1738    RegisterVT = getRegisterType(Context, IntermediateVT);1739    return NumIntermediates;1740  }1741 1742  // FIXME: We don't support non-power-of-2-sized vectors for now.  Ideally1743  // we could break down into LHS/RHS like LegalizeDAG does.1744  if (!isPowerOf2_32(EltCnt.getKnownMinValue())) {1745    NumVectorRegs = EltCnt.getKnownMinValue();1746    EltCnt = ElementCount::getFixed(1);1747  }1748 1749  // Divide the input until we get to a supported size.  This will always1750  // end with a scalar if the target doesn't support vectors.1751  while (EltCnt.getKnownMinValue() > 1 &&1752         !isTypeLegal(EVT::getVectorVT(Context, EltTy, EltCnt))) {1753    EltCnt = EltCnt.divideCoefficientBy(2);1754    NumVectorRegs <<= 1;1755  }1756 1757  NumIntermediates = NumVectorRegs;1758 1759  EVT NewVT = EVT::getVectorVT(Context, EltTy, EltCnt);1760  if (!isTypeLegal(NewVT))1761    NewVT = EltTy;1762  IntermediateVT = NewVT;1763 1764  MVT DestVT = getRegisterType(Context, NewVT);1765  RegisterVT = DestVT;1766 1767  if (EVT(DestVT).bitsLT(NewVT)) {  // Value is expanded, e.g. i64 -> i16.1768    TypeSize NewVTSize = NewVT.getSizeInBits();1769    // Convert sizes such as i33 to i64.1770    if (!llvm::has_single_bit<uint32_t>(NewVTSize.getKnownMinValue()))1771      NewVTSize = NewVTSize.coefficientNextPowerOf2();1772    return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits());1773  }1774 1775  // Otherwise, promotion or legal types use the same number of registers as1776  // the vector decimated to the appropriate level.1777  return NumVectorRegs;1778}1779 1780bool TargetLoweringBase::isSuitableForJumpTable(const SwitchInst *SI,1781                                                uint64_t NumCases,1782                                                uint64_t Range,1783                                                ProfileSummaryInfo *PSI,1784                                                BlockFrequencyInfo *BFI) const {1785  // FIXME: This function check the maximum table size and density, but the1786  // minimum size is not checked. It would be nice if the minimum size is1787  // also combined within this function. Currently, the minimum size check is1788  // performed in findJumpTable() in SelectionDAGBuiler and1789  // getEstimatedNumberOfCaseClusters() in BasicTTIImpl.1790  const bool OptForSize =1791      llvm::shouldOptimizeForSize(SI->getParent(), PSI, BFI);1792  const unsigned MinDensity = getMinimumJumpTableDensity(OptForSize);1793  const unsigned MaxJumpTableSize = getMaximumJumpTableSize();1794 1795  // Check whether the number of cases is small enough and1796  // the range is dense enough for a jump table.1797  return (OptForSize || Range <= MaxJumpTableSize) &&1798         (NumCases * 100 >= Range * MinDensity);1799}1800 1801MVT TargetLoweringBase::getPreferredSwitchConditionType(LLVMContext &Context,1802                                                        EVT ConditionVT) const {1803  return getRegisterType(Context, ConditionVT);1804}1805 1806/// Get the EVTs and ArgFlags collections that represent the legalized return1807/// type of the given function.  This does not require a DAG or a return value,1808/// and is suitable for use before any DAGs for the function are constructed.1809/// TODO: Move this out of TargetLowering.cpp.1810void llvm::GetReturnInfo(CallingConv::ID CC, Type *ReturnType,1811                         AttributeList attr,1812                         SmallVectorImpl<ISD::OutputArg> &Outs,1813                         const TargetLowering &TLI, const DataLayout &DL) {1814  SmallVector<Type *, 4> Types;1815  ComputeValueTypes(DL, ReturnType, Types);1816  unsigned NumValues = Types.size();1817  if (NumValues == 0) return;1818 1819  for (Type *Ty : Types) {1820    EVT VT = TLI.getValueType(DL, Ty);1821    ISD::NodeType ExtendKind = ISD::ANY_EXTEND;1822 1823    if (attr.hasRetAttr(Attribute::SExt))1824      ExtendKind = ISD::SIGN_EXTEND;1825    else if (attr.hasRetAttr(Attribute::ZExt))1826      ExtendKind = ISD::ZERO_EXTEND;1827 1828    if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())1829      VT = TLI.getTypeForExtReturn(ReturnType->getContext(), VT, ExtendKind);1830 1831    unsigned NumParts =1832        TLI.getNumRegistersForCallingConv(ReturnType->getContext(), CC, VT);1833    MVT PartVT =1834        TLI.getRegisterTypeForCallingConv(ReturnType->getContext(), CC, VT);1835 1836    // 'inreg' on function refers to return value1837    ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();1838    if (attr.hasRetAttr(Attribute::InReg))1839      Flags.setInReg();1840 1841    // Propagate extension type if any1842    if (attr.hasRetAttr(Attribute::SExt))1843      Flags.setSExt();1844    else if (attr.hasRetAttr(Attribute::ZExt))1845      Flags.setZExt();1846 1847    for (unsigned i = 0; i < NumParts; ++i)1848      Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, Ty, 0, 0));1849  }1850}1851 1852Align TargetLoweringBase::getByValTypeAlignment(Type *Ty,1853                                                const DataLayout &DL) const {1854  return DL.getABITypeAlign(Ty);1855}1856 1857bool TargetLoweringBase::allowsMemoryAccessForAlignment(1858    LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,1859    Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const {1860  // Check if the specified alignment is sufficient based on the data layout.1861  // TODO: While using the data layout works in practice, a better solution1862  // would be to implement this check directly (make this a virtual function).1863  // For example, the ABI alignment may change based on software platform while1864  // this function should only be affected by hardware implementation.1865  Type *Ty = VT.getTypeForEVT(Context);1866  if (VT.isZeroSized() || Alignment >= DL.getABITypeAlign(Ty)) {1867    // Assume that an access that meets the ABI-specified alignment is fast.1868    if (Fast != nullptr)1869      *Fast = 1;1870    return true;1871  }1872 1873  // This is a misaligned access.1874  return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags, Fast);1875}1876 1877bool TargetLoweringBase::allowsMemoryAccessForAlignment(1878    LLVMContext &Context, const DataLayout &DL, EVT VT,1879    const MachineMemOperand &MMO, unsigned *Fast) const {1880  return allowsMemoryAccessForAlignment(Context, DL, VT, MMO.getAddrSpace(),1881                                        MMO.getAlign(), MMO.getFlags(), Fast);1882}1883 1884bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,1885                                            const DataLayout &DL, EVT VT,1886                                            unsigned AddrSpace, Align Alignment,1887                                            MachineMemOperand::Flags Flags,1888                                            unsigned *Fast) const {1889  return allowsMemoryAccessForAlignment(Context, DL, VT, AddrSpace, Alignment,1890                                        Flags, Fast);1891}1892 1893bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,1894                                            const DataLayout &DL, EVT VT,1895                                            const MachineMemOperand &MMO,1896                                            unsigned *Fast) const {1897  return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(),1898                            MMO.getFlags(), Fast);1899}1900 1901bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,1902                                            const DataLayout &DL, LLT Ty,1903                                            const MachineMemOperand &MMO,1904                                            unsigned *Fast) const {1905  EVT VT = getApproximateEVTForLLT(Ty, Context);1906  return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(),1907                            MMO.getFlags(), Fast);1908}1909 1910//===----------------------------------------------------------------------===//1911//  TargetTransformInfo Helpers1912//===----------------------------------------------------------------------===//1913 1914int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode) const {1915  enum InstructionOpcodes {1916#define HANDLE_INST(NUM, OPCODE, CLASS) OPCODE = NUM,1917#define LAST_OTHER_INST(NUM) InstructionOpcodesCount = NUM1918#include "llvm/IR/Instruction.def"1919  };1920  switch (static_cast<InstructionOpcodes>(Opcode)) {1921  case Ret:            return 0;1922  case Br:             return 0;1923  case Switch:         return 0;1924  case IndirectBr:     return 0;1925  case Invoke:         return 0;1926  case CallBr:         return 0;1927  case Resume:         return 0;1928  case Unreachable:    return 0;1929  case CleanupRet:     return 0;1930  case CatchRet:       return 0;1931  case CatchPad:       return 0;1932  case CatchSwitch:    return 0;1933  case CleanupPad:     return 0;1934  case FNeg:           return ISD::FNEG;1935  case Add:            return ISD::ADD;1936  case FAdd:           return ISD::FADD;1937  case Sub:            return ISD::SUB;1938  case FSub:           return ISD::FSUB;1939  case Mul:            return ISD::MUL;1940  case FMul:           return ISD::FMUL;1941  case UDiv:           return ISD::UDIV;1942  case SDiv:           return ISD::SDIV;1943  case FDiv:           return ISD::FDIV;1944  case URem:           return ISD::UREM;1945  case SRem:           return ISD::SREM;1946  case FRem:           return ISD::FREM;1947  case Shl:            return ISD::SHL;1948  case LShr:           return ISD::SRL;1949  case AShr:           return ISD::SRA;1950  case And:            return ISD::AND;1951  case Or:             return ISD::OR;1952  case Xor:            return ISD::XOR;1953  case Alloca:         return 0;1954  case Load:           return ISD::LOAD;1955  case Store:          return ISD::STORE;1956  case GetElementPtr:  return 0;1957  case Fence:          return 0;1958  case AtomicCmpXchg:  return 0;1959  case AtomicRMW:      return 0;1960  case Trunc:          return ISD::TRUNCATE;1961  case ZExt:           return ISD::ZERO_EXTEND;1962  case SExt:           return ISD::SIGN_EXTEND;1963  case FPToUI:         return ISD::FP_TO_UINT;1964  case FPToSI:         return ISD::FP_TO_SINT;1965  case UIToFP:         return ISD::UINT_TO_FP;1966  case SIToFP:         return ISD::SINT_TO_FP;1967  case FPTrunc:        return ISD::FP_ROUND;1968  case FPExt:          return ISD::FP_EXTEND;1969  case PtrToAddr:      return ISD::BITCAST;1970  case PtrToInt:       return ISD::BITCAST;1971  case IntToPtr:       return ISD::BITCAST;1972  case BitCast:        return ISD::BITCAST;1973  case AddrSpaceCast:  return ISD::ADDRSPACECAST;1974  case ICmp:           return ISD::SETCC;1975  case FCmp:           return ISD::SETCC;1976  case PHI:            return 0;1977  case Call:           return 0;1978  case Select:         return ISD::SELECT;1979  case UserOp1:        return 0;1980  case UserOp2:        return 0;1981  case VAArg:          return 0;1982  case ExtractElement: return ISD::EXTRACT_VECTOR_ELT;1983  case InsertElement:  return ISD::INSERT_VECTOR_ELT;1984  case ShuffleVector:  return ISD::VECTOR_SHUFFLE;1985  case ExtractValue:   return ISD::MERGE_VALUES;1986  case InsertValue:    return ISD::MERGE_VALUES;1987  case LandingPad:     return 0;1988  case Freeze:         return ISD::FREEZE;1989  }1990 1991  llvm_unreachable("Unknown instruction type encountered!");1992}1993 1994int TargetLoweringBase::IntrinsicIDToISD(Intrinsic::ID ID) const {1995  switch (ID) {1996  case Intrinsic::exp:1997    return ISD::FEXP;1998  case Intrinsic::exp2:1999    return ISD::FEXP2;2000  case Intrinsic::log:2001    return ISD::FLOG;2002  default:2003    return ISD::DELETED_NODE;2004  }2005}2006 2007Value *2008TargetLoweringBase::getDefaultSafeStackPointerLocation(IRBuilderBase &IRB,2009                                                       bool UseTLS) const {2010  // compiler-rt provides a variable with a magic name.  Targets that do not2011  // link with compiler-rt may also provide such a variable.2012  Module *M = IRB.GetInsertBlock()->getParent()->getParent();2013  const char *UnsafeStackPtrVar = "__safestack_unsafe_stack_ptr";2014  auto UnsafeStackPtr =2015      dyn_cast_or_null<GlobalVariable>(M->getNamedValue(UnsafeStackPtrVar));2016 2017  const DataLayout &DL = M->getDataLayout();2018  PointerType *StackPtrTy = DL.getAllocaPtrType(M->getContext());2019 2020  if (!UnsafeStackPtr) {2021    auto TLSModel = UseTLS ?2022        GlobalValue::InitialExecTLSModel :2023        GlobalValue::NotThreadLocal;2024    // The global variable is not defined yet, define it ourselves.2025    // We use the initial-exec TLS model because we do not support the2026    // variable living anywhere other than in the main executable.2027    UnsafeStackPtr = new GlobalVariable(2028        *M, StackPtrTy, false, GlobalValue::ExternalLinkage, nullptr,2029        UnsafeStackPtrVar, nullptr, TLSModel);2030  } else {2031    // The variable exists, check its type and attributes.2032    //2033    // FIXME: Move to IR verifier.2034    if (UnsafeStackPtr->getValueType() != StackPtrTy)2035      report_fatal_error(Twine(UnsafeStackPtrVar) + " must have void* type");2036    if (UseTLS != UnsafeStackPtr->isThreadLocal())2037      report_fatal_error(Twine(UnsafeStackPtrVar) + " must " +2038                         (UseTLS ? "" : "not ") + "be thread-local");2039  }2040  return UnsafeStackPtr;2041}2042 2043Value *2044TargetLoweringBase::getSafeStackPointerLocation(IRBuilderBase &IRB) const {2045  // FIXME: Can this triple check be replaced with SAFESTACK_POINTER_ADDRESS2046  // being available?2047  if (!TM.getTargetTriple().isAndroid())2048    return getDefaultSafeStackPointerLocation(IRB, true);2049 2050  Module *M = IRB.GetInsertBlock()->getParent()->getParent();2051  auto *PtrTy = PointerType::getUnqual(M->getContext());2052 2053  const char *SafestackPointerAddressName =2054      getLibcallName(RTLIB::SAFESTACK_POINTER_ADDRESS);2055  if (!SafestackPointerAddressName) {2056    M->getContext().emitError(2057        "no libcall available for safestack pointer address");2058    return PoisonValue::get(PtrTy);2059  }2060 2061  // Android provides a libc function to retrieve the address of the current2062  // thread's unsafe stack pointer.2063  FunctionCallee Fn =2064      M->getOrInsertFunction(SafestackPointerAddressName, PtrTy);2065  return IRB.CreateCall(Fn);2066}2067 2068//===----------------------------------------------------------------------===//2069//  Loop Strength Reduction hooks2070//===----------------------------------------------------------------------===//2071 2072/// isLegalAddressingMode - Return true if the addressing mode represented2073/// by AM is legal for this target, for a load/store of the specified type.2074bool TargetLoweringBase::isLegalAddressingMode(const DataLayout &DL,2075                                               const AddrMode &AM, Type *Ty,2076                                               unsigned AS, Instruction *I) const {2077  // The default implementation of this implements a conservative RISCy, r+r and2078  // r+i addr mode.2079 2080  // Scalable offsets not supported2081  if (AM.ScalableOffset)2082    return false;2083 2084  // Allows a sign-extended 16-bit immediate field.2085  if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1)2086    return false;2087 2088  // No global is ever allowed as a base.2089  if (AM.BaseGV)2090    return false;2091 2092  // Only support r+r,2093  switch (AM.Scale) {2094  case 0:  // "r+i" or just "i", depending on HasBaseReg.2095    break;2096  case 1:2097    if (AM.HasBaseReg && AM.BaseOffs)  // "r+r+i" is not allowed.2098      return false;2099    // Otherwise we have r+r or r+i.2100    break;2101  case 2:2102    if (AM.HasBaseReg || AM.BaseOffs)  // 2*r+r  or  2*r+i is not allowed.2103      return false;2104    // Allow 2*r as r+r.2105    break;2106  default: // Don't allow n * r2107    return false;2108  }2109 2110  return true;2111}2112 2113//===----------------------------------------------------------------------===//2114//  Stack Protector2115//===----------------------------------------------------------------------===//2116 2117// For OpenBSD return its special guard variable. Otherwise return nullptr,2118// so that SelectionDAG handle SSP.2119Value *TargetLoweringBase::getIRStackGuard(IRBuilderBase &IRB) const {2120  RTLIB::LibcallImpl GuardLocalImpl = getLibcallImpl(RTLIB::STACK_CHECK_GUARD);2121  if (GuardLocalImpl != RTLIB::impl___guard_local)2122    return nullptr;2123 2124  Module &M = *IRB.GetInsertBlock()->getParent()->getParent();2125  const DataLayout &DL = M.getDataLayout();2126  PointerType *PtrTy =2127      PointerType::get(M.getContext(), DL.getDefaultGlobalsAddressSpace());2128  GlobalVariable *G =2129      M.getOrInsertGlobal(getLibcallImplName(GuardLocalImpl), PtrTy);2130  G->setVisibility(GlobalValue::HiddenVisibility);2131  return G;2132}2133 2134// Currently only support "standard" __stack_chk_guard.2135// TODO: add LOAD_STACK_GUARD support.2136void TargetLoweringBase::insertSSPDeclarations(Module &M) const {2137  RTLIB::LibcallImpl StackGuardImpl = getLibcallImpl(RTLIB::STACK_CHECK_GUARD);2138  if (StackGuardImpl == RTLIB::Unsupported)2139    return;2140 2141  StringRef StackGuardVarName = getLibcallImplName(StackGuardImpl);2142  M.getOrInsertGlobal(2143      StackGuardVarName, PointerType::getUnqual(M.getContext()), [=, &M]() {2144        auto *GV = new GlobalVariable(M, PointerType::getUnqual(M.getContext()),2145                                      false, GlobalVariable::ExternalLinkage,2146                                      nullptr, StackGuardVarName);2147 2148        // FreeBSD has "__stack_chk_guard" defined externally on libc.so2149        if (M.getDirectAccessExternalData() &&2150            !TM.getTargetTriple().isOSCygMing() &&2151            !(TM.getTargetTriple().isPPC64() &&2152              TM.getTargetTriple().isOSFreeBSD()) &&2153            (!TM.getTargetTriple().isOSDarwin() ||2154             TM.getRelocationModel() == Reloc::Static))2155          GV->setDSOLocal(true);2156 2157        return GV;2158      });2159}2160 2161// Currently only support "standard" __stack_chk_guard.2162// TODO: add LOAD_STACK_GUARD support.2163Value *TargetLoweringBase::getSDagStackGuard(const Module &M) const {2164  RTLIB::LibcallImpl GuardVarImpl = getLibcallImpl(RTLIB::STACK_CHECK_GUARD);2165  if (GuardVarImpl == RTLIB::Unsupported)2166    return nullptr;2167  return M.getNamedValue(getLibcallImplName(GuardVarImpl));2168}2169 2170Function *TargetLoweringBase::getSSPStackGuardCheck(const Module &M) const {2171  // MSVC CRT has a function to validate security cookie.2172  RTLIB::LibcallImpl SecurityCheckCookieLibcall =2173      getLibcallImpl(RTLIB::SECURITY_CHECK_COOKIE);2174  if (SecurityCheckCookieLibcall != RTLIB::Unsupported)2175    return M.getFunction(getLibcallImplName(SecurityCheckCookieLibcall));2176  return nullptr;2177}2178 2179unsigned TargetLoweringBase::getMinimumJumpTableEntries() const {2180  return MinimumJumpTableEntries;2181}2182 2183void TargetLoweringBase::setMinimumJumpTableEntries(unsigned Val) {2184  MinimumJumpTableEntries = Val;2185}2186 2187unsigned TargetLoweringBase::getMinimumJumpTableDensity(bool OptForSize) const {2188  return OptForSize ? OptsizeJumpTableDensity : JumpTableDensity;2189}2190 2191unsigned TargetLoweringBase::getMaximumJumpTableSize() const {2192  return MaximumJumpTableSize;2193}2194 2195void TargetLoweringBase::setMaximumJumpTableSize(unsigned Val) {2196  MaximumJumpTableSize = Val;2197}2198 2199bool TargetLoweringBase::isJumpTableRelative() const {2200  return getTargetMachine().isPositionIndependent();2201}2202 2203unsigned TargetLoweringBase::getMinimumBitTestCmps() const {2204  return MinimumBitTestCmps;2205}2206 2207void TargetLoweringBase::setMinimumBitTestCmps(unsigned Val) {2208  MinimumBitTestCmps = Val;2209}2210 2211Align TargetLoweringBase::getPrefLoopAlignment(MachineLoop *ML) const {2212  if (TM.Options.LoopAlignment)2213    return Align(TM.Options.LoopAlignment);2214  return PrefLoopAlignment;2215}2216 2217unsigned TargetLoweringBase::getMaxPermittedBytesForAlignment(2218    MachineBasicBlock *MBB) const {2219  return MaxBytesForAlignment;2220}2221 2222//===----------------------------------------------------------------------===//2223//  Reciprocal Estimates2224//===----------------------------------------------------------------------===//2225 2226/// Get the reciprocal estimate attribute string for a function that will2227/// override the target defaults.2228static StringRef getRecipEstimateForFunc(MachineFunction &MF) {2229  const Function &F = MF.getFunction();2230  return F.getFnAttribute("reciprocal-estimates").getValueAsString();2231}2232 2233/// Construct a string for the given reciprocal operation of the given type.2234/// This string should match the corresponding option to the front-end's2235/// "-mrecip" flag assuming those strings have been passed through in an2236/// attribute string. For example, "vec-divf" for a division of a vXf32.2237static std::string getReciprocalOpName(bool IsSqrt, EVT VT) {2238  std::string Name = VT.isVector() ? "vec-" : "";2239 2240  Name += IsSqrt ? "sqrt" : "div";2241 2242  // TODO: Handle other float types?2243  if (VT.getScalarType() == MVT::f64) {2244    Name += "d";2245  } else if (VT.getScalarType() == MVT::f16) {2246    Name += "h";2247  } else {2248    assert(VT.getScalarType() == MVT::f32 &&2249           "Unexpected FP type for reciprocal estimate");2250    Name += "f";2251  }2252 2253  return Name;2254}2255 2256/// Return the character position and value (a single numeric character) of a2257/// customized refinement operation in the input string if it exists. Return2258/// false if there is no customized refinement step count.2259static bool parseRefinementStep(StringRef In, size_t &Position,2260                                uint8_t &Value) {2261  const char RefStepToken = ':';2262  Position = In.find(RefStepToken);2263  if (Position == StringRef::npos)2264    return false;2265 2266  StringRef RefStepString = In.substr(Position + 1);2267  // Allow exactly one numeric character for the additional refinement2268  // step parameter.2269  if (RefStepString.size() == 1) {2270    char RefStepChar = RefStepString[0];2271    if (isDigit(RefStepChar)) {2272      Value = RefStepChar - '0';2273      return true;2274    }2275  }2276  report_fatal_error("Invalid refinement step for -recip.");2277}2278 2279/// For the input attribute string, return one of the ReciprocalEstimate enum2280/// status values (enabled, disabled, or not specified) for this operation on2281/// the specified data type.2282static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override) {2283  if (Override.empty())2284    return TargetLoweringBase::ReciprocalEstimate::Unspecified;2285 2286  SmallVector<StringRef, 4> OverrideVector;2287  Override.split(OverrideVector, ',');2288  unsigned NumArgs = OverrideVector.size();2289 2290  // Check if "all", "none", or "default" was specified.2291  if (NumArgs == 1) {2292    // Look for an optional setting of the number of refinement steps needed2293    // for this type of reciprocal operation.2294    size_t RefPos;2295    uint8_t RefSteps;2296    if (parseRefinementStep(Override, RefPos, RefSteps)) {2297      // Split the string for further processing.2298      Override = Override.substr(0, RefPos);2299    }2300 2301    // All reciprocal types are enabled.2302    if (Override == "all")2303      return TargetLoweringBase::ReciprocalEstimate::Enabled;2304 2305    // All reciprocal types are disabled.2306    if (Override == "none")2307      return TargetLoweringBase::ReciprocalEstimate::Disabled;2308 2309    // Target defaults for enablement are used.2310    if (Override == "default")2311      return TargetLoweringBase::ReciprocalEstimate::Unspecified;2312  }2313 2314  // The attribute string may omit the size suffix ('f'/'d').2315  std::string VTName = getReciprocalOpName(IsSqrt, VT);2316  std::string VTNameNoSize = VTName;2317  VTNameNoSize.pop_back();2318  static const char DisabledPrefix = '!';2319 2320  for (StringRef RecipType : OverrideVector) {2321    size_t RefPos;2322    uint8_t RefSteps;2323    if (parseRefinementStep(RecipType, RefPos, RefSteps))2324      RecipType = RecipType.substr(0, RefPos);2325 2326    // Ignore the disablement token for string matching.2327    bool IsDisabled = RecipType[0] == DisabledPrefix;2328    if (IsDisabled)2329      RecipType = RecipType.substr(1);2330 2331    if (RecipType == VTName || RecipType == VTNameNoSize)2332      return IsDisabled ? TargetLoweringBase::ReciprocalEstimate::Disabled2333                        : TargetLoweringBase::ReciprocalEstimate::Enabled;2334  }2335 2336  return TargetLoweringBase::ReciprocalEstimate::Unspecified;2337}2338 2339/// For the input attribute string, return the customized refinement step count2340/// for this operation on the specified data type. If the step count does not2341/// exist, return the ReciprocalEstimate enum value for unspecified.2342static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override) {2343  if (Override.empty())2344    return TargetLoweringBase::ReciprocalEstimate::Unspecified;2345 2346  SmallVector<StringRef, 4> OverrideVector;2347  Override.split(OverrideVector, ',');2348  unsigned NumArgs = OverrideVector.size();2349 2350  // Check if "all", "default", or "none" was specified.2351  if (NumArgs == 1) {2352    // Look for an optional setting of the number of refinement steps needed2353    // for this type of reciprocal operation.2354    size_t RefPos;2355    uint8_t RefSteps;2356    if (!parseRefinementStep(Override, RefPos, RefSteps))2357      return TargetLoweringBase::ReciprocalEstimate::Unspecified;2358 2359    // Split the string for further processing.2360    Override = Override.substr(0, RefPos);2361    assert(Override != "none" &&2362           "Disabled reciprocals, but specifed refinement steps?");2363 2364    // If this is a general override, return the specified number of steps.2365    if (Override == "all" || Override == "default")2366      return RefSteps;2367  }2368 2369  // The attribute string may omit the size suffix ('f'/'d').2370  std::string VTName = getReciprocalOpName(IsSqrt, VT);2371  std::string VTNameNoSize = VTName;2372  VTNameNoSize.pop_back();2373 2374  for (StringRef RecipType : OverrideVector) {2375    size_t RefPos;2376    uint8_t RefSteps;2377    if (!parseRefinementStep(RecipType, RefPos, RefSteps))2378      continue;2379 2380    RecipType = RecipType.substr(0, RefPos);2381    if (RecipType == VTName || RecipType == VTNameNoSize)2382      return RefSteps;2383  }2384 2385  return TargetLoweringBase::ReciprocalEstimate::Unspecified;2386}2387 2388int TargetLoweringBase::getRecipEstimateSqrtEnabled(EVT VT,2389                                                    MachineFunction &MF) const {2390  return getOpEnabled(true, VT, getRecipEstimateForFunc(MF));2391}2392 2393int TargetLoweringBase::getRecipEstimateDivEnabled(EVT VT,2394                                                   MachineFunction &MF) const {2395  return getOpEnabled(false, VT, getRecipEstimateForFunc(MF));2396}2397 2398int TargetLoweringBase::getSqrtRefinementSteps(EVT VT,2399                                               MachineFunction &MF) const {2400  return getOpRefinementSteps(true, VT, getRecipEstimateForFunc(MF));2401}2402 2403int TargetLoweringBase::getDivRefinementSteps(EVT VT,2404                                              MachineFunction &MF) const {2405  return getOpRefinementSteps(false, VT, getRecipEstimateForFunc(MF));2406}2407 2408bool TargetLoweringBase::isLoadBitCastBeneficial(2409    EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG,2410    const MachineMemOperand &MMO) const {2411  // Single-element vectors are scalarized, so we should generally avoid having2412  // any memory operations on such types, as they would get scalarized too.2413  if (LoadVT.isFixedLengthVector() && BitcastVT.isFixedLengthVector() &&2414      BitcastVT.getVectorNumElements() == 1)2415    return false;2416 2417  // Don't do if we could do an indexed load on the original type, but not on2418  // the new one.2419  if (!LoadVT.isSimple() || !BitcastVT.isSimple())2420    return true;2421 2422  MVT LoadMVT = LoadVT.getSimpleVT();2423 2424  // Don't bother doing this if it's just going to be promoted again later, as2425  // doing so might interfere with other combines.2426  if (getOperationAction(ISD::LOAD, LoadMVT) == Promote &&2427      getTypeToPromoteTo(ISD::LOAD, LoadMVT) == BitcastVT.getSimpleVT())2428    return false;2429 2430  unsigned Fast = 0;2431  return allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), BitcastVT,2432                            MMO, &Fast) &&2433         Fast;2434}2435 2436void TargetLoweringBase::finalizeLowering(MachineFunction &MF) const {2437  MF.getRegInfo().freezeReservedRegs();2438}2439 2440MachineMemOperand::Flags TargetLoweringBase::getLoadMemOperandFlags(2441    const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC,2442    const TargetLibraryInfo *LibInfo) const {2443  MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad;2444  if (LI.isVolatile())2445    Flags |= MachineMemOperand::MOVolatile;2446 2447  if (LI.hasMetadata(LLVMContext::MD_nontemporal))2448    Flags |= MachineMemOperand::MONonTemporal;2449 2450  if (LI.hasMetadata(LLVMContext::MD_invariant_load))2451    Flags |= MachineMemOperand::MOInvariant;2452 2453  if (isDereferenceableAndAlignedPointer(LI.getPointerOperand(), LI.getType(),2454                                         LI.getAlign(), DL, &LI, AC,2455                                         /*DT=*/nullptr, LibInfo))2456    Flags |= MachineMemOperand::MODereferenceable;2457 2458  Flags |= getTargetMMOFlags(LI);2459  return Flags;2460}2461 2462MachineMemOperand::Flags2463TargetLoweringBase::getStoreMemOperandFlags(const StoreInst &SI,2464                                            const DataLayout &DL) const {2465  MachineMemOperand::Flags Flags = MachineMemOperand::MOStore;2466 2467  if (SI.isVolatile())2468    Flags |= MachineMemOperand::MOVolatile;2469 2470  if (SI.hasMetadata(LLVMContext::MD_nontemporal))2471    Flags |= MachineMemOperand::MONonTemporal;2472 2473  // FIXME: Not preserving dereferenceable2474  Flags |= getTargetMMOFlags(SI);2475  return Flags;2476}2477 2478MachineMemOperand::Flags2479TargetLoweringBase::getAtomicMemOperandFlags(const Instruction &AI,2480                                             const DataLayout &DL) const {2481  auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;2482 2483  if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(&AI)) {2484    if (RMW->isVolatile())2485      Flags |= MachineMemOperand::MOVolatile;2486  } else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(&AI)) {2487    if (CmpX->isVolatile())2488      Flags |= MachineMemOperand::MOVolatile;2489  } else2490    llvm_unreachable("not an atomic instruction");2491 2492  // FIXME: Not preserving dereferenceable2493  Flags |= getTargetMMOFlags(AI);2494  return Flags;2495}2496 2497MachineMemOperand::Flags TargetLoweringBase::getVPIntrinsicMemOperandFlags(2498    const VPIntrinsic &VPIntrin) const {2499  MachineMemOperand::Flags Flags = MachineMemOperand::MONone;2500  Intrinsic::ID IntrinID = VPIntrin.getIntrinsicID();2501 2502  switch (IntrinID) {2503  default:2504    llvm_unreachable("unexpected intrinsic. Existing code may be appropriate "2505                     "for it, but support must be explicitly enabled");2506  case Intrinsic::vp_load:2507  case Intrinsic::vp_gather:2508  case Intrinsic::experimental_vp_strided_load:2509    Flags = MachineMemOperand::MOLoad;2510    break;2511  case Intrinsic::vp_store:2512  case Intrinsic::vp_scatter:2513  case Intrinsic::experimental_vp_strided_store:2514    Flags = MachineMemOperand::MOStore;2515    break;2516  }2517 2518  if (VPIntrin.hasMetadata(LLVMContext::MD_nontemporal))2519    Flags |= MachineMemOperand::MONonTemporal;2520 2521  Flags |= getTargetMMOFlags(VPIntrin);2522  return Flags;2523}2524 2525Instruction *TargetLoweringBase::emitLeadingFence(IRBuilderBase &Builder,2526                                                  Instruction *Inst,2527                                                  AtomicOrdering Ord) const {2528  if (isReleaseOrStronger(Ord) && Inst->hasAtomicStore())2529    return Builder.CreateFence(Ord);2530  else2531    return nullptr;2532}2533 2534Instruction *TargetLoweringBase::emitTrailingFence(IRBuilderBase &Builder,2535                                                   Instruction *Inst,2536                                                   AtomicOrdering Ord) const {2537  if (isAcquireOrStronger(Ord))2538    return Builder.CreateFence(Ord);2539  else2540    return nullptr;2541}2542 2543//===----------------------------------------------------------------------===//2544//  GlobalISel Hooks2545//===----------------------------------------------------------------------===//2546 2547bool TargetLoweringBase::shouldLocalize(const MachineInstr &MI,2548                                        const TargetTransformInfo *TTI) const {2549  auto &MF = *MI.getMF();2550  auto &MRI = MF.getRegInfo();2551  // Assuming a spill and reload of a value has a cost of 1 instruction each,2552  // this helper function computes the maximum number of uses we should consider2553  // for remat. E.g. on arm64 global addresses take 2 insts to materialize. We2554  // break even in terms of code size when the original MI has 2 users vs2555  // choosing to potentially spill. Any more than 2 users we we have a net code2556  // size increase. This doesn't take into account register pressure though.2557  auto maxUses = [](unsigned RematCost) {2558    // A cost of 1 means remats are basically free.2559    if (RematCost == 1)2560      return std::numeric_limits<unsigned>::max();2561    if (RematCost == 2)2562      return 2U;2563 2564    // Remat is too expensive, only sink if there's one user.2565    if (RematCost > 2)2566      return 1U;2567    llvm_unreachable("Unexpected remat cost");2568  };2569 2570  switch (MI.getOpcode()) {2571  default:2572    return false;2573  // Constants-like instructions should be close to their users.2574  // We don't want long live-ranges for them.2575  case TargetOpcode::G_CONSTANT:2576  case TargetOpcode::G_FCONSTANT:2577  case TargetOpcode::G_FRAME_INDEX:2578  case TargetOpcode::G_INTTOPTR:2579    return true;2580  case TargetOpcode::G_GLOBAL_VALUE: {2581    unsigned RematCost = TTI->getGISelRematGlobalCost();2582    Register Reg = MI.getOperand(0).getReg();2583    unsigned MaxUses = maxUses(RematCost);2584    if (MaxUses == UINT_MAX)2585      return true; // Remats are "free" so always localize.2586    return MRI.hasAtMostUserInstrs(Reg, MaxUses);2587  }2588  }2589}2590