brintos / llvm-project-archived public Read only

1//===--- InterpBuiltin.cpp - Interpreter for the constexpr VM ---*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8#include "../ExprConstShared.h"9#include "Boolean.h"10#include "EvalEmitter.h"11#include "InterpBuiltinBitCast.h"12#include "InterpHelpers.h"13#include "PrimType.h"14#include "Program.h"15#include "clang/AST/InferAlloc.h"16#include "clang/AST/OSLog.h"17#include "clang/AST/RecordLayout.h"18#include "clang/Basic/Builtins.h"19#include "clang/Basic/TargetBuiltins.h"20#include "clang/Basic/TargetInfo.h"21#include "llvm/ADT/StringExtras.h"22#include "llvm/Support/AllocToken.h"23#include "llvm/Support/ErrorHandling.h"24#include "llvm/Support/SipHash.h"25 26namespace clang {27namespace interp {28 29[[maybe_unused]] static bool isNoopBuiltin(unsigned ID) {30  switch (ID) {31  case Builtin::BIas_const:32  case Builtin::BIforward:33  case Builtin::BIforward_like:34  case Builtin::BImove:35  case Builtin::BImove_if_noexcept:36  case Builtin::BIaddressof:37  case Builtin::BI__addressof:38  case Builtin::BI__builtin_addressof:39  case Builtin::BI__builtin_launder:40    return true;41  default:42    return false;43  }44  return false;45}46 47static void discard(InterpStack &Stk, PrimType T) {48  TYPE_SWITCH(T, { Stk.discard<T>(); });49}50 51static APSInt popToAPSInt(InterpStack &Stk, PrimType T) {52  INT_TYPE_SWITCH(T, return Stk.pop<T>().toAPSInt());53}54 55static APSInt popToAPSInt(InterpState &S, const Expr *E) {56  return popToAPSInt(S.Stk, *S.getContext().classify(E->getType()));57}58static APSInt popToAPSInt(InterpState &S, QualType T) {59  return popToAPSInt(S.Stk, *S.getContext().classify(T));60}61 62/// Pushes \p Val on the stack as the type given by \p QT.63static void pushInteger(InterpState &S, const APSInt &Val, QualType QT) {64  assert(QT->isSignedIntegerOrEnumerationType() ||65         QT->isUnsignedIntegerOrEnumerationType());66  OptPrimType T = S.getContext().classify(QT);67  assert(T);68 69  unsigned BitWidth = S.getASTContext().getTypeSize(QT);70 71  if (T == PT_IntAPS) {72    auto Result = S.allocAP<IntegralAP<true>>(BitWidth);73    Result.copy(Val);74    S.Stk.push<IntegralAP<true>>(Result);75    return;76  }77 78  if (T == PT_IntAP) {79    auto Result = S.allocAP<IntegralAP<false>>(BitWidth);80    Result.copy(Val);81    S.Stk.push<IntegralAP<false>>(Result);82    return;83  }84 85  if (QT->isSignedIntegerOrEnumerationType()) {86    int64_t V = Val.getSExtValue();87    INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V, BitWidth)); });88  } else {89    assert(QT->isUnsignedIntegerOrEnumerationType());90    uint64_t V = Val.getZExtValue();91    INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V, BitWidth)); });92  }93}94 95template <typename T>96static void pushInteger(InterpState &S, T Val, QualType QT) {97  if constexpr (std::is_same_v<T, APInt>)98    pushInteger(S, APSInt(Val, !std::is_signed_v<T>), QT);99  else if constexpr (std::is_same_v<T, APSInt>)100    pushInteger(S, Val, QT);101  else102    pushInteger(S,103                APSInt(APInt(sizeof(T) * 8, static_cast<uint64_t>(Val),104                             std::is_signed_v<T>),105                       !std::is_signed_v<T>),106                QT);107}108 109static void assignInteger(InterpState &S, const Pointer &Dest, PrimType ValueT,110                          const APSInt &Value) {111 112  if (ValueT == PT_IntAPS) {113    Dest.deref<IntegralAP<true>>() =114        S.allocAP<IntegralAP<true>>(Value.getBitWidth());115    Dest.deref<IntegralAP<true>>().copy(Value);116  } else if (ValueT == PT_IntAP) {117    Dest.deref<IntegralAP<false>>() =118        S.allocAP<IntegralAP<false>>(Value.getBitWidth());119    Dest.deref<IntegralAP<false>>().copy(Value);120  } else {121    INT_TYPE_SWITCH_NO_BOOL(122        ValueT, { Dest.deref<T>() = T::from(static_cast<T>(Value)); });123  }124}125 126static QualType getElemType(const Pointer &P) {127  const Descriptor *Desc = P.getFieldDesc();128  QualType T = Desc->getType();129  if (Desc->isPrimitive())130    return T;131  if (T->isPointerType())132    return T->getAs<PointerType>()->getPointeeType();133  if (Desc->isArray())134    return Desc->getElemQualType();135  if (const auto *AT = T->getAsArrayTypeUnsafe())136    return AT->getElementType();137  return T;138}139 140static void diagnoseNonConstexprBuiltin(InterpState &S, CodePtr OpPC,141                                        unsigned ID) {142  if (!S.diagnosing())143    return;144 145  auto Loc = S.Current->getSource(OpPC);146  if (S.getLangOpts().CPlusPlus11)147    S.CCEDiag(Loc, diag::note_constexpr_invalid_function)148        << /*isConstexpr=*/0 << /*isConstructor=*/0149        << S.getASTContext().BuiltinInfo.getQuotedName(ID);150  else151    S.CCEDiag(Loc, diag::note_invalid_subexpr_in_const_expr);152}153 154static llvm::APSInt convertBoolVectorToInt(const Pointer &Val) {155  assert(Val.getFieldDesc()->isPrimitiveArray() &&156         Val.getFieldDesc()->getElemQualType()->isBooleanType() &&157         "Not a boolean vector");158  unsigned NumElems = Val.getNumElems();159 160  // Each element is one bit, so create an integer with NumElts bits.161  llvm::APSInt Result(NumElems, 0);162  for (unsigned I = 0; I != NumElems; ++I) {163    if (Val.elem<bool>(I))164      Result.setBit(I);165  }166 167  return Result;168}169 170static bool interp__builtin_is_constant_evaluated(InterpState &S, CodePtr OpPC,171                                                  const InterpFrame *Frame,172                                                  const CallExpr *Call) {173  unsigned Depth = S.Current->getDepth();174  auto isStdCall = [](const FunctionDecl *F) -> bool {175    return F && F->isInStdNamespace() && F->getIdentifier() &&176           F->getIdentifier()->isStr("is_constant_evaluated");177  };178  const InterpFrame *Caller = Frame->Caller;179  // The current frame is the one for __builtin_is_constant_evaluated.180  // The one above that, potentially the one for std::is_constant_evaluated().181  if (S.inConstantContext() && !S.checkingPotentialConstantExpression() &&182      S.getEvalStatus().Diag &&183      (Depth == 0 || (Depth == 1 && isStdCall(Frame->getCallee())))) {184    if (Caller && isStdCall(Frame->getCallee())) {185      const Expr *E = Caller->getExpr(Caller->getRetPC());186      S.report(E->getExprLoc(),187               diag::warn_is_constant_evaluated_always_true_constexpr)188          << "std::is_constant_evaluated" << E->getSourceRange();189    } else {190      S.report(Call->getExprLoc(),191               diag::warn_is_constant_evaluated_always_true_constexpr)192          << "__builtin_is_constant_evaluated" << Call->getSourceRange();193    }194  }195 196  S.Stk.push<Boolean>(Boolean::from(S.inConstantContext()));197  return true;198}199 200// __builtin_assume(int)201static bool interp__builtin_assume(InterpState &S, CodePtr OpPC,202                                   const InterpFrame *Frame,203                                   const CallExpr *Call) {204  assert(Call->getNumArgs() == 1);205  discard(S.Stk, *S.getContext().classify(Call->getArg(0)));206  return true;207}208 209static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,210                                   const InterpFrame *Frame,211                                   const CallExpr *Call, unsigned ID) {212  uint64_t Limit = ~static_cast<uint64_t>(0);213  if (ID == Builtin::BIstrncmp || ID == Builtin::BI__builtin_strncmp ||214      ID == Builtin::BIwcsncmp || ID == Builtin::BI__builtin_wcsncmp)215    Limit = popToAPSInt(S.Stk, *S.getContext().classify(Call->getArg(2)))216                .getZExtValue();217 218  const Pointer &B = S.Stk.pop<Pointer>();219  const Pointer &A = S.Stk.pop<Pointer>();220  if (ID == Builtin::BIstrcmp || ID == Builtin::BIstrncmp ||221      ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp)222    diagnoseNonConstexprBuiltin(S, OpPC, ID);223 224  if (Limit == 0) {225    pushInteger(S, 0, Call->getType());226    return true;227  }228 229  if (!CheckLive(S, OpPC, A, AK_Read) || !CheckLive(S, OpPC, B, AK_Read))230    return false;231 232  if (A.isDummy() || B.isDummy())233    return false;234  if (!A.isBlockPointer() || !B.isBlockPointer())235    return false;236 237  bool IsWide = ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp ||238                ID == Builtin::BI__builtin_wcscmp ||239                ID == Builtin::BI__builtin_wcsncmp;240  assert(A.getFieldDesc()->isPrimitiveArray());241  assert(B.getFieldDesc()->isPrimitiveArray());242 243  // Different element types shouldn't happen, but with casts they can.244  if (!S.getASTContext().hasSameUnqualifiedType(getElemType(A), getElemType(B)))245    return false;246 247  PrimType ElemT = *S.getContext().classify(getElemType(A));248 249  auto returnResult = [&](int V) -> bool {250    pushInteger(S, V, Call->getType());251    return true;252  };253 254  unsigned IndexA = A.getIndex();255  unsigned IndexB = B.getIndex();256  uint64_t Steps = 0;257  for (;; ++IndexA, ++IndexB, ++Steps) {258 259    if (Steps >= Limit)260      break;261    const Pointer &PA = A.atIndex(IndexA);262    const Pointer &PB = B.atIndex(IndexB);263    if (!CheckRange(S, OpPC, PA, AK_Read) ||264        !CheckRange(S, OpPC, PB, AK_Read)) {265      return false;266    }267 268    if (IsWide) {269      INT_TYPE_SWITCH(ElemT, {270        T CA = PA.deref<T>();271        T CB = PB.deref<T>();272        if (CA > CB)273          return returnResult(1);274        if (CA < CB)275          return returnResult(-1);276        if (CA.isZero() || CB.isZero())277          return returnResult(0);278      });279      continue;280    }281 282    uint8_t CA = PA.deref<uint8_t>();283    uint8_t CB = PB.deref<uint8_t>();284 285    if (CA > CB)286      return returnResult(1);287    if (CA < CB)288      return returnResult(-1);289    if (CA == 0 || CB == 0)290      return returnResult(0);291  }292 293  return returnResult(0);294}295 296static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC,297                                   const InterpFrame *Frame,298                                   const CallExpr *Call, unsigned ID) {299  const Pointer &StrPtr = S.Stk.pop<Pointer>().expand();300 301  if (ID == Builtin::BIstrlen || ID == Builtin::BIwcslen)302    diagnoseNonConstexprBuiltin(S, OpPC, ID);303 304  if (!CheckArray(S, OpPC, StrPtr))305    return false;306 307  if (!CheckLive(S, OpPC, StrPtr, AK_Read))308    return false;309 310  if (!CheckDummy(S, OpPC, StrPtr.block(), AK_Read))311    return false;312 313  if (!StrPtr.getFieldDesc()->isPrimitiveArray())314    return false;315 316  assert(StrPtr.getFieldDesc()->isPrimitiveArray());317  unsigned ElemSize = StrPtr.getFieldDesc()->getElemSize();318 319  if (ID == Builtin::BI__builtin_wcslen || ID == Builtin::BIwcslen) {320    [[maybe_unused]] const ASTContext &AC = S.getASTContext();321    assert(ElemSize == AC.getTypeSizeInChars(AC.getWCharType()).getQuantity());322  }323 324  size_t Len = 0;325  for (size_t I = StrPtr.getIndex();; ++I, ++Len) {326    const Pointer &ElemPtr = StrPtr.atIndex(I);327 328    if (!CheckRange(S, OpPC, ElemPtr, AK_Read))329      return false;330 331    uint32_t Val;332    switch (ElemSize) {333    case 1:334      Val = ElemPtr.deref<uint8_t>();335      break;336    case 2:337      Val = ElemPtr.deref<uint16_t>();338      break;339    case 4:340      Val = ElemPtr.deref<uint32_t>();341      break;342    default:343      llvm_unreachable("Unsupported char size");344    }345    if (Val == 0)346      break;347  }348 349  pushInteger(S, Len, Call->getType());350 351  return true;352}353 354static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,355                                const InterpFrame *Frame, const CallExpr *Call,356                                bool Signaling) {357  const Pointer &Arg = S.Stk.pop<Pointer>();358 359  if (!CheckLoad(S, OpPC, Arg))360    return false;361 362  assert(Arg.getFieldDesc()->isPrimitiveArray());363 364  // Convert the given string to an integer using StringRef's API.365  llvm::APInt Fill;366  std::string Str;367  assert(Arg.getNumElems() >= 1);368  for (unsigned I = 0;; ++I) {369    const Pointer &Elem = Arg.atIndex(I);370 371    if (!CheckLoad(S, OpPC, Elem))372      return false;373 374    if (Elem.deref<int8_t>() == 0)375      break;376 377    Str += Elem.deref<char>();378  }379 380  // Treat empty strings as if they were zero.381  if (Str.empty())382    Fill = llvm::APInt(32, 0);383  else if (StringRef(Str).getAsInteger(0, Fill))384    return false;385 386  const llvm::fltSemantics &TargetSemantics =387      S.getASTContext().getFloatTypeSemantics(388          Call->getDirectCallee()->getReturnType());389 390  Floating Result = S.allocFloat(TargetSemantics);391  if (S.getASTContext().getTargetInfo().isNan2008()) {392    if (Signaling)393      Result.copy(394          llvm::APFloat::getSNaN(TargetSemantics, /*Negative=*/false, &Fill));395    else396      Result.copy(397          llvm::APFloat::getQNaN(TargetSemantics, /*Negative=*/false, &Fill));398  } else {399    // Prior to IEEE 754-2008, architectures were allowed to choose whether400    // the first bit of their significand was set for qNaN or sNaN. MIPS chose401    // a different encoding to what became a standard in 2008, and for pre-402    // 2008 revisions, MIPS interpreted sNaN-2008 as qNan and qNaN-2008 as403    // sNaN. This is now known as "legacy NaN" encoding.404    if (Signaling)405      Result.copy(406          llvm::APFloat::getQNaN(TargetSemantics, /*Negative=*/false, &Fill));407    else408      Result.copy(409          llvm::APFloat::getSNaN(TargetSemantics, /*Negative=*/false, &Fill));410  }411 412  S.Stk.push<Floating>(Result);413  return true;414}415 416static bool interp__builtin_inf(InterpState &S, CodePtr OpPC,417                                const InterpFrame *Frame,418                                const CallExpr *Call) {419  const llvm::fltSemantics &TargetSemantics =420      S.getASTContext().getFloatTypeSemantics(421          Call->getDirectCallee()->getReturnType());422 423  Floating Result = S.allocFloat(TargetSemantics);424  Result.copy(APFloat::getInf(TargetSemantics));425  S.Stk.push<Floating>(Result);426  return true;427}428 429static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC,430                                     const InterpFrame *Frame) {431  const Floating &Arg2 = S.Stk.pop<Floating>();432  const Floating &Arg1 = S.Stk.pop<Floating>();433  Floating Result = S.allocFloat(Arg1.getSemantics());434 435  APFloat Copy = Arg1.getAPFloat();436  Copy.copySign(Arg2.getAPFloat());437  Result.copy(Copy);438  S.Stk.push<Floating>(Result);439 440  return true;441}442 443static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC,444                                 const InterpFrame *Frame, bool IsNumBuiltin) {445  const Floating &RHS = S.Stk.pop<Floating>();446  const Floating &LHS = S.Stk.pop<Floating>();447  Floating Result = S.allocFloat(LHS.getSemantics());448 449  if (IsNumBuiltin)450    Result.copy(llvm::minimumnum(LHS.getAPFloat(), RHS.getAPFloat()));451  else452    Result.copy(minnum(LHS.getAPFloat(), RHS.getAPFloat()));453  S.Stk.push<Floating>(Result);454  return true;455}456 457static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC,458                                 const InterpFrame *Frame, bool IsNumBuiltin) {459  const Floating &RHS = S.Stk.pop<Floating>();460  const Floating &LHS = S.Stk.pop<Floating>();461  Floating Result = S.allocFloat(LHS.getSemantics());462 463  if (IsNumBuiltin)464    Result.copy(llvm::maximumnum(LHS.getAPFloat(), RHS.getAPFloat()));465  else466    Result.copy(maxnum(LHS.getAPFloat(), RHS.getAPFloat()));467  S.Stk.push<Floating>(Result);468  return true;469}470 471/// Defined as __builtin_isnan(...), to accommodate the fact that it can472/// take a float, double, long double, etc.473/// But for us, that's all a Floating anyway.474static bool interp__builtin_isnan(InterpState &S, CodePtr OpPC,475                                  const InterpFrame *Frame,476                                  const CallExpr *Call) {477  const Floating &Arg = S.Stk.pop<Floating>();478 479  pushInteger(S, Arg.isNan(), Call->getType());480  return true;481}482 483static bool interp__builtin_issignaling(InterpState &S, CodePtr OpPC,484                                        const InterpFrame *Frame,485                                        const CallExpr *Call) {486  const Floating &Arg = S.Stk.pop<Floating>();487 488  pushInteger(S, Arg.isSignaling(), Call->getType());489  return true;490}491 492static bool interp__builtin_isinf(InterpState &S, CodePtr OpPC,493                                  const InterpFrame *Frame, bool CheckSign,494                                  const CallExpr *Call) {495  const Floating &Arg = S.Stk.pop<Floating>();496  APFloat F = Arg.getAPFloat();497  bool IsInf = F.isInfinity();498 499  if (CheckSign)500    pushInteger(S, IsInf ? (F.isNegative() ? -1 : 1) : 0, Call->getType());501  else502    pushInteger(S, IsInf, Call->getType());503  return true;504}505 506static bool interp__builtin_isfinite(InterpState &S, CodePtr OpPC,507                                     const InterpFrame *Frame,508                                     const CallExpr *Call) {509  const Floating &Arg = S.Stk.pop<Floating>();510 511  pushInteger(S, Arg.isFinite(), Call->getType());512  return true;513}514 515static bool interp__builtin_isnormal(InterpState &S, CodePtr OpPC,516                                     const InterpFrame *Frame,517                                     const CallExpr *Call) {518  const Floating &Arg = S.Stk.pop<Floating>();519 520  pushInteger(S, Arg.isNormal(), Call->getType());521  return true;522}523 524static bool interp__builtin_issubnormal(InterpState &S, CodePtr OpPC,525                                        const InterpFrame *Frame,526                                        const CallExpr *Call) {527  const Floating &Arg = S.Stk.pop<Floating>();528 529  pushInteger(S, Arg.isDenormal(), Call->getType());530  return true;531}532 533static bool interp__builtin_iszero(InterpState &S, CodePtr OpPC,534                                   const InterpFrame *Frame,535                                   const CallExpr *Call) {536  const Floating &Arg = S.Stk.pop<Floating>();537 538  pushInteger(S, Arg.isZero(), Call->getType());539  return true;540}541 542static bool interp__builtin_signbit(InterpState &S, CodePtr OpPC,543                                    const InterpFrame *Frame,544                                    const CallExpr *Call) {545  const Floating &Arg = S.Stk.pop<Floating>();546 547  pushInteger(S, Arg.isNegative(), Call->getType());548  return true;549}550 551static bool interp_floating_comparison(InterpState &S, CodePtr OpPC,552                                       const CallExpr *Call, unsigned ID) {553  const Floating &RHS = S.Stk.pop<Floating>();554  const Floating &LHS = S.Stk.pop<Floating>();555 556  pushInteger(557      S,558      [&] {559        switch (ID) {560        case Builtin::BI__builtin_isgreater:561          return LHS > RHS;562        case Builtin::BI__builtin_isgreaterequal:563          return LHS >= RHS;564        case Builtin::BI__builtin_isless:565          return LHS < RHS;566        case Builtin::BI__builtin_islessequal:567          return LHS <= RHS;568        case Builtin::BI__builtin_islessgreater: {569          ComparisonCategoryResult Cmp = LHS.compare(RHS);570          return Cmp == ComparisonCategoryResult::Less ||571                 Cmp == ComparisonCategoryResult::Greater;572        }573        case Builtin::BI__builtin_isunordered:574          return LHS.compare(RHS) == ComparisonCategoryResult::Unordered;575        default:576          llvm_unreachable("Unexpected builtin ID: Should be a floating point "577                           "comparison function");578        }579      }(),580      Call->getType());581  return true;582}583 584/// First parameter to __builtin_isfpclass is the floating value, the585/// second one is an integral value.586static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC,587                                      const InterpFrame *Frame,588                                      const CallExpr *Call) {589  APSInt FPClassArg = popToAPSInt(S, Call->getArg(1));590  const Floating &F = S.Stk.pop<Floating>();591 592  int32_t Result = static_cast<int32_t>(593      (F.classify() & std::move(FPClassArg)).getZExtValue());594  pushInteger(S, Result, Call->getType());595 596  return true;597}598 599/// Five int values followed by one floating value.600/// __builtin_fpclassify(int, int, int, int, int, float)601static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,602                                       const InterpFrame *Frame,603                                       const CallExpr *Call) {604  const Floating &Val = S.Stk.pop<Floating>();605 606  PrimType IntT = *S.getContext().classify(Call->getArg(0));607  APSInt Values[5];608  for (unsigned I = 0; I != 5; ++I)609    Values[4 - I] = popToAPSInt(S.Stk, IntT);610 611  unsigned Index;612  switch (Val.getCategory()) {613  case APFloat::fcNaN:614    Index = 0;615    break;616  case APFloat::fcInfinity:617    Index = 1;618    break;619  case APFloat::fcNormal:620    Index = Val.isDenormal() ? 3 : 2;621    break;622  case APFloat::fcZero:623    Index = 4;624    break;625  }626 627  // The last argument is first on the stack.628  assert(Index <= 4);629 630  pushInteger(S, Values[Index], Call->getType());631  return true;632}633 634static inline Floating abs(InterpState &S, const Floating &In) {635  if (!In.isNegative())636    return In;637 638  Floating Output = S.allocFloat(In.getSemantics());639  APFloat New = In.getAPFloat();640  New.changeSign();641  Output.copy(New);642  return Output;643}644 645// The C standard says "fabs raises no floating-point exceptions,646// even if x is a signaling NaN. The returned value is independent of647// the current rounding direction mode."  Therefore constant folding can648// proceed without regard to the floating point settings.649// Reference, WG14 N2478 F.10.4.3650static bool interp__builtin_fabs(InterpState &S, CodePtr OpPC,651                                 const InterpFrame *Frame) {652  const Floating &Val = S.Stk.pop<Floating>();653  S.Stk.push<Floating>(abs(S, Val));654  return true;655}656 657static bool interp__builtin_abs(InterpState &S, CodePtr OpPC,658                                const InterpFrame *Frame,659                                const CallExpr *Call) {660  APSInt Val = popToAPSInt(S, Call->getArg(0));661  if (Val ==662      APSInt(APInt::getSignedMinValue(Val.getBitWidth()), /*IsUnsigned=*/false))663    return false;664  if (Val.isNegative())665    Val.negate();666  pushInteger(S, Val, Call->getType());667  return true;668}669 670static bool interp__builtin_popcount(InterpState &S, CodePtr OpPC,671                                     const InterpFrame *Frame,672                                     const CallExpr *Call) {673  APSInt Val;674  if (Call->getArg(0)->getType()->isExtVectorBoolType()) {675    const Pointer &Arg = S.Stk.pop<Pointer>();676    Val = convertBoolVectorToInt(Arg);677  } else {678    Val = popToAPSInt(S, Call->getArg(0));679  }680  pushInteger(S, Val.popcount(), Call->getType());681  return true;682}683 684static bool interp__builtin_classify_type(InterpState &S, CodePtr OpPC,685                                          const InterpFrame *Frame,686                                          const CallExpr *Call) {687  // This is an unevaluated call, so there are no arguments on the stack.688  assert(Call->getNumArgs() == 1);689  const Expr *Arg = Call->getArg(0);690 691  GCCTypeClass ResultClass =692      EvaluateBuiltinClassifyType(Arg->getType(), S.getLangOpts());693  int32_t ReturnVal = static_cast<int32_t>(ResultClass);694  pushInteger(S, ReturnVal, Call->getType());695  return true;696}697 698// __builtin_expect(long, long)699// __builtin_expect_with_probability(long, long, double)700static bool interp__builtin_expect(InterpState &S, CodePtr OpPC,701                                   const InterpFrame *Frame,702                                   const CallExpr *Call) {703  // The return value is simply the value of the first parameter.704  // We ignore the probability.705  unsigned NumArgs = Call->getNumArgs();706  assert(NumArgs == 2 || NumArgs == 3);707 708  PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());709  if (NumArgs == 3)710    S.Stk.discard<Floating>();711  discard(S.Stk, ArgT);712 713  APSInt Val = popToAPSInt(S.Stk, ArgT);714  pushInteger(S, Val, Call->getType());715  return true;716}717 718static bool interp__builtin_addressof(InterpState &S, CodePtr OpPC,719                                      const InterpFrame *Frame,720                                      const CallExpr *Call) {721#ifndef NDEBUG722  assert(Call->getArg(0)->isLValue());723  PrimType PtrT = S.getContext().classify(Call->getArg(0)).value_or(PT_Ptr);724  assert(PtrT == PT_Ptr &&725         "Unsupported pointer type passed to __builtin_addressof()");726#endif727  return true;728}729 730static bool interp__builtin_move(InterpState &S, CodePtr OpPC,731                                 const InterpFrame *Frame,732                                 const CallExpr *Call) {733  return Call->getDirectCallee()->isConstexpr();734}735 736static bool interp__builtin_eh_return_data_regno(InterpState &S, CodePtr OpPC,737                                                 const InterpFrame *Frame,738                                                 const CallExpr *Call) {739  APSInt Arg = popToAPSInt(S, Call->getArg(0));740 741  int Result = S.getASTContext().getTargetInfo().getEHDataRegisterNumber(742      Arg.getZExtValue());743  pushInteger(S, Result, Call->getType());744  return true;745}746 747// Two integral values followed by a pointer (lhs, rhs, resultOut)748static bool interp__builtin_overflowop(InterpState &S, CodePtr OpPC,749                                       const CallExpr *Call,750                                       unsigned BuiltinOp) {751  const Pointer &ResultPtr = S.Stk.pop<Pointer>();752  if (ResultPtr.isDummy() || !ResultPtr.isBlockPointer())753    return false;754 755  PrimType RHST = *S.getContext().classify(Call->getArg(1)->getType());756  PrimType LHST = *S.getContext().classify(Call->getArg(0)->getType());757  APSInt RHS = popToAPSInt(S.Stk, RHST);758  APSInt LHS = popToAPSInt(S.Stk, LHST);759  QualType ResultType = Call->getArg(2)->getType()->getPointeeType();760  PrimType ResultT = *S.getContext().classify(ResultType);761  bool Overflow;762 763  APSInt Result;764  if (BuiltinOp == Builtin::BI__builtin_add_overflow ||765      BuiltinOp == Builtin::BI__builtin_sub_overflow ||766      BuiltinOp == Builtin::BI__builtin_mul_overflow) {767    bool IsSigned = LHS.isSigned() || RHS.isSigned() ||768                    ResultType->isSignedIntegerOrEnumerationType();769    bool AllSigned = LHS.isSigned() && RHS.isSigned() &&770                     ResultType->isSignedIntegerOrEnumerationType();771    uint64_t LHSSize = LHS.getBitWidth();772    uint64_t RHSSize = RHS.getBitWidth();773    uint64_t ResultSize = S.getASTContext().getTypeSize(ResultType);774    uint64_t MaxBits = std::max(std::max(LHSSize, RHSSize), ResultSize);775 776    // Add an additional bit if the signedness isn't uniformly agreed to. We777    // could do this ONLY if there is a signed and an unsigned that both have778    // MaxBits, but the code to check that is pretty nasty.  The issue will be779    // caught in the shrink-to-result later anyway.780    if (IsSigned && !AllSigned)781      ++MaxBits;782 783    LHS = APSInt(LHS.extOrTrunc(MaxBits), !IsSigned);784    RHS = APSInt(RHS.extOrTrunc(MaxBits), !IsSigned);785    Result = APSInt(MaxBits, !IsSigned);786  }787 788  // Find largest int.789  switch (BuiltinOp) {790  default:791    llvm_unreachable("Invalid value for BuiltinOp");792  case Builtin::BI__builtin_add_overflow:793  case Builtin::BI__builtin_sadd_overflow:794  case Builtin::BI__builtin_saddl_overflow:795  case Builtin::BI__builtin_saddll_overflow:796  case Builtin::BI__builtin_uadd_overflow:797  case Builtin::BI__builtin_uaddl_overflow:798  case Builtin::BI__builtin_uaddll_overflow:799    Result = LHS.isSigned() ? LHS.sadd_ov(RHS, Overflow)800                            : LHS.uadd_ov(RHS, Overflow);801    break;802  case Builtin::BI__builtin_sub_overflow:803  case Builtin::BI__builtin_ssub_overflow:804  case Builtin::BI__builtin_ssubl_overflow:805  case Builtin::BI__builtin_ssubll_overflow:806  case Builtin::BI__builtin_usub_overflow:807  case Builtin::BI__builtin_usubl_overflow:808  case Builtin::BI__builtin_usubll_overflow:809    Result = LHS.isSigned() ? LHS.ssub_ov(RHS, Overflow)810                            : LHS.usub_ov(RHS, Overflow);811    break;812  case Builtin::BI__builtin_mul_overflow:813  case Builtin::BI__builtin_smul_overflow:814  case Builtin::BI__builtin_smull_overflow:815  case Builtin::BI__builtin_smulll_overflow:816  case Builtin::BI__builtin_umul_overflow:817  case Builtin::BI__builtin_umull_overflow:818  case Builtin::BI__builtin_umulll_overflow:819    Result = LHS.isSigned() ? LHS.smul_ov(RHS, Overflow)820                            : LHS.umul_ov(RHS, Overflow);821    break;822  }823 824  // In the case where multiple sizes are allowed, truncate and see if825  // the values are the same.826  if (BuiltinOp == Builtin::BI__builtin_add_overflow ||827      BuiltinOp == Builtin::BI__builtin_sub_overflow ||828      BuiltinOp == Builtin::BI__builtin_mul_overflow) {829    // APSInt doesn't have a TruncOrSelf, so we use extOrTrunc instead,830    // since it will give us the behavior of a TruncOrSelf in the case where831    // its parameter <= its size.  We previously set Result to be at least the832    // type-size of the result, so getTypeSize(ResultType) <= Resu833    APSInt Temp = Result.extOrTrunc(S.getASTContext().getTypeSize(ResultType));834    Temp.setIsSigned(ResultType->isSignedIntegerOrEnumerationType());835 836    if (!APSInt::isSameValue(Temp, Result))837      Overflow = true;838    Result = std::move(Temp);839  }840 841  // Write Result to ResultPtr and put Overflow on the stack.842  assignInteger(S, ResultPtr, ResultT, Result);843  if (ResultPtr.canBeInitialized())844    ResultPtr.initialize();845 846  assert(Call->getDirectCallee()->getReturnType()->isBooleanType());847  S.Stk.push<Boolean>(Overflow);848  return true;849}850 851/// Three integral values followed by a pointer (lhs, rhs, carry, carryOut).852static bool interp__builtin_carryop(InterpState &S, CodePtr OpPC,853                                    const InterpFrame *Frame,854                                    const CallExpr *Call, unsigned BuiltinOp) {855  const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();856  PrimType LHST = *S.getContext().classify(Call->getArg(0)->getType());857  PrimType RHST = *S.getContext().classify(Call->getArg(1)->getType());858  APSInt CarryIn = popToAPSInt(S.Stk, LHST);859  APSInt RHS = popToAPSInt(S.Stk, RHST);860  APSInt LHS = popToAPSInt(S.Stk, LHST);861 862  if (CarryOutPtr.isDummy() || !CarryOutPtr.isBlockPointer())863    return false;864 865  APSInt CarryOut;866 867  APSInt Result;868  // Copy the number of bits and sign.869  Result = LHS;870  CarryOut = LHS;871 872  bool FirstOverflowed = false;873  bool SecondOverflowed = false;874  switch (BuiltinOp) {875  default:876    llvm_unreachable("Invalid value for BuiltinOp");877  case Builtin::BI__builtin_addcb:878  case Builtin::BI__builtin_addcs:879  case Builtin::BI__builtin_addc:880  case Builtin::BI__builtin_addcl:881  case Builtin::BI__builtin_addcll:882    Result =883        LHS.uadd_ov(RHS, FirstOverflowed).uadd_ov(CarryIn, SecondOverflowed);884    break;885  case Builtin::BI__builtin_subcb:886  case Builtin::BI__builtin_subcs:887  case Builtin::BI__builtin_subc:888  case Builtin::BI__builtin_subcl:889  case Builtin::BI__builtin_subcll:890    Result =891        LHS.usub_ov(RHS, FirstOverflowed).usub_ov(CarryIn, SecondOverflowed);892    break;893  }894  // It is possible for both overflows to happen but CGBuiltin uses an OR so895  // this is consistent.896  CarryOut = (uint64_t)(FirstOverflowed | SecondOverflowed);897 898  QualType CarryOutType = Call->getArg(3)->getType()->getPointeeType();899  PrimType CarryOutT = *S.getContext().classify(CarryOutType);900  assignInteger(S, CarryOutPtr, CarryOutT, CarryOut);901  CarryOutPtr.initialize();902 903  assert(Call->getType() == Call->getArg(0)->getType());904  pushInteger(S, Result, Call->getType());905  return true;906}907 908static bool interp__builtin_clz(InterpState &S, CodePtr OpPC,909                                const InterpFrame *Frame, const CallExpr *Call,910                                unsigned BuiltinOp) {911 912  std::optional<APSInt> Fallback;913  if (BuiltinOp == Builtin::BI__builtin_clzg && Call->getNumArgs() == 2)914    Fallback = popToAPSInt(S, Call->getArg(1));915 916  APSInt Val;917  if (Call->getArg(0)->getType()->isExtVectorBoolType()) {918    const Pointer &Arg = S.Stk.pop<Pointer>();919    Val = convertBoolVectorToInt(Arg);920  } else {921    Val = popToAPSInt(S, Call->getArg(0));922  }923 924  // When the argument is 0, the result of GCC builtins is undefined, whereas925  // for Microsoft intrinsics, the result is the bit-width of the argument.926  bool ZeroIsUndefined = BuiltinOp != Builtin::BI__lzcnt16 &&927                         BuiltinOp != Builtin::BI__lzcnt &&928                         BuiltinOp != Builtin::BI__lzcnt64;929 930  if (Val == 0) {931    if (Fallback) {932      pushInteger(S, *Fallback, Call->getType());933      return true;934    }935 936    if (ZeroIsUndefined)937      return false;938  }939 940  pushInteger(S, Val.countl_zero(), Call->getType());941  return true;942}943 944static bool interp__builtin_ctz(InterpState &S, CodePtr OpPC,945                                const InterpFrame *Frame, const CallExpr *Call,946                                unsigned BuiltinID) {947  std::optional<APSInt> Fallback;948  if (BuiltinID == Builtin::BI__builtin_ctzg && Call->getNumArgs() == 2)949    Fallback = popToAPSInt(S, Call->getArg(1));950 951  APSInt Val;952  if (Call->getArg(0)->getType()->isExtVectorBoolType()) {953    const Pointer &Arg = S.Stk.pop<Pointer>();954    Val = convertBoolVectorToInt(Arg);955  } else {956    Val = popToAPSInt(S, Call->getArg(0));957  }958 959  if (Val == 0) {960    if (Fallback) {961      pushInteger(S, *Fallback, Call->getType());962      return true;963    }964    return false;965  }966 967  pushInteger(S, Val.countr_zero(), Call->getType());968  return true;969}970 971static bool interp__builtin_bswap(InterpState &S, CodePtr OpPC,972                                  const InterpFrame *Frame,973                                  const CallExpr *Call) {974  const APSInt &Val = popToAPSInt(S, Call->getArg(0));975  if (Val.getBitWidth() == 8)976    pushInteger(S, Val, Call->getType());977  else978    pushInteger(S, Val.byteSwap(), Call->getType());979  return true;980}981 982/// bool __atomic_always_lock_free(size_t, void const volatile*)983/// bool __atomic_is_lock_free(size_t, void const volatile*)984static bool interp__builtin_atomic_lock_free(InterpState &S, CodePtr OpPC,985                                             const InterpFrame *Frame,986                                             const CallExpr *Call,987                                             unsigned BuiltinOp) {988  auto returnBool = [&S](bool Value) -> bool {989    S.Stk.push<Boolean>(Value);990    return true;991  };992 993  const Pointer &Ptr = S.Stk.pop<Pointer>();994  const APSInt &SizeVal = popToAPSInt(S, Call->getArg(0));995 996  // For __atomic_is_lock_free(sizeof(_Atomic(T))), if the size is a power997  // of two less than or equal to the maximum inline atomic width, we know it998  // is lock-free.  If the size isn't a power of two, or greater than the999  // maximum alignment where we promote atomics, we know it is not lock-free1000  // (at least not in the sense of atomic_is_lock_free).  Otherwise,1001  // the answer can only be determined at runtime; for example, 16-byte1002  // atomics have lock-free implementations on some, but not all,1003  // x86-64 processors.1004 1005  // Check power-of-two.1006  CharUnits Size = CharUnits::fromQuantity(SizeVal.getZExtValue());1007  if (Size.isPowerOfTwo()) {1008    // Check against inlining width.1009    unsigned InlineWidthBits =1010        S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();1011    if (Size <= S.getASTContext().toCharUnitsFromBits(InlineWidthBits)) {1012 1013      // OK, we will inline appropriately-aligned operations of this size,1014      // and _Atomic(T) is appropriately-aligned.1015      if (Size == CharUnits::One())1016        return returnBool(true);1017 1018      // Same for null pointers.1019      assert(BuiltinOp != Builtin::BI__c11_atomic_is_lock_free);1020      if (Ptr.isZero())1021        return returnBool(true);1022 1023      if (Ptr.isIntegralPointer()) {1024        uint64_t IntVal = Ptr.getIntegerRepresentation();1025        if (APSInt(APInt(64, IntVal, false), true).isAligned(Size.getAsAlign()))1026          return returnBool(true);1027      }1028 1029      const Expr *PtrArg = Call->getArg(1);1030      // Otherwise, check if the type's alignment against Size.1031      if (const auto *ICE = dyn_cast<ImplicitCastExpr>(PtrArg)) {1032        // Drop the potential implicit-cast to 'const volatile void*', getting1033        // the underlying type.1034        if (ICE->getCastKind() == CK_BitCast)1035          PtrArg = ICE->getSubExpr();1036      }1037 1038      if (const auto *PtrTy = PtrArg->getType()->getAs<PointerType>()) {1039        QualType PointeeType = PtrTy->getPointeeType();1040        if (!PointeeType->isIncompleteType() &&1041            S.getASTContext().getTypeAlignInChars(PointeeType) >= Size) {1042          // OK, we will inline operations on this object.1043          return returnBool(true);1044        }1045      }1046    }1047  }1048 1049  if (BuiltinOp == Builtin::BI__atomic_always_lock_free)1050    return returnBool(false);1051 1052  return false;1053}1054 1055/// bool __c11_atomic_is_lock_free(size_t)1056static bool interp__builtin_c11_atomic_is_lock_free(InterpState &S,1057                                                    CodePtr OpPC,1058                                                    const InterpFrame *Frame,1059                                                    const CallExpr *Call) {1060  const APSInt &SizeVal = popToAPSInt(S, Call->getArg(0));1061 1062  CharUnits Size = CharUnits::fromQuantity(SizeVal.getZExtValue());1063  if (Size.isPowerOfTwo()) {1064    // Check against inlining width.1065    unsigned InlineWidthBits =1066        S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();1067    if (Size <= S.getASTContext().toCharUnitsFromBits(InlineWidthBits)) {1068      S.Stk.push<Boolean>(true);1069      return true;1070    }1071  }1072 1073  return false; // returnBool(false);1074}1075 1076/// __builtin_complex(Float A, float B);1077static bool interp__builtin_complex(InterpState &S, CodePtr OpPC,1078                                    const InterpFrame *Frame,1079                                    const CallExpr *Call) {1080  const Floating &Arg2 = S.Stk.pop<Floating>();1081  const Floating &Arg1 = S.Stk.pop<Floating>();1082  Pointer &Result = S.Stk.peek<Pointer>();1083 1084  Result.elem<Floating>(0) = Arg1;1085  Result.elem<Floating>(1) = Arg2;1086  Result.initializeAllElements();1087 1088  return true;1089}1090 1091/// __builtin_is_aligned()1092/// __builtin_align_up()1093/// __builtin_align_down()1094/// The first parameter is either an integer or a pointer.1095/// The second parameter is the requested alignment as an integer.1096static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,1097                                               const InterpFrame *Frame,1098                                               const CallExpr *Call,1099                                               unsigned BuiltinOp) {1100  const APSInt &Alignment = popToAPSInt(S, Call->getArg(1));1101 1102  if (Alignment < 0 || !Alignment.isPowerOf2()) {1103    S.FFDiag(Call, diag::note_constexpr_invalid_alignment) << Alignment;1104    return false;1105  }1106  unsigned SrcWidth = S.getASTContext().getIntWidth(Call->getArg(0)->getType());1107  APSInt MaxValue(APInt::getOneBitSet(SrcWidth, SrcWidth - 1));1108  if (APSInt::compareValues(Alignment, MaxValue) > 0) {1109    S.FFDiag(Call, diag::note_constexpr_alignment_too_big)1110        << MaxValue << Call->getArg(0)->getType() << Alignment;1111    return false;1112  }1113 1114  // The first parameter is either an integer or a pointer.1115  PrimType FirstArgT = *S.Ctx.classify(Call->getArg(0));1116 1117  if (isIntegralType(FirstArgT)) {1118    const APSInt &Src = popToAPSInt(S.Stk, FirstArgT);1119    APInt AlignMinusOne = Alignment.extOrTrunc(Src.getBitWidth()) - 1;1120    if (BuiltinOp == Builtin::BI__builtin_align_up) {1121      APSInt AlignedVal =1122          APSInt((Src + AlignMinusOne) & ~AlignMinusOne, Src.isUnsigned());1123      pushInteger(S, AlignedVal, Call->getType());1124    } else if (BuiltinOp == Builtin::BI__builtin_align_down) {1125      APSInt AlignedVal = APSInt(Src & ~AlignMinusOne, Src.isUnsigned());1126      pushInteger(S, AlignedVal, Call->getType());1127    } else {1128      assert(*S.Ctx.classify(Call->getType()) == PT_Bool);1129      S.Stk.push<Boolean>((Src & AlignMinusOne) == 0);1130    }1131    return true;1132  }1133  assert(FirstArgT == PT_Ptr);1134  const Pointer &Ptr = S.Stk.pop<Pointer>();1135  if (!Ptr.isBlockPointer())1136    return false;1137 1138  unsigned PtrOffset = Ptr.getIndex();1139  CharUnits BaseAlignment =1140      S.getASTContext().getDeclAlign(Ptr.getDeclDesc()->asValueDecl());1141  CharUnits PtrAlign =1142      BaseAlignment.alignmentAtOffset(CharUnits::fromQuantity(PtrOffset));1143 1144  if (BuiltinOp == Builtin::BI__builtin_is_aligned) {1145    if (PtrAlign.getQuantity() >= Alignment) {1146      S.Stk.push<Boolean>(true);1147      return true;1148    }1149    // If the alignment is not known to be sufficient, some cases could still1150    // be aligned at run time. However, if the requested alignment is less or1151    // equal to the base alignment and the offset is not aligned, we know that1152    // the run-time value can never be aligned.1153    if (BaseAlignment.getQuantity() >= Alignment &&1154        PtrAlign.getQuantity() < Alignment) {1155      S.Stk.push<Boolean>(false);1156      return true;1157    }1158 1159    S.FFDiag(Call->getArg(0), diag::note_constexpr_alignment_compute)1160        << Alignment;1161    return false;1162  }1163 1164  assert(BuiltinOp == Builtin::BI__builtin_align_down ||1165         BuiltinOp == Builtin::BI__builtin_align_up);1166 1167  // For align_up/align_down, we can return the same value if the alignment1168  // is known to be greater or equal to the requested value.1169  if (PtrAlign.getQuantity() >= Alignment) {1170    S.Stk.push<Pointer>(Ptr);1171    return true;1172  }1173 1174  // The alignment could be greater than the minimum at run-time, so we cannot1175  // infer much about the resulting pointer value. One case is possible:1176  // For `_Alignas(32) char buf[N]; __builtin_align_down(&buf[idx], 32)` we1177  // can infer the correct index if the requested alignment is smaller than1178  // the base alignment so we can perform the computation on the offset.1179  if (BaseAlignment.getQuantity() >= Alignment) {1180    assert(Alignment.getBitWidth() <= 64 &&1181           "Cannot handle > 64-bit address-space");1182    uint64_t Alignment64 = Alignment.getZExtValue();1183    CharUnits NewOffset =1184        CharUnits::fromQuantity(BuiltinOp == Builtin::BI__builtin_align_down1185                                    ? llvm::alignDown(PtrOffset, Alignment64)1186                                    : llvm::alignTo(PtrOffset, Alignment64));1187 1188    S.Stk.push<Pointer>(Ptr.atIndex(NewOffset.getQuantity()));1189    return true;1190  }1191 1192  // Otherwise, we cannot constant-evaluate the result.1193  S.FFDiag(Call->getArg(0), diag::note_constexpr_alignment_adjust) << Alignment;1194  return false;1195}1196 1197/// __builtin_assume_aligned(Ptr, Alignment[, ExtraOffset])1198static bool interp__builtin_assume_aligned(InterpState &S, CodePtr OpPC,1199                                           const InterpFrame *Frame,1200                                           const CallExpr *Call) {1201  assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);1202 1203  std::optional<APSInt> ExtraOffset;1204  if (Call->getNumArgs() == 3)1205    ExtraOffset = popToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(2)));1206 1207  APSInt Alignment = popToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(1)));1208  const Pointer &Ptr = S.Stk.pop<Pointer>();1209 1210  CharUnits Align = CharUnits::fromQuantity(Alignment.getZExtValue());1211 1212  // If there is a base object, then it must have the correct alignment.1213  if (Ptr.isBlockPointer()) {1214    CharUnits BaseAlignment;1215    if (const auto *VD = Ptr.getDeclDesc()->asValueDecl())1216      BaseAlignment = S.getASTContext().getDeclAlign(VD);1217    else if (const auto *E = Ptr.getDeclDesc()->asExpr())1218      BaseAlignment = GetAlignOfExpr(S.getASTContext(), E, UETT_AlignOf);1219 1220    if (BaseAlignment < Align) {1221      S.CCEDiag(Call->getArg(0),1222                diag::note_constexpr_baa_insufficient_alignment)1223          << 0 << BaseAlignment.getQuantity() << Align.getQuantity();1224      return false;1225    }1226  }1227 1228  APValue AV = Ptr.toAPValue(S.getASTContext());1229  CharUnits AVOffset = AV.getLValueOffset();1230  if (ExtraOffset)1231    AVOffset -= CharUnits::fromQuantity(ExtraOffset->getZExtValue());1232  if (AVOffset.alignTo(Align) != AVOffset) {1233    if (Ptr.isBlockPointer())1234      S.CCEDiag(Call->getArg(0),1235                diag::note_constexpr_baa_insufficient_alignment)1236          << 1 << AVOffset.getQuantity() << Align.getQuantity();1237    else1238      S.CCEDiag(Call->getArg(0),1239                diag::note_constexpr_baa_value_insufficient_alignment)1240          << AVOffset.getQuantity() << Align.getQuantity();1241    return false;1242  }1243 1244  S.Stk.push<Pointer>(Ptr);1245  return true;1246}1247 1248/// (CarryIn, LHS, RHS, Result)1249static bool interp__builtin_ia32_addcarry_subborrow(InterpState &S,1250                                                    CodePtr OpPC,1251                                                    const InterpFrame *Frame,1252                                                    const CallExpr *Call,1253                                                    unsigned BuiltinOp) {1254  if (Call->getNumArgs() != 4 || !Call->getArg(0)->getType()->isIntegerType() ||1255      !Call->getArg(1)->getType()->isIntegerType() ||1256      !Call->getArg(2)->getType()->isIntegerType())1257    return false;1258 1259  const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();1260 1261  APSInt RHS = popToAPSInt(S, Call->getArg(2));1262  APSInt LHS = popToAPSInt(S, Call->getArg(1));1263  APSInt CarryIn = popToAPSInt(S, Call->getArg(0));1264 1265  bool IsAdd = BuiltinOp == clang::X86::BI__builtin_ia32_addcarryx_u32 ||1266               BuiltinOp == clang::X86::BI__builtin_ia32_addcarryx_u64;1267 1268  unsigned BitWidth = LHS.getBitWidth();1269  unsigned CarryInBit = CarryIn.ugt(0) ? 1 : 0;1270  APInt ExResult =1271      IsAdd ? (LHS.zext(BitWidth + 1) + (RHS.zext(BitWidth + 1) + CarryInBit))1272            : (LHS.zext(BitWidth + 1) - (RHS.zext(BitWidth + 1) + CarryInBit));1273 1274  APInt Result = ExResult.extractBits(BitWidth, 0);1275  APSInt CarryOut =1276      APSInt(ExResult.extractBits(1, BitWidth), /*IsUnsigned=*/true);1277 1278  QualType CarryOutType = Call->getArg(3)->getType()->getPointeeType();1279  PrimType CarryOutT = *S.getContext().classify(CarryOutType);1280  assignInteger(S, CarryOutPtr, CarryOutT, APSInt(std::move(Result), true));1281 1282  pushInteger(S, CarryOut, Call->getType());1283 1284  return true;1285}1286 1287static bool interp__builtin_os_log_format_buffer_size(InterpState &S,1288                                                      CodePtr OpPC,1289                                                      const InterpFrame *Frame,1290                                                      const CallExpr *Call) {1291  analyze_os_log::OSLogBufferLayout Layout;1292  analyze_os_log::computeOSLogBufferLayout(S.getASTContext(), Call, Layout);1293  pushInteger(S, Layout.size().getQuantity(), Call->getType());1294  return true;1295}1296 1297static bool1298interp__builtin_ptrauth_string_discriminator(InterpState &S, CodePtr OpPC,1299                                             const InterpFrame *Frame,1300                                             const CallExpr *Call) {1301  const auto &Ptr = S.Stk.pop<Pointer>();1302  assert(Ptr.getFieldDesc()->isPrimitiveArray());1303 1304  // This should be created for a StringLiteral, so should alway shold at least1305  // one array element.1306  assert(Ptr.getFieldDesc()->getNumElems() >= 1);1307  StringRef R(&Ptr.deref<char>(), Ptr.getFieldDesc()->getNumElems() - 1);1308  uint64_t Result = getPointerAuthStableSipHash(R);1309  pushInteger(S, Result, Call->getType());1310  return true;1311}1312 1313static bool interp__builtin_infer_alloc_token(InterpState &S, CodePtr OpPC,1314                                              const InterpFrame *Frame,1315                                              const CallExpr *Call) {1316  const ASTContext &ASTCtx = S.getASTContext();1317  uint64_t BitWidth = ASTCtx.getTypeSize(ASTCtx.getSizeType());1318  auto Mode =1319      ASTCtx.getLangOpts().AllocTokenMode.value_or(llvm::DefaultAllocTokenMode);1320  auto MaxTokensOpt = ASTCtx.getLangOpts().AllocTokenMax;1321  uint64_t MaxTokens =1322      MaxTokensOpt.value_or(0) ? *MaxTokensOpt : (~0ULL >> (64 - BitWidth));1323 1324  // We do not read any of the arguments; discard them.1325  for (int I = Call->getNumArgs() - 1; I >= 0; --I)1326    discard(S.Stk, *S.getContext().classify(Call->getArg(I)));1327 1328  // Note: Type inference from a surrounding cast is not supported in1329  // constexpr evaluation.1330  QualType AllocType = infer_alloc::inferPossibleType(Call, ASTCtx, nullptr);1331  if (AllocType.isNull()) {1332    S.CCEDiag(Call,1333              diag::note_constexpr_infer_alloc_token_type_inference_failed);1334    return false;1335  }1336 1337  auto ATMD = infer_alloc::getAllocTokenMetadata(AllocType, ASTCtx);1338  if (!ATMD) {1339    S.CCEDiag(Call, diag::note_constexpr_infer_alloc_token_no_metadata);1340    return false;1341  }1342 1343  auto MaybeToken = llvm::getAllocToken(Mode, *ATMD, MaxTokens);1344  if (!MaybeToken) {1345    S.CCEDiag(Call, diag::note_constexpr_infer_alloc_token_stateful_mode);1346    return false;1347  }1348 1349  pushInteger(S, llvm::APInt(BitWidth, *MaybeToken), ASTCtx.getSizeType());1350  return true;1351}1352 1353static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,1354                                         const InterpFrame *Frame,1355                                         const CallExpr *Call) {1356  // A call to __operator_new is only valid within std::allocate<>::allocate.1357  // Walk up the call stack to find the appropriate caller and get the1358  // element type from it.1359  auto [NewCall, ElemType] = S.getStdAllocatorCaller("allocate");1360 1361  if (ElemType.isNull()) {1362    S.FFDiag(Call, S.getLangOpts().CPlusPlus201363                       ? diag::note_constexpr_new_untyped1364                       : diag::note_constexpr_new);1365    return false;1366  }1367  assert(NewCall);1368 1369  if (ElemType->isIncompleteType() || ElemType->isFunctionType()) {1370    S.FFDiag(Call, diag::note_constexpr_new_not_complete_object_type)1371        << (ElemType->isIncompleteType() ? 0 : 1) << ElemType;1372    return false;1373  }1374 1375  // We only care about the first parameter (the size), so discard all the1376  // others.1377  {1378    unsigned NumArgs = Call->getNumArgs();1379    assert(NumArgs >= 1);1380 1381    // The std::nothrow_t arg never gets put on the stack.1382    if (Call->getArg(NumArgs - 1)->getType()->isNothrowT())1383      --NumArgs;1384    auto Args = ArrayRef(Call->getArgs(), Call->getNumArgs());1385    // First arg is needed.1386    Args = Args.drop_front();1387 1388    // Discard the rest.1389    for (const Expr *Arg : Args)1390      discard(S.Stk, *S.getContext().classify(Arg));1391  }1392 1393  APSInt Bytes = popToAPSInt(S, Call->getArg(0));1394  CharUnits ElemSize = S.getASTContext().getTypeSizeInChars(ElemType);1395  assert(!ElemSize.isZero());1396  // Divide the number of bytes by sizeof(ElemType), so we get the number of1397  // elements we should allocate.1398  APInt NumElems, Remainder;1399  APInt ElemSizeAP(Bytes.getBitWidth(), ElemSize.getQuantity());1400  APInt::udivrem(Bytes, ElemSizeAP, NumElems, Remainder);1401  if (Remainder != 0) {1402    // This likely indicates a bug in the implementation of 'std::allocator'.1403    S.FFDiag(Call, diag::note_constexpr_operator_new_bad_size)1404        << Bytes << APSInt(ElemSizeAP, true) << ElemType;1405    return false;1406  }1407 1408  // NB: The same check we're using in CheckArraySize()1409  if (NumElems.getActiveBits() >1410          ConstantArrayType::getMaxSizeBits(S.getASTContext()) ||1411      NumElems.ugt(Descriptor::MaxArrayElemBytes / ElemSize.getQuantity())) {1412    // FIXME: NoThrow check?1413    const SourceInfo &Loc = S.Current->getSource(OpPC);1414    S.FFDiag(Loc, diag::note_constexpr_new_too_large)1415        << NumElems.getZExtValue();1416    return false;1417  }1418 1419  if (!CheckArraySize(S, OpPC, NumElems.getZExtValue()))1420    return false;1421 1422  bool IsArray = NumElems.ugt(1);1423  OptPrimType ElemT = S.getContext().classify(ElemType);1424  DynamicAllocator &Allocator = S.getAllocator();1425  if (ElemT) {1426    Block *B =1427        Allocator.allocate(NewCall, *ElemT, NumElems.getZExtValue(),1428                           S.Ctx.getEvalID(), DynamicAllocator::Form::Operator);1429    assert(B);1430    S.Stk.push<Pointer>(Pointer(B).atIndex(0));1431    return true;1432  }1433 1434  assert(!ElemT);1435 1436  // Composite arrays1437  if (IsArray) {1438    const Descriptor *Desc =1439        S.P.createDescriptor(NewCall, ElemType.getTypePtr(), std::nullopt);1440    Block *B =1441        Allocator.allocate(Desc, NumElems.getZExtValue(), S.Ctx.getEvalID(),1442                           DynamicAllocator::Form::Operator);1443    assert(B);1444    S.Stk.push<Pointer>(Pointer(B).atIndex(0).narrow());1445    return true;1446  }1447 1448  // Records. Still allocate them as single-element arrays.1449  QualType AllocType = S.getASTContext().getConstantArrayType(1450      ElemType, NumElems, nullptr, ArraySizeModifier::Normal, 0);1451 1452  const Descriptor *Desc = S.P.createDescriptor(NewCall, AllocType.getTypePtr(),1453                                                Descriptor::InlineDescMD);1454  Block *B = Allocator.allocate(Desc, S.getContext().getEvalID(),1455                                DynamicAllocator::Form::Operator);1456  assert(B);1457  S.Stk.push<Pointer>(Pointer(B).atIndex(0).narrow());1458  return true;1459}1460 1461static bool interp__builtin_operator_delete(InterpState &S, CodePtr OpPC,1462                                            const InterpFrame *Frame,1463                                            const CallExpr *Call) {1464  const Expr *Source = nullptr;1465  const Block *BlockToDelete = nullptr;1466 1467  if (S.checkingPotentialConstantExpression()) {1468    S.Stk.discard<Pointer>();1469    return false;1470  }1471 1472  // This is permitted only within a call to std::allocator<T>::deallocate.1473  if (!S.getStdAllocatorCaller("deallocate")) {1474    S.FFDiag(Call);1475    S.Stk.discard<Pointer>();1476    return true;1477  }1478 1479  {1480    const Pointer &Ptr = S.Stk.pop<Pointer>();1481 1482    if (Ptr.isZero()) {1483      S.CCEDiag(Call, diag::note_constexpr_deallocate_null);1484      return true;1485    }1486 1487    Source = Ptr.getDeclDesc()->asExpr();1488    BlockToDelete = Ptr.block();1489 1490    if (!BlockToDelete->isDynamic()) {1491      S.FFDiag(Call, diag::note_constexpr_delete_not_heap_alloc)1492          << Ptr.toDiagnosticString(S.getASTContext());1493      if (const auto *D = Ptr.getFieldDesc()->asDecl())1494        S.Note(D->getLocation(), diag::note_declared_at);1495    }1496  }1497  assert(BlockToDelete);1498 1499  DynamicAllocator &Allocator = S.getAllocator();1500  const Descriptor *BlockDesc = BlockToDelete->getDescriptor();1501  std::optional<DynamicAllocator::Form> AllocForm =1502      Allocator.getAllocationForm(Source);1503 1504  if (!Allocator.deallocate(Source, BlockToDelete, S)) {1505    // Nothing has been deallocated, this must be a double-delete.1506    const SourceInfo &Loc = S.Current->getSource(OpPC);1507    S.FFDiag(Loc, diag::note_constexpr_double_delete);1508    return false;1509  }1510  assert(AllocForm);1511 1512  return CheckNewDeleteForms(1513      S, OpPC, *AllocForm, DynamicAllocator::Form::Operator, BlockDesc, Source);1514}1515 1516static bool interp__builtin_arithmetic_fence(InterpState &S, CodePtr OpPC,1517                                             const InterpFrame *Frame,1518                                             const CallExpr *Call) {1519  const Floating &Arg0 = S.Stk.pop<Floating>();1520  S.Stk.push<Floating>(Arg0);1521  return true;1522}1523 1524static bool interp__builtin_vector_reduce(InterpState &S, CodePtr OpPC,1525                                          const CallExpr *Call, unsigned ID) {1526  const Pointer &Arg = S.Stk.pop<Pointer>();1527  assert(Arg.getFieldDesc()->isPrimitiveArray());1528 1529  QualType ElemType = Arg.getFieldDesc()->getElemQualType();1530  assert(Call->getType() == ElemType);1531  PrimType ElemT = *S.getContext().classify(ElemType);1532  unsigned NumElems = Arg.getNumElems();1533 1534  INT_TYPE_SWITCH_NO_BOOL(ElemT, {1535    T Result = Arg.elem<T>(0);1536    unsigned BitWidth = Result.bitWidth();1537    for (unsigned I = 1; I != NumElems; ++I) {1538      T Elem = Arg.elem<T>(I);1539      T PrevResult = Result;1540 1541      if (ID == Builtin::BI__builtin_reduce_add) {1542        if (T::add(Result, Elem, BitWidth, &Result)) {1543          unsigned OverflowBits = BitWidth + 1;1544          (void)handleOverflow(S, OpPC,1545                               (PrevResult.toAPSInt(OverflowBits) +1546                                Elem.toAPSInt(OverflowBits)));1547          return false;1548        }1549      } else if (ID == Builtin::BI__builtin_reduce_mul) {1550        if (T::mul(Result, Elem, BitWidth, &Result)) {1551          unsigned OverflowBits = BitWidth * 2;1552          (void)handleOverflow(S, OpPC,1553                               (PrevResult.toAPSInt(OverflowBits) *1554                                Elem.toAPSInt(OverflowBits)));1555          return false;1556        }1557 1558      } else if (ID == Builtin::BI__builtin_reduce_and) {1559        (void)T::bitAnd(Result, Elem, BitWidth, &Result);1560      } else if (ID == Builtin::BI__builtin_reduce_or) {1561        (void)T::bitOr(Result, Elem, BitWidth, &Result);1562      } else if (ID == Builtin::BI__builtin_reduce_xor) {1563        (void)T::bitXor(Result, Elem, BitWidth, &Result);1564      } else if (ID == Builtin::BI__builtin_reduce_min) {1565        if (Elem < Result)1566          Result = Elem;1567      } else if (ID == Builtin::BI__builtin_reduce_max) {1568        if (Elem > Result)1569          Result = Elem;1570      } else {1571        llvm_unreachable("Unhandled vector reduce builtin");1572      }1573    }1574    pushInteger(S, Result.toAPSInt(), Call->getType());1575  });1576 1577  return true;1578}1579 1580static bool interp__builtin_elementwise_abs(InterpState &S, CodePtr OpPC,1581                                            const InterpFrame *Frame,1582                                            const CallExpr *Call,1583                                            unsigned BuiltinID) {1584  assert(Call->getNumArgs() == 1);1585  QualType Ty = Call->getArg(0)->getType();1586  if (Ty->isIntegerType()) {1587    APSInt Val = popToAPSInt(S, Call->getArg(0));1588    pushInteger(S, Val.abs(), Call->getType());1589    return true;1590  }1591 1592  if (Ty->isFloatingType()) {1593    Floating Val = S.Stk.pop<Floating>();1594    Floating Result = abs(S, Val);1595    S.Stk.push<Floating>(Result);1596    return true;1597  }1598 1599  // Otherwise, the argument must be a vector.1600  assert(Call->getArg(0)->getType()->isVectorType());1601  const Pointer &Arg = S.Stk.pop<Pointer>();1602  assert(Arg.getFieldDesc()->isPrimitiveArray());1603  const Pointer &Dst = S.Stk.peek<Pointer>();1604  assert(Dst.getFieldDesc()->isPrimitiveArray());1605  assert(Arg.getFieldDesc()->getNumElems() ==1606         Dst.getFieldDesc()->getNumElems());1607 1608  QualType ElemType = Arg.getFieldDesc()->getElemQualType();1609  PrimType ElemT = *S.getContext().classify(ElemType);1610  unsigned NumElems = Arg.getNumElems();1611  // we can either have a vector of integer or a vector of floating point1612  for (unsigned I = 0; I != NumElems; ++I) {1613    if (ElemType->isIntegerType()) {1614      INT_TYPE_SWITCH_NO_BOOL(ElemT, {1615        Dst.elem<T>(I) = T::from(static_cast<T>(1616            APSInt(Arg.elem<T>(I).toAPSInt().abs(),1617                   ElemType->isUnsignedIntegerOrEnumerationType())));1618      });1619    } else {1620      Floating Val = Arg.elem<Floating>(I);1621      Dst.elem<Floating>(I) = abs(S, Val);1622    }1623  }1624  Dst.initializeAllElements();1625 1626  return true;1627}1628 1629/// Can be called with an integer or vector as the first and only parameter.1630static bool interp__builtin_elementwise_countzeroes(InterpState &S,1631                                                    CodePtr OpPC,1632                                                    const InterpFrame *Frame,1633                                                    const CallExpr *Call,1634                                                    unsigned BuiltinID) {1635  bool HasZeroArg = Call->getNumArgs() == 2;1636  bool IsCTTZ = BuiltinID == Builtin::BI__builtin_elementwise_ctzg;1637  assert(Call->getNumArgs() == 1 || HasZeroArg);1638  if (Call->getArg(0)->getType()->isIntegerType()) {1639    PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());1640    APSInt Val = popToAPSInt(S.Stk, ArgT);1641    std::optional<APSInt> ZeroVal;1642    if (HasZeroArg) {1643      ZeroVal = Val;1644      Val = popToAPSInt(S.Stk, ArgT);1645    }1646 1647    if (Val.isZero()) {1648      if (ZeroVal) {1649        pushInteger(S, *ZeroVal, Call->getType());1650        return true;1651      }1652      // If we haven't been provided the second argument, the result is1653      // undefined1654      S.FFDiag(S.Current->getSource(OpPC),1655               diag::note_constexpr_countzeroes_zero)1656          << /*IsTrailing=*/IsCTTZ;1657      return false;1658    }1659 1660    if (BuiltinID == Builtin::BI__builtin_elementwise_clzg) {1661      pushInteger(S, Val.countLeadingZeros(), Call->getType());1662    } else {1663      pushInteger(S, Val.countTrailingZeros(), Call->getType());1664    }1665    return true;1666  }1667  // Otherwise, the argument must be a vector.1668  const ASTContext &ASTCtx = S.getASTContext();1669  Pointer ZeroArg;1670  if (HasZeroArg) {1671    assert(Call->getArg(1)->getType()->isVectorType() &&1672           ASTCtx.hasSameUnqualifiedType(Call->getArg(0)->getType(),1673                                         Call->getArg(1)->getType()));1674    (void)ASTCtx;1675    ZeroArg = S.Stk.pop<Pointer>();1676    assert(ZeroArg.getFieldDesc()->isPrimitiveArray());1677  }1678  assert(Call->getArg(0)->getType()->isVectorType());1679  const Pointer &Arg = S.Stk.pop<Pointer>();1680  assert(Arg.getFieldDesc()->isPrimitiveArray());1681  const Pointer &Dst = S.Stk.peek<Pointer>();1682  assert(Dst.getFieldDesc()->isPrimitiveArray());1683  assert(Arg.getFieldDesc()->getNumElems() ==1684         Dst.getFieldDesc()->getNumElems());1685 1686  QualType ElemType = Arg.getFieldDesc()->getElemQualType();1687  PrimType ElemT = *S.getContext().classify(ElemType);1688  unsigned NumElems = Arg.getNumElems();1689 1690  // FIXME: Reading from uninitialized vector elements?1691  for (unsigned I = 0; I != NumElems; ++I) {1692    INT_TYPE_SWITCH_NO_BOOL(ElemT, {1693      APInt EltVal = Arg.atIndex(I).deref<T>().toAPSInt();1694      if (EltVal.isZero()) {1695        if (HasZeroArg) {1696          Dst.atIndex(I).deref<T>() = ZeroArg.atIndex(I).deref<T>();1697        } else {1698          // If we haven't been provided the second argument, the result is1699          // undefined1700          S.FFDiag(S.Current->getSource(OpPC),1701                   diag::note_constexpr_countzeroes_zero)1702              << /*IsTrailing=*/IsCTTZ;1703          return false;1704        }1705      } else if (IsCTTZ) {1706        Dst.atIndex(I).deref<T>() = T::from(EltVal.countTrailingZeros());1707      } else {1708        Dst.atIndex(I).deref<T>() = T::from(EltVal.countLeadingZeros());1709      }1710      Dst.atIndex(I).initialize();1711    });1712  }1713 1714  return true;1715}1716 1717static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,1718                                   const InterpFrame *Frame,1719                                   const CallExpr *Call, unsigned ID) {1720  assert(Call->getNumArgs() == 3);1721  const ASTContext &ASTCtx = S.getASTContext();1722  APSInt Size = popToAPSInt(S, Call->getArg(2));1723  Pointer SrcPtr = S.Stk.pop<Pointer>().expand();1724  Pointer DestPtr = S.Stk.pop<Pointer>().expand();1725 1726  assert(!Size.isSigned() && "memcpy and friends take an unsigned size");1727 1728  if (ID == Builtin::BImemcpy || ID == Builtin::BImemmove)1729    diagnoseNonConstexprBuiltin(S, OpPC, ID);1730 1731  bool Move =1732      (ID == Builtin::BI__builtin_memmove || ID == Builtin::BImemmove ||1733       ID == Builtin::BI__builtin_wmemmove || ID == Builtin::BIwmemmove);1734  bool WChar = ID == Builtin::BIwmemcpy || ID == Builtin::BIwmemmove ||1735               ID == Builtin::BI__builtin_wmemcpy ||1736               ID == Builtin::BI__builtin_wmemmove;1737 1738  // If the size is zero, we treat this as always being a valid no-op.1739  if (Size.isZero()) {1740    S.Stk.push<Pointer>(DestPtr);1741    return true;1742  }1743 1744  if (SrcPtr.isZero() || DestPtr.isZero()) {1745    Pointer DiagPtr = (SrcPtr.isZero() ? SrcPtr : DestPtr);1746    S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_null)1747        << /*IsMove=*/Move << /*IsWchar=*/WChar << !SrcPtr.isZero()1748        << DiagPtr.toDiagnosticString(ASTCtx);1749    return false;1750  }1751 1752  // Diagnose integral src/dest pointers specially.1753  if (SrcPtr.isIntegralPointer() || DestPtr.isIntegralPointer()) {1754    std::string DiagVal = "(void *)";1755    DiagVal += SrcPtr.isIntegralPointer()1756                   ? std::to_string(SrcPtr.getIntegerRepresentation())1757                   : std::to_string(DestPtr.getIntegerRepresentation());1758    S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_null)1759        << Move << WChar << DestPtr.isIntegralPointer() << DiagVal;1760    return false;1761  }1762 1763  // Can't read from dummy pointers.1764  if (DestPtr.isDummy() || SrcPtr.isDummy())1765    return false;1766 1767  if (DestPtr.getType()->isIncompleteType()) {1768    S.FFDiag(S.Current->getSource(OpPC),1769             diag::note_constexpr_memcpy_incomplete_type)1770        << Move << DestPtr.getType();1771    return false;1772  }1773  if (SrcPtr.getType()->isIncompleteType()) {1774    S.FFDiag(S.Current->getSource(OpPC),1775             diag::note_constexpr_memcpy_incomplete_type)1776        << Move << SrcPtr.getType();1777    return false;1778  }1779 1780  QualType DestElemType = getElemType(DestPtr);1781  if (DestElemType->isIncompleteType()) {1782    S.FFDiag(S.Current->getSource(OpPC),1783             diag::note_constexpr_memcpy_incomplete_type)1784        << Move << DestElemType;1785    return false;1786  }1787 1788  size_t RemainingDestElems;1789  if (DestPtr.getFieldDesc()->isArray()) {1790    RemainingDestElems = DestPtr.isUnknownSizeArray()1791                             ? 01792                             : (DestPtr.getNumElems() - DestPtr.getIndex());1793  } else {1794    RemainingDestElems = 1;1795  }1796  unsigned DestElemSize = ASTCtx.getTypeSizeInChars(DestElemType).getQuantity();1797 1798  if (WChar) {1799    uint64_t WCharSize =1800        ASTCtx.getTypeSizeInChars(ASTCtx.getWCharType()).getQuantity();1801    Size *= APSInt(APInt(Size.getBitWidth(), WCharSize, /*IsSigned=*/false),1802                   /*IsUnsigend=*/true);1803  }1804 1805  if (Size.urem(DestElemSize) != 0) {1806    S.FFDiag(S.Current->getSource(OpPC),1807             diag::note_constexpr_memcpy_unsupported)1808        << Move << WChar << 0 << DestElemType << Size << DestElemSize;1809    return false;1810  }1811 1812  QualType SrcElemType = getElemType(SrcPtr);1813  size_t RemainingSrcElems;1814  if (SrcPtr.getFieldDesc()->isArray()) {1815    RemainingSrcElems = SrcPtr.isUnknownSizeArray()1816                            ? 01817                            : (SrcPtr.getNumElems() - SrcPtr.getIndex());1818  } else {1819    RemainingSrcElems = 1;1820  }1821  unsigned SrcElemSize = ASTCtx.getTypeSizeInChars(SrcElemType).getQuantity();1822 1823  if (!ASTCtx.hasSameUnqualifiedType(DestElemType, SrcElemType)) {1824    S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_type_pun)1825        << Move << SrcElemType << DestElemType;1826    return false;1827  }1828 1829  if (!DestElemType.isTriviallyCopyableType(ASTCtx)) {1830    S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_nontrivial)1831        << Move << DestElemType;1832    return false;1833  }1834 1835  // Check if we have enough elements to read from and write to.1836  size_t RemainingDestBytes = RemainingDestElems * DestElemSize;1837  size_t RemainingSrcBytes = RemainingSrcElems * SrcElemSize;1838  if (Size.ugt(RemainingDestBytes) || Size.ugt(RemainingSrcBytes)) {1839    APInt N = Size.udiv(DestElemSize);1840    S.FFDiag(S.Current->getSource(OpPC),1841             diag::note_constexpr_memcpy_unsupported)1842        << Move << WChar << (Size.ugt(RemainingSrcBytes) ? 1 : 2)1843        << DestElemType << toString(N, 10, /*Signed=*/false);1844    return false;1845  }1846 1847  // Check for overlapping memory regions.1848  if (!Move && Pointer::pointToSameBlock(SrcPtr, DestPtr)) {1849    // Remove base casts.1850    Pointer SrcP = SrcPtr;1851    while (SrcP.isBaseClass())1852      SrcP = SrcP.getBase();1853 1854    Pointer DestP = DestPtr;1855    while (DestP.isBaseClass())1856      DestP = DestP.getBase();1857 1858    unsigned SrcIndex = SrcP.expand().getIndex() * SrcP.elemSize();1859    unsigned DstIndex = DestP.expand().getIndex() * DestP.elemSize();1860    unsigned N = Size.getZExtValue();1861 1862    if ((SrcIndex <= DstIndex && (SrcIndex + N) > DstIndex) ||1863        (DstIndex <= SrcIndex && (DstIndex + N) > SrcIndex)) {1864      S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_overlap)1865          << /*IsWChar=*/false;1866      return false;1867    }1868  }1869 1870  assert(Size.getZExtValue() % DestElemSize == 0);1871  if (!DoMemcpy(S, OpPC, SrcPtr, DestPtr, Bytes(Size.getZExtValue()).toBits()))1872    return false;1873 1874  S.Stk.push<Pointer>(DestPtr);1875  return true;1876}1877 1878/// Determine if T is a character type for which we guarantee that1879/// sizeof(T) == 1.1880static bool isOneByteCharacterType(QualType T) {1881  return T->isCharType() || T->isChar8Type();1882}1883 1884static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC,1885                                   const InterpFrame *Frame,1886                                   const CallExpr *Call, unsigned ID) {1887  assert(Call->getNumArgs() == 3);1888  const APSInt &Size = popToAPSInt(S, Call->getArg(2));1889  const Pointer &PtrB = S.Stk.pop<Pointer>();1890  const Pointer &PtrA = S.Stk.pop<Pointer>();1891 1892  if (ID == Builtin::BImemcmp || ID == Builtin::BIbcmp ||1893      ID == Builtin::BIwmemcmp)1894    diagnoseNonConstexprBuiltin(S, OpPC, ID);1895 1896  if (Size.isZero()) {1897    pushInteger(S, 0, Call->getType());1898    return true;1899  }1900 1901  if (!PtrA.isBlockPointer() || !PtrB.isBlockPointer())1902    return false;1903 1904  bool IsWide =1905      (ID == Builtin::BIwmemcmp || ID == Builtin::BI__builtin_wmemcmp);1906 1907  const ASTContext &ASTCtx = S.getASTContext();1908  QualType ElemTypeA = getElemType(PtrA);1909  QualType ElemTypeB = getElemType(PtrB);1910  // FIXME: This is an arbitrary limitation the current constant interpreter1911  // had. We could remove this.1912  if (!IsWide && (!isOneByteCharacterType(ElemTypeA) ||1913                  !isOneByteCharacterType(ElemTypeB))) {1914    S.FFDiag(S.Current->getSource(OpPC),1915             diag::note_constexpr_memcmp_unsupported)1916        << ASTCtx.BuiltinInfo.getQuotedName(ID) << PtrA.getType()1917        << PtrB.getType();1918    return false;1919  }1920 1921  if (PtrA.isDummy() || PtrB.isDummy())1922    return false;1923 1924  if (!CheckRange(S, OpPC, PtrA, AK_Read) ||1925      !CheckRange(S, OpPC, PtrB, AK_Read))1926    return false;1927 1928  // Now, read both pointers to a buffer and compare those.1929  BitcastBuffer BufferA(1930      Bits(ASTCtx.getTypeSize(ElemTypeA) * PtrA.getNumElems()));1931  readPointerToBuffer(S.getContext(), PtrA, BufferA, false);1932  // FIXME: The swapping here is UNDOING something we do when reading the1933  // data into the buffer.1934  if (ASTCtx.getTargetInfo().isBigEndian())1935    swapBytes(BufferA.Data.get(), BufferA.byteSize().getQuantity());1936 1937  BitcastBuffer BufferB(1938      Bits(ASTCtx.getTypeSize(ElemTypeB) * PtrB.getNumElems()));1939  readPointerToBuffer(S.getContext(), PtrB, BufferB, false);1940  // FIXME: The swapping here is UNDOING something we do when reading the1941  // data into the buffer.1942  if (ASTCtx.getTargetInfo().isBigEndian())1943    swapBytes(BufferB.Data.get(), BufferB.byteSize().getQuantity());1944 1945  size_t MinBufferSize = std::min(BufferA.byteSize().getQuantity(),1946                                  BufferB.byteSize().getQuantity());1947 1948  unsigned ElemSize = 1;1949  if (IsWide)1950    ElemSize = ASTCtx.getTypeSizeInChars(ASTCtx.getWCharType()).getQuantity();1951  // The Size given for the wide variants is in wide-char units. Convert it1952  // to bytes.1953  size_t ByteSize = Size.getZExtValue() * ElemSize;1954  size_t CmpSize = std::min(MinBufferSize, ByteSize);1955 1956  for (size_t I = 0; I != CmpSize; I += ElemSize) {1957    if (IsWide) {1958      INT_TYPE_SWITCH(*S.getContext().classify(ASTCtx.getWCharType()), {1959        T A = *reinterpret_cast<T *>(BufferA.atByte(I));1960        T B = *reinterpret_cast<T *>(BufferB.atByte(I));1961        if (A < B) {1962          pushInteger(S, -1, Call->getType());1963          return true;1964        }1965        if (A > B) {1966          pushInteger(S, 1, Call->getType());1967          return true;1968        }1969      });1970    } else {1971      std::byte A = BufferA.deref<std::byte>(Bytes(I));1972      std::byte B = BufferB.deref<std::byte>(Bytes(I));1973 1974      if (A < B) {1975        pushInteger(S, -1, Call->getType());1976        return true;1977      }1978      if (A > B) {1979        pushInteger(S, 1, Call->getType());1980        return true;1981      }1982    }1983  }1984 1985  // We compared CmpSize bytes above. If the limiting factor was the Size1986  // passed, we're done and the result is equality (0).1987  if (ByteSize <= CmpSize) {1988    pushInteger(S, 0, Call->getType());1989    return true;1990  }1991 1992  // However, if we read all the available bytes but were instructed to read1993  // even more, diagnose this as a "read of dereferenced one-past-the-end1994  // pointer". This is what would happen if we called CheckLoad() on every array1995  // element.1996  S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_access_past_end)1997      << AK_Read << S.Current->getRange(OpPC);1998  return false;1999}2000 2001// __builtin_memchr(ptr, int, int)2002// __builtin_strchr(ptr, int)2003static bool interp__builtin_memchr(InterpState &S, CodePtr OpPC,2004                                   const CallExpr *Call, unsigned ID) {2005  if (ID == Builtin::BImemchr || ID == Builtin::BIwcschr ||2006      ID == Builtin::BIstrchr || ID == Builtin::BIwmemchr)2007    diagnoseNonConstexprBuiltin(S, OpPC, ID);2008 2009  std::optional<APSInt> MaxLength;2010  if (Call->getNumArgs() == 3)2011    MaxLength = popToAPSInt(S, Call->getArg(2));2012 2013  APSInt Desired = popToAPSInt(S, Call->getArg(1));2014  const Pointer &Ptr = S.Stk.pop<Pointer>();2015 2016  if (MaxLength && MaxLength->isZero()) {2017    S.Stk.push<Pointer>();2018    return true;2019  }2020 2021  if (Ptr.isDummy()) {2022    if (Ptr.getType()->isIncompleteType())2023      S.FFDiag(S.Current->getSource(OpPC),2024               diag::note_constexpr_ltor_incomplete_type)2025          << Ptr.getType();2026    return false;2027  }2028 2029  // Null is only okay if the given size is 0.2030  if (Ptr.isZero()) {2031    S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_access_null)2032        << AK_Read;2033    return false;2034  }2035 2036  QualType ElemTy = Ptr.getFieldDesc()->isArray()2037                        ? Ptr.getFieldDesc()->getElemQualType()2038                        : Ptr.getFieldDesc()->getType();2039  bool IsRawByte = ID == Builtin::BImemchr || ID == Builtin::BI__builtin_memchr;2040 2041  // Give up on byte-oriented matching against multibyte elements.2042  if (IsRawByte && !isOneByteCharacterType(ElemTy)) {2043    S.FFDiag(S.Current->getSource(OpPC),2044             diag::note_constexpr_memchr_unsupported)2045        << S.getASTContext().BuiltinInfo.getQuotedName(ID) << ElemTy;2046    return false;2047  }2048 2049  if (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr) {2050    int64_t DesiredTrunc;2051    if (S.getASTContext().CharTy->isSignedIntegerType())2052      DesiredTrunc =2053          Desired.trunc(S.getASTContext().getCharWidth()).getSExtValue();2054    else2055      DesiredTrunc =2056          Desired.trunc(S.getASTContext().getCharWidth()).getZExtValue();2057    // strchr compares directly to the passed integer, and therefore2058    // always fails if given an int that is not a char.2059    if (Desired != DesiredTrunc) {2060      S.Stk.push<Pointer>();2061      return true;2062    }2063  }2064 2065  uint64_t DesiredVal;2066  if (ID == Builtin::BIwmemchr || ID == Builtin::BI__builtin_wmemchr ||2067      ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr) {2068    // wcschr and wmemchr are given a wchar_t to look for. Just use it.2069    DesiredVal = Desired.getZExtValue();2070  } else {2071    DesiredVal = Desired.trunc(S.getASTContext().getCharWidth()).getZExtValue();2072  }2073 2074  bool StopAtZero =2075      (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr ||2076       ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr);2077 2078  PrimType ElemT =2079      IsRawByte ? PT_Sint8 : *S.getContext().classify(getElemType(Ptr));2080 2081  size_t Index = Ptr.getIndex();2082  size_t Step = 0;2083  for (;;) {2084    const Pointer &ElemPtr =2085        (Index + Step) > 0 ? Ptr.atIndex(Index + Step) : Ptr;2086 2087    if (!CheckLoad(S, OpPC, ElemPtr))2088      return false;2089 2090    uint64_t V;2091    INT_TYPE_SWITCH_NO_BOOL(2092        ElemT, { V = static_cast<uint64_t>(ElemPtr.deref<T>().toUnsigned()); });2093 2094    if (V == DesiredVal) {2095      S.Stk.push<Pointer>(ElemPtr);2096      return true;2097    }2098 2099    if (StopAtZero && V == 0)2100      break;2101 2102    ++Step;2103    if (MaxLength && Step == MaxLength->getZExtValue())2104      break;2105  }2106 2107  S.Stk.push<Pointer>();2108  return true;2109}2110 2111static std::optional<unsigned> computeFullDescSize(const ASTContext &ASTCtx,2112                                                   const Descriptor *Desc) {2113  if (Desc->isPrimitive())2114    return ASTCtx.getTypeSizeInChars(Desc->getType()).getQuantity();2115  if (Desc->isArray())2116    return ASTCtx.getTypeSizeInChars(Desc->getElemQualType()).getQuantity() *2117           Desc->getNumElems();2118  if (Desc->isRecord()) {2119    // Can't use Descriptor::getType() as that may return a pointer type. Look2120    // at the decl directly.2121    return ASTCtx2122        .getTypeSizeInChars(2123            ASTCtx.getCanonicalTagType(Desc->ElemRecord->getDecl()))2124        .getQuantity();2125  }2126 2127  return std::nullopt;2128}2129 2130/// Compute the byte offset of \p Ptr in the full declaration.2131static unsigned computePointerOffset(const ASTContext &ASTCtx,2132                                     const Pointer &Ptr) {2133  unsigned Result = 0;2134 2135  Pointer P = Ptr;2136  while (P.isField() || P.isArrayElement()) {2137    P = P.expand();2138    const Descriptor *D = P.getFieldDesc();2139 2140    if (P.isArrayElement()) {2141      unsigned ElemSize =2142          ASTCtx.getTypeSizeInChars(D->getElemQualType()).getQuantity();2143      if (P.isOnePastEnd())2144        Result += ElemSize * P.getNumElems();2145      else2146        Result += ElemSize * P.getIndex();2147      P = P.expand().getArray();2148    } else if (P.isBaseClass()) {2149      const auto *RD = cast<CXXRecordDecl>(D->asDecl());2150      bool IsVirtual = Ptr.isVirtualBaseClass();2151      P = P.getBase();2152      const Record *BaseRecord = P.getRecord();2153 2154      const ASTRecordLayout &Layout =2155          ASTCtx.getASTRecordLayout(cast<CXXRecordDecl>(BaseRecord->getDecl()));2156      if (IsVirtual)2157        Result += Layout.getVBaseClassOffset(RD).getQuantity();2158      else2159        Result += Layout.getBaseClassOffset(RD).getQuantity();2160    } else if (P.isField()) {2161      const FieldDecl *FD = P.getField();2162      const ASTRecordLayout &Layout =2163          ASTCtx.getASTRecordLayout(FD->getParent());2164      unsigned FieldIndex = FD->getFieldIndex();2165      uint64_t FieldOffset =2166          ASTCtx.toCharUnitsFromBits(Layout.getFieldOffset(FieldIndex))2167              .getQuantity();2168      Result += FieldOffset;2169      P = P.getBase();2170    } else2171      llvm_unreachable("Unhandled descriptor type");2172  }2173 2174  return Result;2175}2176 2177/// Does Ptr point to the last subobject?2178static bool pointsToLastObject(const Pointer &Ptr) {2179  Pointer P = Ptr;2180  while (!P.isRoot()) {2181 2182    if (P.isArrayElement()) {2183      P = P.expand().getArray();2184      continue;2185    }2186    if (P.isBaseClass()) {2187      if (P.getRecord()->getNumFields() > 0)2188        return false;2189      P = P.getBase();2190      continue;2191    }2192 2193    Pointer Base = P.getBase();2194    if (const Record *R = Base.getRecord()) {2195      assert(P.getField());2196      if (P.getField()->getFieldIndex() != R->getNumFields() - 1)2197        return false;2198    }2199    P = Base;2200  }2201 2202  return true;2203}2204 2205/// Does Ptr point to the last object AND to a flexible array member?2206static bool isUserWritingOffTheEnd(const ASTContext &Ctx, const Pointer &Ptr) {2207  auto isFlexibleArrayMember = [&](const Descriptor *FieldDesc) {2208    using FAMKind = LangOptions::StrictFlexArraysLevelKind;2209    FAMKind StrictFlexArraysLevel =2210        Ctx.getLangOpts().getStrictFlexArraysLevel();2211 2212    if (StrictFlexArraysLevel == FAMKind::Default)2213      return true;2214 2215    unsigned NumElems = FieldDesc->getNumElems();2216    if (NumElems == 0 && StrictFlexArraysLevel != FAMKind::IncompleteOnly)2217      return true;2218 2219    if (NumElems == 1 && StrictFlexArraysLevel == FAMKind::OneZeroOrIncomplete)2220      return true;2221    return false;2222  };2223 2224  const Descriptor *FieldDesc = Ptr.getFieldDesc();2225  if (!FieldDesc->isArray())2226    return false;2227 2228  return Ptr.isDummy() && pointsToLastObject(Ptr) &&2229         isFlexibleArrayMember(FieldDesc);2230}2231 2232static bool interp__builtin_object_size(InterpState &S, CodePtr OpPC,2233                                        const InterpFrame *Frame,2234                                        const CallExpr *Call) {2235  const ASTContext &ASTCtx = S.getASTContext();2236  // From the GCC docs:2237  // Kind is an integer constant from 0 to 3. If the least significant bit is2238  // clear, objects are whole variables. If it is set, a closest surrounding2239  // subobject is considered the object a pointer points to. The second bit2240  // determines if maximum or minimum of remaining bytes is computed.2241  unsigned Kind = popToAPSInt(S, Call->getArg(1)).getZExtValue();2242  assert(Kind <= 3 && "unexpected kind");2243  bool UseFieldDesc = (Kind & 1u);2244  bool ReportMinimum = (Kind & 2u);2245  const Pointer &Ptr = S.Stk.pop<Pointer>();2246 2247  if (Call->getArg(0)->HasSideEffects(ASTCtx)) {2248    // "If there are any side effects in them, it returns (size_t) -12249    // for type 0 or 1 and (size_t) 0 for type 2 or 3."2250    pushInteger(S, Kind <= 1 ? -1 : 0, Call->getType());2251    return true;2252  }2253 2254  if (Ptr.isZero() || !Ptr.isBlockPointer())2255    return false;2256 2257  // We can't load through pointers.2258  if (Ptr.isDummy() && Ptr.getType()->isPointerType())2259    return false;2260 2261  bool DetermineForCompleteObject = Ptr.getFieldDesc() == Ptr.getDeclDesc();2262  const Descriptor *DeclDesc = Ptr.getDeclDesc();2263  assert(DeclDesc);2264 2265  if (!UseFieldDesc || DetermineForCompleteObject) {2266    // Lower bound, so we can't fall back to this.2267    if (ReportMinimum && !DetermineForCompleteObject)2268      return false;2269 2270    // Can't read beyond the pointer decl desc.2271    if (!UseFieldDesc && !ReportMinimum && DeclDesc->getType()->isPointerType())2272      return false;2273  } else {2274    if (isUserWritingOffTheEnd(ASTCtx, Ptr.expand())) {2275      // If we cannot determine the size of the initial allocation, then we2276      // can't given an accurate upper-bound. However, we are still able to give2277      // conservative lower-bounds for Type=3.2278      if (Kind == 1)2279        return false;2280    }2281  }2282 2283  const Descriptor *Desc = UseFieldDesc ? Ptr.getFieldDesc() : DeclDesc;2284  assert(Desc);2285 2286  std::optional<unsigned> FullSize = computeFullDescSize(ASTCtx, Desc);2287  if (!FullSize)2288    return false;2289 2290  unsigned ByteOffset;2291  if (UseFieldDesc) {2292    if (Ptr.isBaseClass())2293      ByteOffset = computePointerOffset(ASTCtx, Ptr.getBase()) -2294                   computePointerOffset(ASTCtx, Ptr);2295    else {2296      if (Ptr.inArray())2297        ByteOffset =2298            computePointerOffset(ASTCtx, Ptr) -2299            computePointerOffset(ASTCtx, Ptr.expand().atIndex(0).narrow());2300      else2301        ByteOffset = 0;2302    }2303  } else2304    ByteOffset = computePointerOffset(ASTCtx, Ptr);2305 2306  assert(ByteOffset <= *FullSize);2307  unsigned Result = *FullSize - ByteOffset;2308 2309  pushInteger(S, Result, Call->getType());2310  return true;2311}2312 2313static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC,2314                                               const CallExpr *Call) {2315 2316  if (!S.inConstantContext())2317    return false;2318 2319  const Pointer &Ptr = S.Stk.pop<Pointer>();2320 2321  auto Error = [&](int Diag) {2322    bool CalledFromStd = false;2323    const auto *Callee = S.Current->getCallee();2324    if (Callee && Callee->isInStdNamespace()) {2325      const IdentifierInfo *Identifier = Callee->getIdentifier();2326      CalledFromStd = Identifier && Identifier->isStr("is_within_lifetime");2327    }2328    S.CCEDiag(CalledFromStd2329                  ? S.Current->Caller->getSource(S.Current->getRetPC())2330                  : S.Current->getSource(OpPC),2331              diag::err_invalid_is_within_lifetime)2332        << (CalledFromStd ? "std::is_within_lifetime"2333                          : "__builtin_is_within_lifetime")2334        << Diag;2335    return false;2336  };2337 2338  if (Ptr.isZero())2339    return Error(0);2340  if (Ptr.isOnePastEnd())2341    return Error(1);2342 2343  bool Result = Ptr.getLifetime() != Lifetime::Ended;2344  if (!Ptr.isActive()) {2345    Result = false;2346  } else {2347    if (!CheckLive(S, OpPC, Ptr, AK_Read))2348      return false;2349    if (!CheckMutable(S, OpPC, Ptr))2350      return false;2351    if (!CheckDummy(S, OpPC, Ptr.block(), AK_Read))2352      return false;2353  }2354 2355  // Check if we're currently running an initializer.2356  if (llvm::is_contained(S.InitializingBlocks, Ptr.block()))2357    return Error(2);2358  if (S.EvaluatingDecl && Ptr.getDeclDesc()->asVarDecl() == S.EvaluatingDecl)2359    return Error(2);2360 2361  pushInteger(S, Result, Call->getType());2362  return true;2363}2364 2365static bool interp__builtin_elementwise_int_unaryop(2366    InterpState &S, CodePtr OpPC, const CallExpr *Call,2367    llvm::function_ref<APInt(const APSInt &)> Fn) {2368  assert(Call->getNumArgs() == 1);2369 2370  // Single integer case.2371  if (!Call->getArg(0)->getType()->isVectorType()) {2372    assert(Call->getType()->isIntegerType());2373    APSInt Src = popToAPSInt(S, Call->getArg(0));2374    APInt Result = Fn(Src);2375    pushInteger(S, APSInt(std::move(Result), !Src.isSigned()), Call->getType());2376    return true;2377  }2378 2379  // Vector case.2380  const Pointer &Arg = S.Stk.pop<Pointer>();2381  assert(Arg.getFieldDesc()->isPrimitiveArray());2382  const Pointer &Dst = S.Stk.peek<Pointer>();2383  assert(Dst.getFieldDesc()->isPrimitiveArray());2384  assert(Arg.getFieldDesc()->getNumElems() ==2385         Dst.getFieldDesc()->getNumElems());2386 2387  QualType ElemType = Arg.getFieldDesc()->getElemQualType();2388  PrimType ElemT = *S.getContext().classify(ElemType);2389  unsigned NumElems = Arg.getNumElems();2390  bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();2391 2392  for (unsigned I = 0; I != NumElems; ++I) {2393    INT_TYPE_SWITCH_NO_BOOL(ElemT, {2394      APSInt Src = Arg.elem<T>(I).toAPSInt();2395      APInt Result = Fn(Src);2396      Dst.elem<T>(I) = static_cast<T>(APSInt(std::move(Result), DestUnsigned));2397    });2398  }2399  Dst.initializeAllElements();2400 2401  return true;2402}2403 2404static bool interp__builtin_elementwise_int_binop(2405    InterpState &S, CodePtr OpPC, const CallExpr *Call,2406    llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {2407  assert(Call->getNumArgs() == 2);2408 2409  // Single integer case.2410  if (!Call->getArg(0)->getType()->isVectorType()) {2411    assert(!Call->getArg(1)->getType()->isVectorType());2412    APSInt RHS = popToAPSInt(S, Call->getArg(1));2413    APSInt LHS = popToAPSInt(S, Call->getArg(0));2414    APInt Result = Fn(LHS, RHS);2415    pushInteger(S, APSInt(std::move(Result), !LHS.isSigned()), Call->getType());2416    return true;2417  }2418 2419  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();2420  assert(VT->getElementType()->isIntegralOrEnumerationType());2421  PrimType ElemT = *S.getContext().classify(VT->getElementType());2422  unsigned NumElems = VT->getNumElements();2423  bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();2424 2425  // Vector + Scalar case.2426  if (!Call->getArg(1)->getType()->isVectorType()) {2427    assert(Call->getArg(1)->getType()->isIntegralOrEnumerationType());2428 2429    APSInt RHS = popToAPSInt(S, Call->getArg(1));2430    const Pointer &LHS = S.Stk.pop<Pointer>();2431    const Pointer &Dst = S.Stk.peek<Pointer>();2432 2433    for (unsigned I = 0; I != NumElems; ++I) {2434      INT_TYPE_SWITCH_NO_BOOL(ElemT, {2435        Dst.elem<T>(I) = static_cast<T>(2436            APSInt(Fn(LHS.elem<T>(I).toAPSInt(), RHS), DestUnsigned));2437      });2438    }2439    Dst.initializeAllElements();2440    return true;2441  }2442 2443  // Vector case.2444  assert(Call->getArg(0)->getType()->isVectorType() &&2445         Call->getArg(1)->getType()->isVectorType());2446  assert(VT->getElementType() ==2447         Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());2448  assert(VT->getNumElements() ==2449         Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());2450  assert(VT->getElementType()->isIntegralOrEnumerationType());2451 2452  const Pointer &RHS = S.Stk.pop<Pointer>();2453  const Pointer &LHS = S.Stk.pop<Pointer>();2454  const Pointer &Dst = S.Stk.peek<Pointer>();2455  for (unsigned I = 0; I != NumElems; ++I) {2456    INT_TYPE_SWITCH_NO_BOOL(ElemT, {2457      APSInt Elem1 = LHS.elem<T>(I).toAPSInt();2458      APSInt Elem2 = RHS.elem<T>(I).toAPSInt();2459      Dst.elem<T>(I) = static_cast<T>(APSInt(Fn(Elem1, Elem2), DestUnsigned));2460    });2461  }2462  Dst.initializeAllElements();2463 2464  return true;2465}2466 2467static bool2468interp__builtin_x86_pack(InterpState &S, CodePtr, const CallExpr *E,2469                         llvm::function_ref<APInt(const APSInt &)> PackFn) {2470  const auto *VT0 = E->getArg(0)->getType()->castAs<VectorType>();2471  [[maybe_unused]] const auto *VT1 =2472      E->getArg(1)->getType()->castAs<VectorType>();2473  assert(VT0 && VT1 && "pack builtin VT0 and VT1 must be VectorType");2474  assert(VT0->getElementType() == VT1->getElementType() &&2475         VT0->getNumElements() == VT1->getNumElements() &&2476         "pack builtin VT0 and VT1 ElementType must be same");2477 2478  const Pointer &RHS = S.Stk.pop<Pointer>();2479  const Pointer &LHS = S.Stk.pop<Pointer>();2480  const Pointer &Dst = S.Stk.peek<Pointer>();2481 2482  const ASTContext &ASTCtx = S.getASTContext();2483  unsigned SrcBits = ASTCtx.getIntWidth(VT0->getElementType());2484  unsigned LHSVecLen = VT0->getNumElements();2485  unsigned SrcPerLane = 128 / SrcBits;2486  unsigned Lanes = LHSVecLen * SrcBits / 128;2487 2488  PrimType SrcT = *S.getContext().classify(VT0->getElementType());2489  PrimType DstT = *S.getContext().classify(getElemType(Dst));2490  bool IsUnsigend = getElemType(Dst)->isUnsignedIntegerType();2491 2492  for (unsigned Lane = 0; Lane != Lanes; ++Lane) {2493    unsigned BaseSrc = Lane * SrcPerLane;2494    unsigned BaseDst = Lane * (2 * SrcPerLane);2495 2496    for (unsigned I = 0; I != SrcPerLane; ++I) {2497      INT_TYPE_SWITCH_NO_BOOL(SrcT, {2498        APSInt A = LHS.elem<T>(BaseSrc + I).toAPSInt();2499        APSInt B = RHS.elem<T>(BaseSrc + I).toAPSInt();2500 2501        assignInteger(S, Dst.atIndex(BaseDst + I), DstT,2502                      APSInt(PackFn(A), IsUnsigend));2503        assignInteger(S, Dst.atIndex(BaseDst + SrcPerLane + I), DstT,2504                      APSInt(PackFn(B), IsUnsigend));2505      });2506    }2507  }2508 2509  Dst.initializeAllElements();2510  return true;2511}2512 2513static bool interp__builtin_elementwise_maxmin(InterpState &S, CodePtr OpPC,2514                                               const CallExpr *Call,2515                                               unsigned BuiltinID) {2516  assert(Call->getNumArgs() == 2);2517 2518  QualType Arg0Type = Call->getArg(0)->getType();2519 2520  // TODO: Support floating-point types.2521  if (!(Arg0Type->isIntegerType() ||2522        (Arg0Type->isVectorType() &&2523         Arg0Type->castAs<VectorType>()->getElementType()->isIntegerType())))2524    return false;2525 2526  if (!Arg0Type->isVectorType()) {2527    assert(!Call->getArg(1)->getType()->isVectorType());2528    APSInt RHS = popToAPSInt(S, Call->getArg(1));2529    APSInt LHS = popToAPSInt(S, Arg0Type);2530    APInt Result;2531    if (BuiltinID == Builtin::BI__builtin_elementwise_max) {2532      Result = std::max(LHS, RHS);2533    } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {2534      Result = std::min(LHS, RHS);2535    } else {2536      llvm_unreachable("Wrong builtin ID");2537    }2538 2539    pushInteger(S, APSInt(Result, !LHS.isSigned()), Call->getType());2540    return true;2541  }2542 2543  // Vector case.2544  assert(Call->getArg(0)->getType()->isVectorType() &&2545         Call->getArg(1)->getType()->isVectorType());2546  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();2547  assert(VT->getElementType() ==2548         Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());2549  assert(VT->getNumElements() ==2550         Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());2551  assert(VT->getElementType()->isIntegralOrEnumerationType());2552 2553  const Pointer &RHS = S.Stk.pop<Pointer>();2554  const Pointer &LHS = S.Stk.pop<Pointer>();2555  const Pointer &Dst = S.Stk.peek<Pointer>();2556  PrimType ElemT = *S.getContext().classify(VT->getElementType());2557  unsigned NumElems = VT->getNumElements();2558  for (unsigned I = 0; I != NumElems; ++I) {2559    APSInt Elem1;2560    APSInt Elem2;2561    INT_TYPE_SWITCH_NO_BOOL(ElemT, {2562      Elem1 = LHS.elem<T>(I).toAPSInt();2563      Elem2 = RHS.elem<T>(I).toAPSInt();2564    });2565 2566    APSInt Result;2567    if (BuiltinID == Builtin::BI__builtin_elementwise_max) {2568      Result = APSInt(std::max(Elem1, Elem2),2569                      Call->getType()->isUnsignedIntegerOrEnumerationType());2570    } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {2571      Result = APSInt(std::min(Elem1, Elem2),2572                      Call->getType()->isUnsignedIntegerOrEnumerationType());2573    } else {2574      llvm_unreachable("Wrong builtin ID");2575    }2576 2577    INT_TYPE_SWITCH_NO_BOOL(ElemT,2578                            { Dst.elem<T>(I) = static_cast<T>(Result); });2579  }2580  Dst.initializeAllElements();2581 2582  return true;2583}2584 2585static bool interp__builtin_ia32_pmul(2586    InterpState &S, CodePtr OpPC, const CallExpr *Call,2587    llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &,2588                             const APSInt &)>2589        Fn) {2590  assert(Call->getArg(0)->getType()->isVectorType() &&2591         Call->getArg(1)->getType()->isVectorType());2592  const Pointer &RHS = S.Stk.pop<Pointer>();2593  const Pointer &LHS = S.Stk.pop<Pointer>();2594  const Pointer &Dst = S.Stk.peek<Pointer>();2595 2596  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();2597  PrimType ElemT = *S.getContext().classify(VT->getElementType());2598  unsigned NumElems = VT->getNumElements();2599  const auto *DestVT = Call->getType()->castAs<VectorType>();2600  PrimType DestElemT = *S.getContext().classify(DestVT->getElementType());2601  bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();2602 2603  unsigned DstElem = 0;2604  for (unsigned I = 0; I != NumElems; I += 2) {2605    APSInt Result;2606    INT_TYPE_SWITCH_NO_BOOL(ElemT, {2607      APSInt LoLHS = LHS.elem<T>(I).toAPSInt();2608      APSInt HiLHS = LHS.elem<T>(I + 1).toAPSInt();2609      APSInt LoRHS = RHS.elem<T>(I).toAPSInt();2610      APSInt HiRHS = RHS.elem<T>(I + 1).toAPSInt();2611      Result = APSInt(Fn(LoLHS, HiLHS, LoRHS, HiRHS), DestUnsigned);2612    });2613 2614    INT_TYPE_SWITCH_NO_BOOL(DestElemT,2615                            { Dst.elem<T>(DstElem) = static_cast<T>(Result); });2616    ++DstElem;2617  }2618 2619  Dst.initializeAllElements();2620  return true;2621}2622 2623static bool interp_builtin_horizontal_int_binop(2624    InterpState &S, CodePtr OpPC, const CallExpr *Call,2625    llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {2626  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();2627  PrimType ElemT = *S.getContext().classify(VT->getElementType());2628  bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();2629 2630  const Pointer &RHS = S.Stk.pop<Pointer>();2631  const Pointer &LHS = S.Stk.pop<Pointer>();2632  const Pointer &Dst = S.Stk.peek<Pointer>();2633  unsigned NumElts = VT->getNumElements();2634  unsigned EltBits = S.getASTContext().getIntWidth(VT->getElementType());2635  unsigned EltsPerLane = 128 / EltBits;2636  unsigned Lanes = NumElts * EltBits / 128;2637  unsigned DestIndex = 0;2638 2639  for (unsigned Lane = 0; Lane < Lanes; ++Lane) {2640    unsigned LaneStart = Lane * EltsPerLane;2641    for (unsigned I = 0; I < EltsPerLane; I += 2) {2642      INT_TYPE_SWITCH_NO_BOOL(ElemT, {2643        APSInt Elem1 = LHS.elem<T>(LaneStart + I).toAPSInt();2644        APSInt Elem2 = LHS.elem<T>(LaneStart + I + 1).toAPSInt();2645        APSInt ResL = APSInt(Fn(Elem1, Elem2), DestUnsigned);2646        Dst.elem<T>(DestIndex++) = static_cast<T>(ResL);2647      });2648    }2649 2650    for (unsigned I = 0; I < EltsPerLane; I += 2) {2651      INT_TYPE_SWITCH_NO_BOOL(ElemT, {2652        APSInt Elem1 = RHS.elem<T>(LaneStart + I).toAPSInt();2653        APSInt Elem2 = RHS.elem<T>(LaneStart + I + 1).toAPSInt();2654        APSInt ResR = APSInt(Fn(Elem1, Elem2), DestUnsigned);2655        Dst.elem<T>(DestIndex++) = static_cast<T>(ResR);2656      });2657    }2658  }2659  Dst.initializeAllElements();2660  return true;2661}2662 2663static bool interp_builtin_horizontal_fp_binop(2664    InterpState &S, CodePtr OpPC, const CallExpr *Call,2665    llvm::function_ref<APFloat(const APFloat &, const APFloat &,2666                               llvm::RoundingMode)>2667        Fn) {2668  const Pointer &RHS = S.Stk.pop<Pointer>();2669  const Pointer &LHS = S.Stk.pop<Pointer>();2670  const Pointer &Dst = S.Stk.peek<Pointer>();2671  FPOptions FPO = Call->getFPFeaturesInEffect(S.Ctx.getLangOpts());2672  llvm::RoundingMode RM = getRoundingMode(FPO);2673  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();2674 2675  unsigned NumElts = VT->getNumElements();2676  unsigned EltBits = S.getASTContext().getTypeSize(VT->getElementType());2677  unsigned NumLanes = NumElts * EltBits / 128;2678  unsigned NumElemsPerLane = NumElts / NumLanes;2679  unsigned HalfElemsPerLane = NumElemsPerLane / 2;2680 2681  for (unsigned L = 0; L != NumElts; L += NumElemsPerLane) {2682    using T = PrimConv<PT_Float>::T;2683    for (unsigned E = 0; E != HalfElemsPerLane; ++E) {2684      APFloat Elem1 = LHS.elem<T>(L + (2 * E) + 0).getAPFloat();2685      APFloat Elem2 = LHS.elem<T>(L + (2 * E) + 1).getAPFloat();2686      Dst.elem<T>(L + E) = static_cast<T>(Fn(Elem1, Elem2, RM));2687    }2688    for (unsigned E = 0; E != HalfElemsPerLane; ++E) {2689      APFloat Elem1 = RHS.elem<T>(L + (2 * E) + 0).getAPFloat();2690      APFloat Elem2 = RHS.elem<T>(L + (2 * E) + 1).getAPFloat();2691      Dst.elem<T>(L + E + HalfElemsPerLane) =2692          static_cast<T>(Fn(Elem1, Elem2, RM));2693    }2694  }2695  Dst.initializeAllElements();2696  return true;2697}2698 2699static bool interp__builtin_ia32_addsub(InterpState &S, CodePtr OpPC,2700                                        const CallExpr *Call) {2701  // Addsub: alternates between subtraction and addition2702  // Result[i] = (i % 2 == 0) ? (a[i] - b[i]) : (a[i] + b[i])2703  const Pointer &RHS = S.Stk.pop<Pointer>();2704  const Pointer &LHS = S.Stk.pop<Pointer>();2705  const Pointer &Dst = S.Stk.peek<Pointer>();2706  FPOptions FPO = Call->getFPFeaturesInEffect(S.Ctx.getLangOpts());2707  llvm::RoundingMode RM = getRoundingMode(FPO);2708  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();2709  unsigned NumElems = VT->getNumElements();2710 2711  using T = PrimConv<PT_Float>::T;2712  for (unsigned I = 0; I != NumElems; ++I) {2713    APFloat LElem = LHS.elem<T>(I).getAPFloat();2714    APFloat RElem = RHS.elem<T>(I).getAPFloat();2715    if (I % 2 == 0) {2716      // Even indices: subtract2717      LElem.subtract(RElem, RM);2718    } else {2719      // Odd indices: add2720      LElem.add(RElem, RM);2721    }2722    Dst.elem<T>(I) = static_cast<T>(LElem);2723  }2724  Dst.initializeAllElements();2725  return true;2726}2727 2728static bool interp__builtin_elementwise_triop_fp(2729    InterpState &S, CodePtr OpPC, const CallExpr *Call,2730    llvm::function_ref<APFloat(const APFloat &, const APFloat &,2731                               const APFloat &, llvm::RoundingMode)>2732        Fn) {2733  assert(Call->getNumArgs() == 3);2734 2735  FPOptions FPO = Call->getFPFeaturesInEffect(S.Ctx.getLangOpts());2736  llvm::RoundingMode RM = getRoundingMode(FPO);2737  QualType Arg1Type = Call->getArg(0)->getType();2738  QualType Arg2Type = Call->getArg(1)->getType();2739  QualType Arg3Type = Call->getArg(2)->getType();2740 2741  // Non-vector floating point types.2742  if (!Arg1Type->isVectorType()) {2743    assert(!Arg2Type->isVectorType());2744    assert(!Arg3Type->isVectorType());2745    (void)Arg2Type;2746    (void)Arg3Type;2747 2748    const Floating &Z = S.Stk.pop<Floating>();2749    const Floating &Y = S.Stk.pop<Floating>();2750    const Floating &X = S.Stk.pop<Floating>();2751    APFloat F = Fn(X.getAPFloat(), Y.getAPFloat(), Z.getAPFloat(), RM);2752    Floating Result = S.allocFloat(X.getSemantics());2753    Result.copy(F);2754    S.Stk.push<Floating>(Result);2755    return true;2756  }2757 2758  // Vector type.2759  assert(Arg1Type->isVectorType() && Arg2Type->isVectorType() &&2760         Arg3Type->isVectorType());2761 2762  const VectorType *VecTy = Arg1Type->castAs<VectorType>();2763  QualType ElemQT = VecTy->getElementType();2764  unsigned NumElems = VecTy->getNumElements();2765 2766  assert(ElemQT == Arg2Type->castAs<VectorType>()->getElementType() &&2767         ElemQT == Arg3Type->castAs<VectorType>()->getElementType());2768  assert(NumElems == Arg2Type->castAs<VectorType>()->getNumElements() &&2769         NumElems == Arg3Type->castAs<VectorType>()->getNumElements());2770  assert(ElemQT->isRealFloatingType());2771  (void)ElemQT;2772 2773  const Pointer &VZ = S.Stk.pop<Pointer>();2774  const Pointer &VY = S.Stk.pop<Pointer>();2775  const Pointer &VX = S.Stk.pop<Pointer>();2776  const Pointer &Dst = S.Stk.peek<Pointer>();2777  for (unsigned I = 0; I != NumElems; ++I) {2778    using T = PrimConv<PT_Float>::T;2779    APFloat X = VX.elem<T>(I).getAPFloat();2780    APFloat Y = VY.elem<T>(I).getAPFloat();2781    APFloat Z = VZ.elem<T>(I).getAPFloat();2782    APFloat F = Fn(X, Y, Z, RM);2783    Dst.elem<Floating>(I) = Floating(F);2784  }2785  Dst.initializeAllElements();2786  return true;2787}2788 2789/// AVX512 predicated move: "Result = Mask[] ? LHS[] : RHS[]".2790static bool interp__builtin_select(InterpState &S, CodePtr OpPC,2791                                   const CallExpr *Call) {2792  const Pointer &RHS = S.Stk.pop<Pointer>();2793  const Pointer &LHS = S.Stk.pop<Pointer>();2794  APSInt Mask = popToAPSInt(S, Call->getArg(0));2795  const Pointer &Dst = S.Stk.peek<Pointer>();2796 2797  assert(LHS.getNumElems() == RHS.getNumElems());2798  assert(LHS.getNumElems() == Dst.getNumElems());2799  unsigned NumElems = LHS.getNumElems();2800  PrimType ElemT = LHS.getFieldDesc()->getPrimType();2801  PrimType DstElemT = Dst.getFieldDesc()->getPrimType();2802 2803  for (unsigned I = 0; I != NumElems; ++I) {2804    if (ElemT == PT_Float) {2805      assert(DstElemT == PT_Float);2806      Dst.elem<Floating>(I) =2807          Mask[I] ? LHS.elem<Floating>(I) : RHS.elem<Floating>(I);2808    } else {2809      APSInt Elem;2810      INT_TYPE_SWITCH(ElemT, {2811        Elem = Mask[I] ? LHS.elem<T>(I).toAPSInt() : RHS.elem<T>(I).toAPSInt();2812      });2813      INT_TYPE_SWITCH_NO_BOOL(DstElemT,2814                              { Dst.elem<T>(I) = static_cast<T>(Elem); });2815    }2816  }2817  Dst.initializeAllElements();2818 2819  return true;2820}2821 2822/// Scalar variant of AVX512 predicated select:2823/// Result[i] = (Mask bit 0) ? LHS[i] : RHS[i], but only element 0 may change.2824/// All other elements are taken from RHS.2825static bool interp__builtin_select_scalar(InterpState &S,2826                                          const CallExpr *Call) {2827  unsigned N =2828      Call->getArg(1)->getType()->getAs<VectorType>()->getNumElements();2829 2830  const Pointer &W = S.Stk.pop<Pointer>();2831  const Pointer &A = S.Stk.pop<Pointer>();2832  APSInt U = popToAPSInt(S, Call->getArg(0));2833  const Pointer &Dst = S.Stk.peek<Pointer>();2834 2835  bool TakeA0 = U.getZExtValue() & 1ULL;2836 2837  for (unsigned I = TakeA0; I != N; ++I)2838    Dst.elem<Floating>(I) = W.elem<Floating>(I);2839  if (TakeA0)2840    Dst.elem<Floating>(0) = A.elem<Floating>(0);2841 2842  Dst.initializeAllElements();2843  return true;2844}2845 2846static bool interp__builtin_blend(InterpState &S, CodePtr OpPC,2847                                  const CallExpr *Call) {2848  APSInt Mask = popToAPSInt(S, Call->getArg(2));2849  const Pointer &TrueVec = S.Stk.pop<Pointer>();2850  const Pointer &FalseVec = S.Stk.pop<Pointer>();2851  const Pointer &Dst = S.Stk.peek<Pointer>();2852 2853  assert(FalseVec.getNumElems() == TrueVec.getNumElems());2854  assert(FalseVec.getNumElems() == Dst.getNumElems());2855  unsigned NumElems = FalseVec.getNumElems();2856  PrimType ElemT = FalseVec.getFieldDesc()->getPrimType();2857  PrimType DstElemT = Dst.getFieldDesc()->getPrimType();2858 2859  for (unsigned I = 0; I != NumElems; ++I) {2860    bool MaskBit = Mask[I % 8];2861    if (ElemT == PT_Float) {2862      assert(DstElemT == PT_Float);2863      Dst.elem<Floating>(I) =2864          MaskBit ? TrueVec.elem<Floating>(I) : FalseVec.elem<Floating>(I);2865    } else {2866      assert(DstElemT == ElemT);2867      INT_TYPE_SWITCH_NO_BOOL(DstElemT, {2868        Dst.elem<T>(I) =2869            static_cast<T>(MaskBit ? TrueVec.elem<T>(I).toAPSInt()2870                                   : FalseVec.elem<T>(I).toAPSInt());2871      });2872    }2873  }2874  Dst.initializeAllElements();2875 2876  return true;2877}2878 2879static bool interp__builtin_ia32_test_op(2880    InterpState &S, CodePtr OpPC, const CallExpr *Call,2881    llvm::function_ref<bool(const APInt &A, const APInt &B)> Fn) {2882  const Pointer &RHS = S.Stk.pop<Pointer>();2883  const Pointer &LHS = S.Stk.pop<Pointer>();2884 2885  assert(LHS.getNumElems() == RHS.getNumElems());2886 2887  unsigned SourceLen = LHS.getNumElems();2888  QualType ElemQT = getElemType(LHS);2889  OptPrimType ElemPT = S.getContext().classify(ElemQT);2890  unsigned LaneWidth = S.getASTContext().getTypeSize(ElemQT);2891 2892  APInt AWide(LaneWidth * SourceLen, 0);2893  APInt BWide(LaneWidth * SourceLen, 0);2894 2895  for (unsigned I = 0; I != SourceLen; ++I) {2896    APInt ALane;2897    APInt BLane;2898 2899    if (ElemQT->isIntegerType()) { // Get value.2900      INT_TYPE_SWITCH_NO_BOOL(*ElemPT, {2901        ALane = LHS.elem<T>(I).toAPSInt();2902        BLane = RHS.elem<T>(I).toAPSInt();2903      });2904    } else if (ElemQT->isFloatingType()) { // Get only sign bit.2905      using T = PrimConv<PT_Float>::T;2906      ALane = LHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();2907      BLane = RHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();2908    } else { // Must be integer or floating type.2909      return false;2910    }2911    AWide.insertBits(ALane, I * LaneWidth);2912    BWide.insertBits(BLane, I * LaneWidth);2913  }2914  pushInteger(S, Fn(AWide, BWide), Call->getType());2915  return true;2916}2917 2918static bool interp__builtin_ia32_movmsk_op(InterpState &S, CodePtr OpPC,2919                                           const CallExpr *Call) {2920  assert(Call->getNumArgs() == 1);2921 2922  const Pointer &Source = S.Stk.pop<Pointer>();2923 2924  unsigned SourceLen = Source.getNumElems();2925  QualType ElemQT = getElemType(Source);2926  OptPrimType ElemT = S.getContext().classify(ElemQT);2927  unsigned ResultLen =2928      S.getASTContext().getTypeSize(Call->getType()); // Always 32-bit integer.2929  APInt Result(ResultLen, 0);2930 2931  for (unsigned I = 0; I != SourceLen; ++I) {2932    APInt Elem;2933    if (ElemQT->isIntegerType()) {2934      INT_TYPE_SWITCH_NO_BOOL(*ElemT, { Elem = Source.elem<T>(I).toAPSInt(); });2935    } else if (ElemQT->isRealFloatingType()) {2936      using T = PrimConv<PT_Float>::T;2937      Elem = Source.elem<T>(I).getAPFloat().bitcastToAPInt();2938    } else {2939      return false;2940    }2941    Result.setBitVal(I, Elem.isNegative());2942  }2943  pushInteger(S, Result, Call->getType());2944  return true;2945}2946 2947static bool interp__builtin_elementwise_triop(2948    InterpState &S, CodePtr OpPC, const CallExpr *Call,2949    llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &)>2950        Fn) {2951  assert(Call->getNumArgs() == 3);2952 2953  QualType Arg0Type = Call->getArg(0)->getType();2954  QualType Arg2Type = Call->getArg(2)->getType();2955  // Non-vector integer types.2956  if (!Arg0Type->isVectorType()) {2957    const APSInt &Op2 = popToAPSInt(S, Arg2Type);2958    const APSInt &Op1 = popToAPSInt(S, Call->getArg(1));2959    const APSInt &Op0 = popToAPSInt(S, Arg0Type);2960    APSInt Result = APSInt(Fn(Op0, Op1, Op2), Op0.isUnsigned());2961    pushInteger(S, Result, Call->getType());2962    return true;2963  }2964 2965  const auto *VecT = Arg0Type->castAs<VectorType>();2966  PrimType ElemT = *S.getContext().classify(VecT->getElementType());2967  unsigned NumElems = VecT->getNumElements();2968  bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();2969 2970  // Vector + Vector + Scalar case.2971  if (!Arg2Type->isVectorType()) {2972    APSInt Op2 = popToAPSInt(S, Arg2Type);2973 2974    const Pointer &Op1 = S.Stk.pop<Pointer>();2975    const Pointer &Op0 = S.Stk.pop<Pointer>();2976    const Pointer &Dst = S.Stk.peek<Pointer>();2977    for (unsigned I = 0; I != NumElems; ++I) {2978      INT_TYPE_SWITCH_NO_BOOL(ElemT, {2979        Dst.elem<T>(I) = static_cast<T>(APSInt(2980            Fn(Op0.elem<T>(I).toAPSInt(), Op1.elem<T>(I).toAPSInt(), Op2),2981            DestUnsigned));2982      });2983    }2984    Dst.initializeAllElements();2985 2986    return true;2987  }2988 2989  // Vector type.2990  const Pointer &Op2 = S.Stk.pop<Pointer>();2991  const Pointer &Op1 = S.Stk.pop<Pointer>();2992  const Pointer &Op0 = S.Stk.pop<Pointer>();2993  const Pointer &Dst = S.Stk.peek<Pointer>();2994  for (unsigned I = 0; I != NumElems; ++I) {2995    APSInt Val0, Val1, Val2;2996    INT_TYPE_SWITCH_NO_BOOL(ElemT, {2997      Val0 = Op0.elem<T>(I).toAPSInt();2998      Val1 = Op1.elem<T>(I).toAPSInt();2999      Val2 = Op2.elem<T>(I).toAPSInt();3000    });3001    APSInt Result = APSInt(Fn(Val0, Val1, Val2), Val0.isUnsigned());3002    INT_TYPE_SWITCH_NO_BOOL(ElemT,3003                            { Dst.elem<T>(I) = static_cast<T>(Result); });3004  }3005  Dst.initializeAllElements();3006 3007  return true;3008}3009 3010static bool interp__builtin_x86_extract_vector(InterpState &S, CodePtr OpPC,3011                                               const CallExpr *Call,3012                                               unsigned ID) {3013  assert(Call->getNumArgs() == 2);3014 3015  APSInt ImmAPS = popToAPSInt(S, Call->getArg(1));3016  uint64_t Index = ImmAPS.getZExtValue();3017 3018  const Pointer &Src = S.Stk.pop<Pointer>();3019  if (!Src.getFieldDesc()->isPrimitiveArray())3020    return false;3021 3022  const Pointer &Dst = S.Stk.peek<Pointer>();3023  if (!Dst.getFieldDesc()->isPrimitiveArray())3024    return false;3025 3026  unsigned SrcElems = Src.getNumElems();3027  unsigned DstElems = Dst.getNumElems();3028 3029  unsigned NumLanes = SrcElems / DstElems;3030  unsigned Lane = static_cast<unsigned>(Index % NumLanes);3031  unsigned ExtractPos = Lane * DstElems;3032 3033  PrimType ElemT = Src.getFieldDesc()->getPrimType();3034 3035  TYPE_SWITCH(ElemT, {3036    for (unsigned I = 0; I != DstElems; ++I) {3037      Dst.elem<T>(I) = Src.elem<T>(ExtractPos + I);3038    }3039  });3040 3041  Dst.initializeAllElements();3042  return true;3043}3044 3045static bool interp__builtin_x86_extract_vector_masked(InterpState &S,3046                                                      CodePtr OpPC,3047                                                      const CallExpr *Call,3048                                                      unsigned ID) {3049  assert(Call->getNumArgs() == 4);3050 3051  APSInt MaskAPS = popToAPSInt(S, Call->getArg(3));3052  const Pointer &Merge = S.Stk.pop<Pointer>();3053  APSInt ImmAPS = popToAPSInt(S, Call->getArg(1));3054  const Pointer &Src = S.Stk.pop<Pointer>();3055 3056  if (!Src.getFieldDesc()->isPrimitiveArray() ||3057      !Merge.getFieldDesc()->isPrimitiveArray())3058    return false;3059 3060  const Pointer &Dst = S.Stk.peek<Pointer>();3061  if (!Dst.getFieldDesc()->isPrimitiveArray())3062    return false;3063 3064  unsigned SrcElems = Src.getNumElems();3065  unsigned DstElems = Dst.getNumElems();3066 3067  unsigned NumLanes = SrcElems / DstElems;3068  unsigned Lane = static_cast<unsigned>(ImmAPS.getZExtValue() % NumLanes);3069  unsigned Base = Lane * DstElems;3070 3071  PrimType ElemT = Src.getFieldDesc()->getPrimType();3072 3073  TYPE_SWITCH(ElemT, {3074    for (unsigned I = 0; I != DstElems; ++I) {3075      if (MaskAPS[I])3076        Dst.elem<T>(I) = Src.elem<T>(Base + I);3077      else3078        Dst.elem<T>(I) = Merge.elem<T>(I);3079    }3080  });3081 3082  Dst.initializeAllElements();3083  return true;3084}3085 3086static bool interp__builtin_x86_insert_subvector(InterpState &S, CodePtr OpPC,3087                                                 const CallExpr *Call,3088                                                 unsigned ID) {3089  assert(Call->getNumArgs() == 3);3090 3091  APSInt ImmAPS = popToAPSInt(S, Call->getArg(2));3092  uint64_t Index = ImmAPS.getZExtValue();3093 3094  const Pointer &SubVec = S.Stk.pop<Pointer>();3095  if (!SubVec.getFieldDesc()->isPrimitiveArray())3096    return false;3097 3098  const Pointer &BaseVec = S.Stk.pop<Pointer>();3099  if (!BaseVec.getFieldDesc()->isPrimitiveArray())3100    return false;3101 3102  const Pointer &Dst = S.Stk.peek<Pointer>();3103 3104  unsigned BaseElements = BaseVec.getNumElems();3105  unsigned SubElements = SubVec.getNumElems();3106 3107  assert(SubElements != 0 && BaseElements != 0 &&3108         (BaseElements % SubElements) == 0);3109 3110  unsigned NumLanes = BaseElements / SubElements;3111  unsigned Lane = static_cast<unsigned>(Index % NumLanes);3112  unsigned InsertPos = Lane * SubElements;3113 3114  PrimType ElemT = BaseVec.getFieldDesc()->getPrimType();3115 3116  TYPE_SWITCH(ElemT, {3117    for (unsigned I = 0; I != BaseElements; ++I)3118      Dst.elem<T>(I) = BaseVec.elem<T>(I);3119    for (unsigned I = 0; I != SubElements; ++I)3120      Dst.elem<T>(InsertPos + I) = SubVec.elem<T>(I);3121  });3122 3123  Dst.initializeAllElements();3124  return true;3125}3126 3127static bool interp__builtin_ia32_phminposuw(InterpState &S, CodePtr OpPC,3128                                            const CallExpr *Call) {3129  assert(Call->getNumArgs() == 1);3130 3131  const Pointer &Source = S.Stk.pop<Pointer>();3132  const Pointer &Dest = S.Stk.peek<Pointer>();3133 3134  unsigned SourceLen = Source.getNumElems();3135  QualType ElemQT = getElemType(Source);3136  OptPrimType ElemT = S.getContext().classify(ElemQT);3137  unsigned ElemBitWidth = S.getASTContext().getTypeSize(ElemQT);3138 3139  bool DestUnsigned = Call->getCallReturnType(S.getASTContext())3140                          ->castAs<VectorType>()3141                          ->getElementType()3142                          ->isUnsignedIntegerOrEnumerationType();3143 3144  INT_TYPE_SWITCH_NO_BOOL(*ElemT, {3145    APSInt MinIndex(ElemBitWidth, DestUnsigned);3146    APSInt MinVal = Source.elem<T>(0).toAPSInt();3147 3148    for (unsigned I = 1; I != SourceLen; ++I) {3149      APSInt Val = Source.elem<T>(I).toAPSInt();3150      if (MinVal.ugt(Val)) {3151        MinVal = Val;3152        MinIndex = I;3153      }3154    }3155 3156    Dest.elem<T>(0) = static_cast<T>(MinVal);3157    Dest.elem<T>(1) = static_cast<T>(MinIndex);3158    for (unsigned I = 2; I != SourceLen; ++I) {3159      Dest.elem<T>(I) = static_cast<T>(APSInt(ElemBitWidth, DestUnsigned));3160    }3161  });3162  Dest.initializeAllElements();3163  return true;3164}3165 3166static bool interp__builtin_ia32_pternlog(InterpState &S, CodePtr OpPC,3167                                          const CallExpr *Call, bool MaskZ) {3168  assert(Call->getNumArgs() == 5);3169 3170  APInt U = popToAPSInt(S, Call->getArg(4));   // Lane mask3171  APInt Imm = popToAPSInt(S, Call->getArg(3)); // Ternary truth table3172  const Pointer &C = S.Stk.pop<Pointer>();3173  const Pointer &B = S.Stk.pop<Pointer>();3174  const Pointer &A = S.Stk.pop<Pointer>();3175  const Pointer &Dst = S.Stk.peek<Pointer>();3176 3177  unsigned DstLen = A.getNumElems();3178  QualType ElemQT = getElemType(A);3179  OptPrimType ElemT = S.getContext().classify(ElemQT);3180  unsigned LaneWidth = S.getASTContext().getTypeSize(ElemQT);3181  bool DstUnsigned = ElemQT->isUnsignedIntegerOrEnumerationType();3182 3183  INT_TYPE_SWITCH_NO_BOOL(*ElemT, {3184    for (unsigned I = 0; I != DstLen; ++I) {3185      APInt ALane = A.elem<T>(I).toAPSInt();3186      APInt BLane = B.elem<T>(I).toAPSInt();3187      APInt CLane = C.elem<T>(I).toAPSInt();3188      APInt RLane(LaneWidth, 0);3189      if (U[I]) { // If lane not masked, compute ternary logic.3190        for (unsigned Bit = 0; Bit != LaneWidth; ++Bit) {3191          unsigned ABit = ALane[Bit];3192          unsigned BBit = BLane[Bit];3193          unsigned CBit = CLane[Bit];3194          unsigned Idx = (ABit << 2) | (BBit << 1) | (CBit);3195          RLane.setBitVal(Bit, Imm[Idx]);3196        }3197        Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));3198      } else if (MaskZ) { // If zero masked, zero the lane.3199        Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));3200      } else { // Just masked, put in A lane.3201        Dst.elem<T>(I) = static_cast<T>(APSInt(ALane, DstUnsigned));3202      }3203    }3204  });3205  Dst.initializeAllElements();3206  return true;3207}3208 3209static bool interp__builtin_vec_ext(InterpState &S, CodePtr OpPC,3210                                    const CallExpr *Call, unsigned ID) {3211  assert(Call->getNumArgs() == 2);3212 3213  APSInt ImmAPS = popToAPSInt(S, Call->getArg(1));3214  const Pointer &Vec = S.Stk.pop<Pointer>();3215  if (!Vec.getFieldDesc()->isPrimitiveArray())3216    return false;3217 3218  unsigned NumElems = Vec.getNumElems();3219  unsigned Index =3220      static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));3221 3222  PrimType ElemT = Vec.getFieldDesc()->getPrimType();3223  // FIXME(#161685): Replace float+int split with a numeric-only type switch3224  if (ElemT == PT_Float) {3225    S.Stk.push<Floating>(Vec.elem<Floating>(Index));3226    return true;3227  }3228  INT_TYPE_SWITCH_NO_BOOL(ElemT, {3229    APSInt V = Vec.elem<T>(Index).toAPSInt();3230    pushInteger(S, V, Call->getType());3231  });3232 3233  return true;3234}3235 3236static bool interp__builtin_vec_set(InterpState &S, CodePtr OpPC,3237                                    const CallExpr *Call, unsigned ID) {3238  assert(Call->getNumArgs() == 3);3239 3240  APSInt ImmAPS = popToAPSInt(S, Call->getArg(2));3241  APSInt ValAPS = popToAPSInt(S, Call->getArg(1));3242 3243  const Pointer &Base = S.Stk.pop<Pointer>();3244  if (!Base.getFieldDesc()->isPrimitiveArray())3245    return false;3246 3247  const Pointer &Dst = S.Stk.peek<Pointer>();3248 3249  unsigned NumElems = Base.getNumElems();3250  unsigned Index =3251      static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));3252 3253  PrimType ElemT = Base.getFieldDesc()->getPrimType();3254  INT_TYPE_SWITCH_NO_BOOL(ElemT, {3255    for (unsigned I = 0; I != NumElems; ++I)3256      Dst.elem<T>(I) = Base.elem<T>(I);3257    Dst.elem<T>(Index) = static_cast<T>(ValAPS);3258  });3259 3260  Dst.initializeAllElements();3261  return true;3262}3263 3264static bool evalICmpImm(uint8_t Imm, const APSInt &A, const APSInt &B,3265                        bool IsUnsigned) {3266  switch (Imm & 0x7) {3267  case 0x00: // _MM_CMPINT_EQ3268    return (A == B);3269  case 0x01: // _MM_CMPINT_LT3270    return IsUnsigned ? A.ult(B) : A.slt(B);3271  case 0x02: // _MM_CMPINT_LE3272    return IsUnsigned ? A.ule(B) : A.sle(B);3273  case 0x03: // _MM_CMPINT_FALSE3274    return false;3275  case 0x04: // _MM_CMPINT_NE3276    return (A != B);3277  case 0x05: // _MM_CMPINT_NLT3278    return IsUnsigned ? A.ugt(B) : A.sgt(B);3279  case 0x06: // _MM_CMPINT_NLE3280    return IsUnsigned ? A.uge(B) : A.sge(B);3281  case 0x07: // _MM_CMPINT_TRUE3282    return true;3283  default:3284    llvm_unreachable("Invalid Op");3285  }3286}3287 3288static bool interp__builtin_ia32_cmp_mask(InterpState &S, CodePtr OpPC,3289                                          const CallExpr *Call, unsigned ID,3290                                          bool IsUnsigned) {3291  assert(Call->getNumArgs() == 4);3292 3293  APSInt Mask = popToAPSInt(S, Call->getArg(3));3294  APSInt Opcode = popToAPSInt(S, Call->getArg(2));3295  unsigned CmpOp = static_cast<unsigned>(Opcode.getZExtValue());3296  const Pointer &RHS = S.Stk.pop<Pointer>();3297  const Pointer &LHS = S.Stk.pop<Pointer>();3298 3299  assert(LHS.getNumElems() == RHS.getNumElems());3300 3301  APInt RetMask = APInt::getZero(LHS.getNumElems());3302  unsigned VectorLen = LHS.getNumElems();3303  PrimType ElemT = LHS.getFieldDesc()->getPrimType();3304 3305  for (unsigned ElemNum = 0; ElemNum < VectorLen; ++ElemNum) {3306    APSInt A, B;3307    INT_TYPE_SWITCH_NO_BOOL(ElemT, {3308      A = LHS.elem<T>(ElemNum).toAPSInt();3309      B = RHS.elem<T>(ElemNum).toAPSInt();3310    });3311    RetMask.setBitVal(ElemNum,3312                      Mask[ElemNum] && evalICmpImm(CmpOp, A, B, IsUnsigned));3313  }3314  pushInteger(S, RetMask, Call->getType());3315  return true;3316}3317 3318static bool interp__builtin_ia32_vpconflict(InterpState &S, CodePtr OpPC,3319                                            const CallExpr *Call) {3320  assert(Call->getNumArgs() == 1);3321 3322  QualType Arg0Type = Call->getArg(0)->getType();3323  const auto *VecT = Arg0Type->castAs<VectorType>();3324  PrimType ElemT = *S.getContext().classify(VecT->getElementType());3325  unsigned NumElems = VecT->getNumElements();3326  bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();3327  const Pointer &Src = S.Stk.pop<Pointer>();3328  const Pointer &Dst = S.Stk.peek<Pointer>();3329 3330  for (unsigned I = 0; I != NumElems; ++I) {3331    INT_TYPE_SWITCH_NO_BOOL(ElemT, {3332      APSInt ElemI = Src.elem<T>(I).toAPSInt();3333      APInt ConflictMask(ElemI.getBitWidth(), 0);3334      for (unsigned J = 0; J != I; ++J) {3335        APSInt ElemJ = Src.elem<T>(J).toAPSInt();3336        ConflictMask.setBitVal(J, ElemI == ElemJ);3337      }3338      Dst.elem<T>(I) = static_cast<T>(APSInt(ConflictMask, DestUnsigned));3339    });3340  }3341  Dst.initializeAllElements();3342  return true;3343}3344 3345static bool interp__builtin_ia32_cvt_vec2mask(InterpState &S, CodePtr OpPC,3346                                              const CallExpr *Call,3347                                              unsigned ID) {3348  assert(Call->getNumArgs() == 1);3349 3350  const Pointer &Vec = S.Stk.pop<Pointer>();3351  unsigned RetWidth = S.getASTContext().getIntWidth(Call->getType());3352  APInt RetMask(RetWidth, 0);3353 3354  unsigned VectorLen = Vec.getNumElems();3355  PrimType ElemT = Vec.getFieldDesc()->getPrimType();3356 3357  for (unsigned ElemNum = 0; ElemNum != VectorLen; ++ElemNum) {3358    APSInt A;3359    INT_TYPE_SWITCH_NO_BOOL(ElemT, { A = Vec.elem<T>(ElemNum).toAPSInt(); });3360    unsigned MSB = A[A.getBitWidth() - 1];3361    RetMask.setBitVal(ElemNum, MSB);3362  }3363  pushInteger(S, RetMask, Call->getType());3364  return true;3365}3366 3367static bool interp__builtin_ia32_shuffle_generic(3368    InterpState &S, CodePtr OpPC, const CallExpr *Call,3369    llvm::function_ref<std::pair<unsigned, int>(unsigned, unsigned)>3370        GetSourceIndex) {3371 3372  assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);3373 3374  unsigned ShuffleMask = 0;3375  Pointer A, MaskVector, B;3376  bool IsVectorMask = false;3377  bool IsSingleOperand = (Call->getNumArgs() == 2);3378 3379  if (IsSingleOperand) {3380    QualType MaskType = Call->getArg(1)->getType();3381    if (MaskType->isVectorType()) {3382      IsVectorMask = true;3383      MaskVector = S.Stk.pop<Pointer>();3384      A = S.Stk.pop<Pointer>();3385      B = A;3386    } else if (MaskType->isIntegerType()) {3387      ShuffleMask = popToAPSInt(S, Call->getArg(1)).getZExtValue();3388      A = S.Stk.pop<Pointer>();3389      B = A;3390    } else {3391      return false;3392    }3393  } else {3394    QualType Arg2Type = Call->getArg(2)->getType();3395    if (Arg2Type->isVectorType()) {3396      IsVectorMask = true;3397      B = S.Stk.pop<Pointer>();3398      MaskVector = S.Stk.pop<Pointer>();3399      A = S.Stk.pop<Pointer>();3400    } else if (Arg2Type->isIntegerType()) {3401      ShuffleMask = popToAPSInt(S, Call->getArg(2)).getZExtValue();3402      B = S.Stk.pop<Pointer>();3403      A = S.Stk.pop<Pointer>();3404    } else {3405      return false;3406    }3407  }3408 3409  QualType Arg0Type = Call->getArg(0)->getType();3410  const auto *VecT = Arg0Type->castAs<VectorType>();3411  PrimType ElemT = *S.getContext().classify(VecT->getElementType());3412  unsigned NumElems = VecT->getNumElements();3413 3414  const Pointer &Dst = S.Stk.peek<Pointer>();3415 3416  PrimType MaskElemT = PT_Uint32;3417  if (IsVectorMask) {3418    QualType Arg1Type = Call->getArg(1)->getType();3419    const auto *MaskVecT = Arg1Type->castAs<VectorType>();3420    QualType MaskElemType = MaskVecT->getElementType();3421    MaskElemT = *S.getContext().classify(MaskElemType);3422  }3423 3424  for (unsigned DstIdx = 0; DstIdx != NumElems; ++DstIdx) {3425    if (IsVectorMask) {3426      INT_TYPE_SWITCH(MaskElemT, {3427        ShuffleMask = static_cast<unsigned>(MaskVector.elem<T>(DstIdx));3428      });3429    }3430 3431    auto [SrcVecIdx, SrcIdx] = GetSourceIndex(DstIdx, ShuffleMask);3432 3433    if (SrcIdx < 0) {3434      // Zero out this element3435      if (ElemT == PT_Float) {3436        Dst.elem<Floating>(DstIdx) = Floating(3437            S.getASTContext().getFloatTypeSemantics(VecT->getElementType()));3438      } else {3439        INT_TYPE_SWITCH_NO_BOOL(ElemT, { Dst.elem<T>(DstIdx) = T::from(0); });3440      }3441    } else {3442      const Pointer &Src = (SrcVecIdx == 0) ? A : B;3443      TYPE_SWITCH(ElemT, { Dst.elem<T>(DstIdx) = Src.elem<T>(SrcIdx); });3444    }3445  }3446  Dst.initializeAllElements();3447 3448  return true;3449}3450 3451static bool interp__builtin_ia32_shift_with_count(3452    InterpState &S, CodePtr OpPC, const CallExpr *Call,3453    llvm::function_ref<APInt(const APInt &, uint64_t)> ShiftOp,3454    llvm::function_ref<APInt(const APInt &, unsigned)> OverflowOp) {3455 3456  assert(Call->getNumArgs() == 2);3457 3458  const Pointer &Count = S.Stk.pop<Pointer>();3459  const Pointer &Source = S.Stk.pop<Pointer>();3460 3461  QualType SourceType = Call->getArg(0)->getType();3462  QualType CountType = Call->getArg(1)->getType();3463  assert(SourceType->isVectorType() && CountType->isVectorType());3464 3465  const auto *SourceVecT = SourceType->castAs<VectorType>();3466  const auto *CountVecT = CountType->castAs<VectorType>();3467  PrimType SourceElemT = *S.getContext().classify(SourceVecT->getElementType());3468  PrimType CountElemT = *S.getContext().classify(CountVecT->getElementType());3469 3470  const Pointer &Dst = S.Stk.peek<Pointer>();3471 3472  unsigned DestEltWidth =3473      S.getASTContext().getTypeSize(SourceVecT->getElementType());3474  bool IsDestUnsigned = SourceVecT->getElementType()->isUnsignedIntegerType();3475  unsigned DestLen = SourceVecT->getNumElements();3476  unsigned CountEltWidth =3477      S.getASTContext().getTypeSize(CountVecT->getElementType());3478  unsigned NumBitsInQWord = 64;3479  unsigned NumCountElts = NumBitsInQWord / CountEltWidth;3480 3481  uint64_t CountLQWord = 0;3482  for (unsigned EltIdx = 0; EltIdx != NumCountElts; ++EltIdx) {3483    uint64_t Elt = 0;3484    INT_TYPE_SWITCH(CountElemT,3485                    { Elt = static_cast<uint64_t>(Count.elem<T>(EltIdx)); });3486    CountLQWord |= (Elt << (EltIdx * CountEltWidth));3487  }3488 3489  for (unsigned EltIdx = 0; EltIdx != DestLen; ++EltIdx) {3490    APSInt Elt;3491    INT_TYPE_SWITCH(SourceElemT, { Elt = Source.elem<T>(EltIdx).toAPSInt(); });3492 3493    APInt Result;3494    if (CountLQWord < DestEltWidth) {3495      Result = ShiftOp(Elt, CountLQWord);3496    } else {3497      Result = OverflowOp(Elt, DestEltWidth);3498    }3499    if (IsDestUnsigned) {3500      INT_TYPE_SWITCH(SourceElemT, {3501        Dst.elem<T>(EltIdx) = T::from(Result.getZExtValue());3502      });3503    } else {3504      INT_TYPE_SWITCH(SourceElemT, {3505        Dst.elem<T>(EltIdx) = T::from(Result.getSExtValue());3506      });3507    }3508  }3509 3510  Dst.initializeAllElements();3511  return true;3512}3513 3514static bool interp__builtin_ia32_shufbitqmb_mask(InterpState &S, CodePtr OpPC,3515                                                 const CallExpr *Call) {3516 3517  assert(Call->getNumArgs() == 3);3518 3519  QualType SourceType = Call->getArg(0)->getType();3520  QualType ShuffleMaskType = Call->getArg(1)->getType();3521  QualType ZeroMaskType = Call->getArg(2)->getType();3522  if (!SourceType->isVectorType() || !ShuffleMaskType->isVectorType() ||3523      !ZeroMaskType->isIntegerType()) {3524    return false;3525  }3526 3527  Pointer Source, ShuffleMask;3528  APSInt ZeroMask = popToAPSInt(S, Call->getArg(2));3529  ShuffleMask = S.Stk.pop<Pointer>();3530  Source = S.Stk.pop<Pointer>();3531 3532  const auto *SourceVecT = SourceType->castAs<VectorType>();3533  const auto *ShuffleMaskVecT = ShuffleMaskType->castAs<VectorType>();3534  assert(SourceVecT->getNumElements() == ShuffleMaskVecT->getNumElements());3535  assert(ZeroMask.getBitWidth() == SourceVecT->getNumElements());3536 3537  PrimType SourceElemT = *S.getContext().classify(SourceVecT->getElementType());3538  PrimType ShuffleMaskElemT =3539      *S.getContext().classify(ShuffleMaskVecT->getElementType());3540 3541  unsigned NumBytesInQWord = 8;3542  unsigned NumBitsInByte = 8;3543  unsigned NumBytes = SourceVecT->getNumElements();3544  unsigned NumQWords = NumBytes / NumBytesInQWord;3545  unsigned RetWidth = ZeroMask.getBitWidth();3546  APSInt RetMask(llvm::APInt(RetWidth, 0), /*isUnsigned=*/true);3547 3548  for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {3549    APInt SourceQWord(64, 0);3550    for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {3551      uint64_t Byte = 0;3552      INT_TYPE_SWITCH(SourceElemT, {3553        Byte = static_cast<uint64_t>(3554            Source.elem<T>(QWordId * NumBytesInQWord + ByteIdx));3555      });3556      SourceQWord.insertBits(APInt(8, Byte & 0xFF), ByteIdx * NumBitsInByte);3557    }3558 3559    for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {3560      unsigned SelIdx = QWordId * NumBytesInQWord + ByteIdx;3561      unsigned M = 0;3562      INT_TYPE_SWITCH(ShuffleMaskElemT, {3563        M = static_cast<unsigned>(ShuffleMask.elem<T>(SelIdx)) & 0x3F;3564      });3565 3566      if (ZeroMask[SelIdx]) {3567        RetMask.setBitVal(SelIdx, SourceQWord[M]);3568      }3569    }3570  }3571 3572  pushInteger(S, RetMask, Call->getType());3573  return true;3574}3575 3576static bool interp__builtin_ia32_vcvtps2ph(InterpState &S, CodePtr OpPC,3577                                           const CallExpr *Call) {3578  // Arguments are: vector of floats, rounding immediate3579  assert(Call->getNumArgs() == 2);3580 3581  APSInt Imm = popToAPSInt(S, Call->getArg(1));3582  const Pointer &Src = S.Stk.pop<Pointer>();3583  const Pointer &Dst = S.Stk.peek<Pointer>();3584 3585  assert(Src.getFieldDesc()->isPrimitiveArray());3586  assert(Dst.getFieldDesc()->isPrimitiveArray());3587 3588  const auto *SrcVTy = Call->getArg(0)->getType()->castAs<VectorType>();3589  unsigned SrcNumElems = SrcVTy->getNumElements();3590  const auto *DstVTy = Call->getType()->castAs<VectorType>();3591  unsigned DstNumElems = DstVTy->getNumElements();3592 3593  const llvm::fltSemantics &HalfSem =3594      S.getASTContext().getFloatTypeSemantics(S.getASTContext().HalfTy);3595 3596  // imm[2] == 1 means use MXCSR rounding mode.3597  // In that case, we can only evaluate if the conversion is exact.3598  int ImmVal = Imm.getZExtValue();3599  bool UseMXCSR = (ImmVal & 4) != 0;3600  bool IsFPConstrained =3601      Call->getFPFeaturesInEffect(S.getASTContext().getLangOpts())3602          .isFPConstrained();3603 3604  llvm::RoundingMode RM;3605  if (!UseMXCSR) {3606    switch (ImmVal & 3) {3607    case 0:3608      RM = llvm::RoundingMode::NearestTiesToEven;3609      break;3610    case 1:3611      RM = llvm::RoundingMode::TowardNegative;3612      break;3613    case 2:3614      RM = llvm::RoundingMode::TowardPositive;3615      break;3616    case 3:3617      RM = llvm::RoundingMode::TowardZero;3618      break;3619    default:3620      llvm_unreachable("Invalid immediate rounding mode");3621    }3622  } else {3623    // For MXCSR, we must check for exactness. We can use any rounding mode3624    // for the trial conversion since the result is the same if it's exact.3625    RM = llvm::RoundingMode::NearestTiesToEven;3626  }3627 3628  QualType DstElemQT = Dst.getFieldDesc()->getElemQualType();3629  PrimType DstElemT = *S.getContext().classify(DstElemQT);3630 3631  for (unsigned I = 0; I != SrcNumElems; ++I) {3632    Floating SrcVal = Src.elem<Floating>(I);3633    APFloat DstVal = SrcVal.getAPFloat();3634 3635    bool LostInfo;3636    APFloat::opStatus St = DstVal.convert(HalfSem, RM, &LostInfo);3637 3638    if (UseMXCSR && IsFPConstrained && St != APFloat::opOK) {3639      S.FFDiag(S.Current->getSource(OpPC),3640               diag::note_constexpr_dynamic_rounding);3641      return false;3642    }3643 3644    INT_TYPE_SWITCH_NO_BOOL(DstElemT, {3645      // Convert the destination value's bit pattern to an unsigned integer,3646      // then reconstruct the element using the target type's 'from' method.3647      uint64_t RawBits = DstVal.bitcastToAPInt().getZExtValue();3648      Dst.elem<T>(I) = T::from(RawBits);3649    });3650  }3651 3652  // Zero out remaining elements if the destination has more elements3653  // (e.g., vcvtps2ph converting 4 floats to 8 shorts).3654  if (DstNumElems > SrcNumElems) {3655    for (unsigned I = SrcNumElems; I != DstNumElems; ++I) {3656      INT_TYPE_SWITCH_NO_BOOL(DstElemT, { Dst.elem<T>(I) = T::from(0); });3657    }3658  }3659 3660  Dst.initializeAllElements();3661  return true;3662}3663 3664static bool interp__builtin_ia32_multishiftqb(InterpState &S, CodePtr OpPC,3665                                              const CallExpr *Call) {3666  assert(Call->getNumArgs() == 2);3667 3668  QualType ATy = Call->getArg(0)->getType();3669  QualType BTy = Call->getArg(1)->getType();3670  if (!ATy->isVectorType() || !BTy->isVectorType()) {3671    return false;3672  }3673 3674  const Pointer &BPtr = S.Stk.pop<Pointer>();3675  const Pointer &APtr = S.Stk.pop<Pointer>();3676  const auto *AVecT = ATy->castAs<VectorType>();3677  assert(AVecT->getNumElements() ==3678         BTy->castAs<VectorType>()->getNumElements());3679 3680  PrimType ElemT = *S.getContext().classify(AVecT->getElementType());3681 3682  unsigned NumBytesInQWord = 8;3683  unsigned NumBitsInByte = 8;3684  unsigned NumBytes = AVecT->getNumElements();3685  unsigned NumQWords = NumBytes / NumBytesInQWord;3686  const Pointer &Dst = S.Stk.peek<Pointer>();3687 3688  for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {3689    APInt BQWord(64, 0);3690    for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {3691      unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;3692      INT_TYPE_SWITCH(ElemT, {3693        uint64_t Byte = static_cast<uint64_t>(BPtr.elem<T>(Idx));3694        BQWord.insertBits(APInt(8, Byte & 0xFF), ByteIdx * NumBitsInByte);3695      });3696    }3697 3698    for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {3699      unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;3700      uint64_t Ctrl = 0;3701      INT_TYPE_SWITCH(3702          ElemT, { Ctrl = static_cast<uint64_t>(APtr.elem<T>(Idx)) & 0x3F; });3703 3704      APInt Byte(8, 0);3705      for (unsigned BitIdx = 0; BitIdx != NumBitsInByte; ++BitIdx) {3706        Byte.setBitVal(BitIdx, BQWord[(Ctrl + BitIdx) & 0x3F]);3707      }3708      INT_TYPE_SWITCH(ElemT,3709                      { Dst.elem<T>(Idx) = T::from(Byte.getZExtValue()); });3710    }3711  }3712 3713  Dst.initializeAllElements();3714 3715  return true;3716}3717 3718bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,3719                      uint32_t BuiltinID) {3720  if (!S.getASTContext().BuiltinInfo.isConstantEvaluated(BuiltinID))3721    return Invalid(S, OpPC);3722 3723  const InterpFrame *Frame = S.Current;3724  switch (BuiltinID) {3725  case Builtin::BI__builtin_is_constant_evaluated:3726    return interp__builtin_is_constant_evaluated(S, OpPC, Frame, Call);3727 3728  case Builtin::BI__builtin_assume:3729  case Builtin::BI__assume:3730    return interp__builtin_assume(S, OpPC, Frame, Call);3731 3732  case Builtin::BI__builtin_strcmp:3733  case Builtin::BIstrcmp:3734  case Builtin::BI__builtin_strncmp:3735  case Builtin::BIstrncmp:3736  case Builtin::BI__builtin_wcsncmp:3737  case Builtin::BIwcsncmp:3738  case Builtin::BI__builtin_wcscmp:3739  case Builtin::BIwcscmp:3740    return interp__builtin_strcmp(S, OpPC, Frame, Call, BuiltinID);3741 3742  case Builtin::BI__builtin_strlen:3743  case Builtin::BIstrlen:3744  case Builtin::BI__builtin_wcslen:3745  case Builtin::BIwcslen:3746    return interp__builtin_strlen(S, OpPC, Frame, Call, BuiltinID);3747 3748  case Builtin::BI__builtin_nan:3749  case Builtin::BI__builtin_nanf:3750  case Builtin::BI__builtin_nanl:3751  case Builtin::BI__builtin_nanf16:3752  case Builtin::BI__builtin_nanf128:3753    return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/false);3754 3755  case Builtin::BI__builtin_nans:3756  case Builtin::BI__builtin_nansf:3757  case Builtin::BI__builtin_nansl:3758  case Builtin::BI__builtin_nansf16:3759  case Builtin::BI__builtin_nansf128:3760    return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/true);3761 3762  case Builtin::BI__builtin_huge_val:3763  case Builtin::BI__builtin_huge_valf:3764  case Builtin::BI__builtin_huge_vall:3765  case Builtin::BI__builtin_huge_valf16:3766  case Builtin::BI__builtin_huge_valf128:3767  case Builtin::BI__builtin_inf:3768  case Builtin::BI__builtin_inff:3769  case Builtin::BI__builtin_infl:3770  case Builtin::BI__builtin_inff16:3771  case Builtin::BI__builtin_inff128:3772    return interp__builtin_inf(S, OpPC, Frame, Call);3773 3774  case Builtin::BI__builtin_copysign:3775  case Builtin::BI__builtin_copysignf:3776  case Builtin::BI__builtin_copysignl:3777  case Builtin::BI__builtin_copysignf128:3778    return interp__builtin_copysign(S, OpPC, Frame);3779 3780  case Builtin::BI__builtin_fmin:3781  case Builtin::BI__builtin_fminf:3782  case Builtin::BI__builtin_fminl:3783  case Builtin::BI__builtin_fminf16:3784  case Builtin::BI__builtin_fminf128:3785    return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/false);3786 3787  case Builtin::BI__builtin_fminimum_num:3788  case Builtin::BI__builtin_fminimum_numf:3789  case Builtin::BI__builtin_fminimum_numl:3790  case Builtin::BI__builtin_fminimum_numf16:3791  case Builtin::BI__builtin_fminimum_numf128:3792    return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/true);3793 3794  case Builtin::BI__builtin_fmax:3795  case Builtin::BI__builtin_fmaxf:3796  case Builtin::BI__builtin_fmaxl:3797  case Builtin::BI__builtin_fmaxf16:3798  case Builtin::BI__builtin_fmaxf128:3799    return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/false);3800 3801  case Builtin::BI__builtin_fmaximum_num:3802  case Builtin::BI__builtin_fmaximum_numf:3803  case Builtin::BI__builtin_fmaximum_numl:3804  case Builtin::BI__builtin_fmaximum_numf16:3805  case Builtin::BI__builtin_fmaximum_numf128:3806    return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/true);3807 3808  case Builtin::BI__builtin_isnan:3809    return interp__builtin_isnan(S, OpPC, Frame, Call);3810 3811  case Builtin::BI__builtin_issignaling:3812    return interp__builtin_issignaling(S, OpPC, Frame, Call);3813 3814  case Builtin::BI__builtin_isinf:3815    return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/false, Call);3816 3817  case Builtin::BI__builtin_isinf_sign:3818    return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/true, Call);3819 3820  case Builtin::BI__builtin_isfinite:3821    return interp__builtin_isfinite(S, OpPC, Frame, Call);3822 3823  case Builtin::BI__builtin_isnormal:3824    return interp__builtin_isnormal(S, OpPC, Frame, Call);3825 3826  case Builtin::BI__builtin_issubnormal:3827    return interp__builtin_issubnormal(S, OpPC, Frame, Call);3828 3829  case Builtin::BI__builtin_iszero:3830    return interp__builtin_iszero(S, OpPC, Frame, Call);3831 3832  case Builtin::BI__builtin_signbit:3833  case Builtin::BI__builtin_signbitf:3834  case Builtin::BI__builtin_signbitl:3835    return interp__builtin_signbit(S, OpPC, Frame, Call);3836 3837  case Builtin::BI__builtin_isgreater:3838  case Builtin::BI__builtin_isgreaterequal:3839  case Builtin::BI__builtin_isless:3840  case Builtin::BI__builtin_islessequal:3841  case Builtin::BI__builtin_islessgreater:3842  case Builtin::BI__builtin_isunordered:3843    return interp_floating_comparison(S, OpPC, Call, BuiltinID);3844 3845  case Builtin::BI__builtin_isfpclass:3846    return interp__builtin_isfpclass(S, OpPC, Frame, Call);3847 3848  case Builtin::BI__builtin_fpclassify:3849    return interp__builtin_fpclassify(S, OpPC, Frame, Call);3850 3851  case Builtin::BI__builtin_fabs:3852  case Builtin::BI__builtin_fabsf:3853  case Builtin::BI__builtin_fabsl:3854  case Builtin::BI__builtin_fabsf128:3855    return interp__builtin_fabs(S, OpPC, Frame);3856 3857  case Builtin::BI__builtin_abs:3858  case Builtin::BI__builtin_labs:3859  case Builtin::BI__builtin_llabs:3860    return interp__builtin_abs(S, OpPC, Frame, Call);3861 3862  case Builtin::BI__builtin_popcount:3863  case Builtin::BI__builtin_popcountl:3864  case Builtin::BI__builtin_popcountll:3865  case Builtin::BI__builtin_popcountg:3866  case Builtin::BI__popcnt16: // Microsoft variants of popcount3867  case Builtin::BI__popcnt:3868  case Builtin::BI__popcnt64:3869    return interp__builtin_popcount(S, OpPC, Frame, Call);3870 3871  case Builtin::BI__builtin_parity:3872  case Builtin::BI__builtin_parityl:3873  case Builtin::BI__builtin_parityll:3874    return interp__builtin_elementwise_int_unaryop(3875        S, OpPC, Call, [](const APSInt &Val) {3876          return APInt(Val.getBitWidth(), Val.popcount() % 2);3877        });3878  case Builtin::BI__builtin_clrsb:3879  case Builtin::BI__builtin_clrsbl:3880  case Builtin::BI__builtin_clrsbll:3881    return interp__builtin_elementwise_int_unaryop(3882        S, OpPC, Call, [](const APSInt &Val) {3883          return APInt(Val.getBitWidth(),3884                       Val.getBitWidth() - Val.getSignificantBits());3885        });3886  case Builtin::BI__builtin_bitreverse8:3887  case Builtin::BI__builtin_bitreverse16:3888  case Builtin::BI__builtin_bitreverse32:3889  case Builtin::BI__builtin_bitreverse64:3890    return interp__builtin_elementwise_int_unaryop(3891        S, OpPC, Call, [](const APSInt &Val) { return Val.reverseBits(); });3892 3893  case Builtin::BI__builtin_classify_type:3894    return interp__builtin_classify_type(S, OpPC, Frame, Call);3895 3896  case Builtin::BI__builtin_expect:3897  case Builtin::BI__builtin_expect_with_probability:3898    return interp__builtin_expect(S, OpPC, Frame, Call);3899 3900  case Builtin::BI__builtin_rotateleft8:3901  case Builtin::BI__builtin_rotateleft16:3902  case Builtin::BI__builtin_rotateleft32:3903  case Builtin::BI__builtin_rotateleft64:3904  case Builtin::BI_rotl8: // Microsoft variants of rotate left3905  case Builtin::BI_rotl16:3906  case Builtin::BI_rotl:3907  case Builtin::BI_lrotl:3908  case Builtin::BI_rotl64:3909    return interp__builtin_elementwise_int_binop(3910        S, OpPC, Call, [](const APSInt &Value, const APSInt &Amount) {3911          return Value.rotl(Amount);3912        });3913 3914  case Builtin::BI__builtin_rotateright8:3915  case Builtin::BI__builtin_rotateright16:3916  case Builtin::BI__builtin_rotateright32:3917  case Builtin::BI__builtin_rotateright64:3918  case Builtin::BI_rotr8: // Microsoft variants of rotate right3919  case Builtin::BI_rotr16:3920  case Builtin::BI_rotr:3921  case Builtin::BI_lrotr:3922  case Builtin::BI_rotr64:3923    return interp__builtin_elementwise_int_binop(3924        S, OpPC, Call, [](const APSInt &Value, const APSInt &Amount) {3925          return Value.rotr(Amount);3926        });3927 3928  case Builtin::BI__builtin_ffs:3929  case Builtin::BI__builtin_ffsl:3930  case Builtin::BI__builtin_ffsll:3931    return interp__builtin_elementwise_int_unaryop(3932        S, OpPC, Call, [](const APSInt &Val) {3933          return APInt(Val.getBitWidth(),3934                       Val.isZero() ? 0u : Val.countTrailingZeros() + 1u);3935        });3936 3937  case Builtin::BIaddressof:3938  case Builtin::BI__addressof:3939  case Builtin::BI__builtin_addressof:3940    assert(isNoopBuiltin(BuiltinID));3941    return interp__builtin_addressof(S, OpPC, Frame, Call);3942 3943  case Builtin::BIas_const:3944  case Builtin::BIforward:3945  case Builtin::BIforward_like:3946  case Builtin::BImove:3947  case Builtin::BImove_if_noexcept:3948    assert(isNoopBuiltin(BuiltinID));3949    return interp__builtin_move(S, OpPC, Frame, Call);3950 3951  case Builtin::BI__builtin_eh_return_data_regno:3952    return interp__builtin_eh_return_data_regno(S, OpPC, Frame, Call);3953 3954  case Builtin::BI__builtin_launder:3955    assert(isNoopBuiltin(BuiltinID));3956    return true;3957 3958  case Builtin::BI__builtin_add_overflow:3959  case Builtin::BI__builtin_sub_overflow:3960  case Builtin::BI__builtin_mul_overflow:3961  case Builtin::BI__builtin_sadd_overflow:3962  case Builtin::BI__builtin_uadd_overflow:3963  case Builtin::BI__builtin_uaddl_overflow:3964  case Builtin::BI__builtin_uaddll_overflow:3965  case Builtin::BI__builtin_usub_overflow:3966  case Builtin::BI__builtin_usubl_overflow:3967  case Builtin::BI__builtin_usubll_overflow:3968  case Builtin::BI__builtin_umul_overflow:3969  case Builtin::BI__builtin_umull_overflow:3970  case Builtin::BI__builtin_umulll_overflow:3971  case Builtin::BI__builtin_saddl_overflow:3972  case Builtin::BI__builtin_saddll_overflow:3973  case Builtin::BI__builtin_ssub_overflow:3974  case Builtin::BI__builtin_ssubl_overflow:3975  case Builtin::BI__builtin_ssubll_overflow:3976  case Builtin::BI__builtin_smul_overflow:3977  case Builtin::BI__builtin_smull_overflow:3978  case Builtin::BI__builtin_smulll_overflow:3979    return interp__builtin_overflowop(S, OpPC, Call, BuiltinID);3980 3981  case Builtin::BI__builtin_addcb:3982  case Builtin::BI__builtin_addcs:3983  case Builtin::BI__builtin_addc:3984  case Builtin::BI__builtin_addcl:3985  case Builtin::BI__builtin_addcll:3986  case Builtin::BI__builtin_subcb:3987  case Builtin::BI__builtin_subcs:3988  case Builtin::BI__builtin_subc:3989  case Builtin::BI__builtin_subcl:3990  case Builtin::BI__builtin_subcll:3991    return interp__builtin_carryop(S, OpPC, Frame, Call, BuiltinID);3992 3993  case Builtin::BI__builtin_clz:3994  case Builtin::BI__builtin_clzl:3995  case Builtin::BI__builtin_clzll:3996  case Builtin::BI__builtin_clzs:3997  case Builtin::BI__builtin_clzg:3998  case Builtin::BI__lzcnt16: // Microsoft variants of count leading-zeroes3999  case Builtin::BI__lzcnt:4000  case Builtin::BI__lzcnt64:4001    return interp__builtin_clz(S, OpPC, Frame, Call, BuiltinID);4002 4003  case Builtin::BI__builtin_ctz:4004  case Builtin::BI__builtin_ctzl:4005  case Builtin::BI__builtin_ctzll:4006  case Builtin::BI__builtin_ctzs:4007  case Builtin::BI__builtin_ctzg:4008    return interp__builtin_ctz(S, OpPC, Frame, Call, BuiltinID);4009 4010  case Builtin::BI__builtin_elementwise_clzg:4011  case Builtin::BI__builtin_elementwise_ctzg:4012    return interp__builtin_elementwise_countzeroes(S, OpPC, Frame, Call,4013                                                   BuiltinID);4014  case Builtin::BI__builtin_bswapg:4015  case Builtin::BI__builtin_bswap16:4016  case Builtin::BI__builtin_bswap32:4017  case Builtin::BI__builtin_bswap64:4018    return interp__builtin_bswap(S, OpPC, Frame, Call);4019 4020  case Builtin::BI__atomic_always_lock_free:4021  case Builtin::BI__atomic_is_lock_free:4022    return interp__builtin_atomic_lock_free(S, OpPC, Frame, Call, BuiltinID);4023 4024  case Builtin::BI__c11_atomic_is_lock_free:4025    return interp__builtin_c11_atomic_is_lock_free(S, OpPC, Frame, Call);4026 4027  case Builtin::BI__builtin_complex:4028    return interp__builtin_complex(S, OpPC, Frame, Call);4029 4030  case Builtin::BI__builtin_is_aligned:4031  case Builtin::BI__builtin_align_up:4032  case Builtin::BI__builtin_align_down:4033    return interp__builtin_is_aligned_up_down(S, OpPC, Frame, Call, BuiltinID);4034 4035  case Builtin::BI__builtin_assume_aligned:4036    return interp__builtin_assume_aligned(S, OpPC, Frame, Call);4037 4038  case clang::X86::BI__builtin_ia32_bextr_u32:4039  case clang::X86::BI__builtin_ia32_bextr_u64:4040  case clang::X86::BI__builtin_ia32_bextri_u32:4041  case clang::X86::BI__builtin_ia32_bextri_u64:4042    return interp__builtin_elementwise_int_binop(4043        S, OpPC, Call, [](const APSInt &Val, const APSInt &Idx) {4044          unsigned BitWidth = Val.getBitWidth();4045          uint64_t Shift = Idx.extractBitsAsZExtValue(8, 0);4046          uint64_t Length = Idx.extractBitsAsZExtValue(8, 8);4047          if (Length > BitWidth) {4048            Length = BitWidth;4049          }4050 4051          // Handle out of bounds cases.4052          if (Length == 0 || Shift >= BitWidth)4053            return APInt(BitWidth, 0);4054 4055          uint64_t Result = Val.getZExtValue() >> Shift;4056          Result &= llvm::maskTrailingOnes<uint64_t>(Length);4057          return APInt(BitWidth, Result);4058        });4059 4060  case clang::X86::BI__builtin_ia32_bzhi_si:4061  case clang::X86::BI__builtin_ia32_bzhi_di:4062    return interp__builtin_elementwise_int_binop(4063        S, OpPC, Call, [](const APSInt &Val, const APSInt &Idx) {4064          unsigned BitWidth = Val.getBitWidth();4065          uint64_t Index = Idx.extractBitsAsZExtValue(8, 0);4066          APSInt Result = Val;4067 4068          if (Index < BitWidth)4069            Result.clearHighBits(BitWidth - Index);4070 4071          return Result;4072        });4073 4074  case clang::X86::BI__builtin_ia32_ktestcqi:4075  case clang::X86::BI__builtin_ia32_ktestchi:4076  case clang::X86::BI__builtin_ia32_ktestcsi:4077  case clang::X86::BI__builtin_ia32_ktestcdi:4078    return interp__builtin_elementwise_int_binop(4079        S, OpPC, Call, [](const APSInt &A, const APSInt &B) {4080          return APInt(sizeof(unsigned char) * 8, (~A & B) == 0);4081        });4082 4083  case clang::X86::BI__builtin_ia32_ktestzqi:4084  case clang::X86::BI__builtin_ia32_ktestzhi:4085  case clang::X86::BI__builtin_ia32_ktestzsi:4086  case clang::X86::BI__builtin_ia32_ktestzdi:4087    return interp__builtin_elementwise_int_binop(4088        S, OpPC, Call, [](const APSInt &A, const APSInt &B) {4089          return APInt(sizeof(unsigned char) * 8, (A & B) == 0);4090        });4091 4092  case clang::X86::BI__builtin_ia32_kortestcqi:4093  case clang::X86::BI__builtin_ia32_kortestchi:4094  case clang::X86::BI__builtin_ia32_kortestcsi:4095  case clang::X86::BI__builtin_ia32_kortestcdi:4096    return interp__builtin_elementwise_int_binop(4097        S, OpPC, Call, [](const APSInt &A, const APSInt &B) {4098          return APInt(sizeof(unsigned char) * 8, ~(A | B) == 0);4099        });4100 4101  case clang::X86::BI__builtin_ia32_kortestzqi:4102  case clang::X86::BI__builtin_ia32_kortestzhi:4103  case clang::X86::BI__builtin_ia32_kortestzsi:4104  case clang::X86::BI__builtin_ia32_kortestzdi:4105    return interp__builtin_elementwise_int_binop(4106        S, OpPC, Call, [](const APSInt &A, const APSInt &B) {4107          return APInt(sizeof(unsigned char) * 8, (A | B) == 0);4108        });4109 4110  case clang::X86::BI__builtin_ia32_lzcnt_u16:4111  case clang::X86::BI__builtin_ia32_lzcnt_u32:4112  case clang::X86::BI__builtin_ia32_lzcnt_u64:4113    return interp__builtin_elementwise_int_unaryop(4114        S, OpPC, Call, [](const APSInt &Src) {4115          return APInt(Src.getBitWidth(), Src.countLeadingZeros());4116        });4117 4118  case clang::X86::BI__builtin_ia32_tzcnt_u16:4119  case clang::X86::BI__builtin_ia32_tzcnt_u32:4120  case clang::X86::BI__builtin_ia32_tzcnt_u64:4121    return interp__builtin_elementwise_int_unaryop(4122        S, OpPC, Call, [](const APSInt &Src) {4123          return APInt(Src.getBitWidth(), Src.countTrailingZeros());4124        });4125 4126  case clang::X86::BI__builtin_ia32_pdep_si:4127  case clang::X86::BI__builtin_ia32_pdep_di:4128    return interp__builtin_elementwise_int_binop(4129        S, OpPC, Call, [](const APSInt &Val, const APSInt &Mask) {4130          unsigned BitWidth = Val.getBitWidth();4131          APInt Result = APInt::getZero(BitWidth);4132 4133          for (unsigned I = 0, P = 0; I != BitWidth; ++I) {4134            if (Mask[I])4135              Result.setBitVal(I, Val[P++]);4136          }4137 4138          return Result;4139        });4140 4141  case clang::X86::BI__builtin_ia32_pext_si:4142  case clang::X86::BI__builtin_ia32_pext_di:4143    return interp__builtin_elementwise_int_binop(4144        S, OpPC, Call, [](const APSInt &Val, const APSInt &Mask) {4145          unsigned BitWidth = Val.getBitWidth();4146          APInt Result = APInt::getZero(BitWidth);4147 4148          for (unsigned I = 0, P = 0; I != BitWidth; ++I) {4149            if (Mask[I])4150              Result.setBitVal(P++, Val[I]);4151          }4152 4153          return Result;4154        });4155 4156  case clang::X86::BI__builtin_ia32_addcarryx_u32:4157  case clang::X86::BI__builtin_ia32_addcarryx_u64:4158  case clang::X86::BI__builtin_ia32_subborrow_u32:4159  case clang::X86::BI__builtin_ia32_subborrow_u64:4160    return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,4161                                                   BuiltinID);4162 4163  case Builtin::BI__builtin_os_log_format_buffer_size:4164    return interp__builtin_os_log_format_buffer_size(S, OpPC, Frame, Call);4165 4166  case Builtin::BI__builtin_ptrauth_string_discriminator:4167    return interp__builtin_ptrauth_string_discriminator(S, OpPC, Frame, Call);4168 4169  case Builtin::BI__builtin_infer_alloc_token:4170    return interp__builtin_infer_alloc_token(S, OpPC, Frame, Call);4171 4172  case Builtin::BI__noop:4173    pushInteger(S, 0, Call->getType());4174    return true;4175 4176  case Builtin::BI__builtin_operator_new:4177    return interp__builtin_operator_new(S, OpPC, Frame, Call);4178 4179  case Builtin::BI__builtin_operator_delete:4180    return interp__builtin_operator_delete(S, OpPC, Frame, Call);4181 4182  case Builtin::BI__arithmetic_fence:4183    return interp__builtin_arithmetic_fence(S, OpPC, Frame, Call);4184 4185  case Builtin::BI__builtin_reduce_add:4186  case Builtin::BI__builtin_reduce_mul:4187  case Builtin::BI__builtin_reduce_and:4188  case Builtin::BI__builtin_reduce_or:4189  case Builtin::BI__builtin_reduce_xor:4190  case Builtin::BI__builtin_reduce_min:4191  case Builtin::BI__builtin_reduce_max:4192    return interp__builtin_vector_reduce(S, OpPC, Call, BuiltinID);4193 4194  case Builtin::BI__builtin_elementwise_popcount:4195    return interp__builtin_elementwise_int_unaryop(4196        S, OpPC, Call, [](const APSInt &Src) {4197          return APInt(Src.getBitWidth(), Src.popcount());4198        });4199  case Builtin::BI__builtin_elementwise_bitreverse:4200    return interp__builtin_elementwise_int_unaryop(4201        S, OpPC, Call, [](const APSInt &Src) { return Src.reverseBits(); });4202 4203  case Builtin::BI__builtin_elementwise_abs:4204    return interp__builtin_elementwise_abs(S, OpPC, Frame, Call, BuiltinID);4205 4206  case Builtin::BI__builtin_memcpy:4207  case Builtin::BImemcpy:4208  case Builtin::BI__builtin_wmemcpy:4209  case Builtin::BIwmemcpy:4210  case Builtin::BI__builtin_memmove:4211  case Builtin::BImemmove:4212  case Builtin::BI__builtin_wmemmove:4213  case Builtin::BIwmemmove:4214    return interp__builtin_memcpy(S, OpPC, Frame, Call, BuiltinID);4215 4216  case Builtin::BI__builtin_memcmp:4217  case Builtin::BImemcmp:4218  case Builtin::BI__builtin_bcmp:4219  case Builtin::BIbcmp:4220  case Builtin::BI__builtin_wmemcmp:4221  case Builtin::BIwmemcmp:4222    return interp__builtin_memcmp(S, OpPC, Frame, Call, BuiltinID);4223 4224  case Builtin::BImemchr:4225  case Builtin::BI__builtin_memchr:4226  case Builtin::BIstrchr:4227  case Builtin::BI__builtin_strchr:4228  case Builtin::BIwmemchr:4229  case Builtin::BI__builtin_wmemchr:4230  case Builtin::BIwcschr:4231  case Builtin::BI__builtin_wcschr:4232  case Builtin::BI__builtin_char_memchr:4233    return interp__builtin_memchr(S, OpPC, Call, BuiltinID);4234 4235  case Builtin::BI__builtin_object_size:4236  case Builtin::BI__builtin_dynamic_object_size:4237    return interp__builtin_object_size(S, OpPC, Frame, Call);4238 4239  case Builtin::BI__builtin_is_within_lifetime:4240    return interp__builtin_is_within_lifetime(S, OpPC, Call);4241 4242  case Builtin::BI__builtin_elementwise_add_sat:4243    return interp__builtin_elementwise_int_binop(4244        S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {4245          return LHS.isSigned() ? LHS.sadd_sat(RHS) : LHS.uadd_sat(RHS);4246        });4247 4248  case Builtin::BI__builtin_elementwise_sub_sat:4249    return interp__builtin_elementwise_int_binop(4250        S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {4251          return LHS.isSigned() ? LHS.ssub_sat(RHS) : LHS.usub_sat(RHS);4252        });4253  case X86::BI__builtin_ia32_extract128i256:4254  case X86::BI__builtin_ia32_vextractf128_pd256:4255  case X86::BI__builtin_ia32_vextractf128_ps256:4256  case X86::BI__builtin_ia32_vextractf128_si256:4257    return interp__builtin_x86_extract_vector(S, OpPC, Call, BuiltinID);4258 4259  case X86::BI__builtin_ia32_extractf32x4_256_mask:4260  case X86::BI__builtin_ia32_extractf32x4_mask:4261  case X86::BI__builtin_ia32_extractf32x8_mask:4262  case X86::BI__builtin_ia32_extractf64x2_256_mask:4263  case X86::BI__builtin_ia32_extractf64x2_512_mask:4264  case X86::BI__builtin_ia32_extractf64x4_mask:4265  case X86::BI__builtin_ia32_extracti32x4_256_mask:4266  case X86::BI__builtin_ia32_extracti32x4_mask:4267  case X86::BI__builtin_ia32_extracti32x8_mask:4268  case X86::BI__builtin_ia32_extracti64x2_256_mask:4269  case X86::BI__builtin_ia32_extracti64x2_512_mask:4270  case X86::BI__builtin_ia32_extracti64x4_mask:4271    return interp__builtin_x86_extract_vector_masked(S, OpPC, Call, BuiltinID);4272 4273  case clang::X86::BI__builtin_ia32_pmulhrsw128:4274  case clang::X86::BI__builtin_ia32_pmulhrsw256:4275  case clang::X86::BI__builtin_ia32_pmulhrsw512:4276    return interp__builtin_elementwise_int_binop(4277        S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {4278          return (llvm::APIntOps::mulsExtended(LHS, RHS).ashr(14) + 1)4279              .extractBits(16, 1);4280        });4281 4282  case clang::X86::BI__builtin_ia32_movmskps:4283  case clang::X86::BI__builtin_ia32_movmskpd:4284  case clang::X86::BI__builtin_ia32_pmovmskb128:4285  case clang::X86::BI__builtin_ia32_pmovmskb256:4286  case clang::X86::BI__builtin_ia32_movmskps256:4287  case clang::X86::BI__builtin_ia32_movmskpd256: {4288    return interp__builtin_ia32_movmsk_op(S, OpPC, Call);4289  }4290 4291  case X86::BI__builtin_ia32_psignb128:4292  case X86::BI__builtin_ia32_psignb256:4293  case X86::BI__builtin_ia32_psignw128:4294  case X86::BI__builtin_ia32_psignw256:4295  case X86::BI__builtin_ia32_psignd128:4296  case X86::BI__builtin_ia32_psignd256:4297    return interp__builtin_elementwise_int_binop(4298        S, OpPC, Call, [](const APInt &AElem, const APInt &BElem) {4299          if (BElem.isZero())4300            return APInt::getZero(AElem.getBitWidth());4301          if (BElem.isNegative())4302            return -AElem;4303          return AElem;4304        });4305 4306  case clang::X86::BI__builtin_ia32_pavgb128:4307  case clang::X86::BI__builtin_ia32_pavgw128:4308  case clang::X86::BI__builtin_ia32_pavgb256:4309  case clang::X86::BI__builtin_ia32_pavgw256:4310  case clang::X86::BI__builtin_ia32_pavgb512:4311  case clang::X86::BI__builtin_ia32_pavgw512:4312    return interp__builtin_elementwise_int_binop(S, OpPC, Call,4313                                                 llvm::APIntOps::avgCeilU);4314 4315  case clang::X86::BI__builtin_ia32_pmaddubsw128:4316  case clang::X86::BI__builtin_ia32_pmaddubsw256:4317  case clang::X86::BI__builtin_ia32_pmaddubsw512:4318    return interp__builtin_ia32_pmul(4319        S, OpPC, Call,4320        [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,4321           const APSInt &HiRHS) {4322          unsigned BitWidth = 2 * LoLHS.getBitWidth();4323          return (LoLHS.zext(BitWidth) * LoRHS.sext(BitWidth))4324              .sadd_sat((HiLHS.zext(BitWidth) * HiRHS.sext(BitWidth)));4325        });4326 4327  case clang::X86::BI__builtin_ia32_pmaddwd128:4328  case clang::X86::BI__builtin_ia32_pmaddwd256:4329  case clang::X86::BI__builtin_ia32_pmaddwd512:4330    return interp__builtin_ia32_pmul(4331        S, OpPC, Call,4332        [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,4333           const APSInt &HiRHS) {4334          unsigned BitWidth = 2 * LoLHS.getBitWidth();4335          return (LoLHS.sext(BitWidth) * LoRHS.sext(BitWidth)) +4336                 (HiLHS.sext(BitWidth) * HiRHS.sext(BitWidth));4337        });4338 4339  case clang::X86::BI__builtin_ia32_pmulhuw128:4340  case clang::X86::BI__builtin_ia32_pmulhuw256:4341  case clang::X86::BI__builtin_ia32_pmulhuw512:4342    return interp__builtin_elementwise_int_binop(S, OpPC, Call,4343                                                 llvm::APIntOps::mulhu);4344 4345  case clang::X86::BI__builtin_ia32_pmulhw128:4346  case clang::X86::BI__builtin_ia32_pmulhw256:4347  case clang::X86::BI__builtin_ia32_pmulhw512:4348    return interp__builtin_elementwise_int_binop(S, OpPC, Call,4349                                                 llvm::APIntOps::mulhs);4350 4351  case clang::X86::BI__builtin_ia32_psllv2di:4352  case clang::X86::BI__builtin_ia32_psllv4di:4353  case clang::X86::BI__builtin_ia32_psllv4si:4354  case clang::X86::BI__builtin_ia32_psllv8di:4355  case clang::X86::BI__builtin_ia32_psllv8hi:4356  case clang::X86::BI__builtin_ia32_psllv8si:4357  case clang::X86::BI__builtin_ia32_psllv16hi:4358  case clang::X86::BI__builtin_ia32_psllv16si:4359  case clang::X86::BI__builtin_ia32_psllv32hi:4360  case clang::X86::BI__builtin_ia32_psllwi128:4361  case clang::X86::BI__builtin_ia32_psllwi256:4362  case clang::X86::BI__builtin_ia32_psllwi512:4363  case clang::X86::BI__builtin_ia32_pslldi128:4364  case clang::X86::BI__builtin_ia32_pslldi256:4365  case clang::X86::BI__builtin_ia32_pslldi512:4366  case clang::X86::BI__builtin_ia32_psllqi128:4367  case clang::X86::BI__builtin_ia32_psllqi256:4368  case clang::X86::BI__builtin_ia32_psllqi512:4369    return interp__builtin_elementwise_int_binop(4370        S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {4371          if (RHS.uge(LHS.getBitWidth())) {4372            return APInt::getZero(LHS.getBitWidth());4373          }4374          return LHS.shl(RHS.getZExtValue());4375        });4376 4377  case clang::X86::BI__builtin_ia32_psrav4si:4378  case clang::X86::BI__builtin_ia32_psrav8di:4379  case clang::X86::BI__builtin_ia32_psrav8hi:4380  case clang::X86::BI__builtin_ia32_psrav8si:4381  case clang::X86::BI__builtin_ia32_psrav16hi:4382  case clang::X86::BI__builtin_ia32_psrav16si:4383  case clang::X86::BI__builtin_ia32_psrav32hi:4384  case clang::X86::BI__builtin_ia32_psravq128:4385  case clang::X86::BI__builtin_ia32_psravq256:4386  case clang::X86::BI__builtin_ia32_psrawi128:4387  case clang::X86::BI__builtin_ia32_psrawi256:4388  case clang::X86::BI__builtin_ia32_psrawi512:4389  case clang::X86::BI__builtin_ia32_psradi128:4390  case clang::X86::BI__builtin_ia32_psradi256:4391  case clang::X86::BI__builtin_ia32_psradi512:4392  case clang::X86::BI__builtin_ia32_psraqi128:4393  case clang::X86::BI__builtin_ia32_psraqi256:4394  case clang::X86::BI__builtin_ia32_psraqi512:4395    return interp__builtin_elementwise_int_binop(4396        S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {4397          if (RHS.uge(LHS.getBitWidth())) {4398            return LHS.ashr(LHS.getBitWidth() - 1);4399          }4400          return LHS.ashr(RHS.getZExtValue());4401        });4402 4403  case clang::X86::BI__builtin_ia32_psrlv2di:4404  case clang::X86::BI__builtin_ia32_psrlv4di:4405  case clang::X86::BI__builtin_ia32_psrlv4si:4406  case clang::X86::BI__builtin_ia32_psrlv8di:4407  case clang::X86::BI__builtin_ia32_psrlv8hi:4408  case clang::X86::BI__builtin_ia32_psrlv8si:4409  case clang::X86::BI__builtin_ia32_psrlv16hi:4410  case clang::X86::BI__builtin_ia32_psrlv16si:4411  case clang::X86::BI__builtin_ia32_psrlv32hi:4412  case clang::X86::BI__builtin_ia32_psrlwi128:4413  case clang::X86::BI__builtin_ia32_psrlwi256:4414  case clang::X86::BI__builtin_ia32_psrlwi512:4415  case clang::X86::BI__builtin_ia32_psrldi128:4416  case clang::X86::BI__builtin_ia32_psrldi256:4417  case clang::X86::BI__builtin_ia32_psrldi512:4418  case clang::X86::BI__builtin_ia32_psrlqi128:4419  case clang::X86::BI__builtin_ia32_psrlqi256:4420  case clang::X86::BI__builtin_ia32_psrlqi512:4421    return interp__builtin_elementwise_int_binop(4422        S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {4423          if (RHS.uge(LHS.getBitWidth())) {4424            return APInt::getZero(LHS.getBitWidth());4425          }4426          return LHS.lshr(RHS.getZExtValue());4427        });4428  case clang::X86::BI__builtin_ia32_packsswb128:4429  case clang::X86::BI__builtin_ia32_packsswb256:4430  case clang::X86::BI__builtin_ia32_packsswb512:4431  case clang::X86::BI__builtin_ia32_packssdw128:4432  case clang::X86::BI__builtin_ia32_packssdw256:4433  case clang::X86::BI__builtin_ia32_packssdw512:4434    return interp__builtin_x86_pack(S, OpPC, Call, [](const APSInt &Src) {4435      return APInt(Src).truncSSat(Src.getBitWidth() / 2);4436    });4437  case clang::X86::BI__builtin_ia32_packusdw128:4438  case clang::X86::BI__builtin_ia32_packusdw256:4439  case clang::X86::BI__builtin_ia32_packusdw512:4440  case clang::X86::BI__builtin_ia32_packuswb128:4441  case clang::X86::BI__builtin_ia32_packuswb256:4442  case clang::X86::BI__builtin_ia32_packuswb512:4443    return interp__builtin_x86_pack(S, OpPC, Call, [](const APSInt &Src) {4444      unsigned DstBits = Src.getBitWidth() / 2;4445      if (Src.isNegative())4446        return APInt::getZero(DstBits);4447      if (Src.isIntN(DstBits))4448        return APInt(Src).trunc(DstBits);4449      return APInt::getAllOnes(DstBits);4450    });4451 4452  case clang::X86::BI__builtin_ia32_selectss_128:4453  case clang::X86::BI__builtin_ia32_selectsd_128:4454  case clang::X86::BI__builtin_ia32_selectsh_128:4455  case clang::X86::BI__builtin_ia32_selectsbf_128:4456    return interp__builtin_select_scalar(S, Call);4457  case clang::X86::BI__builtin_ia32_vprotbi:4458  case clang::X86::BI__builtin_ia32_vprotdi:4459  case clang::X86::BI__builtin_ia32_vprotqi:4460  case clang::X86::BI__builtin_ia32_vprotwi:4461  case clang::X86::BI__builtin_ia32_prold128:4462  case clang::X86::BI__builtin_ia32_prold256:4463  case clang::X86::BI__builtin_ia32_prold512:4464  case clang::X86::BI__builtin_ia32_prolq128:4465  case clang::X86::BI__builtin_ia32_prolq256:4466  case clang::X86::BI__builtin_ia32_prolq512:4467    return interp__builtin_elementwise_int_binop(4468        S, OpPC, Call,4469        [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotl(RHS); });4470 4471  case clang::X86::BI__builtin_ia32_prord128:4472  case clang::X86::BI__builtin_ia32_prord256:4473  case clang::X86::BI__builtin_ia32_prord512:4474  case clang::X86::BI__builtin_ia32_prorq128:4475  case clang::X86::BI__builtin_ia32_prorq256:4476  case clang::X86::BI__builtin_ia32_prorq512:4477    return interp__builtin_elementwise_int_binop(4478        S, OpPC, Call,4479        [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotr(RHS); });4480 4481  case Builtin::BI__builtin_elementwise_max:4482  case Builtin::BI__builtin_elementwise_min:4483    return interp__builtin_elementwise_maxmin(S, OpPC, Call, BuiltinID);4484 4485  case clang::X86::BI__builtin_ia32_phaddw128:4486  case clang::X86::BI__builtin_ia32_phaddw256:4487  case clang::X86::BI__builtin_ia32_phaddd128:4488  case clang::X86::BI__builtin_ia32_phaddd256:4489    return interp_builtin_horizontal_int_binop(4490        S, OpPC, Call,4491        [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });4492  case clang::X86::BI__builtin_ia32_phaddsw128:4493  case clang::X86::BI__builtin_ia32_phaddsw256:4494    return interp_builtin_horizontal_int_binop(4495        S, OpPC, Call,4496        [](const APSInt &LHS, const APSInt &RHS) { return LHS.sadd_sat(RHS); });4497  case clang::X86::BI__builtin_ia32_phsubw128:4498  case clang::X86::BI__builtin_ia32_phsubw256:4499  case clang::X86::BI__builtin_ia32_phsubd128:4500  case clang::X86::BI__builtin_ia32_phsubd256:4501    return interp_builtin_horizontal_int_binop(4502        S, OpPC, Call,4503        [](const APSInt &LHS, const APSInt &RHS) { return LHS - RHS; });4504  case clang::X86::BI__builtin_ia32_phsubsw128:4505  case clang::X86::BI__builtin_ia32_phsubsw256:4506    return interp_builtin_horizontal_int_binop(4507        S, OpPC, Call,4508        [](const APSInt &LHS, const APSInt &RHS) { return LHS.ssub_sat(RHS); });4509  case clang::X86::BI__builtin_ia32_haddpd:4510  case clang::X86::BI__builtin_ia32_haddps:4511  case clang::X86::BI__builtin_ia32_haddpd256:4512  case clang::X86::BI__builtin_ia32_haddps256:4513    return interp_builtin_horizontal_fp_binop(4514        S, OpPC, Call,4515        [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {4516          APFloat F = LHS;4517          F.add(RHS, RM);4518          return F;4519        });4520  case clang::X86::BI__builtin_ia32_hsubpd:4521  case clang::X86::BI__builtin_ia32_hsubps:4522  case clang::X86::BI__builtin_ia32_hsubpd256:4523  case clang::X86::BI__builtin_ia32_hsubps256:4524    return interp_builtin_horizontal_fp_binop(4525        S, OpPC, Call,4526        [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {4527          APFloat F = LHS;4528          F.subtract(RHS, RM);4529          return F;4530        });4531  case clang::X86::BI__builtin_ia32_addsubpd:4532  case clang::X86::BI__builtin_ia32_addsubps:4533  case clang::X86::BI__builtin_ia32_addsubpd256:4534  case clang::X86::BI__builtin_ia32_addsubps256:4535    return interp__builtin_ia32_addsub(S, OpPC, Call);4536 4537  case clang::X86::BI__builtin_ia32_pmuldq128:4538  case clang::X86::BI__builtin_ia32_pmuldq256:4539  case clang::X86::BI__builtin_ia32_pmuldq512:4540    return interp__builtin_ia32_pmul(4541        S, OpPC, Call,4542        [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,4543           const APSInt &HiRHS) {4544          return llvm::APIntOps::mulsExtended(LoLHS, LoRHS);4545        });4546 4547  case clang::X86::BI__builtin_ia32_pmuludq128:4548  case clang::X86::BI__builtin_ia32_pmuludq256:4549  case clang::X86::BI__builtin_ia32_pmuludq512:4550    return interp__builtin_ia32_pmul(4551        S, OpPC, Call,4552        [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,4553           const APSInt &HiRHS) {4554          return llvm::APIntOps::muluExtended(LoLHS, LoRHS);4555        });4556 4557  case Builtin::BI__builtin_elementwise_fma:4558    return interp__builtin_elementwise_triop_fp(4559        S, OpPC, Call,4560        [](const APFloat &X, const APFloat &Y, const APFloat &Z,4561           llvm::RoundingMode RM) {4562          APFloat F = X;4563          F.fusedMultiplyAdd(Y, Z, RM);4564          return F;4565        });4566 4567  case X86::BI__builtin_ia32_vpmadd52luq128:4568  case X86::BI__builtin_ia32_vpmadd52luq256:4569  case X86::BI__builtin_ia32_vpmadd52luq512:4570    return interp__builtin_elementwise_triop(4571        S, OpPC, Call, [](const APSInt &A, const APSInt &B, const APSInt &C) {4572          return A + (B.trunc(52) * C.trunc(52)).zext(64);4573        });4574  case X86::BI__builtin_ia32_vpmadd52huq128:4575  case X86::BI__builtin_ia32_vpmadd52huq256:4576  case X86::BI__builtin_ia32_vpmadd52huq512:4577    return interp__builtin_elementwise_triop(4578        S, OpPC, Call, [](const APSInt &A, const APSInt &B, const APSInt &C) {4579          return A + llvm::APIntOps::mulhu(B.trunc(52), C.trunc(52)).zext(64);4580        });4581 4582  case X86::BI__builtin_ia32_vpshldd128:4583  case X86::BI__builtin_ia32_vpshldd256:4584  case X86::BI__builtin_ia32_vpshldd512:4585  case X86::BI__builtin_ia32_vpshldq128:4586  case X86::BI__builtin_ia32_vpshldq256:4587  case X86::BI__builtin_ia32_vpshldq512:4588  case X86::BI__builtin_ia32_vpshldw128:4589  case X86::BI__builtin_ia32_vpshldw256:4590  case X86::BI__builtin_ia32_vpshldw512:4591    return interp__builtin_elementwise_triop(4592        S, OpPC, Call,4593        [](const APSInt &Hi, const APSInt &Lo, const APSInt &Amt) {4594          return llvm::APIntOps::fshl(Hi, Lo, Amt);4595        });4596 4597  case X86::BI__builtin_ia32_vpshrdd128:4598  case X86::BI__builtin_ia32_vpshrdd256:4599  case X86::BI__builtin_ia32_vpshrdd512:4600  case X86::BI__builtin_ia32_vpshrdq128:4601  case X86::BI__builtin_ia32_vpshrdq256:4602  case X86::BI__builtin_ia32_vpshrdq512:4603  case X86::BI__builtin_ia32_vpshrdw128:4604  case X86::BI__builtin_ia32_vpshrdw256:4605  case X86::BI__builtin_ia32_vpshrdw512:4606    // NOTE: Reversed Hi/Lo operands.4607    return interp__builtin_elementwise_triop(4608        S, OpPC, Call,4609        [](const APSInt &Lo, const APSInt &Hi, const APSInt &Amt) {4610          return llvm::APIntOps::fshr(Hi, Lo, Amt);4611        });4612  case X86::BI__builtin_ia32_vpconflictsi_128:4613  case X86::BI__builtin_ia32_vpconflictsi_256:4614  case X86::BI__builtin_ia32_vpconflictsi_512:4615  case X86::BI__builtin_ia32_vpconflictdi_128:4616  case X86::BI__builtin_ia32_vpconflictdi_256:4617  case X86::BI__builtin_ia32_vpconflictdi_512:4618    return interp__builtin_ia32_vpconflict(S, OpPC, Call);4619  case clang::X86::BI__builtin_ia32_blendpd:4620  case clang::X86::BI__builtin_ia32_blendpd256:4621  case clang::X86::BI__builtin_ia32_blendps:4622  case clang::X86::BI__builtin_ia32_blendps256:4623  case clang::X86::BI__builtin_ia32_pblendw128:4624  case clang::X86::BI__builtin_ia32_pblendw256:4625  case clang::X86::BI__builtin_ia32_pblendd128:4626  case clang::X86::BI__builtin_ia32_pblendd256:4627    return interp__builtin_blend(S, OpPC, Call);4628 4629  case clang::X86::BI__builtin_ia32_blendvpd:4630  case clang::X86::BI__builtin_ia32_blendvpd256:4631  case clang::X86::BI__builtin_ia32_blendvps:4632  case clang::X86::BI__builtin_ia32_blendvps256:4633    return interp__builtin_elementwise_triop_fp(4634        S, OpPC, Call,4635        [](const APFloat &F, const APFloat &T, const APFloat &C,4636           llvm::RoundingMode) { return C.isNegative() ? T : F; });4637 4638  case clang::X86::BI__builtin_ia32_pblendvb128:4639  case clang::X86::BI__builtin_ia32_pblendvb256:4640    return interp__builtin_elementwise_triop(4641        S, OpPC, Call, [](const APSInt &F, const APSInt &T, const APSInt &C) {4642          return ((APInt)C).isNegative() ? T : F;4643        });4644  case X86::BI__builtin_ia32_ptestz128:4645  case X86::BI__builtin_ia32_ptestz256:4646  case X86::BI__builtin_ia32_vtestzps:4647  case X86::BI__builtin_ia32_vtestzps256:4648  case X86::BI__builtin_ia32_vtestzpd:4649  case X86::BI__builtin_ia32_vtestzpd256:4650    return interp__builtin_ia32_test_op(4651        S, OpPC, Call,4652        [](const APInt &A, const APInt &B) { return (A & B) == 0; });4653  case X86::BI__builtin_ia32_ptestc128:4654  case X86::BI__builtin_ia32_ptestc256:4655  case X86::BI__builtin_ia32_vtestcps:4656  case X86::BI__builtin_ia32_vtestcps256:4657  case X86::BI__builtin_ia32_vtestcpd:4658  case X86::BI__builtin_ia32_vtestcpd256:4659    return interp__builtin_ia32_test_op(4660        S, OpPC, Call,4661        [](const APInt &A, const APInt &B) { return (~A & B) == 0; });4662  case X86::BI__builtin_ia32_ptestnzc128:4663  case X86::BI__builtin_ia32_ptestnzc256:4664  case X86::BI__builtin_ia32_vtestnzcps:4665  case X86::BI__builtin_ia32_vtestnzcps256:4666  case X86::BI__builtin_ia32_vtestnzcpd:4667  case X86::BI__builtin_ia32_vtestnzcpd256:4668    return interp__builtin_ia32_test_op(4669        S, OpPC, Call, [](const APInt &A, const APInt &B) {4670          return ((A & B) != 0) && ((~A & B) != 0);4671        });4672  case X86::BI__builtin_ia32_selectb_128:4673  case X86::BI__builtin_ia32_selectb_256:4674  case X86::BI__builtin_ia32_selectb_512:4675  case X86::BI__builtin_ia32_selectw_128:4676  case X86::BI__builtin_ia32_selectw_256:4677  case X86::BI__builtin_ia32_selectw_512:4678  case X86::BI__builtin_ia32_selectd_128:4679  case X86::BI__builtin_ia32_selectd_256:4680  case X86::BI__builtin_ia32_selectd_512:4681  case X86::BI__builtin_ia32_selectq_128:4682  case X86::BI__builtin_ia32_selectq_256:4683  case X86::BI__builtin_ia32_selectq_512:4684  case X86::BI__builtin_ia32_selectph_128:4685  case X86::BI__builtin_ia32_selectph_256:4686  case X86::BI__builtin_ia32_selectph_512:4687  case X86::BI__builtin_ia32_selectpbf_128:4688  case X86::BI__builtin_ia32_selectpbf_256:4689  case X86::BI__builtin_ia32_selectpbf_512:4690  case X86::BI__builtin_ia32_selectps_128:4691  case X86::BI__builtin_ia32_selectps_256:4692  case X86::BI__builtin_ia32_selectps_512:4693  case X86::BI__builtin_ia32_selectpd_128:4694  case X86::BI__builtin_ia32_selectpd_256:4695  case X86::BI__builtin_ia32_selectpd_512:4696    return interp__builtin_select(S, OpPC, Call);4697 4698  case X86::BI__builtin_ia32_shufps:4699  case X86::BI__builtin_ia32_shufps256:4700  case X86::BI__builtin_ia32_shufps512:4701    return interp__builtin_ia32_shuffle_generic(4702        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4703          unsigned NumElemPerLane = 4;4704          unsigned NumSelectableElems = NumElemPerLane / 2;4705          unsigned BitsPerElem = 2;4706          unsigned IndexMask = 0x3;4707          unsigned MaskBits = 8;4708          unsigned Lane = DstIdx / NumElemPerLane;4709          unsigned ElemInLane = DstIdx % NumElemPerLane;4710          unsigned LaneOffset = Lane * NumElemPerLane;4711          unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;4712          unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;4713          unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;4714          return std::pair<unsigned, int>{SrcIdx,4715                                          static_cast<int>(LaneOffset + Index)};4716        });4717  case X86::BI__builtin_ia32_shufpd:4718  case X86::BI__builtin_ia32_shufpd256:4719  case X86::BI__builtin_ia32_shufpd512:4720    return interp__builtin_ia32_shuffle_generic(4721        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4722          unsigned NumElemPerLane = 2;4723          unsigned NumSelectableElems = NumElemPerLane / 2;4724          unsigned BitsPerElem = 1;4725          unsigned IndexMask = 0x1;4726          unsigned MaskBits = 8;4727          unsigned Lane = DstIdx / NumElemPerLane;4728          unsigned ElemInLane = DstIdx % NumElemPerLane;4729          unsigned LaneOffset = Lane * NumElemPerLane;4730          unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;4731          unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;4732          unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;4733          return std::pair<unsigned, int>{SrcIdx,4734                                          static_cast<int>(LaneOffset + Index)};4735        });4736  case X86::BI__builtin_ia32_insertps128:4737    return interp__builtin_ia32_shuffle_generic(4738        S, OpPC, Call, [](unsigned DstIdx, unsigned Mask) {4739          // Bits [3:0]: zero mask - if bit is set, zero this element4740          if ((Mask & (1 << DstIdx)) != 0) {4741            return std::pair<unsigned, int>{0, -1};4742          }4743          // Bits [7:6]: select element from source vector Y (0-3)4744          // Bits [5:4]: select destination position (0-3)4745          unsigned SrcElem = (Mask >> 6) & 0x3;4746          unsigned DstElem = (Mask >> 4) & 0x3;4747          if (DstIdx == DstElem) {4748            // Insert element from source vector (B) at this position4749            return std::pair<unsigned, int>{1, static_cast<int>(SrcElem)};4750          } else {4751            // Copy from destination vector (A)4752            return std::pair<unsigned, int>{0, static_cast<int>(DstIdx)};4753          }4754        });4755  case X86::BI__builtin_ia32_permvarsi256:4756  case X86::BI__builtin_ia32_permvarsf256:4757  case X86::BI__builtin_ia32_permvardf512:4758  case X86::BI__builtin_ia32_permvardi512:4759  case X86::BI__builtin_ia32_permvarhi128:4760    return interp__builtin_ia32_shuffle_generic(4761        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4762          int Offset = ShuffleMask & 0x7;4763          return std::pair<unsigned, int>{0, Offset};4764        });4765  case X86::BI__builtin_ia32_permvarqi128:4766  case X86::BI__builtin_ia32_permvarhi256:4767  case X86::BI__builtin_ia32_permvarsi512:4768  case X86::BI__builtin_ia32_permvarsf512:4769    return interp__builtin_ia32_shuffle_generic(4770        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4771          int Offset = ShuffleMask & 0xF;4772          return std::pair<unsigned, int>{0, Offset};4773        });4774  case X86::BI__builtin_ia32_permvardi256:4775  case X86::BI__builtin_ia32_permvardf256:4776    return interp__builtin_ia32_shuffle_generic(4777        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4778          int Offset = ShuffleMask & 0x3;4779          return std::pair<unsigned, int>{0, Offset};4780        });4781  case X86::BI__builtin_ia32_permvarqi256:4782  case X86::BI__builtin_ia32_permvarhi512:4783    return interp__builtin_ia32_shuffle_generic(4784        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4785          int Offset = ShuffleMask & 0x1F;4786          return std::pair<unsigned, int>{0, Offset};4787        });4788  case X86::BI__builtin_ia32_permvarqi512:4789    return interp__builtin_ia32_shuffle_generic(4790        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4791          int Offset = ShuffleMask & 0x3F;4792          return std::pair<unsigned, int>{0, Offset};4793        });4794  case X86::BI__builtin_ia32_vpermi2varq128:4795  case X86::BI__builtin_ia32_vpermi2varpd128:4796    return interp__builtin_ia32_shuffle_generic(4797        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4798          int Offset = ShuffleMask & 0x1;4799          unsigned SrcIdx = (ShuffleMask >> 1) & 0x1;4800          return std::pair<unsigned, int>{SrcIdx, Offset};4801        });4802  case X86::BI__builtin_ia32_vpermi2vard128:4803  case X86::BI__builtin_ia32_vpermi2varps128:4804  case X86::BI__builtin_ia32_vpermi2varq256:4805  case X86::BI__builtin_ia32_vpermi2varpd256:4806    return interp__builtin_ia32_shuffle_generic(4807        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4808          int Offset = ShuffleMask & 0x3;4809          unsigned SrcIdx = (ShuffleMask >> 2) & 0x1;4810          return std::pair<unsigned, int>{SrcIdx, Offset};4811        });4812  case X86::BI__builtin_ia32_vpermi2varhi128:4813  case X86::BI__builtin_ia32_vpermi2vard256:4814  case X86::BI__builtin_ia32_vpermi2varps256:4815  case X86::BI__builtin_ia32_vpermi2varq512:4816  case X86::BI__builtin_ia32_vpermi2varpd512:4817    return interp__builtin_ia32_shuffle_generic(4818        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4819          int Offset = ShuffleMask & 0x7;4820          unsigned SrcIdx = (ShuffleMask >> 3) & 0x1;4821          return std::pair<unsigned, int>{SrcIdx, Offset};4822        });4823  case X86::BI__builtin_ia32_vpermi2varqi128:4824  case X86::BI__builtin_ia32_vpermi2varhi256:4825  case X86::BI__builtin_ia32_vpermi2vard512:4826  case X86::BI__builtin_ia32_vpermi2varps512:4827    return interp__builtin_ia32_shuffle_generic(4828        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4829          int Offset = ShuffleMask & 0xF;4830          unsigned SrcIdx = (ShuffleMask >> 4) & 0x1;4831          return std::pair<unsigned, int>{SrcIdx, Offset};4832        });4833  case X86::BI__builtin_ia32_vpermi2varqi256:4834  case X86::BI__builtin_ia32_vpermi2varhi512:4835    return interp__builtin_ia32_shuffle_generic(4836        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4837          int Offset = ShuffleMask & 0x1F;4838          unsigned SrcIdx = (ShuffleMask >> 5) & 0x1;4839          return std::pair<unsigned, int>{SrcIdx, Offset};4840        });4841  case X86::BI__builtin_ia32_vpermi2varqi512:4842    return interp__builtin_ia32_shuffle_generic(4843        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4844          int Offset = ShuffleMask & 0x3F;4845          unsigned SrcIdx = (ShuffleMask >> 6) & 0x1;4846          return std::pair<unsigned, int>{SrcIdx, Offset};4847        });4848  case X86::BI__builtin_ia32_pshufb128:4849  case X86::BI__builtin_ia32_pshufb256:4850  case X86::BI__builtin_ia32_pshufb512:4851    return interp__builtin_ia32_shuffle_generic(4852        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4853          uint8_t Ctlb = static_cast<uint8_t>(ShuffleMask);4854          if (Ctlb & 0x80)4855            return std::make_pair(0, -1);4856 4857          unsigned LaneBase = (DstIdx / 16) * 16;4858          unsigned SrcOffset = Ctlb & 0x0F;4859          unsigned SrcIdx = LaneBase + SrcOffset;4860          return std::make_pair(0, static_cast<int>(SrcIdx));4861        });4862 4863  case X86::BI__builtin_ia32_pshuflw:4864  case X86::BI__builtin_ia32_pshuflw256:4865  case X86::BI__builtin_ia32_pshuflw512:4866    return interp__builtin_ia32_shuffle_generic(4867        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4868          unsigned LaneBase = (DstIdx / 8) * 8;4869          unsigned LaneIdx = DstIdx % 8;4870          if (LaneIdx < 4) {4871            unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;4872            return std::make_pair(0, static_cast<int>(LaneBase + Sel));4873          }4874 4875          return std::make_pair(0, static_cast<int>(DstIdx));4876        });4877 4878  case X86::BI__builtin_ia32_pshufhw:4879  case X86::BI__builtin_ia32_pshufhw256:4880  case X86::BI__builtin_ia32_pshufhw512:4881    return interp__builtin_ia32_shuffle_generic(4882        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4883          unsigned LaneBase = (DstIdx / 8) * 8;4884          unsigned LaneIdx = DstIdx % 8;4885          if (LaneIdx >= 4) {4886            unsigned Sel = (ShuffleMask >> (2 * (LaneIdx - 4))) & 0x3;4887            return std::make_pair(0, static_cast<int>(LaneBase + 4 + Sel));4888          }4889 4890          return std::make_pair(0, static_cast<int>(DstIdx));4891        });4892 4893  case X86::BI__builtin_ia32_pshufd:4894  case X86::BI__builtin_ia32_pshufd256:4895  case X86::BI__builtin_ia32_pshufd512:4896  case X86::BI__builtin_ia32_vpermilps:4897  case X86::BI__builtin_ia32_vpermilps256:4898  case X86::BI__builtin_ia32_vpermilps512:4899    return interp__builtin_ia32_shuffle_generic(4900        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4901          unsigned LaneBase = (DstIdx / 4) * 4;4902          unsigned LaneIdx = DstIdx % 4;4903          unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;4904          return std::make_pair(0, static_cast<int>(LaneBase + Sel));4905        });4906 4907  case X86::BI__builtin_ia32_vpermilvarpd:4908  case X86::BI__builtin_ia32_vpermilvarpd256:4909  case X86::BI__builtin_ia32_vpermilvarpd512:4910    return interp__builtin_ia32_shuffle_generic(4911        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4912          unsigned NumElemPerLane = 2;4913          unsigned Lane = DstIdx / NumElemPerLane;4914          unsigned Offset = ShuffleMask & 0b10 ? 1 : 0;4915          return std::make_pair(4916              0, static_cast<int>(Lane * NumElemPerLane + Offset));4917        });4918 4919  case X86::BI__builtin_ia32_vpermilvarps:4920  case X86::BI__builtin_ia32_vpermilvarps256:4921  case X86::BI__builtin_ia32_vpermilvarps512:4922    return interp__builtin_ia32_shuffle_generic(4923        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {4924          unsigned NumElemPerLane = 4;4925          unsigned Lane = DstIdx / NumElemPerLane;4926          unsigned Offset = ShuffleMask & 0b11;4927          return std::make_pair(4928              0, static_cast<int>(Lane * NumElemPerLane + Offset));4929        });4930 4931  case X86::BI__builtin_ia32_vpermilpd:4932  case X86::BI__builtin_ia32_vpermilpd256:4933  case X86::BI__builtin_ia32_vpermilpd512:4934    return interp__builtin_ia32_shuffle_generic(4935        S, OpPC, Call, [](unsigned DstIdx, unsigned Control) {4936          unsigned NumElemPerLane = 2;4937          unsigned BitsPerElem = 1;4938          unsigned MaskBits = 8;4939          unsigned IndexMask = 0x1;4940          unsigned Lane = DstIdx / NumElemPerLane;4941          unsigned LaneOffset = Lane * NumElemPerLane;4942          unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;4943          unsigned Index = (Control >> BitIndex) & IndexMask;4944          return std::make_pair(0, static_cast<int>(LaneOffset + Index));4945        });4946 4947  case X86::BI__builtin_ia32_vpmultishiftqb128:4948  case X86::BI__builtin_ia32_vpmultishiftqb256:4949  case X86::BI__builtin_ia32_vpmultishiftqb512:4950    return interp__builtin_ia32_multishiftqb(S, OpPC, Call);4951  case X86::BI__builtin_ia32_kandqi:4952  case X86::BI__builtin_ia32_kandhi:4953  case X86::BI__builtin_ia32_kandsi:4954  case X86::BI__builtin_ia32_kanddi:4955    return interp__builtin_elementwise_int_binop(4956        S, OpPC, Call,4957        [](const APSInt &LHS, const APSInt &RHS) { return LHS & RHS; });4958 4959  case X86::BI__builtin_ia32_kandnqi:4960  case X86::BI__builtin_ia32_kandnhi:4961  case X86::BI__builtin_ia32_kandnsi:4962  case X86::BI__builtin_ia32_kandndi:4963    return interp__builtin_elementwise_int_binop(4964        S, OpPC, Call,4965        [](const APSInt &LHS, const APSInt &RHS) { return ~LHS & RHS; });4966 4967  case X86::BI__builtin_ia32_korqi:4968  case X86::BI__builtin_ia32_korhi:4969  case X86::BI__builtin_ia32_korsi:4970  case X86::BI__builtin_ia32_kordi:4971    return interp__builtin_elementwise_int_binop(4972        S, OpPC, Call,4973        [](const APSInt &LHS, const APSInt &RHS) { return LHS | RHS; });4974 4975  case X86::BI__builtin_ia32_kxnorqi:4976  case X86::BI__builtin_ia32_kxnorhi:4977  case X86::BI__builtin_ia32_kxnorsi:4978  case X86::BI__builtin_ia32_kxnordi:4979    return interp__builtin_elementwise_int_binop(4980        S, OpPC, Call,4981        [](const APSInt &LHS, const APSInt &RHS) { return ~(LHS ^ RHS); });4982 4983  case X86::BI__builtin_ia32_kxorqi:4984  case X86::BI__builtin_ia32_kxorhi:4985  case X86::BI__builtin_ia32_kxorsi:4986  case X86::BI__builtin_ia32_kxordi:4987    return interp__builtin_elementwise_int_binop(4988        S, OpPC, Call,4989        [](const APSInt &LHS, const APSInt &RHS) { return LHS ^ RHS; });4990 4991  case X86::BI__builtin_ia32_knotqi:4992  case X86::BI__builtin_ia32_knothi:4993  case X86::BI__builtin_ia32_knotsi:4994  case X86::BI__builtin_ia32_knotdi:4995    return interp__builtin_elementwise_int_unaryop(4996        S, OpPC, Call, [](const APSInt &Src) { return ~Src; });4997 4998  case X86::BI__builtin_ia32_kaddqi:4999  case X86::BI__builtin_ia32_kaddhi:5000  case X86::BI__builtin_ia32_kaddsi:5001  case X86::BI__builtin_ia32_kadddi:5002    return interp__builtin_elementwise_int_binop(5003        S, OpPC, Call,5004        [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });5005 5006  case X86::BI__builtin_ia32_kmovb:5007  case X86::BI__builtin_ia32_kmovw:5008  case X86::BI__builtin_ia32_kmovd:5009  case X86::BI__builtin_ia32_kmovq:5010    return interp__builtin_elementwise_int_unaryop(5011        S, OpPC, Call, [](const APSInt &Src) { return Src; });5012 5013  case X86::BI__builtin_ia32_kunpckhi:5014  case X86::BI__builtin_ia32_kunpckdi:5015  case X86::BI__builtin_ia32_kunpcksi:5016    return interp__builtin_elementwise_int_binop(5017        S, OpPC, Call, [](const APSInt &A, const APSInt &B) {5018          // Generic kunpack: extract lower half of each operand and concatenate5019          // Result = A[HalfWidth-1:0] concat B[HalfWidth-1:0]5020          unsigned BW = A.getBitWidth();5021          return APSInt(A.trunc(BW / 2).concat(B.trunc(BW / 2)),5022                        A.isUnsigned());5023        });5024 5025  case X86::BI__builtin_ia32_phminposuw128:5026    return interp__builtin_ia32_phminposuw(S, OpPC, Call);5027 5028  case X86::BI__builtin_ia32_psraq128:5029  case X86::BI__builtin_ia32_psraq256:5030  case X86::BI__builtin_ia32_psraq512:5031  case X86::BI__builtin_ia32_psrad128:5032  case X86::BI__builtin_ia32_psrad256:5033  case X86::BI__builtin_ia32_psrad512:5034  case X86::BI__builtin_ia32_psraw128:5035  case X86::BI__builtin_ia32_psraw256:5036  case X86::BI__builtin_ia32_psraw512:5037    return interp__builtin_ia32_shift_with_count(5038        S, OpPC, Call,5039        [](const APInt &Elt, uint64_t Count) { return Elt.ashr(Count); },5040        [](const APInt &Elt, unsigned Width) { return Elt.ashr(Width - 1); });5041 5042  case X86::BI__builtin_ia32_psllq128:5043  case X86::BI__builtin_ia32_psllq256:5044  case X86::BI__builtin_ia32_psllq512:5045  case X86::BI__builtin_ia32_pslld128:5046  case X86::BI__builtin_ia32_pslld256:5047  case X86::BI__builtin_ia32_pslld512:5048  case X86::BI__builtin_ia32_psllw128:5049  case X86::BI__builtin_ia32_psllw256:5050  case X86::BI__builtin_ia32_psllw512:5051    return interp__builtin_ia32_shift_with_count(5052        S, OpPC, Call,5053        [](const APInt &Elt, uint64_t Count) { return Elt.shl(Count); },5054        [](const APInt &Elt, unsigned Width) { return APInt::getZero(Width); });5055 5056  case X86::BI__builtin_ia32_psrlq128:5057  case X86::BI__builtin_ia32_psrlq256:5058  case X86::BI__builtin_ia32_psrlq512:5059  case X86::BI__builtin_ia32_psrld128:5060  case X86::BI__builtin_ia32_psrld256:5061  case X86::BI__builtin_ia32_psrld512:5062  case X86::BI__builtin_ia32_psrlw128:5063  case X86::BI__builtin_ia32_psrlw256:5064  case X86::BI__builtin_ia32_psrlw512:5065    return interp__builtin_ia32_shift_with_count(5066        S, OpPC, Call,5067        [](const APInt &Elt, uint64_t Count) { return Elt.lshr(Count); },5068        [](const APInt &Elt, unsigned Width) { return APInt::getZero(Width); });5069 5070  case X86::BI__builtin_ia32_pternlogd128_mask:5071  case X86::BI__builtin_ia32_pternlogd256_mask:5072  case X86::BI__builtin_ia32_pternlogd512_mask:5073  case X86::BI__builtin_ia32_pternlogq128_mask:5074  case X86::BI__builtin_ia32_pternlogq256_mask:5075  case X86::BI__builtin_ia32_pternlogq512_mask:5076    return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/false);5077  case X86::BI__builtin_ia32_pternlogd128_maskz:5078  case X86::BI__builtin_ia32_pternlogd256_maskz:5079  case X86::BI__builtin_ia32_pternlogd512_maskz:5080  case X86::BI__builtin_ia32_pternlogq128_maskz:5081  case X86::BI__builtin_ia32_pternlogq256_maskz:5082  case X86::BI__builtin_ia32_pternlogq512_maskz:5083    return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/true);5084  case Builtin::BI__builtin_elementwise_fshl:5085    return interp__builtin_elementwise_triop(S, OpPC, Call,5086                                             llvm::APIntOps::fshl);5087  case Builtin::BI__builtin_elementwise_fshr:5088    return interp__builtin_elementwise_triop(S, OpPC, Call,5089                                             llvm::APIntOps::fshr);5090 5091  case X86::BI__builtin_ia32_shuf_f32x4_256:5092  case X86::BI__builtin_ia32_shuf_i32x4_256:5093  case X86::BI__builtin_ia32_shuf_f64x2_256:5094  case X86::BI__builtin_ia32_shuf_i64x2_256:5095  case X86::BI__builtin_ia32_shuf_f32x4:5096  case X86::BI__builtin_ia32_shuf_i32x4:5097  case X86::BI__builtin_ia32_shuf_f64x2:5098  case X86::BI__builtin_ia32_shuf_i64x2: {5099    // Destination and sources A, B all have the same type.5100    QualType VecQT = Call->getArg(0)->getType();5101    const auto *VecT = VecQT->castAs<VectorType>();5102    unsigned NumElems = VecT->getNumElements();5103    unsigned ElemBits = S.getASTContext().getTypeSize(VecT->getElementType());5104    unsigned LaneBits = 128u;5105    unsigned NumLanes = (NumElems * ElemBits) / LaneBits;5106    unsigned NumElemsPerLane = LaneBits / ElemBits;5107 5108    return interp__builtin_ia32_shuffle_generic(5109        S, OpPC, Call,5110        [NumLanes, NumElemsPerLane](unsigned DstIdx, unsigned ShuffleMask) {5111          // DstIdx determines source. ShuffleMask selects lane in source.5112          unsigned BitsPerElem = NumLanes / 2;5113          unsigned IndexMask = (1u << BitsPerElem) - 1;5114          unsigned Lane = DstIdx / NumElemsPerLane;5115          unsigned SrcIdx = (Lane < NumLanes / 2) ? 0 : 1;5116          unsigned BitIdx = BitsPerElem * Lane;5117          unsigned SrcLaneIdx = (ShuffleMask >> BitIdx) & IndexMask;5118          unsigned ElemInLane = DstIdx % NumElemsPerLane;5119          unsigned IdxToPick = SrcLaneIdx * NumElemsPerLane + ElemInLane;5120          return std::pair<unsigned, int>{SrcIdx, IdxToPick};5121        });5122  }5123 5124  case X86::BI__builtin_ia32_insertf32x4_256:5125  case X86::BI__builtin_ia32_inserti32x4_256:5126  case X86::BI__builtin_ia32_insertf64x2_256:5127  case X86::BI__builtin_ia32_inserti64x2_256:5128  case X86::BI__builtin_ia32_insertf32x4:5129  case X86::BI__builtin_ia32_inserti32x4:5130  case X86::BI__builtin_ia32_insertf64x2_512:5131  case X86::BI__builtin_ia32_inserti64x2_512:5132  case X86::BI__builtin_ia32_insertf32x8:5133  case X86::BI__builtin_ia32_inserti32x8:5134  case X86::BI__builtin_ia32_insertf64x4:5135  case X86::BI__builtin_ia32_inserti64x4:5136  case X86::BI__builtin_ia32_vinsertf128_ps256:5137  case X86::BI__builtin_ia32_vinsertf128_pd256:5138  case X86::BI__builtin_ia32_vinsertf128_si256:5139  case X86::BI__builtin_ia32_insert128i256:5140    return interp__builtin_x86_insert_subvector(S, OpPC, Call, BuiltinID);5141 5142  case clang::X86::BI__builtin_ia32_vcvtps2ph:5143  case clang::X86::BI__builtin_ia32_vcvtps2ph256:5144    return interp__builtin_ia32_vcvtps2ph(S, OpPC, Call);5145 5146  case X86::BI__builtin_ia32_vec_ext_v4hi:5147  case X86::BI__builtin_ia32_vec_ext_v16qi:5148  case X86::BI__builtin_ia32_vec_ext_v8hi:5149  case X86::BI__builtin_ia32_vec_ext_v4si:5150  case X86::BI__builtin_ia32_vec_ext_v2di:5151  case X86::BI__builtin_ia32_vec_ext_v32qi:5152  case X86::BI__builtin_ia32_vec_ext_v16hi:5153  case X86::BI__builtin_ia32_vec_ext_v8si:5154  case X86::BI__builtin_ia32_vec_ext_v4di:5155  case X86::BI__builtin_ia32_vec_ext_v4sf:5156    return interp__builtin_vec_ext(S, OpPC, Call, BuiltinID);5157 5158  case X86::BI__builtin_ia32_vec_set_v4hi:5159  case X86::BI__builtin_ia32_vec_set_v16qi:5160  case X86::BI__builtin_ia32_vec_set_v8hi:5161  case X86::BI__builtin_ia32_vec_set_v4si:5162  case X86::BI__builtin_ia32_vec_set_v2di:5163  case X86::BI__builtin_ia32_vec_set_v32qi:5164  case X86::BI__builtin_ia32_vec_set_v16hi:5165  case X86::BI__builtin_ia32_vec_set_v8si:5166  case X86::BI__builtin_ia32_vec_set_v4di:5167    return interp__builtin_vec_set(S, OpPC, Call, BuiltinID);5168 5169  case X86::BI__builtin_ia32_cvtb2mask128:5170  case X86::BI__builtin_ia32_cvtb2mask256:5171  case X86::BI__builtin_ia32_cvtb2mask512:5172  case X86::BI__builtin_ia32_cvtw2mask128:5173  case X86::BI__builtin_ia32_cvtw2mask256:5174  case X86::BI__builtin_ia32_cvtw2mask512:5175  case X86::BI__builtin_ia32_cvtd2mask128:5176  case X86::BI__builtin_ia32_cvtd2mask256:5177  case X86::BI__builtin_ia32_cvtd2mask512:5178  case X86::BI__builtin_ia32_cvtq2mask128:5179  case X86::BI__builtin_ia32_cvtq2mask256:5180  case X86::BI__builtin_ia32_cvtq2mask512:5181    return interp__builtin_ia32_cvt_vec2mask(S, OpPC, Call, BuiltinID);5182 5183  case X86::BI__builtin_ia32_cmpb128_mask:5184  case X86::BI__builtin_ia32_cmpw128_mask:5185  case X86::BI__builtin_ia32_cmpd128_mask:5186  case X86::BI__builtin_ia32_cmpq128_mask:5187  case X86::BI__builtin_ia32_cmpb256_mask:5188  case X86::BI__builtin_ia32_cmpw256_mask:5189  case X86::BI__builtin_ia32_cmpd256_mask:5190  case X86::BI__builtin_ia32_cmpq256_mask:5191  case X86::BI__builtin_ia32_cmpb512_mask:5192  case X86::BI__builtin_ia32_cmpw512_mask:5193  case X86::BI__builtin_ia32_cmpd512_mask:5194  case X86::BI__builtin_ia32_cmpq512_mask:5195    return interp__builtin_ia32_cmp_mask(S, OpPC, Call, BuiltinID,5196                                         /*IsUnsigned=*/false);5197 5198  case X86::BI__builtin_ia32_ucmpb128_mask:5199  case X86::BI__builtin_ia32_ucmpw128_mask:5200  case X86::BI__builtin_ia32_ucmpd128_mask:5201  case X86::BI__builtin_ia32_ucmpq128_mask:5202  case X86::BI__builtin_ia32_ucmpb256_mask:5203  case X86::BI__builtin_ia32_ucmpw256_mask:5204  case X86::BI__builtin_ia32_ucmpd256_mask:5205  case X86::BI__builtin_ia32_ucmpq256_mask:5206  case X86::BI__builtin_ia32_ucmpb512_mask:5207  case X86::BI__builtin_ia32_ucmpw512_mask:5208  case X86::BI__builtin_ia32_ucmpd512_mask:5209  case X86::BI__builtin_ia32_ucmpq512_mask:5210    return interp__builtin_ia32_cmp_mask(S, OpPC, Call, BuiltinID,5211                                         /*IsUnsigned=*/true);5212 5213  case X86::BI__builtin_ia32_vpshufbitqmb128_mask:5214  case X86::BI__builtin_ia32_vpshufbitqmb256_mask:5215  case X86::BI__builtin_ia32_vpshufbitqmb512_mask:5216    return interp__builtin_ia32_shufbitqmb_mask(S, OpPC, Call);5217 5218  case X86::BI__builtin_ia32_pslldqi128_byteshift:5219  case X86::BI__builtin_ia32_pslldqi256_byteshift:5220  case X86::BI__builtin_ia32_pslldqi512_byteshift:5221    // These SLLDQ intrinsics always operate on byte elements (8 bits).5222    // The lane width is hardcoded to 16 to match the SIMD register size,5223    // but the algorithm processes one byte per iteration,5224    // so APInt(8, ...) is correct and intentional.5225    return interp__builtin_ia32_shuffle_generic(5226        S, OpPC, Call,5227        [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {5228          unsigned LaneBase = (DstIdx / 16) * 16;5229          unsigned LaneIdx = DstIdx % 16;5230          if (LaneIdx < Shift)5231            return std::make_pair(0, -1);5232 5233          return std::make_pair(0,5234                                static_cast<int>(LaneBase + LaneIdx - Shift));5235        });5236 5237  case X86::BI__builtin_ia32_psrldqi128_byteshift:5238  case X86::BI__builtin_ia32_psrldqi256_byteshift:5239  case X86::BI__builtin_ia32_psrldqi512_byteshift:5240    // These SRLDQ intrinsics always operate on byte elements (8 bits).5241    // The lane width is hardcoded to 16 to match the SIMD register size,5242    // but the algorithm processes one byte per iteration,5243    // so APInt(8, ...) is correct and intentional.5244    return interp__builtin_ia32_shuffle_generic(5245        S, OpPC, Call,5246        [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {5247          unsigned LaneBase = (DstIdx / 16) * 16;5248          unsigned LaneIdx = DstIdx % 16;5249          if (LaneIdx + Shift < 16)5250            return std::make_pair(0,5251                                  static_cast<int>(LaneBase + LaneIdx + Shift));5252 5253          return std::make_pair(0, -1);5254        });5255 5256  case X86::BI__builtin_ia32_palignr128:5257  case X86::BI__builtin_ia32_palignr256:5258  case X86::BI__builtin_ia32_palignr512:5259    return interp__builtin_ia32_shuffle_generic(5260        S, OpPC, Call, [](unsigned DstIdx, unsigned Shift) {5261          // Default to -1 → zero-fill this destination element5262          unsigned VecIdx = 1;5263          int ElemIdx = -1;5264 5265          int Lane = DstIdx / 16;5266          int Offset = DstIdx % 16;5267 5268          // Elements come from VecB first, then VecA after the shift boundary5269          unsigned ShiftedIdx = Offset + (Shift & 0xFF);5270          if (ShiftedIdx < 16) { // from VecB5271            ElemIdx = ShiftedIdx + (Lane * 16);5272          } else if (ShiftedIdx < 32) { // from VecA5273            VecIdx = 0;5274            ElemIdx = (ShiftedIdx - 16) + (Lane * 16);5275          }5276 5277          return std::pair<unsigned, int>{VecIdx, ElemIdx};5278        });5279 5280  case X86::BI__builtin_ia32_alignd128:5281  case X86::BI__builtin_ia32_alignd256:5282  case X86::BI__builtin_ia32_alignd512:5283  case X86::BI__builtin_ia32_alignq128:5284  case X86::BI__builtin_ia32_alignq256:5285  case X86::BI__builtin_ia32_alignq512: {5286    unsigned NumElems = Call->getType()->castAs<VectorType>()->getNumElements();5287    return interp__builtin_ia32_shuffle_generic(5288        S, OpPC, Call, [NumElems](unsigned DstIdx, unsigned Shift) {5289          unsigned Imm = Shift & 0xFF;5290          unsigned EffectiveShift = Imm & (NumElems - 1);5291          unsigned SourcePos = DstIdx + EffectiveShift;5292          unsigned VecIdx = SourcePos < NumElems ? 1u : 0u;5293          unsigned ElemIdx = SourcePos & (NumElems - 1);5294          return std::pair<unsigned, int>{VecIdx, static_cast<int>(ElemIdx)};5295        });5296  }5297 5298  default:5299    S.FFDiag(S.Current->getLocation(OpPC),5300             diag::note_invalid_subexpr_in_const_expr)5301        << S.Current->getRange(OpPC);5302 5303    return false;5304  }5305 5306  llvm_unreachable("Unhandled builtin ID");5307}5308 5309bool InterpretOffsetOf(InterpState &S, CodePtr OpPC, const OffsetOfExpr *E,5310                       ArrayRef<int64_t> ArrayIndices, int64_t &IntResult) {5311  CharUnits Result;5312  unsigned N = E->getNumComponents();5313  assert(N > 0);5314 5315  unsigned ArrayIndex = 0;5316  QualType CurrentType = E->getTypeSourceInfo()->getType();5317  for (unsigned I = 0; I != N; ++I) {5318    const OffsetOfNode &Node = E->getComponent(I);5319    switch (Node.getKind()) {5320    case OffsetOfNode::Field: {5321      const FieldDecl *MemberDecl = Node.getField();5322      const auto *RD = CurrentType->getAsRecordDecl();5323      if (!RD || RD->isInvalidDecl())5324        return false;5325      const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(RD);5326      unsigned FieldIndex = MemberDecl->getFieldIndex();5327      assert(FieldIndex < RL.getFieldCount() && "offsetof field in wrong type");5328      Result +=5329          S.getASTContext().toCharUnitsFromBits(RL.getFieldOffset(FieldIndex));5330      CurrentType = MemberDecl->getType().getNonReferenceType();5331      break;5332    }5333    case OffsetOfNode::Array: {5334      // When generating bytecode, we put all the index expressions as Sint64 on5335      // the stack.5336      int64_t Index = ArrayIndices[ArrayIndex];5337      const ArrayType *AT = S.getASTContext().getAsArrayType(CurrentType);5338      if (!AT)5339        return false;5340      CurrentType = AT->getElementType();5341      CharUnits ElementSize = S.getASTContext().getTypeSizeInChars(CurrentType);5342      Result += Index * ElementSize;5343      ++ArrayIndex;5344      break;5345    }5346    case OffsetOfNode::Base: {5347      const CXXBaseSpecifier *BaseSpec = Node.getBase();5348      if (BaseSpec->isVirtual())5349        return false;5350 5351      // Find the layout of the class whose base we are looking into.5352      const auto *RD = CurrentType->getAsCXXRecordDecl();5353      if (!RD || RD->isInvalidDecl())5354        return false;5355      const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(RD);5356 5357      // Find the base class itself.5358      CurrentType = BaseSpec->getType();5359      const auto *BaseRD = CurrentType->getAsCXXRecordDecl();5360      if (!BaseRD)5361        return false;5362 5363      // Add the offset to the base.5364      Result += RL.getBaseClassOffset(BaseRD);5365      break;5366    }5367    case OffsetOfNode::Identifier:5368      llvm_unreachable("Dependent OffsetOfExpr?");5369    }5370  }5371 5372  IntResult = Result.getQuantity();5373 5374  return true;5375}5376 5377bool SetThreeWayComparisonField(InterpState &S, CodePtr OpPC,5378                                const Pointer &Ptr, const APSInt &IntValue) {5379 5380  const Record *R = Ptr.getRecord();5381  assert(R);5382  assert(R->getNumFields() == 1);5383 5384  unsigned FieldOffset = R->getField(0u)->Offset;5385  const Pointer &FieldPtr = Ptr.atField(FieldOffset);5386  PrimType FieldT = *S.getContext().classify(FieldPtr.getType());5387 5388  INT_TYPE_SWITCH(FieldT,5389                  FieldPtr.deref<T>() = T::from(IntValue.getSExtValue()));5390  FieldPtr.initialize();5391  return true;5392}5393 5394static void zeroAll(Pointer &Dest) {5395  const Descriptor *Desc = Dest.getFieldDesc();5396 5397  if (Desc->isPrimitive()) {5398    TYPE_SWITCH(Desc->getPrimType(), {5399      Dest.deref<T>().~T();5400      new (&Dest.deref<T>()) T();5401    });5402    return;5403  }5404 5405  if (Desc->isRecord()) {5406    const Record *R = Desc->ElemRecord;5407    for (const Record::Field &F : R->fields()) {5408      Pointer FieldPtr = Dest.atField(F.Offset);5409      zeroAll(FieldPtr);5410    }5411    return;5412  }5413 5414  if (Desc->isPrimitiveArray()) {5415    for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {5416      TYPE_SWITCH(Desc->getPrimType(), {5417        Dest.deref<T>().~T();5418        new (&Dest.deref<T>()) T();5419      });5420    }5421    return;5422  }5423 5424  if (Desc->isCompositeArray()) {5425    for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {5426      Pointer ElemPtr = Dest.atIndex(I).narrow();5427      zeroAll(ElemPtr);5428    }5429    return;5430  }5431}5432 5433static bool copyComposite(InterpState &S, CodePtr OpPC, const Pointer &Src,5434                          Pointer &Dest, bool Activate);5435static bool copyRecord(InterpState &S, CodePtr OpPC, const Pointer &Src,5436                       Pointer &Dest, bool Activate = false) {5437  [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();5438  const Descriptor *DestDesc = Dest.getFieldDesc();5439 5440  auto copyField = [&](const Record::Field &F, bool Activate) -> bool {5441    Pointer DestField = Dest.atField(F.Offset);5442    if (OptPrimType FT = S.Ctx.classify(F.Decl->getType())) {5443      TYPE_SWITCH(*FT, {5444        DestField.deref<T>() = Src.atField(F.Offset).deref<T>();5445        if (Src.atField(F.Offset).isInitialized())5446          DestField.initialize();5447        if (Activate)5448          DestField.activate();5449      });5450      return true;5451    }5452    // Composite field.5453    return copyComposite(S, OpPC, Src.atField(F.Offset), DestField, Activate);5454  };5455 5456  assert(SrcDesc->isRecord());5457  assert(SrcDesc->ElemRecord == DestDesc->ElemRecord);5458  const Record *R = DestDesc->ElemRecord;5459  for (const Record::Field &F : R->fields()) {5460    if (R->isUnion()) {5461      // For unions, only copy the active field. Zero all others.5462      const Pointer &SrcField = Src.atField(F.Offset);5463      if (SrcField.isActive()) {5464        if (!copyField(F, /*Activate=*/true))5465          return false;5466      } else {5467        if (!CheckMutable(S, OpPC, Src.atField(F.Offset)))5468          return false;5469        Pointer DestField = Dest.atField(F.Offset);5470        zeroAll(DestField);5471      }5472    } else {5473      if (!copyField(F, Activate))5474        return false;5475    }5476  }5477 5478  for (const Record::Base &B : R->bases()) {5479    Pointer DestBase = Dest.atField(B.Offset);5480    if (!copyRecord(S, OpPC, Src.atField(B.Offset), DestBase, Activate))5481      return false;5482  }5483 5484  Dest.initialize();5485  return true;5486}5487 5488static bool copyComposite(InterpState &S, CodePtr OpPC, const Pointer &Src,5489                          Pointer &Dest, bool Activate = false) {5490  assert(Src.isLive() && Dest.isLive());5491 5492  [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();5493  const Descriptor *DestDesc = Dest.getFieldDesc();5494 5495  assert(!DestDesc->isPrimitive() && !SrcDesc->isPrimitive());5496 5497  if (DestDesc->isPrimitiveArray()) {5498    assert(SrcDesc->isPrimitiveArray());5499    assert(SrcDesc->getNumElems() == DestDesc->getNumElems());5500    PrimType ET = DestDesc->getPrimType();5501    for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {5502      Pointer DestElem = Dest.atIndex(I);5503      TYPE_SWITCH(ET, {5504        DestElem.deref<T>() = Src.elem<T>(I);5505        DestElem.initialize();5506      });5507    }5508    return true;5509  }5510 5511  if (DestDesc->isCompositeArray()) {5512    assert(SrcDesc->isCompositeArray());5513    assert(SrcDesc->getNumElems() == DestDesc->getNumElems());5514    for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {5515      const Pointer &SrcElem = Src.atIndex(I).narrow();5516      Pointer DestElem = Dest.atIndex(I).narrow();5517      if (!copyComposite(S, OpPC, SrcElem, DestElem, Activate))5518        return false;5519    }5520    return true;5521  }5522 5523  if (DestDesc->isRecord())5524    return copyRecord(S, OpPC, Src, Dest, Activate);5525  return Invalid(S, OpPC);5526}5527 5528bool DoMemcpy(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest) {5529  return copyComposite(S, OpPC, Src, Dest);5530}5531 5532} // namespace interp5533} // namespace clang5534