brintos / llvm-project-archived public Read only

1//===-- NumericalStabilitySanitizer.cpp -----------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains the instrumentation pass for the numerical sanitizer.10// Conceptually the pass injects shadow computations using higher precision11// types and inserts consistency checks. For details see the paper12// https://arxiv.org/abs/2102.12782.13//14//===----------------------------------------------------------------------===//15 16#include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h"17 18#include "llvm/ADT/DenseMap.h"19#include "llvm/ADT/SmallVector.h"20#include "llvm/ADT/Statistic.h"21#include "llvm/ADT/StringExtras.h"22#include "llvm/Analysis/TargetLibraryInfo.h"23#include "llvm/Analysis/ValueTracking.h"24#include "llvm/IR/DataLayout.h"25#include "llvm/IR/Function.h"26#include "llvm/IR/IRBuilder.h"27#include "llvm/IR/IntrinsicInst.h"28#include "llvm/IR/Intrinsics.h"29#include "llvm/IR/LLVMContext.h"30#include "llvm/IR/MDBuilder.h"31#include "llvm/IR/Metadata.h"32#include "llvm/IR/Module.h"33#include "llvm/IR/Type.h"34#include "llvm/Support/CommandLine.h"35#include "llvm/Support/Debug.h"36#include "llvm/Support/Regex.h"37#include "llvm/Support/raw_ostream.h"38#include "llvm/Transforms/Utils/BasicBlockUtils.h"39#include "llvm/Transforms/Utils/Instrumentation.h"40#include "llvm/Transforms/Utils/Local.h"41#include "llvm/Transforms/Utils/ModuleUtils.h"42 43#include <cstdint>44 45using namespace llvm;46 47#define DEBUG_TYPE "nsan"48 49STATISTIC(NumInstrumentedFTLoads,50          "Number of instrumented floating-point loads");51 52STATISTIC(NumInstrumentedFTCalls,53          "Number of instrumented floating-point calls");54STATISTIC(NumInstrumentedFTRets,55          "Number of instrumented floating-point returns");56STATISTIC(NumInstrumentedFTStores,57          "Number of instrumented floating-point stores");58STATISTIC(NumInstrumentedNonFTStores,59          "Number of instrumented non floating-point stores");60STATISTIC(61    NumInstrumentedNonFTMemcpyStores,62    "Number of instrumented non floating-point stores with memcpy semantics");63STATISTIC(NumInstrumentedFCmp, "Number of instrumented fcmps");64 65// Using smaller shadow types types can help improve speed. For example, `dlq`66// is 3x slower to 5x faster in opt mode and 2-6x faster in dbg mode compared to67// `dqq`.68static cl::opt<std::string> ClShadowMapping(69    "nsan-shadow-type-mapping", cl::init("dqq"),70    cl::desc("One shadow type id for each of `float`, `double`, `long double`. "71             "`d`,`l`,`q`,`e` mean double, x86_fp80, fp128 (quad) and "72             "ppc_fp128 (extended double) respectively. The default is to "73             "shadow `float` as `double`, and `double` and `x86_fp80` as "74             "`fp128`"),75    cl::Hidden);76 77static cl::opt<bool>78    ClInstrumentFCmp("nsan-instrument-fcmp", cl::init(true),79                     cl::desc("Instrument floating-point comparisons"),80                     cl::Hidden);81 82static cl::opt<std::string> ClCheckFunctionsFilter(83    "check-functions-filter",84    cl::desc("Only emit checks for arguments of functions "85             "whose names match the given regular expression"),86    cl::value_desc("regex"));87 88static cl::opt<bool> ClTruncateFCmpEq(89    "nsan-truncate-fcmp-eq", cl::init(true),90    cl::desc(91        "This flag controls the behaviour of fcmp equality comparisons."92        "For equality comparisons such as `x == 0.0f`, we can perform the "93        "shadow check in the shadow (`x_shadow == 0.0) == (x == 0.0f)`) or app "94        " domain (`(trunc(x_shadow) == 0.0f) == (x == 0.0f)`). This helps "95        "catch the case when `x_shadow` is accurate enough (and therefore "96        "close enough to zero) so that `trunc(x_shadow)` is zero even though "97        "both `x` and `x_shadow` are not"),98    cl::Hidden);99 100// When there is external, uninstrumented code writing to memory, the shadow101// memory can get out of sync with the application memory. Enabling this flag102// emits consistency checks for loads to catch this situation.103// When everything is instrumented, this is not strictly necessary because any104// load should have a corresponding store, but can help debug cases when the105// framework did a bad job at tracking shadow memory modifications by failing on106// load rather than store.107// TODO: provide a way to resume computations from the FT value when the load108// is inconsistent. This ensures that further computations are not polluted.109static cl::opt<bool> ClCheckLoads("nsan-check-loads",110                                  cl::desc("Check floating-point load"),111                                  cl::Hidden);112 113static cl::opt<bool> ClCheckStores("nsan-check-stores", cl::init(true),114                                   cl::desc("Check floating-point stores"),115                                   cl::Hidden);116 117static cl::opt<bool> ClCheckRet("nsan-check-ret", cl::init(true),118                                cl::desc("Check floating-point return values"),119                                cl::Hidden);120 121// LLVM may store constant floats as bitcasted ints.122// It's not really necessary to shadow such stores,123// if the shadow value is unknown the framework will re-extend it on load124// anyway. Moreover, because of size collisions (e.g. bf16 vs f16) it is125// impossible to determine the floating-point type based on the size.126// However, for debugging purposes it can be useful to model such stores.127static cl::opt<bool> ClPropagateNonFTConstStoresAsFT(128    "nsan-propagate-non-ft-const-stores-as-ft",129    cl::desc(130        "Propagate non floating-point const stores as floating point values."131        "For debugging purposes only"),132    cl::Hidden);133 134constexpr StringLiteral kNsanModuleCtorName("nsan.module_ctor");135constexpr StringLiteral kNsanInitName("__nsan_init");136 137// The following values must be kept in sync with the runtime.138constexpr int kShadowScale = 2;139constexpr int kMaxVectorWidth = 8;140constexpr int kMaxNumArgs = 128;141constexpr int kMaxShadowTypeSizeBytes = 16; // fp128142 143namespace {144 145// Defines the characteristics (type id, type, and floating-point semantics)146// attached for all possible shadow types.147class ShadowTypeConfig {148public:149  static std::unique_ptr<ShadowTypeConfig> fromNsanTypeId(char TypeId);150 151  // The LLVM Type corresponding to the shadow type.152  virtual Type *getType(LLVMContext &Context) const = 0;153 154  // The nsan type id of the shadow type (`d`, `l`, `q`, ...).155  virtual char getNsanTypeId() const = 0;156 157  virtual ~ShadowTypeConfig() = default;158};159 160template <char NsanTypeId>161class ShadowTypeConfigImpl : public ShadowTypeConfig {162public:163  char getNsanTypeId() const override { return NsanTypeId; }164  static constexpr char kNsanTypeId = NsanTypeId;165};166 167// `double` (`d`) shadow type.168class F64ShadowConfig : public ShadowTypeConfigImpl<'d'> {169  Type *getType(LLVMContext &Context) const override {170    return Type::getDoubleTy(Context);171  }172};173 174// `x86_fp80` (`l`) shadow type: X86 long double.175class F80ShadowConfig : public ShadowTypeConfigImpl<'l'> {176  Type *getType(LLVMContext &Context) const override {177    return Type::getX86_FP80Ty(Context);178  }179};180 181// `fp128` (`q`) shadow type.182class F128ShadowConfig : public ShadowTypeConfigImpl<'q'> {183  Type *getType(LLVMContext &Context) const override {184    return Type::getFP128Ty(Context);185  }186};187 188// `ppc_fp128` (`e`) shadow type: IBM extended double with 106 bits of mantissa.189class PPC128ShadowConfig : public ShadowTypeConfigImpl<'e'> {190  Type *getType(LLVMContext &Context) const override {191    return Type::getPPC_FP128Ty(Context);192  }193};194 195// Creates a ShadowTypeConfig given its type id.196std::unique_ptr<ShadowTypeConfig>197ShadowTypeConfig::fromNsanTypeId(const char TypeId) {198  switch (TypeId) {199  case F64ShadowConfig::kNsanTypeId:200    return std::make_unique<F64ShadowConfig>();201  case F80ShadowConfig::kNsanTypeId:202    return std::make_unique<F80ShadowConfig>();203  case F128ShadowConfig::kNsanTypeId:204    return std::make_unique<F128ShadowConfig>();205  case PPC128ShadowConfig::kNsanTypeId:206    return std::make_unique<PPC128ShadowConfig>();207  }208  report_fatal_error("nsan: invalid shadow type id '" + Twine(TypeId) + "'");209}210 211// An enum corresponding to shadow value types. Used as indices in arrays, so212// not an `enum class`.213enum FTValueType { kFloat, kDouble, kLongDouble, kNumValueTypes };214 215// If `FT` corresponds to a primitive FTValueType, return it.216static std::optional<FTValueType> ftValueTypeFromType(Type *FT) {217  if (FT->isFloatTy())218    return kFloat;219  if (FT->isDoubleTy())220    return kDouble;221  if (FT->isX86_FP80Ty())222    return kLongDouble;223  return {};224}225 226// Returns the LLVM type for an FTValueType.227static Type *typeFromFTValueType(FTValueType VT, LLVMContext &Context) {228  switch (VT) {229  case kFloat:230    return Type::getFloatTy(Context);231  case kDouble:232    return Type::getDoubleTy(Context);233  case kLongDouble:234    return Type::getX86_FP80Ty(Context);235  case kNumValueTypes:236    return nullptr;237  }238  llvm_unreachable("Unhandled FTValueType enum");239}240 241// Returns the type name for an FTValueType.242static const char *typeNameFromFTValueType(FTValueType VT) {243  switch (VT) {244  case kFloat:245    return "float";246  case kDouble:247    return "double";248  case kLongDouble:249    return "longdouble";250  case kNumValueTypes:251    return nullptr;252  }253  llvm_unreachable("Unhandled FTValueType enum");254}255 256// A specific mapping configuration of application type to shadow type for nsan257// (see -nsan-shadow-mapping flag).258class MappingConfig {259public:260  explicit MappingConfig(LLVMContext &C) : Context(C) {261    if (ClShadowMapping.size() != 3)262      report_fatal_error("Invalid nsan mapping: " + Twine(ClShadowMapping));263    unsigned ShadowTypeSizeBits[kNumValueTypes];264    for (int VT = 0; VT < kNumValueTypes; ++VT) {265      auto Config = ShadowTypeConfig::fromNsanTypeId(ClShadowMapping[VT]);266      if (!Config)267        report_fatal_error("Failed to get ShadowTypeConfig for " +268                           Twine(ClShadowMapping[VT]));269      const unsigned AppTypeSize =270          typeFromFTValueType(static_cast<FTValueType>(VT), Context)271              ->getScalarSizeInBits();272      const unsigned ShadowTypeSize =273          Config->getType(Context)->getScalarSizeInBits();274      // Check that the shadow type size is at most kShadowScale times the275      // application type size, so that shadow memory compoutations are valid.276      if (ShadowTypeSize > kShadowScale * AppTypeSize)277        report_fatal_error("Invalid nsan mapping f" + Twine(AppTypeSize) +278                           "->f" + Twine(ShadowTypeSize) +279                           ": The shadow type size should be at most " +280                           Twine(kShadowScale) +281                           " times the application type size");282      ShadowTypeSizeBits[VT] = ShadowTypeSize;283      Configs[VT] = std::move(Config);284    }285 286    // Check that the mapping is monotonous. This is required because if one287    // does an fpextend of `float->long double` in application code, nsan is288    // going to do an fpextend of `shadow(float) -> shadow(long double)` in289    // shadow code. This will fail in `qql` mode, since nsan would be290    // fpextending `f128->long`, which is invalid.291    // TODO: Relax this.292    if (ShadowTypeSizeBits[kFloat] > ShadowTypeSizeBits[kDouble] ||293        ShadowTypeSizeBits[kDouble] > ShadowTypeSizeBits[kLongDouble])294      report_fatal_error("Invalid nsan mapping: { float->f" +295                         Twine(ShadowTypeSizeBits[kFloat]) + "; double->f" +296                         Twine(ShadowTypeSizeBits[kDouble]) +297                         "; long double->f" +298                         Twine(ShadowTypeSizeBits[kLongDouble]) + " }");299  }300 301  const ShadowTypeConfig &byValueType(FTValueType VT) const {302    assert(VT < FTValueType::kNumValueTypes && "invalid value type");303    return *Configs[VT];304  }305 306  // Returns the extended shadow type for a given application type.307  Type *getExtendedFPType(Type *FT) const {308    if (const auto VT = ftValueTypeFromType(FT))309      return Configs[*VT]->getType(Context);310    if (FT->isVectorTy()) {311      auto *VecTy = cast<VectorType>(FT);312      // TODO: add support for scalable vector types.313      if (VecTy->isScalableTy())314        return nullptr;315      Type *ExtendedScalar = getExtendedFPType(VecTy->getElementType());316      return ExtendedScalar317                 ? VectorType::get(ExtendedScalar, VecTy->getElementCount())318                 : nullptr;319    }320    return nullptr;321  }322 323private:324  LLVMContext &Context;325  std::unique_ptr<ShadowTypeConfig> Configs[FTValueType::kNumValueTypes];326};327 328// The memory extents of a type specifies how many elements of a given329// FTValueType needs to be stored when storing this type.330struct MemoryExtents {331  FTValueType ValueType;332  uint64_t NumElts;333};334 335static MemoryExtents getMemoryExtentsOrDie(Type *FT) {336  if (const auto VT = ftValueTypeFromType(FT))337    return {*VT, 1};338  if (auto *VecTy = dyn_cast<VectorType>(FT)) {339    const auto ScalarExtents = getMemoryExtentsOrDie(VecTy->getElementType());340    return {ScalarExtents.ValueType,341            ScalarExtents.NumElts * VecTy->getElementCount().getFixedValue()};342  }343  llvm_unreachable("invalid value type");344}345 346// The location of a check. Passed as parameters to runtime checking functions.347class CheckLoc {348public:349  // Creates a location that references an application memory location.350  static CheckLoc makeStore(Value *Address) {351    CheckLoc Result(kStore);352    Result.Address = Address;353    return Result;354  }355  static CheckLoc makeLoad(Value *Address) {356    CheckLoc Result(kLoad);357    Result.Address = Address;358    return Result;359  }360 361  // Creates a location that references an argument, given by id.362  static CheckLoc makeArg(int ArgId) {363    CheckLoc Result(kArg);364    Result.ArgId = ArgId;365    return Result;366  }367 368  // Creates a location that references the return value of a function.369  static CheckLoc makeRet() { return CheckLoc(kRet); }370 371  // Creates a location that references a vector insert.372  static CheckLoc makeInsert() { return CheckLoc(kInsert); }373 374  // Returns the CheckType of location this refers to, as an integer-typed LLVM375  // IR value.376  Value *getType(LLVMContext &C) const {377    return ConstantInt::get(Type::getInt32Ty(C), static_cast<int>(CheckTy));378  }379 380  // Returns a CheckType-specific value representing details of the location381  // (e.g. application address for loads or stores), as an `IntptrTy`-typed LLVM382  // IR value.383  Value *getValue(Type *IntptrTy, IRBuilder<> &Builder) const {384    switch (CheckTy) {385    case kUnknown:386      llvm_unreachable("unknown type");387    case kRet:388    case kInsert:389      return ConstantInt::get(IntptrTy, 0);390    case kArg:391      return ConstantInt::get(IntptrTy, ArgId);392    case kLoad:393    case kStore:394      return Builder.CreatePtrToInt(Address, IntptrTy);395    }396    llvm_unreachable("Unhandled CheckType enum");397  }398 399private:400  // Must be kept in sync with the runtime,401  // see compiler-rt/lib/nsan/nsan_stats.h402  enum CheckType {403    kUnknown = 0,404    kRet,405    kArg,406    kLoad,407    kStore,408    kInsert,409  };410  explicit CheckLoc(CheckType CheckTy) : CheckTy(CheckTy) {}411 412  Value *Address = nullptr;413  const CheckType CheckTy;414  int ArgId = -1;415};416 417// A map of LLVM IR values to shadow LLVM IR values.418class ValueToShadowMap {419public:420  explicit ValueToShadowMap(const MappingConfig &Config) : Config(Config) {}421 422  ValueToShadowMap(const ValueToShadowMap &) = delete;423  ValueToShadowMap &operator=(const ValueToShadowMap &) = delete;424 425  // Sets the shadow value for a value. Asserts that the value does not already426  // have a value.427  void setShadow(Value &V, Value &Shadow) {428    [[maybe_unused]] const bool Inserted = Map.try_emplace(&V, &Shadow).second;429    LLVM_DEBUG({430      if (!Inserted) {431        if (auto *I = dyn_cast<Instruction>(&V))432          errs() << I->getFunction()->getName() << ": ";433        errs() << "duplicate shadow (" << &V << "): ";434        V.dump();435      }436    });437    assert(Inserted && "duplicate shadow");438  }439 440  // Returns true if the value already has a shadow (including if the value is a441  // constant). If true, calling getShadow() is valid.442  bool hasShadow(Value *V) const { return isa<Constant>(V) || Map.contains(V); }443 444  // Returns the shadow value for a given value. Asserts that the value has445  // a shadow value. Lazily creates shadows for constant values.446  Value *getShadow(Value *V) const {447    if (Constant *C = dyn_cast<Constant>(V))448      return getShadowConstant(C);449    return Map.find(V)->second;450  }451 452  bool empty() const { return Map.empty(); }453 454private:455  // Extends a constant application value to its shadow counterpart.456  APFloat extendConstantFP(APFloat CV, const fltSemantics &To) const {457    bool LosesInfo = false;458    CV.convert(To, APFloatBase::rmTowardZero, &LosesInfo);459    return CV;460  }461 462  // Returns the shadow constant for the given application constant.463  Constant *getShadowConstant(Constant *C) const {464    if (UndefValue *U = dyn_cast<UndefValue>(C)) {465      return UndefValue::get(Config.getExtendedFPType(U->getType()));466    }467    if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) {468      // Floating-point constants.469      Type *Ty = Config.getExtendedFPType(CFP->getType());470      return ConstantFP::get(471          Ty, extendConstantFP(CFP->getValueAPF(),472                               Ty->getScalarType()->getFltSemantics()));473    }474    // Vector, array, or aggregate constants.475    if (C->getType()->isVectorTy()) {476      SmallVector<Constant *, 8> Elements;477      for (int I = 0, E = cast<VectorType>(C->getType())478                              ->getElementCount()479                              .getFixedValue();480           I < E; ++I)481        Elements.push_back(getShadowConstant(C->getAggregateElement(I)));482      return ConstantVector::get(Elements);483    }484    llvm_unreachable("unimplemented");485  }486 487  const MappingConfig &Config;488  DenseMap<Value *, Value *> Map;489};490 491class NsanMemOpFn {492public:493  NsanMemOpFn(Module &M, ArrayRef<StringRef> Sized, StringRef Fallback,494              size_t NumArgs);495  FunctionCallee getFunctionFor(uint64_t MemOpSize) const;496  FunctionCallee getFallback() const;497 498private:499  SmallVector<FunctionCallee> Funcs;500  size_t NumSizedFuncs;501};502 503NsanMemOpFn::NsanMemOpFn(Module &M, ArrayRef<StringRef> Sized,504                         StringRef Fallback, size_t NumArgs) {505  LLVMContext &Ctx = M.getContext();506  AttributeList Attr;507  Attr = Attr.addFnAttribute(Ctx, Attribute::NoUnwind);508  Type *PtrTy = PointerType::getUnqual(Ctx);509  Type *VoidTy = Type::getVoidTy(Ctx);510  IntegerType *IntptrTy = M.getDataLayout().getIntPtrType(Ctx);511  FunctionType *SizedFnTy = nullptr;512 513  NumSizedFuncs = Sized.size();514 515  // First entry is fallback function516  if (NumArgs == 3) {517    Funcs.push_back(518        M.getOrInsertFunction(Fallback, Attr, VoidTy, PtrTy, PtrTy, IntptrTy));519    SizedFnTy = FunctionType::get(VoidTy, {PtrTy, PtrTy}, false);520  } else if (NumArgs == 2) {521    Funcs.push_back(522        M.getOrInsertFunction(Fallback, Attr, VoidTy, PtrTy, IntptrTy));523    SizedFnTy = FunctionType::get(VoidTy, {PtrTy}, false);524  } else {525    llvm_unreachable("Unexpected value of sized functions arguments");526  }527 528  for (size_t i = 0; i < NumSizedFuncs; ++i)529    Funcs.push_back(M.getOrInsertFunction(Sized[i], SizedFnTy, Attr));530}531 532FunctionCallee NsanMemOpFn::getFunctionFor(uint64_t MemOpSize) const {533  // Now `getFunctionFor` operates on `Funcs` of size 4 (at least) and the534  // following code assumes that the number of functions in `Func` is sufficient535  assert(NumSizedFuncs >= 3 && "Unexpected number of sized functions");536 537  size_t Idx =538      MemOpSize == 4 ? 1 : (MemOpSize == 8 ? 2 : (MemOpSize == 16 ? 3 : 0));539 540  return Funcs[Idx];541}542 543FunctionCallee NsanMemOpFn::getFallback() const { return Funcs[0]; }544 545/// Instantiating NumericalStabilitySanitizer inserts the nsan runtime library546/// API function declarations into the module if they don't exist already.547/// Instantiating ensures the __nsan_init function is in the list of global548/// constructors for the module.549class NumericalStabilitySanitizer {550public:551  NumericalStabilitySanitizer(Module &M);552  bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI);553 554private:555  bool instrumentMemIntrinsic(MemIntrinsic *MI);556  void maybeAddSuffixForNsanInterface(CallBase *CI);557  bool addrPointsToConstantData(Value *Addr);558  void maybeCreateShadowValue(Instruction &Root, const TargetLibraryInfo &TLI,559                              ValueToShadowMap &Map);560  Value *createShadowValueWithOperandsAvailable(Instruction &Inst,561                                                const TargetLibraryInfo &TLI,562                                                const ValueToShadowMap &Map);563  PHINode *maybeCreateShadowPhi(PHINode &Phi, const TargetLibraryInfo &TLI);564  void createShadowArguments(Function &F, const TargetLibraryInfo &TLI,565                             ValueToShadowMap &Map);566 567  void populateShadowStack(CallBase &CI, const TargetLibraryInfo &TLI,568                           const ValueToShadowMap &Map);569 570  void propagateShadowValues(Instruction &Inst, const TargetLibraryInfo &TLI,571                             const ValueToShadowMap &Map);572  Value *emitCheck(Value *V, Value *ShadowV, IRBuilder<> &Builder,573                   CheckLoc Loc);574  Value *emitCheckInternal(Value *V, Value *ShadowV, IRBuilder<> &Builder,575                           CheckLoc Loc);576  void emitFCmpCheck(FCmpInst &FCmp, const ValueToShadowMap &Map);577 578  // Value creation handlers.579  Value *handleLoad(LoadInst &Load, Type *VT, Type *ExtendedVT);580  Value *handleCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,581                        const TargetLibraryInfo &TLI,582                        const ValueToShadowMap &Map, IRBuilder<> &Builder);583  Value *maybeHandleKnownCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,584                                  const TargetLibraryInfo &TLI,585                                  const ValueToShadowMap &Map,586                                  IRBuilder<> &Builder);587  Value *handleTrunc(const FPTruncInst &Trunc, Type *VT, Type *ExtendedVT,588                     const ValueToShadowMap &Map, IRBuilder<> &Builder);589  Value *handleExt(const FPExtInst &Ext, Type *VT, Type *ExtendedVT,590                   const ValueToShadowMap &Map, IRBuilder<> &Builder);591 592  // Value propagation handlers.593  void propagateFTStore(StoreInst &Store, Type *VT, Type *ExtendedVT,594                        const ValueToShadowMap &Map);595  void propagateNonFTStore(StoreInst &Store, Type *VT,596                           const ValueToShadowMap &Map);597 598  const DataLayout &DL;599  LLVMContext &Context;600  MappingConfig Config;601  IntegerType *IntptrTy = nullptr;602 603  // TODO: Use std::array instead?604  FunctionCallee NsanGetShadowPtrForStore[FTValueType::kNumValueTypes] = {};605  FunctionCallee NsanGetShadowPtrForLoad[FTValueType::kNumValueTypes] = {};606  FunctionCallee NsanCheckValue[FTValueType::kNumValueTypes] = {};607  FunctionCallee NsanFCmpFail[FTValueType::kNumValueTypes] = {};608 609  NsanMemOpFn NsanCopyFns;610  NsanMemOpFn NsanSetUnknownFns;611 612  FunctionCallee NsanGetRawShadowTypePtr;613  FunctionCallee NsanGetRawShadowPtr;614  GlobalValue *NsanShadowRetTag = nullptr;615 616  Type *NsanShadowRetType = nullptr;617  GlobalValue *NsanShadowRetPtr = nullptr;618 619  GlobalValue *NsanShadowArgsTag = nullptr;620 621  Type *NsanShadowArgsType = nullptr;622  GlobalValue *NsanShadowArgsPtr = nullptr;623 624  std::optional<Regex> CheckFunctionsFilter;625};626} // end anonymous namespace627 628PreservedAnalyses629NumericalStabilitySanitizerPass::run(Module &M, ModuleAnalysisManager &MAM) {630  getOrCreateSanitizerCtorAndInitFunctions(631      M, kNsanModuleCtorName, kNsanInitName, /*InitArgTypes=*/{},632      /*InitArgs=*/{},633      // This callback is invoked when the functions are created the first634      // time. Hook them into the global ctors list in that case:635      [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, Ctor, 0); });636 637  NumericalStabilitySanitizer Nsan(M);638  auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();639  for (Function &F : M)640    Nsan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F));641 642  return PreservedAnalyses::none();643}644 645static GlobalValue *createThreadLocalGV(const char *Name, Module &M, Type *Ty) {646  return M.getOrInsertGlobal(Name, Ty, [&M, Ty, Name] {647    return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,648                              nullptr, Name, nullptr,649                              GlobalVariable::InitialExecTLSModel);650  });651}652 653NumericalStabilitySanitizer::NumericalStabilitySanitizer(Module &M)654    : DL(M.getDataLayout()), Context(M.getContext()), Config(Context),655      NsanCopyFns(M, {"__nsan_copy_4", "__nsan_copy_8", "__nsan_copy_16"},656                  "__nsan_copy_values", /*NumArgs=*/3),657      NsanSetUnknownFns(M,658                        {"__nsan_set_value_unknown_4",659                         "__nsan_set_value_unknown_8",660                         "__nsan_set_value_unknown_16"},661                        "__nsan_set_value_unknown", /*NumArgs=*/2) {662  IntptrTy = DL.getIntPtrType(Context);663  Type *PtrTy = PointerType::getUnqual(Context);664  Type *Int32Ty = Type::getInt32Ty(Context);665  Type *Int1Ty = Type::getInt1Ty(Context);666  Type *VoidTy = Type::getVoidTy(Context);667 668  AttributeList Attr;669  Attr = Attr.addFnAttribute(Context, Attribute::NoUnwind);670  // Initialize the runtime values (functions and global variables).671  for (int I = 0; I < kNumValueTypes; ++I) {672    const FTValueType VT = static_cast<FTValueType>(I);673    const char *VTName = typeNameFromFTValueType(VT);674    Type *VTTy = typeFromFTValueType(VT, Context);675 676    // Load/store.677    const std::string GetterPrefix =678        std::string("__nsan_get_shadow_ptr_for_") + VTName;679    NsanGetShadowPtrForStore[VT] = M.getOrInsertFunction(680        GetterPrefix + "_store", Attr, PtrTy, PtrTy, IntptrTy);681    NsanGetShadowPtrForLoad[VT] = M.getOrInsertFunction(682        GetterPrefix + "_load", Attr, PtrTy, PtrTy, IntptrTy);683 684    // Check.685    const auto &ShadowConfig = Config.byValueType(VT);686    Type *ShadowTy = ShadowConfig.getType(Context);687    NsanCheckValue[VT] =688        M.getOrInsertFunction(std::string("__nsan_internal_check_") + VTName +689                                  "_" + ShadowConfig.getNsanTypeId(),690                              Attr, Int32Ty, VTTy, ShadowTy, Int32Ty, IntptrTy);691    NsanFCmpFail[VT] = M.getOrInsertFunction(692        std::string("__nsan_fcmp_fail_") + VTName + "_" +693            ShadowConfig.getNsanTypeId(),694        Attr, VoidTy, VTTy, VTTy, ShadowTy, ShadowTy, Int32Ty, Int1Ty, Int1Ty);695  }696 697  // TODO: Add attributes nofree, nosync, readnone, readonly,698  NsanGetRawShadowTypePtr = M.getOrInsertFunction(699      "__nsan_internal_get_raw_shadow_type_ptr", Attr, PtrTy, PtrTy);700  NsanGetRawShadowPtr = M.getOrInsertFunction(701      "__nsan_internal_get_raw_shadow_ptr", Attr, PtrTy, PtrTy);702 703  NsanShadowRetTag = createThreadLocalGV("__nsan_shadow_ret_tag", M, IntptrTy);704 705  NsanShadowRetType = ArrayType::get(Type::getInt8Ty(Context),706                                     kMaxVectorWidth * kMaxShadowTypeSizeBytes);707  NsanShadowRetPtr =708      createThreadLocalGV("__nsan_shadow_ret_ptr", M, NsanShadowRetType);709 710  NsanShadowArgsTag =711      createThreadLocalGV("__nsan_shadow_args_tag", M, IntptrTy);712 713  NsanShadowArgsType =714      ArrayType::get(Type::getInt8Ty(Context),715                     kMaxVectorWidth * kMaxNumArgs * kMaxShadowTypeSizeBytes);716 717  NsanShadowArgsPtr =718      createThreadLocalGV("__nsan_shadow_args_ptr", M, NsanShadowArgsType);719 720  if (!ClCheckFunctionsFilter.empty()) {721    Regex R = Regex(ClCheckFunctionsFilter);722    std::string RegexError;723    assert(R.isValid(RegexError));724    CheckFunctionsFilter = std::move(R);725  }726}727 728// Returns true if the given LLVM Value points to constant data (typically, a729// global variable reference).730bool NumericalStabilitySanitizer::addrPointsToConstantData(Value *Addr) {731  // If this is a GEP, just analyze its pointer operand.732  if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr))733    Addr = GEP->getPointerOperand();734 735  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr))736    return GV->isConstant();737  return false;738}739 740// This instruments the function entry to create shadow arguments.741// Pseudocode:742//   if (this_fn_ptr == __nsan_shadow_args_tag) {743//     s(arg0) = LOAD<sizeof(arg0)>(__nsan_shadow_args);744//     s(arg1) = LOAD<sizeof(arg1)>(__nsan_shadow_args + sizeof(arg0));745//     ...746//     __nsan_shadow_args_tag = 0;747//   } else {748//     s(arg0) = fext(arg0);749//     s(arg1) = fext(arg1);750//     ...751//   }752void NumericalStabilitySanitizer::createShadowArguments(753    Function &F, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) {754  assert(!F.getIntrinsicID() && "found a definition of an intrinsic");755 756  // Do not bother if there are no FP args.757  if (all_of(F.args(), [this](const Argument &Arg) {758        return Config.getExtendedFPType(Arg.getType()) == nullptr;759      }))760    return;761 762  IRBuilder<> Builder(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHIIt());763  // The function has shadow args if the shadow args tag matches the function764  // address.765  Value *HasShadowArgs = Builder.CreateICmpEQ(766      Builder.CreateLoad(IntptrTy, NsanShadowArgsTag, /*isVolatile=*/false),767      Builder.CreatePtrToInt(&F, IntptrTy));768 769  unsigned ShadowArgsOffsetBytes = 0;770  for (Argument &Arg : F.args()) {771    Type *VT = Arg.getType();772    Type *ExtendedVT = Config.getExtendedFPType(VT);773    if (ExtendedVT == nullptr)774      continue; // Not an FT value.775    Value *L = Builder.CreateAlignedLoad(776        ExtendedVT,777        Builder.CreateConstGEP2_64(NsanShadowArgsType, NsanShadowArgsPtr, 0,778                                   ShadowArgsOffsetBytes),779        Align(1), /*isVolatile=*/false);780    Value *Shadow = Builder.CreateSelect(HasShadowArgs, L,781                                         Builder.CreateFPExt(&Arg, ExtendedVT));782    Map.setShadow(Arg, *Shadow);783    TypeSize SlotSize = DL.getTypeStoreSize(ExtendedVT);784    assert(!SlotSize.isScalable() && "unsupported");785    ShadowArgsOffsetBytes += SlotSize;786  }787  Builder.CreateStore(ConstantInt::get(IntptrTy, 0), NsanShadowArgsTag);788}789 790// Returns true if the instrumentation should emit code to check arguments791// before a function call.792static bool shouldCheckArgs(CallBase &CI, const TargetLibraryInfo &TLI,793                            const std::optional<Regex> &CheckFunctionsFilter) {794 795  Function *Fn = CI.getCalledFunction();796 797  if (CheckFunctionsFilter) {798    // Skip checking args of indirect calls.799    if (Fn == nullptr)800      return false;801    if (CheckFunctionsFilter->match(Fn->getName()))802      return true;803    return false;804  }805 806  if (Fn == nullptr)807    return true; // Always check args of indirect calls.808 809  // Never check nsan functions, the user called them for a reason.810  if (Fn->getName().starts_with("__nsan_"))811    return false;812 813  const auto ID = Fn->getIntrinsicID();814  LibFunc LFunc = LibFunc::NotLibFunc;815  // Always check args of unknown functions.816  if (ID == Intrinsic::ID() && !TLI.getLibFunc(*Fn, LFunc))817    return true;818 819  // Do not check args of an `fabs` call that is used for a comparison.820  // This is typically used for `fabs(a-b) < tolerance`, where what matters is821  // the result of the comparison, which is already caught be the fcmp checks.822  if (ID == Intrinsic::fabs || LFunc == LibFunc_fabsf ||823      LFunc == LibFunc_fabs || LFunc == LibFunc_fabsl)824    for (const auto &U : CI.users())825      if (isa<CmpInst>(U))826        return false;827 828  return true; // Default is check.829}830 831// Populates the shadow call stack (which contains shadow values for every832// floating-point parameter to the function).833void NumericalStabilitySanitizer::populateShadowStack(834    CallBase &CI, const TargetLibraryInfo &TLI, const ValueToShadowMap &Map) {835  // Do not create a shadow stack for inline asm.836  if (CI.isInlineAsm())837    return;838 839  // Do not bother if there are no FP args.840  if (all_of(CI.operands(), [this](const Value *Arg) {841        return Config.getExtendedFPType(Arg->getType()) == nullptr;842      }))843    return;844 845  IRBuilder<> Builder(&CI);846  SmallVector<Value *, 8> ArgShadows;847  const bool ShouldCheckArgs = shouldCheckArgs(CI, TLI, CheckFunctionsFilter);848  for (auto [ArgIdx, Arg] : enumerate(CI.operands())) {849    if (Config.getExtendedFPType(Arg->getType()) == nullptr)850      continue; // Not an FT value.851    Value *ArgShadow = Map.getShadow(Arg);852    ArgShadows.push_back(ShouldCheckArgs ? emitCheck(Arg, ArgShadow, Builder,853                                                     CheckLoc::makeArg(ArgIdx))854                                         : ArgShadow);855  }856 857  // Do not create shadow stacks for intrinsics/known lib funcs.858  if (Function *Fn = CI.getCalledFunction()) {859    LibFunc LFunc;860    if (Fn->isIntrinsic() || TLI.getLibFunc(*Fn, LFunc))861      return;862  }863 864  // Set the shadow stack tag.865  Builder.CreateStore(CI.getCalledOperand(), NsanShadowArgsTag);866  TypeSize ShadowArgsOffsetBytes = TypeSize::getFixed(0);867 868  unsigned ShadowArgId = 0;869  for (const Value *Arg : CI.operands()) {870    Type *VT = Arg->getType();871    Type *ExtendedVT = Config.getExtendedFPType(VT);872    if (ExtendedVT == nullptr)873      continue; // Not an FT value.874    Builder.CreateAlignedStore(875        ArgShadows[ShadowArgId++],876        Builder.CreateConstGEP2_64(NsanShadowArgsType, NsanShadowArgsPtr, 0,877                                   ShadowArgsOffsetBytes),878        Align(1), /*isVolatile=*/false);879    TypeSize SlotSize = DL.getTypeStoreSize(ExtendedVT);880    assert(!SlotSize.isScalable() && "unsupported");881    ShadowArgsOffsetBytes += SlotSize;882  }883}884 885// Internal part of emitCheck(). Returns a value that indicates whether886// computation should continue with the shadow or resume by re-fextending the887// value.888enum class ContinuationType { // Keep in sync with runtime.889  ContinueWithShadow = 0,890  ResumeFromValue = 1,891};892 893Value *NumericalStabilitySanitizer::emitCheckInternal(Value *V, Value *ShadowV,894                                                      IRBuilder<> &Builder,895                                                      CheckLoc Loc) {896  // Do not emit checks for constant values, this is redundant.897  if (isa<Constant>(V))898    return ConstantInt::get(899        Builder.getInt32Ty(),900        static_cast<int>(ContinuationType::ContinueWithShadow));901 902  Type *Ty = V->getType();903  if (const auto VT = ftValueTypeFromType(Ty))904    return Builder.CreateCall(905        NsanCheckValue[*VT],906        {V, ShadowV, Loc.getType(Context), Loc.getValue(IntptrTy, Builder)});907 908  if (Ty->isVectorTy()) {909    auto *VecTy = cast<VectorType>(Ty);910    // We currently skip scalable vector types in MappingConfig,911    // thus we should not encounter any such types here.912    assert(!VecTy->isScalableTy() &&913           "Scalable vector types are not supported yet");914    Value *CheckResult = nullptr;915    for (int I = 0, E = VecTy->getElementCount().getFixedValue(); I < E; ++I) {916      // We resume if any element resumes. Another option would be to create a917      // vector shuffle with the array of ContinueWithShadow, but that is too918      // complex.919      Value *ExtractV = Builder.CreateExtractElement(V, I);920      Value *ExtractShadowV = Builder.CreateExtractElement(ShadowV, I);921      Value *ComponentCheckResult =922          emitCheckInternal(ExtractV, ExtractShadowV, Builder, Loc);923      CheckResult = CheckResult924                        ? Builder.CreateOr(CheckResult, ComponentCheckResult)925                        : ComponentCheckResult;926    }927    return CheckResult;928  }929  if (Ty->isArrayTy()) {930    Value *CheckResult = nullptr;931    for (auto I : seq(Ty->getArrayNumElements())) {932      Value *ExtractV = Builder.CreateExtractElement(V, I);933      Value *ExtractShadowV = Builder.CreateExtractElement(ShadowV, I);934      Value *ComponentCheckResult =935          emitCheckInternal(ExtractV, ExtractShadowV, Builder, Loc);936      CheckResult = CheckResult937                        ? Builder.CreateOr(CheckResult, ComponentCheckResult)938                        : ComponentCheckResult;939    }940    return CheckResult;941  }942  if (Ty->isStructTy()) {943    Value *CheckResult = nullptr;944    for (auto I : seq(Ty->getStructNumElements())) {945      if (Config.getExtendedFPType(Ty->getStructElementType(I)) == nullptr)946        continue; // Only check FT values.947      Value *ExtractV = Builder.CreateExtractValue(V, I);948      Value *ExtractShadowV = Builder.CreateExtractElement(ShadowV, I);949      Value *ComponentCheckResult =950          emitCheckInternal(ExtractV, ExtractShadowV, Builder, Loc);951      CheckResult = CheckResult952                        ? Builder.CreateOr(CheckResult, ComponentCheckResult)953                        : ComponentCheckResult;954    }955    if (!CheckResult)956      return ConstantInt::get(957          Builder.getInt32Ty(),958          static_cast<int>(ContinuationType::ContinueWithShadow));959    return CheckResult;960  }961 962  llvm_unreachable("not implemented");963}964 965// Inserts a runtime check of V against its shadow value ShadowV.966// We check values whenever they escape: on return, call, stores, and967// insertvalue.968// Returns the shadow value that should be used to continue the computations,969// depending on the answer from the runtime.970// TODO: Should we check on select ? phi ?971Value *NumericalStabilitySanitizer::emitCheck(Value *V, Value *ShadowV,972                                              IRBuilder<> &Builder,973                                              CheckLoc Loc) {974  // Do not emit checks for constant values, this is redundant.975  if (isa<Constant>(V))976    return ShadowV;977 978  if (Instruction *Inst = dyn_cast<Instruction>(V)) {979    Function *F = Inst->getFunction();980    if (CheckFunctionsFilter && !CheckFunctionsFilter->match(F->getName())) {981      return ShadowV;982    }983  }984 985  Value *CheckResult = emitCheckInternal(V, ShadowV, Builder, Loc);986  Value *ICmpEQ = Builder.CreateICmpEQ(987      CheckResult,988      ConstantInt::get(Builder.getInt32Ty(),989                       static_cast<int>(ContinuationType::ResumeFromValue)));990  return Builder.CreateSelect(991      ICmpEQ, Builder.CreateFPExt(V, Config.getExtendedFPType(V->getType())),992      ShadowV);993}994 995// Inserts a check that fcmp on shadow values are consistent with that on base996// values.997void NumericalStabilitySanitizer::emitFCmpCheck(FCmpInst &FCmp,998                                                const ValueToShadowMap &Map) {999  if (!ClInstrumentFCmp)1000    return;1001 1002  Function *F = FCmp.getFunction();1003  if (CheckFunctionsFilter && !CheckFunctionsFilter->match(F->getName()))1004    return;1005 1006  Value *LHS = FCmp.getOperand(0);1007  if (Config.getExtendedFPType(LHS->getType()) == nullptr)1008    return;1009  Value *RHS = FCmp.getOperand(1);1010 1011  // Split the basic block. On mismatch, we'll jump to the new basic block with1012  // a call to the runtime for error reporting.1013  BasicBlock *FCmpBB = FCmp.getParent();1014  BasicBlock *NextBB = FCmpBB->splitBasicBlock(FCmp.getNextNode());1015  // Remove the newly created terminator unconditional branch.1016  FCmpBB->back().eraseFromParent();1017  BasicBlock *FailBB =1018      BasicBlock::Create(Context, "", FCmpBB->getParent(), NextBB);1019 1020  // Create the shadow fcmp and comparison between the fcmps.1021  IRBuilder<> FCmpBuilder(FCmpBB);1022  FCmpBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());1023  Value *ShadowLHS = Map.getShadow(LHS);1024  Value *ShadowRHS = Map.getShadow(RHS);1025  // See comment on ClTruncateFCmpEq.1026  if (FCmp.isEquality() && ClTruncateFCmpEq) {1027    Type *Ty = ShadowLHS->getType();1028    ShadowLHS = FCmpBuilder.CreateFPExt(1029        FCmpBuilder.CreateFPTrunc(ShadowLHS, LHS->getType()), Ty);1030    ShadowRHS = FCmpBuilder.CreateFPExt(1031        FCmpBuilder.CreateFPTrunc(ShadowRHS, RHS->getType()), Ty);1032  }1033  Value *ShadowFCmp =1034      FCmpBuilder.CreateFCmp(FCmp.getPredicate(), ShadowLHS, ShadowRHS);1035  Value *OriginalAndShadowFcmpMatch =1036      FCmpBuilder.CreateICmpEQ(&FCmp, ShadowFCmp);1037 1038  if (OriginalAndShadowFcmpMatch->getType()->isVectorTy()) {1039    // If we have a vector type, `OriginalAndShadowFcmpMatch` is a vector of i1,1040    // where an element is true if the corresponding elements in original and1041    // shadow are the same. We want all elements to be 1.1042    OriginalAndShadowFcmpMatch =1043        FCmpBuilder.CreateAndReduce(OriginalAndShadowFcmpMatch);1044  }1045 1046  // Use MDBuilder(*C).createLikelyBranchWeights() because "match" is the common1047  // case.1048  FCmpBuilder.CreateCondBr(OriginalAndShadowFcmpMatch, NextBB, FailBB,1049                           MDBuilder(Context).createLikelyBranchWeights());1050 1051  // Fill in FailBB.1052  IRBuilder<> FailBuilder(FailBB);1053  FailBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());1054 1055  const auto EmitFailCall = [this, &FCmp, &FCmpBuilder,1056                             &FailBuilder](Value *L, Value *R, Value *ShadowL,1057                                           Value *ShadowR, Value *Result,1058                                           Value *ShadowResult) {1059    Type *FT = L->getType();1060    FunctionCallee *Callee = nullptr;1061    if (FT->isFloatTy()) {1062      Callee = &(NsanFCmpFail[kFloat]);1063    } else if (FT->isDoubleTy()) {1064      Callee = &(NsanFCmpFail[kDouble]);1065    } else if (FT->isX86_FP80Ty()) {1066      // TODO: make NsanFCmpFailLongDouble work.1067      Callee = &(NsanFCmpFail[kDouble]);1068      L = FailBuilder.CreateFPTrunc(L, Type::getDoubleTy(Context));1069      R = FailBuilder.CreateFPTrunc(L, Type::getDoubleTy(Context));1070    } else {1071      llvm_unreachable("not implemented");1072    }1073    FailBuilder.CreateCall(*Callee, {L, R, ShadowL, ShadowR,1074                                     ConstantInt::get(FCmpBuilder.getInt32Ty(),1075                                                      FCmp.getPredicate()),1076                                     Result, ShadowResult});1077  };1078  if (LHS->getType()->isVectorTy()) {1079    for (int I = 0, E = cast<VectorType>(LHS->getType())1080                            ->getElementCount()1081                            .getFixedValue();1082         I < E; ++I) {1083      Value *ExtractLHS = FailBuilder.CreateExtractElement(LHS, I);1084      Value *ExtractRHS = FailBuilder.CreateExtractElement(RHS, I);1085      Value *ExtractShaodwLHS = FailBuilder.CreateExtractElement(ShadowLHS, I);1086      Value *ExtractShaodwRHS = FailBuilder.CreateExtractElement(ShadowRHS, I);1087      Value *ExtractFCmp = FailBuilder.CreateExtractElement(&FCmp, I);1088      Value *ExtractShadowFCmp =1089          FailBuilder.CreateExtractElement(ShadowFCmp, I);1090      EmitFailCall(ExtractLHS, ExtractRHS, ExtractShaodwLHS, ExtractShaodwRHS,1091                   ExtractFCmp, ExtractShadowFCmp);1092    }1093  } else {1094    EmitFailCall(LHS, RHS, ShadowLHS, ShadowRHS, &FCmp, ShadowFCmp);1095  }1096  FailBuilder.CreateBr(NextBB);1097 1098  ++NumInstrumentedFCmp;1099}1100 1101// Creates a shadow phi value for any phi that defines a value of FT type.1102PHINode *NumericalStabilitySanitizer::maybeCreateShadowPhi(1103    PHINode &Phi, const TargetLibraryInfo &TLI) {1104  Type *VT = Phi.getType();1105  Type *ExtendedVT = Config.getExtendedFPType(VT);1106  if (ExtendedVT == nullptr)1107    return nullptr; // Not an FT value.1108  // The phi operands are shadow values and are not available when the phi is1109  // created. They will be populated in a final phase, once all shadow values1110  // have been created.1111  PHINode *Shadow = PHINode::Create(ExtendedVT, Phi.getNumIncomingValues());1112  Shadow->insertAfter(Phi.getIterator());1113  return Shadow;1114}1115 1116Value *NumericalStabilitySanitizer::handleLoad(LoadInst &Load, Type *VT,1117                                               Type *ExtendedVT) {1118  IRBuilder<> Builder(Load.getNextNode());1119  Builder.SetCurrentDebugLocation(Load.getDebugLoc());1120  if (addrPointsToConstantData(Load.getPointerOperand())) {1121    // No need to look into the shadow memory, the value is a constant. Just1122    // convert from FT to 2FT.1123    return Builder.CreateFPExt(&Load, ExtendedVT);1124  }1125 1126  // if (%shadowptr == &)1127  //    %shadow = fpext %v1128  // else1129  //    %shadow = load (ptrcast %shadow_ptr))1130  // Considered options here:1131  //  - Have `NsanGetShadowPtrForLoad` return a fixed address1132  //    &__nsan_unknown_value_shadow_address that is valid to load from, and1133  //    use a select. This has the advantage that the generated IR is simpler.1134  //  - Have `NsanGetShadowPtrForLoad` return nullptr.  Because `select` does1135  //    not short-circuit, dereferencing the returned pointer is no longer an1136  //    option, have to split and create a separate basic block. This has the1137  //    advantage of being easier to debug because it crashes if we ever mess1138  //    up.1139 1140  const auto Extents = getMemoryExtentsOrDie(VT);1141  Value *ShadowPtr = Builder.CreateCall(1142      NsanGetShadowPtrForLoad[Extents.ValueType],1143      {Load.getPointerOperand(), ConstantInt::get(IntptrTy, Extents.NumElts)});1144  ++NumInstrumentedFTLoads;1145 1146  // Split the basic block.1147  BasicBlock *LoadBB = Load.getParent();1148  BasicBlock *NextBB = LoadBB->splitBasicBlock(Builder.GetInsertPoint());1149  // Create the two options for creating the shadow value.1150  BasicBlock *ShadowLoadBB =1151      BasicBlock::Create(Context, "", LoadBB->getParent(), NextBB);1152  BasicBlock *FExtBB =1153      BasicBlock::Create(Context, "", LoadBB->getParent(), NextBB);1154 1155  // Replace the newly created terminator unconditional branch by a conditional1156  // branch to one of the options.1157  {1158    LoadBB->back().eraseFromParent();1159    IRBuilder<> LoadBBBuilder(LoadBB); // The old builder has been invalidated.1160    LoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());1161    LoadBBBuilder.CreateCondBr(LoadBBBuilder.CreateIsNull(ShadowPtr), FExtBB,1162                               ShadowLoadBB);1163  }1164 1165  // Fill in ShadowLoadBB.1166  IRBuilder<> ShadowLoadBBBuilder(ShadowLoadBB);1167  ShadowLoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());1168  Value *ShadowLoad = ShadowLoadBBBuilder.CreateAlignedLoad(1169      ExtendedVT, ShadowPtr, Align(1), Load.isVolatile());1170  if (ClCheckLoads) {1171    ShadowLoad = emitCheck(&Load, ShadowLoad, ShadowLoadBBBuilder,1172                           CheckLoc::makeLoad(Load.getPointerOperand()));1173  }1174  ShadowLoadBBBuilder.CreateBr(NextBB);1175 1176  // Fill in FExtBB.1177  IRBuilder<> FExtBBBuilder(FExtBB);1178  FExtBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());1179  Value *FExt = FExtBBBuilder.CreateFPExt(&Load, ExtendedVT);1180  FExtBBBuilder.CreateBr(NextBB);1181 1182  // The shadow value come from any of the options.1183  IRBuilder<> NextBBBuilder(&*NextBB->begin());1184  NextBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());1185  PHINode *ShadowPhi = NextBBBuilder.CreatePHI(ExtendedVT, 2);1186  ShadowPhi->addIncoming(ShadowLoad, ShadowLoadBB);1187  ShadowPhi->addIncoming(FExt, FExtBB);1188  return ShadowPhi;1189}1190 1191Value *NumericalStabilitySanitizer::handleTrunc(const FPTruncInst &Trunc,1192                                                Type *VT, Type *ExtendedVT,1193                                                const ValueToShadowMap &Map,1194                                                IRBuilder<> &Builder) {1195  Value *OrigSource = Trunc.getOperand(0);1196  Type *OrigSourceTy = OrigSource->getType();1197  Type *ExtendedSourceTy = Config.getExtendedFPType(OrigSourceTy);1198 1199  // When truncating:1200  //  - (A) If the source has a shadow, we truncate from the shadow, else we1201  //    truncate from the original source.1202  //  - (B) If the shadow of the source is larger than the shadow of the dest,1203  //    we still need a truncate. Else, the shadow of the source is the same1204  //    type as the shadow of the dest (because mappings are non-decreasing), so1205  //   we don't need to emit a truncate.1206  // Examples,1207  //   with a mapping of {f32->f64;f64->f80;f80->f128}1208  //     fptrunc double   %1 to float     ->  fptrunc x86_fp80 s(%1) to double1209  //     fptrunc x86_fp80 %1 to float     ->  fptrunc fp128    s(%1) to double1210  //     fptrunc fp128    %1 to float     ->  fptrunc fp128    %1    to double1211  //     fptrunc x86_fp80 %1 to double    ->  x86_fp80 s(%1)1212  //     fptrunc fp128    %1 to double    ->  fptrunc fp128 %1 to x86_fp801213  //     fptrunc fp128    %1 to x86_fp80  ->  fp128 %11214  //   with a mapping of {f32->f64;f64->f128;f80->f128}1215  //     fptrunc double   %1 to float     ->  fptrunc fp128    s(%1) to double1216  //     fptrunc x86_fp80 %1 to float     ->  fptrunc fp128    s(%1) to double1217  //     fptrunc fp128    %1 to float     ->  fptrunc fp128    %1    to double1218  //     fptrunc x86_fp80 %1 to double    ->  fp128 %11219  //     fptrunc fp128    %1 to double    ->  fp128 %11220  //     fptrunc fp128    %1 to x86_fp80  ->  fp128 %11221  //   with a mapping of {f32->f32;f64->f32;f80->f64}1222  //     fptrunc double   %1 to float     ->  float s(%1)1223  //     fptrunc x86_fp80 %1 to float     ->  fptrunc double    s(%1) to float1224  //     fptrunc fp128    %1 to float     ->  fptrunc fp128     %1    to float1225  //     fptrunc x86_fp80 %1 to double    ->  fptrunc double    s(%1) to float1226  //     fptrunc fp128    %1 to double    ->  fptrunc fp128     %1    to float1227  //     fptrunc fp128    %1 to x86_fp80  ->  fptrunc fp128     %1    to double1228 1229  // See (A) above.1230  Value *Source = ExtendedSourceTy ? Map.getShadow(OrigSource) : OrigSource;1231  Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;1232  // See (B) above.1233  if (SourceTy == ExtendedVT)1234    return Source;1235 1236  return Builder.CreateFPTrunc(Source, ExtendedVT);1237}1238 1239Value *NumericalStabilitySanitizer::handleExt(const FPExtInst &Ext, Type *VT,1240                                              Type *ExtendedVT,1241                                              const ValueToShadowMap &Map,1242                                              IRBuilder<> &Builder) {1243  Value *OrigSource = Ext.getOperand(0);1244  Type *OrigSourceTy = OrigSource->getType();1245  Type *ExtendedSourceTy = Config.getExtendedFPType(OrigSourceTy);1246  // When extending:1247  //  - (A) If the source has a shadow, we extend from the shadow, else we1248  //    extend from the original source.1249  //  - (B) If the shadow of the dest is larger than the shadow of the source,1250  //    we still need an extend. Else, the shadow of the source is the same1251  //    type as the shadow of the dest (because mappings are non-decreasing), so1252  //    we don't need to emit an extend.1253  // Examples,1254  //   with a mapping of {f32->f64;f64->f80;f80->f128}1255  //     fpext half    %1 to float     ->  fpext half     %1    to double1256  //     fpext half    %1 to double    ->  fpext half     %1    to x86_fp801257  //     fpext half    %1 to x86_fp80  ->  fpext half     %1    to fp1281258  //     fpext float   %1 to double    ->  double s(%1)1259  //     fpext float   %1 to x86_fp80  ->  fpext double   s(%1) to fp1281260  //     fpext double  %1 to x86_fp80  ->  fpext x86_fp80 s(%1) to fp1281261  //   with a mapping of {f32->f64;f64->f128;f80->f128}1262  //     fpext half    %1 to float     ->  fpext half     %1    to double1263  //     fpext half    %1 to double    ->  fpext half     %1    to fp1281264  //     fpext half    %1 to x86_fp80  ->  fpext half     %1    to fp1281265  //     fpext float   %1 to double    ->  fpext double   s(%1) to fp1281266  //     fpext float   %1 to x86_fp80  ->  fpext double   s(%1) to fp1281267  //     fpext double  %1 to x86_fp80  ->  fp128 s(%1)1268  //   with a mapping of {f32->f32;f64->f32;f80->f64}1269  //     fpext half    %1 to float     ->  fpext half     %1    to float1270  //     fpext half    %1 to double    ->  fpext half     %1    to float1271  //     fpext half    %1 to x86_fp80  ->  fpext half     %1    to double1272  //     fpext float   %1 to double    ->  s(%1)1273  //     fpext float   %1 to x86_fp80  ->  fpext float    s(%1) to double1274  //     fpext double  %1 to x86_fp80  ->  fpext float    s(%1) to double1275 1276  // See (A) above.1277  Value *Source = ExtendedSourceTy ? Map.getShadow(OrigSource) : OrigSource;1278  Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;1279  // See (B) above.1280  if (SourceTy == ExtendedVT)1281    return Source;1282 1283  return Builder.CreateFPExt(Source, ExtendedVT);1284}1285 1286namespace {1287// TODO: This should be tablegen-ed.1288struct KnownIntrinsic {1289  struct WidenedIntrinsic {1290    const char *NarrowName;1291    Intrinsic::ID ID; // wide id.1292    using FnTypeFactory = FunctionType *(*)(LLVMContext &);1293    FnTypeFactory MakeFnTy;1294  };1295 1296  static const char *get(LibFunc LFunc);1297 1298  // Given an intrinsic with an `FT` argument, try to find a wider intrinsic1299  // that applies the same operation on the shadow argument.1300  // Options are:1301  //  - pass in the ID and full function type,1302  //  - pass in the name, which includes the function type through mangling.1303  static const WidenedIntrinsic *widen(StringRef Name);1304 1305private:1306  struct LFEntry {1307    LibFunc LFunc;1308    const char *IntrinsicName;1309  };1310  static const LFEntry kLibfuncIntrinsics[];1311 1312  static const WidenedIntrinsic kWidenedIntrinsics[];1313};1314} // namespace1315 1316static FunctionType *makeDoubleDouble(LLVMContext &C) {1317  return FunctionType::get(Type::getDoubleTy(C), {Type::getDoubleTy(C)}, false);1318}1319 1320static FunctionType *makeX86FP80X86FP80(LLVMContext &C) {1321  return FunctionType::get(Type::getX86_FP80Ty(C), {Type::getX86_FP80Ty(C)},1322                           false);1323}1324 1325static FunctionType *makeDoubleDoubleI32(LLVMContext &C) {1326  return FunctionType::get(Type::getDoubleTy(C),1327                           {Type::getDoubleTy(C), Type::getInt32Ty(C)}, false);1328}1329 1330static FunctionType *makeX86FP80X86FP80I32(LLVMContext &C) {1331  return FunctionType::get(Type::getX86_FP80Ty(C),1332                           {Type::getX86_FP80Ty(C), Type::getInt32Ty(C)},1333                           false);1334}1335 1336static FunctionType *makeDoubleDoubleDouble(LLVMContext &C) {1337  return FunctionType::get(Type::getDoubleTy(C),1338                           {Type::getDoubleTy(C), Type::getDoubleTy(C)}, false);1339}1340 1341static FunctionType *makeX86FP80X86FP80X86FP80(LLVMContext &C) {1342  return FunctionType::get(Type::getX86_FP80Ty(C),1343                           {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)},1344                           false);1345}1346 1347static FunctionType *makeDoubleDoubleDoubleDouble(LLVMContext &C) {1348  return FunctionType::get(1349      Type::getDoubleTy(C),1350      {Type::getDoubleTy(C), Type::getDoubleTy(C), Type::getDoubleTy(C)},1351      false);1352}1353 1354static FunctionType *makeX86FP80X86FP80X86FP80X86FP80(LLVMContext &C) {1355  return FunctionType::get(1356      Type::getX86_FP80Ty(C),1357      {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)},1358      false);1359}1360 1361const KnownIntrinsic::WidenedIntrinsic KnownIntrinsic::kWidenedIntrinsics[] = {1362    // TODO: Right now we ignore vector intrinsics.1363    // This is hard because we have to model the semantics of the intrinsics,1364    // e.g. llvm.x86.sse2.min.sd means extract first element, min, insert back.1365    // Intrinsics that take any non-vector FT types:1366    // NOTE: Right now because of1367    // https://github.com/llvm/llvm-project/issues/447441368    // for f128 we need to use makeX86FP80X86FP80 (go to a lower precision and1369    // come back).1370    {"llvm.sqrt.f32", Intrinsic::sqrt, makeDoubleDouble},1371    {"llvm.sqrt.f64", Intrinsic::sqrt, makeX86FP80X86FP80},1372    {"llvm.sqrt.f80", Intrinsic::sqrt, makeX86FP80X86FP80},1373    {"llvm.powi.f32", Intrinsic::powi, makeDoubleDoubleI32},1374    {"llvm.powi.f64", Intrinsic::powi, makeX86FP80X86FP80I32},1375    {"llvm.powi.f80", Intrinsic::powi, makeX86FP80X86FP80I32},1376    {"llvm.sin.f32", Intrinsic::sin, makeDoubleDouble},1377    {"llvm.sin.f64", Intrinsic::sin, makeX86FP80X86FP80},1378    {"llvm.sin.f80", Intrinsic::sin, makeX86FP80X86FP80},1379    {"llvm.cos.f32", Intrinsic::cos, makeDoubleDouble},1380    {"llvm.cos.f64", Intrinsic::cos, makeX86FP80X86FP80},1381    {"llvm.cos.f80", Intrinsic::cos, makeX86FP80X86FP80},1382    {"llvm.pow.f32", Intrinsic::pow, makeDoubleDoubleDouble},1383    {"llvm.pow.f64", Intrinsic::pow, makeX86FP80X86FP80X86FP80},1384    {"llvm.pow.f80", Intrinsic::pow, makeX86FP80X86FP80X86FP80},1385    {"llvm.exp.f32", Intrinsic::exp, makeDoubleDouble},1386    {"llvm.exp.f64", Intrinsic::exp, makeX86FP80X86FP80},1387    {"llvm.exp.f80", Intrinsic::exp, makeX86FP80X86FP80},1388    {"llvm.exp2.f32", Intrinsic::exp2, makeDoubleDouble},1389    {"llvm.exp2.f64", Intrinsic::exp2, makeX86FP80X86FP80},1390    {"llvm.exp2.f80", Intrinsic::exp2, makeX86FP80X86FP80},1391    {"llvm.log.f32", Intrinsic::log, makeDoubleDouble},1392    {"llvm.log.f64", Intrinsic::log, makeX86FP80X86FP80},1393    {"llvm.log.f80", Intrinsic::log, makeX86FP80X86FP80},1394    {"llvm.log10.f32", Intrinsic::log10, makeDoubleDouble},1395    {"llvm.log10.f64", Intrinsic::log10, makeX86FP80X86FP80},1396    {"llvm.log10.f80", Intrinsic::log10, makeX86FP80X86FP80},1397    {"llvm.log2.f32", Intrinsic::log2, makeDoubleDouble},1398    {"llvm.log2.f64", Intrinsic::log2, makeX86FP80X86FP80},1399    {"llvm.log2.f80", Intrinsic::log2, makeX86FP80X86FP80},1400    {"llvm.fma.f32", Intrinsic::fma, makeDoubleDoubleDoubleDouble},1401 1402    {"llvm.fmuladd.f32", Intrinsic::fmuladd, makeDoubleDoubleDoubleDouble},1403 1404    {"llvm.fma.f64", Intrinsic::fma, makeX86FP80X86FP80X86FP80X86FP80},1405 1406    {"llvm.fmuladd.f64", Intrinsic::fma, makeX86FP80X86FP80X86FP80X86FP80},1407 1408    {"llvm.fma.f80", Intrinsic::fma, makeX86FP80X86FP80X86FP80X86FP80},1409    {"llvm.fabs.f32", Intrinsic::fabs, makeDoubleDouble},1410    {"llvm.fabs.f64", Intrinsic::fabs, makeX86FP80X86FP80},1411    {"llvm.fabs.f80", Intrinsic::fabs, makeX86FP80X86FP80},1412    {"llvm.minnum.f32", Intrinsic::minnum, makeDoubleDoubleDouble},1413    {"llvm.minnum.f64", Intrinsic::minnum, makeX86FP80X86FP80X86FP80},1414    {"llvm.minnum.f80", Intrinsic::minnum, makeX86FP80X86FP80X86FP80},1415    {"llvm.maxnum.f32", Intrinsic::maxnum, makeDoubleDoubleDouble},1416    {"llvm.maxnum.f64", Intrinsic::maxnum, makeX86FP80X86FP80X86FP80},1417    {"llvm.maxnum.f80", Intrinsic::maxnum, makeX86FP80X86FP80X86FP80},1418    {"llvm.minimum.f32", Intrinsic::minimum, makeDoubleDoubleDouble},1419    {"llvm.minimum.f64", Intrinsic::minimum, makeX86FP80X86FP80X86FP80},1420    {"llvm.minimum.f80", Intrinsic::minimum, makeX86FP80X86FP80X86FP80},1421    {"llvm.maximum.f32", Intrinsic::maximum, makeDoubleDoubleDouble},1422    {"llvm.maximum.f64", Intrinsic::maximum, makeX86FP80X86FP80X86FP80},1423    {"llvm.maximum.f80", Intrinsic::maximum, makeX86FP80X86FP80X86FP80},1424    {"llvm.copysign.f32", Intrinsic::copysign, makeDoubleDoubleDouble},1425    {"llvm.copysign.f64", Intrinsic::copysign, makeX86FP80X86FP80X86FP80},1426    {"llvm.copysign.f80", Intrinsic::copysign, makeX86FP80X86FP80X86FP80},1427    {"llvm.floor.f32", Intrinsic::floor, makeDoubleDouble},1428    {"llvm.floor.f64", Intrinsic::floor, makeX86FP80X86FP80},1429    {"llvm.floor.f80", Intrinsic::floor, makeX86FP80X86FP80},1430    {"llvm.ceil.f32", Intrinsic::ceil, makeDoubleDouble},1431    {"llvm.ceil.f64", Intrinsic::ceil, makeX86FP80X86FP80},1432    {"llvm.ceil.f80", Intrinsic::ceil, makeX86FP80X86FP80},1433    {"llvm.trunc.f32", Intrinsic::trunc, makeDoubleDouble},1434    {"llvm.trunc.f64", Intrinsic::trunc, makeX86FP80X86FP80},1435    {"llvm.trunc.f80", Intrinsic::trunc, makeX86FP80X86FP80},1436    {"llvm.rint.f32", Intrinsic::rint, makeDoubleDouble},1437    {"llvm.rint.f64", Intrinsic::rint, makeX86FP80X86FP80},1438    {"llvm.rint.f80", Intrinsic::rint, makeX86FP80X86FP80},1439    {"llvm.nearbyint.f32", Intrinsic::nearbyint, makeDoubleDouble},1440    {"llvm.nearbyint.f64", Intrinsic::nearbyint, makeX86FP80X86FP80},1441    {"llvm.nearbyin80f64", Intrinsic::nearbyint, makeX86FP80X86FP80},1442    {"llvm.round.f32", Intrinsic::round, makeDoubleDouble},1443    {"llvm.round.f64", Intrinsic::round, makeX86FP80X86FP80},1444    {"llvm.round.f80", Intrinsic::round, makeX86FP80X86FP80},1445    {"llvm.lround.f32", Intrinsic::lround, makeDoubleDouble},1446    {"llvm.lround.f64", Intrinsic::lround, makeX86FP80X86FP80},1447    {"llvm.lround.f80", Intrinsic::lround, makeX86FP80X86FP80},1448    {"llvm.llround.f32", Intrinsic::llround, makeDoubleDouble},1449    {"llvm.llround.f64", Intrinsic::llround, makeX86FP80X86FP80},1450    {"llvm.llround.f80", Intrinsic::llround, makeX86FP80X86FP80},1451    {"llvm.lrint.f32", Intrinsic::lrint, makeDoubleDouble},1452    {"llvm.lrint.f64", Intrinsic::lrint, makeX86FP80X86FP80},1453    {"llvm.lrint.f80", Intrinsic::lrint, makeX86FP80X86FP80},1454    {"llvm.llrint.f32", Intrinsic::llrint, makeDoubleDouble},1455    {"llvm.llrint.f64", Intrinsic::llrint, makeX86FP80X86FP80},1456    {"llvm.llrint.f80", Intrinsic::llrint, makeX86FP80X86FP80},1457};1458 1459const KnownIntrinsic::LFEntry KnownIntrinsic::kLibfuncIntrinsics[] = {1460    {LibFunc_sqrtf, "llvm.sqrt.f32"},1461    {LibFunc_sqrt, "llvm.sqrt.f64"},1462    {LibFunc_sqrtl, "llvm.sqrt.f80"},1463    {LibFunc_sinf, "llvm.sin.f32"},1464    {LibFunc_sin, "llvm.sin.f64"},1465    {LibFunc_sinl, "llvm.sin.f80"},1466    {LibFunc_cosf, "llvm.cos.f32"},1467    {LibFunc_cos, "llvm.cos.f64"},1468    {LibFunc_cosl, "llvm.cos.f80"},1469    {LibFunc_powf, "llvm.pow.f32"},1470    {LibFunc_pow, "llvm.pow.f64"},1471    {LibFunc_powl, "llvm.pow.f80"},1472    {LibFunc_expf, "llvm.exp.f32"},1473    {LibFunc_exp, "llvm.exp.f64"},1474    {LibFunc_expl, "llvm.exp.f80"},1475    {LibFunc_exp2f, "llvm.exp2.f32"},1476    {LibFunc_exp2, "llvm.exp2.f64"},1477    {LibFunc_exp2l, "llvm.exp2.f80"},1478    {LibFunc_logf, "llvm.log.f32"},1479    {LibFunc_log, "llvm.log.f64"},1480    {LibFunc_logl, "llvm.log.f80"},1481    {LibFunc_log10f, "llvm.log10.f32"},1482    {LibFunc_log10, "llvm.log10.f64"},1483    {LibFunc_log10l, "llvm.log10.f80"},1484    {LibFunc_log2f, "llvm.log2.f32"},1485    {LibFunc_log2, "llvm.log2.f64"},1486    {LibFunc_log2l, "llvm.log2.f80"},1487    {LibFunc_fabsf, "llvm.fabs.f32"},1488    {LibFunc_fabs, "llvm.fabs.f64"},1489    {LibFunc_fabsl, "llvm.fabs.f80"},1490    {LibFunc_copysignf, "llvm.copysign.f32"},1491    {LibFunc_copysign, "llvm.copysign.f64"},1492    {LibFunc_copysignl, "llvm.copysign.f80"},1493    {LibFunc_floorf, "llvm.floor.f32"},1494    {LibFunc_floor, "llvm.floor.f64"},1495    {LibFunc_floorl, "llvm.floor.f80"},1496    {LibFunc_fmaxf, "llvm.maxnum.f32"},1497    {LibFunc_fmax, "llvm.maxnum.f64"},1498    {LibFunc_fmaxl, "llvm.maxnum.f80"},1499    {LibFunc_fminf, "llvm.minnum.f32"},1500    {LibFunc_fmin, "llvm.minnum.f64"},1501    {LibFunc_fminl, "llvm.minnum.f80"},1502    {LibFunc_ceilf, "llvm.ceil.f32"},1503    {LibFunc_ceil, "llvm.ceil.f64"},1504    {LibFunc_ceill, "llvm.ceil.f80"},1505    {LibFunc_truncf, "llvm.trunc.f32"},1506    {LibFunc_trunc, "llvm.trunc.f64"},1507    {LibFunc_truncl, "llvm.trunc.f80"},1508    {LibFunc_rintf, "llvm.rint.f32"},1509    {LibFunc_rint, "llvm.rint.f64"},1510    {LibFunc_rintl, "llvm.rint.f80"},1511    {LibFunc_nearbyintf, "llvm.nearbyint.f32"},1512    {LibFunc_nearbyint, "llvm.nearbyint.f64"},1513    {LibFunc_nearbyintl, "llvm.nearbyint.f80"},1514    {LibFunc_roundf, "llvm.round.f32"},1515    {LibFunc_round, "llvm.round.f64"},1516    {LibFunc_roundl, "llvm.round.f80"},1517};1518 1519const char *KnownIntrinsic::get(LibFunc LFunc) {1520  for (const auto &E : kLibfuncIntrinsics) {1521    if (E.LFunc == LFunc)1522      return E.IntrinsicName;1523  }1524  return nullptr;1525}1526 1527const KnownIntrinsic::WidenedIntrinsic *KnownIntrinsic::widen(StringRef Name) {1528  for (const auto &E : kWidenedIntrinsics) {1529    if (E.NarrowName == Name)1530      return &E;1531  }1532  return nullptr;1533}1534 1535// Returns the name of the LLVM intrinsic corresponding to the given function.1536static const char *getIntrinsicFromLibfunc(Function &Fn, Type *VT,1537                                           const TargetLibraryInfo &TLI) {1538  LibFunc LFunc;1539  if (!TLI.getLibFunc(Fn, LFunc))1540    return nullptr;1541 1542  if (const char *Name = KnownIntrinsic::get(LFunc))1543    return Name;1544 1545  LLVM_DEBUG(errs() << "TODO: LibFunc: " << TLI.getName(LFunc) << "\n");1546  return nullptr;1547}1548 1549// Try to handle a known function call.1550Value *NumericalStabilitySanitizer::maybeHandleKnownCallBase(1551    CallBase &Call, Type *VT, Type *ExtendedVT, const TargetLibraryInfo &TLI,1552    const ValueToShadowMap &Map, IRBuilder<> &Builder) {1553  Function *Fn = Call.getCalledFunction();1554  if (Fn == nullptr)1555    return nullptr;1556 1557  Intrinsic::ID WidenedId = Intrinsic::ID();1558  FunctionType *WidenedFnTy = nullptr;1559  if (const auto ID = Fn->getIntrinsicID()) {1560    const auto *Widened = KnownIntrinsic::widen(Fn->getName());1561    if (Widened) {1562      WidenedId = Widened->ID;1563      WidenedFnTy = Widened->MakeFnTy(Context);1564    } else {1565      // If we don't know how to widen the intrinsic, we have no choice but to1566      // call the non-wide version on a truncated shadow and extend again1567      // afterwards.1568      WidenedId = ID;1569      WidenedFnTy = Fn->getFunctionType();1570    }1571  } else if (const char *Name = getIntrinsicFromLibfunc(*Fn, VT, TLI)) {1572    // We might have a call to a library function that we can replace with a1573    // wider Intrinsic.1574    const auto *Widened = KnownIntrinsic::widen(Name);1575    assert(Widened && "make sure KnownIntrinsic entries are consistent");1576    WidenedId = Widened->ID;1577    WidenedFnTy = Widened->MakeFnTy(Context);1578  } else {1579    // This is not a known library function or intrinsic.1580    return nullptr;1581  }1582 1583  // Check that the widened intrinsic is valid.1584  SmallVector<Intrinsic::IITDescriptor, 8> Table;1585  getIntrinsicInfoTableEntries(WidenedId, Table);1586  SmallVector<Type *, 4> ArgTys;1587  ArrayRef<Intrinsic::IITDescriptor> TableRef = Table;1588  [[maybe_unused]] Intrinsic::MatchIntrinsicTypesResult MatchResult =1589      Intrinsic::matchIntrinsicSignature(WidenedFnTy, TableRef, ArgTys);1590  assert(MatchResult == Intrinsic::MatchIntrinsicTypes_Match &&1591         "invalid widened intrinsic");1592  // For known intrinsic functions, we create a second call to the same1593  // intrinsic with a different type.1594  SmallVector<Value *, 4> Args;1595  // The last operand is the intrinsic itself, skip it.1596  for (unsigned I = 0, E = Call.getNumOperands() - 1; I < E; ++I) {1597    Value *Arg = Call.getOperand(I);1598    Type *OrigArgTy = Arg->getType();1599    Type *IntrinsicArgTy = WidenedFnTy->getParamType(I);1600    if (OrigArgTy == IntrinsicArgTy) {1601      Args.push_back(Arg); // The arg is passed as is.1602      continue;1603    }1604    Type *ShadowArgTy = Config.getExtendedFPType(Arg->getType());1605    assert(ShadowArgTy &&1606           "don't know how to get the shadow value for a non-FT");1607    Value *Shadow = Map.getShadow(Arg);1608    if (ShadowArgTy == IntrinsicArgTy) {1609      // The shadow is the right type for the intrinsic.1610      assert(Shadow->getType() == ShadowArgTy);1611      Args.push_back(Shadow);1612      continue;1613    }1614    // There is no intrinsic with his level of precision, truncate the shadow.1615    Args.push_back(Builder.CreateFPTrunc(Shadow, IntrinsicArgTy));1616  }1617  Value *IntrinsicCall = Builder.CreateIntrinsic(WidenedId, ArgTys, Args);1618  return WidenedFnTy->getReturnType() == ExtendedVT1619             ? IntrinsicCall1620             : Builder.CreateFPExt(IntrinsicCall, ExtendedVT);1621}1622 1623// Handle a CallBase, i.e. a function call, an inline asm sequence, or an1624// invoke.1625Value *NumericalStabilitySanitizer::handleCallBase(CallBase &Call, Type *VT,1626                                                   Type *ExtendedVT,1627                                                   const TargetLibraryInfo &TLI,1628                                                   const ValueToShadowMap &Map,1629                                                   IRBuilder<> &Builder) {1630  // We cannot look inside inline asm, just expand the result again.1631  if (Call.isInlineAsm())1632    return Builder.CreateFPExt(&Call, ExtendedVT);1633 1634  // Intrinsics and library functions (e.g. sin, exp) are handled1635  // specifically, because we know their semantics and can do better than1636  // blindly calling them (e.g. compute the sinus in the actual shadow domain).1637  if (Value *V =1638          maybeHandleKnownCallBase(Call, VT, ExtendedVT, TLI, Map, Builder))1639    return V;1640 1641  // If the return tag matches that of the called function, read the extended1642  // return value from the shadow ret ptr. Else, just extend the return value.1643  Value *L =1644      Builder.CreateLoad(IntptrTy, NsanShadowRetTag, /*isVolatile=*/false);1645  Value *HasShadowRet = Builder.CreateICmpEQ(1646      L, Builder.CreatePtrToInt(Call.getCalledOperand(), IntptrTy));1647 1648  Value *ShadowRetVal = Builder.CreateLoad(1649      ExtendedVT,1650      Builder.CreateConstGEP2_64(NsanShadowRetType, NsanShadowRetPtr, 0, 0),1651      /*isVolatile=*/false);1652  Value *Shadow = Builder.CreateSelect(HasShadowRet, ShadowRetVal,1653                                       Builder.CreateFPExt(&Call, ExtendedVT));1654  ++NumInstrumentedFTCalls;1655  return Shadow;1656}1657 1658// Creates a shadow value for the given FT value. At that point all operands are1659// guaranteed to be available.1660Value *NumericalStabilitySanitizer::createShadowValueWithOperandsAvailable(1661    Instruction &Inst, const TargetLibraryInfo &TLI,1662    const ValueToShadowMap &Map) {1663  Type *VT = Inst.getType();1664  Type *ExtendedVT = Config.getExtendedFPType(VT);1665  assert(ExtendedVT != nullptr && "trying to create a shadow for a non-FT");1666 1667  if (auto *Load = dyn_cast<LoadInst>(&Inst))1668    return handleLoad(*Load, VT, ExtendedVT);1669 1670  if (auto *Call = dyn_cast<CallInst>(&Inst)) {1671    // Insert after the call.1672    BasicBlock::iterator It(Inst);1673    IRBuilder<> Builder(Call->getParent(), ++It);1674    Builder.SetCurrentDebugLocation(Call->getDebugLoc());1675    return handleCallBase(*Call, VT, ExtendedVT, TLI, Map, Builder);1676  }1677 1678  if (auto *Invoke = dyn_cast<InvokeInst>(&Inst)) {1679    // The Invoke terminates the basic block, create a new basic block in1680    // between the successful invoke and the next block.1681    BasicBlock *InvokeBB = Invoke->getParent();1682    BasicBlock *NextBB = Invoke->getNormalDest();1683    BasicBlock *NewBB =1684        BasicBlock::Create(Context, "", NextBB->getParent(), NextBB);1685    Inst.replaceSuccessorWith(NextBB, NewBB);1686 1687    IRBuilder<> Builder(NewBB);1688    Builder.SetCurrentDebugLocation(Invoke->getDebugLoc());1689    Value *Shadow = handleCallBase(*Invoke, VT, ExtendedVT, TLI, Map, Builder);1690    Builder.CreateBr(NextBB);1691    NewBB->replaceSuccessorsPhiUsesWith(InvokeBB, NewBB);1692    return Shadow;1693  }1694 1695  IRBuilder<> Builder(Inst.getNextNode());1696  Builder.SetCurrentDebugLocation(Inst.getDebugLoc());1697 1698  if (auto *Trunc = dyn_cast<FPTruncInst>(&Inst))1699    return handleTrunc(*Trunc, VT, ExtendedVT, Map, Builder);1700  if (auto *Ext = dyn_cast<FPExtInst>(&Inst))1701    return handleExt(*Ext, VT, ExtendedVT, Map, Builder);1702 1703  if (auto *UnaryOp = dyn_cast<UnaryOperator>(&Inst))1704    return Builder.CreateUnOp(UnaryOp->getOpcode(),1705                              Map.getShadow(UnaryOp->getOperand(0)));1706 1707  if (auto *BinOp = dyn_cast<BinaryOperator>(&Inst))1708    return Builder.CreateBinOp(BinOp->getOpcode(),1709                               Map.getShadow(BinOp->getOperand(0)),1710                               Map.getShadow(BinOp->getOperand(1)));1711 1712  if (isa<UIToFPInst>(&Inst) || isa<SIToFPInst>(&Inst)) {1713    auto *Cast = cast<CastInst>(&Inst);1714    return Builder.CreateCast(Cast->getOpcode(), Cast->getOperand(0),1715                              ExtendedVT);1716  }1717 1718  if (auto *S = dyn_cast<SelectInst>(&Inst))1719    return Builder.CreateSelect(S->getCondition(),1720                                Map.getShadow(S->getTrueValue()),1721                                Map.getShadow(S->getFalseValue()));1722 1723  if (auto *Freeze = dyn_cast<FreezeInst>(&Inst))1724    return Builder.CreateFreeze(Map.getShadow(Freeze->getOperand(0)));1725 1726  if (auto *Extract = dyn_cast<ExtractElementInst>(&Inst))1727    return Builder.CreateExtractElement(1728        Map.getShadow(Extract->getVectorOperand()), Extract->getIndexOperand());1729 1730  if (auto *Insert = dyn_cast<InsertElementInst>(&Inst))1731    return Builder.CreateInsertElement(Map.getShadow(Insert->getOperand(0)),1732                                       Map.getShadow(Insert->getOperand(1)),1733                                       Insert->getOperand(2));1734 1735  if (auto *Shuffle = dyn_cast<ShuffleVectorInst>(&Inst))1736    return Builder.CreateShuffleVector(Map.getShadow(Shuffle->getOperand(0)),1737                                       Map.getShadow(Shuffle->getOperand(1)),1738                                       Shuffle->getShuffleMask());1739  // TODO: We could make aggregate object first class citizens. For now we1740  // just extend the extracted value.1741  if (auto *Extract = dyn_cast<ExtractValueInst>(&Inst))1742    return Builder.CreateFPExt(Extract, ExtendedVT);1743 1744  if (auto *BC = dyn_cast<BitCastInst>(&Inst))1745    return Builder.CreateFPExt(BC, ExtendedVT);1746 1747  report_fatal_error("Unimplemented support for " +1748                     Twine(Inst.getOpcodeName()));1749}1750 1751// Creates a shadow value for an instruction that defines a value of FT type.1752// FT operands that do not already have shadow values are created recursively.1753// The DFS is guaranteed to not loop as phis and arguments already have1754// shadows.1755void NumericalStabilitySanitizer::maybeCreateShadowValue(1756    Instruction &Root, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) {1757  Type *VT = Root.getType();1758  Type *ExtendedVT = Config.getExtendedFPType(VT);1759  if (ExtendedVT == nullptr)1760    return; // Not an FT value.1761 1762  if (Map.hasShadow(&Root))1763    return; // Shadow already exists.1764 1765  assert(!isa<PHINode>(Root) && "phi nodes should already have shadows");1766 1767  std::vector<Instruction *> DfsStack(1, &Root);1768  while (!DfsStack.empty()) {1769    // Ensure that all operands to the instruction have shadows before1770    // proceeding.1771    Instruction *I = DfsStack.back();1772    // The shadow for the instruction might have been created deeper in the DFS,1773    // see `forward_use_with_two_uses` test.1774    if (Map.hasShadow(I)) {1775      DfsStack.pop_back();1776      continue;1777    }1778 1779    bool MissingShadow = false;1780    for (Value *Op : I->operands()) {1781      Type *VT = Op->getType();1782      if (!Config.getExtendedFPType(VT))1783        continue; // Not an FT value.1784      if (Map.hasShadow(Op))1785        continue; // Shadow is already available.1786      MissingShadow = true;1787      DfsStack.push_back(cast<Instruction>(Op));1788    }1789    if (MissingShadow)1790      continue; // Process operands and come back to this instruction later.1791 1792    // All operands have shadows. Create a shadow for the current value.1793    Value *Shadow = createShadowValueWithOperandsAvailable(*I, TLI, Map);1794    Map.setShadow(*I, *Shadow);1795    DfsStack.pop_back();1796  }1797}1798 1799// A floating-point store needs its value and type written to shadow memory.1800void NumericalStabilitySanitizer::propagateFTStore(1801    StoreInst &Store, Type *VT, Type *ExtendedVT, const ValueToShadowMap &Map) {1802  Value *StoredValue = Store.getValueOperand();1803  IRBuilder<> Builder(&Store);1804  Builder.SetCurrentDebugLocation(Store.getDebugLoc());1805  const auto Extents = getMemoryExtentsOrDie(VT);1806  Value *ShadowPtr = Builder.CreateCall(1807      NsanGetShadowPtrForStore[Extents.ValueType],1808      {Store.getPointerOperand(), ConstantInt::get(IntptrTy, Extents.NumElts)});1809 1810  Value *StoredShadow = Map.getShadow(StoredValue);1811  if (!Store.getParent()->getParent()->hasOptNone()) {1812    // Only check stores when optimizing, because non-optimized code generates1813    // too many stores to the stack, creating false positives.1814    if (ClCheckStores) {1815      StoredShadow = emitCheck(StoredValue, StoredShadow, Builder,1816                               CheckLoc::makeStore(Store.getPointerOperand()));1817      ++NumInstrumentedFTStores;1818    }1819  }1820 1821  Builder.CreateAlignedStore(StoredShadow, ShadowPtr, Align(1),1822                             Store.isVolatile());1823}1824 1825// A non-ft store needs to invalidate shadow memory. Exceptions are:1826//   - memory transfers of floating-point data through other pointer types (llvm1827//     optimization passes transform `*(float*)a = *(float*)b` into1828//     `*(i32*)a = *(i32*)b` ). These have the same semantics as memcpy.1829//   - Writes of FT-sized constants. LLVM likes to do float stores as bitcasted1830//     ints. Note that this is not really necessary because if the value is1831//     unknown the framework will re-extend it on load anyway. It just felt1832//     easier to debug tests with vectors of FTs.1833void NumericalStabilitySanitizer::propagateNonFTStore(1834    StoreInst &Store, Type *VT, const ValueToShadowMap &Map) {1835  Value *PtrOp = Store.getPointerOperand();1836  IRBuilder<> Builder(Store.getNextNode());1837  Builder.SetCurrentDebugLocation(Store.getDebugLoc());1838  Value *Dst = PtrOp;1839  TypeSize SlotSize = DL.getTypeStoreSize(VT);1840  assert(!SlotSize.isScalable() && "unsupported");1841  const auto LoadSizeBytes = SlotSize.getFixedValue();1842  Value *ValueSize = Constant::getIntegerValue(1843      IntptrTy, APInt(IntptrTy->getPrimitiveSizeInBits(), LoadSizeBytes));1844 1845  ++NumInstrumentedNonFTStores;1846  Value *StoredValue = Store.getValueOperand();1847  if (LoadInst *Load = dyn_cast<LoadInst>(StoredValue)) {1848    // TODO: Handle the case when the value is from a phi.1849    // This is a memory transfer with memcpy semantics. Copy the type and1850    // value from the source. Note that we cannot use __nsan_copy_values()1851    // here, because that will not work when there is a write to memory in1852    // between the load and the store, e.g. in the case of a swap.1853    Type *ShadowTypeIntTy = Type::getIntNTy(Context, 8 * LoadSizeBytes);1854    Type *ShadowValueIntTy =1855        Type::getIntNTy(Context, 8 * kShadowScale * LoadSizeBytes);1856    IRBuilder<> LoadBuilder(Load->getNextNode());1857    Builder.SetCurrentDebugLocation(Store.getDebugLoc());1858    Value *LoadSrc = Load->getPointerOperand();1859    // Read the shadow type and value at load time. The type has the same size1860    // as the FT value, the value has twice its size.1861    // TODO: cache them to avoid re-creating them when a load is used by1862    // several stores. Maybe create them like the FT shadows when a load is1863    // encountered.1864    Value *RawShadowType = LoadBuilder.CreateAlignedLoad(1865        ShadowTypeIntTy,1866        LoadBuilder.CreateCall(NsanGetRawShadowTypePtr, {LoadSrc}), Align(1),1867        /*isVolatile=*/false);1868    Value *RawShadowValue = LoadBuilder.CreateAlignedLoad(1869        ShadowValueIntTy,1870        LoadBuilder.CreateCall(NsanGetRawShadowPtr, {LoadSrc}), Align(1),1871        /*isVolatile=*/false);1872 1873    // Write back the shadow type and value at store time.1874    Builder.CreateAlignedStore(1875        RawShadowType, Builder.CreateCall(NsanGetRawShadowTypePtr, {Dst}),1876        Align(1),1877        /*isVolatile=*/false);1878    Builder.CreateAlignedStore(RawShadowValue,1879                               Builder.CreateCall(NsanGetRawShadowPtr, {Dst}),1880                               Align(1),1881                               /*isVolatile=*/false);1882 1883    ++NumInstrumentedNonFTMemcpyStores;1884    return;1885  }1886  // ClPropagateNonFTConstStoresAsFT is by default false.1887  if (Constant *C; ClPropagateNonFTConstStoresAsFT &&1888                   (C = dyn_cast<Constant>(StoredValue))) {1889    // This might be a fp constant stored as an int. Bitcast and store if it has1890    // appropriate size.1891    Type *BitcastTy = nullptr; // The FT type to bitcast to.1892    if (auto *CInt = dyn_cast<ConstantInt>(C)) {1893      switch (CInt->getType()->getScalarSizeInBits()) {1894      case 32:1895        BitcastTy = Type::getFloatTy(Context);1896        break;1897      case 64:1898        BitcastTy = Type::getDoubleTy(Context);1899        break;1900      case 80:1901        BitcastTy = Type::getX86_FP80Ty(Context);1902        break;1903      default:1904        break;1905      }1906    } else if (auto *CDV = dyn_cast<ConstantDataVector>(C)) {1907      const int NumElements =1908          cast<VectorType>(CDV->getType())->getElementCount().getFixedValue();1909      switch (CDV->getType()->getScalarSizeInBits()) {1910      case 32:1911        BitcastTy =1912            VectorType::get(Type::getFloatTy(Context), NumElements, false);1913        break;1914      case 64:1915        BitcastTy =1916            VectorType::get(Type::getDoubleTy(Context), NumElements, false);1917        break;1918      case 80:1919        BitcastTy =1920            VectorType::get(Type::getX86_FP80Ty(Context), NumElements, false);1921        break;1922      default:1923        break;1924      }1925    }1926    if (BitcastTy) {1927      const MemoryExtents Extents = getMemoryExtentsOrDie(BitcastTy);1928      Value *ShadowPtr = Builder.CreateCall(1929          NsanGetShadowPtrForStore[Extents.ValueType],1930          {PtrOp, ConstantInt::get(IntptrTy, Extents.NumElts)});1931      // Bitcast the integer value to the appropriate FT type and extend to 2FT.1932      Type *ExtVT = Config.getExtendedFPType(BitcastTy);1933      Value *Shadow =1934          Builder.CreateFPExt(Builder.CreateBitCast(C, BitcastTy), ExtVT);1935      Builder.CreateAlignedStore(Shadow, ShadowPtr, Align(1),1936                                 Store.isVolatile());1937      return;1938    }1939  }1940  // All other stores just reset the shadow value to unknown.1941  Builder.CreateCall(NsanSetUnknownFns.getFallback(), {Dst, ValueSize});1942}1943 1944void NumericalStabilitySanitizer::propagateShadowValues(1945    Instruction &Inst, const TargetLibraryInfo &TLI,1946    const ValueToShadowMap &Map) {1947  if (auto *Store = dyn_cast<StoreInst>(&Inst)) {1948    Value *StoredValue = Store->getValueOperand();1949    Type *VT = StoredValue->getType();1950    Type *ExtendedVT = Config.getExtendedFPType(VT);1951    if (ExtendedVT == nullptr)1952      return propagateNonFTStore(*Store, VT, Map);1953    return propagateFTStore(*Store, VT, ExtendedVT, Map);1954  }1955 1956  if (auto *FCmp = dyn_cast<FCmpInst>(&Inst)) {1957    emitFCmpCheck(*FCmp, Map);1958    return;1959  }1960 1961  if (auto *CB = dyn_cast<CallBase>(&Inst)) {1962    maybeAddSuffixForNsanInterface(CB);1963    if (CallInst *CI = dyn_cast<CallInst>(&Inst))1964      maybeMarkSanitizerLibraryCallNoBuiltin(CI, &TLI);1965    if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(&Inst)) {1966      instrumentMemIntrinsic(MI);1967      return;1968    }1969    populateShadowStack(*CB, TLI, Map);1970    return;1971  }1972 1973  if (auto *RetInst = dyn_cast<ReturnInst>(&Inst)) {1974    if (!ClCheckRet)1975      return;1976 1977    Value *RV = RetInst->getReturnValue();1978    if (RV == nullptr)1979      return; // This is a `ret void`.1980    Type *VT = RV->getType();1981    Type *ExtendedVT = Config.getExtendedFPType(VT);1982    if (ExtendedVT == nullptr)1983      return; // Not an FT ret.1984    Value *RVShadow = Map.getShadow(RV);1985    IRBuilder<> Builder(RetInst);1986 1987    RVShadow = emitCheck(RV, RVShadow, Builder, CheckLoc::makeRet());1988    ++NumInstrumentedFTRets;1989    // Store tag.1990    Value *FnAddr =1991        Builder.CreatePtrToInt(Inst.getParent()->getParent(), IntptrTy);1992    Builder.CreateStore(FnAddr, NsanShadowRetTag);1993    // Store value.1994    Value *ShadowRetValPtr =1995        Builder.CreateConstGEP2_64(NsanShadowRetType, NsanShadowRetPtr, 0, 0);1996    Builder.CreateStore(RVShadow, ShadowRetValPtr);1997    return;1998  }1999 2000  if (InsertValueInst *Insert = dyn_cast<InsertValueInst>(&Inst)) {2001    Value *V = Insert->getOperand(1);2002    Type *VT = V->getType();2003    Type *ExtendedVT = Config.getExtendedFPType(VT);2004    if (ExtendedVT == nullptr)2005      return;2006    IRBuilder<> Builder(Insert);2007    emitCheck(V, Map.getShadow(V), Builder, CheckLoc::makeInsert());2008    return;2009  }2010}2011 2012// Moves fast math flags from the function to individual instructions, and2013// removes the attribute from the function.2014// TODO: Make this controllable with a flag.2015static void moveFastMathFlags(Function &F,2016                              std::vector<Instruction *> &Instructions) {2017  FastMathFlags FMF;2018#define MOVE_FLAG(attr, setter)                                                \2019  if (F.getFnAttribute(attr).getValueAsString() == "true") {                   \2020    F.removeFnAttr(attr);                                                      \2021    FMF.set##setter();                                                         \2022  }2023  MOVE_FLAG("no-infs-fp-math", NoInfs)2024  MOVE_FLAG("no-nans-fp-math", NoNaNs)2025  MOVE_FLAG("no-signed-zeros-fp-math", NoSignedZeros)2026#undef MOVE_FLAG2027 2028  for (Instruction *I : Instructions)2029    if (isa<FPMathOperator>(I))2030      I->setFastMathFlags(FMF);2031}2032 2033bool NumericalStabilitySanitizer::sanitizeFunction(2034    Function &F, const TargetLibraryInfo &TLI) {2035  if (!F.hasFnAttribute(Attribute::SanitizeNumericalStability) ||2036      F.isDeclaration())2037    return false;2038 2039  // This is required to prevent instrumenting call to __nsan_init from within2040  // the module constructor.2041  if (F.getName() == kNsanModuleCtorName)2042    return false;2043 2044  // The instrumentation maintains:2045  //  - for each IR value `v` of floating-point (or vector floating-point) type2046  //    FT, a shadow IR value `s(v)` with twice the precision 2FT (e.g.2047  //    double for float and f128 for double).2048  //  - A shadow memory, which stores `s(v)` for any `v` that has been stored,2049  //    along with a shadow memory tag, which stores whether the value in the2050  //    corresponding shadow memory is valid. Note that this might be2051  //    incorrect if a non-instrumented function stores to memory, or if2052  //    memory is stored to through a char pointer.2053  //  - A shadow stack, which holds `s(v)` for any floating-point argument `v`2054  //    of a call to an instrumented function. This allows2055  //    instrumented functions to retrieve the shadow values for their2056  //    arguments.2057  //    Because instrumented functions can be called from non-instrumented2058  //    functions, the stack needs to include a tag so that the instrumented2059  //    function knows whether shadow values are available for their2060  //    parameters (i.e. whether is was called by an instrumented function).2061  //    When shadow arguments are not available, they have to be recreated by2062  //    extending the precision of the non-shadow arguments to the non-shadow2063  //    value. Non-instrumented functions do not modify (or even know about) the2064  //    shadow stack. The shadow stack pointer is __nsan_shadow_args. The shadow2065  //    stack tag is __nsan_shadow_args_tag. The tag is any unique identifier2066  //    for the function (we use the address of the function). Both variables2067  //    are thread local.2068  //    Example:2069  //     calls                             shadow stack tag      shadow stack2070  //     =======================================================================2071  //     non_instrumented_1()              0                     02072  //             |2073  //             v2074  //     instrumented_2(float a)           0                     02075  //             |2076  //             v2077  //     instrumented_3(float b, double c) &instrumented_3       s(b),s(c)2078  //             |2079  //             v2080  //     instrumented_4(float d)           &instrumented_4       s(d)2081  //             |2082  //             v2083  //     non_instrumented_5(float e)       &non_instrumented_5   s(e)2084  //             |2085  //             v2086  //     instrumented_6(float f)           &non_instrumented_5   s(e)2087  //2088  //   On entry, instrumented_2 checks whether the tag corresponds to its2089  //   function ptr.2090  //   Note that functions reset the tag to 0 after reading shadow parameters.2091  //   This ensures that the function does not erroneously read invalid data if2092  //   called twice in the same stack, once from an instrumented function and2093  //   once from an uninstrumented one. For example, in the following example,2094  //   resetting the tag in (A) ensures that (B) does not reuse the same the2095  //   shadow arguments (which would be incorrect).2096  //      instrumented_1(float a)2097  //             |2098  //             v2099  //      instrumented_2(float b)  (A)2100  //             |2101  //             v2102  //      non_instrumented_3()2103  //             |2104  //             v2105  //      instrumented_2(float b)  (B)2106  //2107  //  - A shadow return slot. Any function that returns a floating-point value2108  //    places a shadow return value in __nsan_shadow_ret_val. Again, because2109  //    we might be calling non-instrumented functions, this value is guarded2110  //    by __nsan_shadow_ret_tag marker indicating which instrumented function2111  //    placed the value in __nsan_shadow_ret_val, so that the caller can check2112  //    that this corresponds to the callee. Both variables are thread local.2113  //2114  //    For example, in the following example, the instrumentation in2115  //    `instrumented_1` rejects the shadow return value from `instrumented_3`2116  //    because is is not tagged as expected (`&instrumented_3` instead of2117  //    `non_instrumented_2`):2118  //2119  //        instrumented_1()2120  //            |2121  //            v2122  //        float non_instrumented_2()2123  //            |2124  //            v2125  //        float instrumented_3()2126  //2127  // Calls of known math functions (sin, cos, exp, ...) are duplicated to call2128  // their overload on the shadow type.2129 2130  // Collect all instructions before processing, as creating shadow values2131  // creates new instructions inside the function.2132  std::vector<Instruction *> OriginalInstructions;2133  for (BasicBlock &BB : F)2134    for (Instruction &Inst : BB)2135      OriginalInstructions.emplace_back(&Inst);2136 2137  moveFastMathFlags(F, OriginalInstructions);2138  ValueToShadowMap ValueToShadow(Config);2139 2140  // In the first pass, we create shadow values for all FT function arguments2141  // and all phis. This ensures that the DFS of the next pass does not have2142  // any loops.2143  std::vector<PHINode *> OriginalPhis;2144  createShadowArguments(F, TLI, ValueToShadow);2145  for (Instruction *I : OriginalInstructions) {2146    if (PHINode *Phi = dyn_cast<PHINode>(I)) {2147      if (PHINode *Shadow = maybeCreateShadowPhi(*Phi, TLI)) {2148        OriginalPhis.push_back(Phi);2149        ValueToShadow.setShadow(*Phi, *Shadow);2150      }2151    }2152  }2153 2154  // Create shadow values for all instructions creating FT values.2155  for (Instruction *I : OriginalInstructions)2156    maybeCreateShadowValue(*I, TLI, ValueToShadow);2157 2158  // Propagate shadow values across stores, calls and rets.2159  for (Instruction *I : OriginalInstructions)2160    propagateShadowValues(*I, TLI, ValueToShadow);2161 2162  // The last pass populates shadow phis with shadow values.2163  for (PHINode *Phi : OriginalPhis) {2164    PHINode *ShadowPhi = cast<PHINode>(ValueToShadow.getShadow(Phi));2165    for (unsigned I : seq(Phi->getNumOperands())) {2166      Value *V = Phi->getOperand(I);2167      Value *Shadow = ValueToShadow.getShadow(V);2168      BasicBlock *IncomingBB = Phi->getIncomingBlock(I);2169      // For some instructions (e.g. invoke), we create the shadow in a separate2170      // block, different from the block where the original value is created.2171      // In that case, the shadow phi might need to refer to this block instead2172      // of the original block.2173      // Note that this can only happen for instructions as constant shadows are2174      // always created in the same block.2175      ShadowPhi->addIncoming(Shadow, IncomingBB);2176    }2177  }2178 2179  return !ValueToShadow.empty();2180}2181 2182static uint64_t GetMemOpSize(Value *V) {2183  uint64_t OpSize = 0;2184  if (Constant *C = dyn_cast<Constant>(V)) {2185    auto *CInt = dyn_cast<ConstantInt>(C);2186    if (CInt && CInt->getValue().getBitWidth() <= 64)2187      OpSize = CInt->getValue().getZExtValue();2188  }2189 2190  return OpSize;2191}2192 2193// Instrument the memory intrinsics so that they properly modify the shadow2194// memory.2195bool NumericalStabilitySanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {2196  IRBuilder<> Builder(MI);2197  if (auto *M = dyn_cast<MemSetInst>(MI)) {2198    FunctionCallee SetUnknownFn =2199        NsanSetUnknownFns.getFunctionFor(GetMemOpSize(M->getArgOperand(2)));2200    if (SetUnknownFn.getFunctionType()->getNumParams() == 1)2201      Builder.CreateCall(SetUnknownFn, {/*Address=*/M->getArgOperand(0)});2202    else2203      Builder.CreateCall(SetUnknownFn,2204                         {/*Address=*/M->getArgOperand(0),2205                          /*Size=*/Builder.CreateIntCast(M->getArgOperand(2),2206                                                         IntptrTy, false)});2207 2208  } else if (auto *M = dyn_cast<MemTransferInst>(MI)) {2209    FunctionCallee CopyFn =2210        NsanCopyFns.getFunctionFor(GetMemOpSize(M->getArgOperand(2)));2211 2212    if (CopyFn.getFunctionType()->getNumParams() == 2)2213      Builder.CreateCall(CopyFn, {/*Destination=*/M->getArgOperand(0),2214                                  /*Source=*/M->getArgOperand(1)});2215    else2216      Builder.CreateCall(CopyFn, {/*Destination=*/M->getArgOperand(0),2217                                  /*Source=*/M->getArgOperand(1),2218                                  /*Size=*/2219                                  Builder.CreateIntCast(M->getArgOperand(2),2220                                                        IntptrTy, false)});2221  }2222  return false;2223}2224 2225void NumericalStabilitySanitizer::maybeAddSuffixForNsanInterface(CallBase *CI) {2226  Function *Fn = CI->getCalledFunction();2227  if (Fn == nullptr)2228    return;2229 2230  if (!Fn->getName().starts_with("__nsan_"))2231    return;2232 2233  if (Fn->getName() == "__nsan_dump_shadow_mem") {2234    assert(CI->arg_size() == 4 &&2235           "invalid prototype for __nsan_dump_shadow_mem");2236    // __nsan_dump_shadow_mem requires an extra parameter with the dynamic2237    // configuration:2238    // (shadow_type_id_for_long_double << 16) | (shadow_type_id_for_double << 8)2239    // | shadow_type_id_for_double2240    const uint64_t shadow_value_type_ids =2241        (static_cast<size_t>(Config.byValueType(kLongDouble).getNsanTypeId())2242         << 16) |2243        (static_cast<size_t>(Config.byValueType(kDouble).getNsanTypeId())2244         << 8) |2245        static_cast<size_t>(Config.byValueType(kFloat).getNsanTypeId());2246    CI->setArgOperand(3, ConstantInt::get(IntptrTy, shadow_value_type_ids));2247  }2248}2249