869 lines · cpp
1//===-- nsan.cc -----------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// NumericalStabilitySanitizer runtime.10//11// This implements:12// - The public nsan interface (include/sanitizer/nsan_interface.h).13// - The private nsan interface (./nsan.h).14// - The internal instrumentation interface. These are function emitted by the15// instrumentation pass:16// * __nsan_get_shadow_ptr_for_{float,double,longdouble}_load17// These return the shadow memory pointer for loading the shadow value,18// after checking that the types are consistent. If the types are not19// consistent, returns nullptr.20// * __nsan_get_shadow_ptr_for_{float,double,longdouble}_store21// Sets the shadow types appropriately and returns the shadow memory22// pointer for storing the shadow value.23// * __nsan_internal_check_{float,double,long double}_{f,d,l} checks the24// accuracy of a value against its shadow and emits a warning depending25// on the runtime configuration. The middle part indicates the type of26// the application value, the suffix (f,d,l) indicates the type of the27// shadow, and depends on the instrumentation configuration.28// * __nsan_fcmp_fail_* emits a warning for a fcmp instruction whose29// corresponding shadow fcmp result differs.30//31//===----------------------------------------------------------------------===//32 33#include "nsan.h"34#include "nsan_flags.h"35#include "nsan_stats.h"36#include "nsan_suppressions.h"37#include "nsan_thread.h"38 39#include <assert.h>40#include <math.h>41#include <stdint.h>42#include <stdio.h>43#include <stdlib.h>44 45#include "sanitizer_common/sanitizer_atomic.h"46#include "sanitizer_common/sanitizer_common.h"47#include "sanitizer_common/sanitizer_libc.h"48#include "sanitizer_common/sanitizer_report_decorator.h"49#include "sanitizer_common/sanitizer_stacktrace.h"50#include "sanitizer_common/sanitizer_symbolizer.h"51 52using namespace __sanitizer;53using namespace __nsan;54 55constexpr int kMaxVectorWidth = 8;56 57// When copying application memory, we also copy its shadow and shadow type.58extern "C" SANITIZER_INTERFACE_ATTRIBUTE void59__nsan_copy_values(const void *daddr, const void *saddr, uptr size) {60 internal_memmove(GetShadowTypeAddrFor(daddr), GetShadowTypeAddrFor(saddr),61 size);62 internal_memmove(GetShadowAddrFor(daddr), GetShadowAddrFor(saddr),63 size * kShadowScale);64}65 66#define NSAN_COPY_VALUES_N(N) \67 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_copy_##N( \68 const u8 *daddr, const u8 *saddr) { \69 __builtin_memmove(GetShadowTypeAddrFor(daddr), \70 GetShadowTypeAddrFor(saddr), N); \71 __builtin_memmove(GetShadowAddrFor(daddr), GetShadowAddrFor(saddr), \72 N *kShadowScale); \73 }74 75NSAN_COPY_VALUES_N(4)76NSAN_COPY_VALUES_N(8)77NSAN_COPY_VALUES_N(16)78 79extern "C" SANITIZER_INTERFACE_ATTRIBUTE void80__nsan_set_value_unknown(const void *addr, uptr size) {81 internal_memset(GetShadowTypeAddrFor(addr), 0, size);82}83 84#define NSAN_SET_VALUE_UNKNOWN_N(N) \85 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_set_value_unknown_##N( \86 const u8 *daddr) { \87 __builtin_memset(GetShadowTypeAddrFor(daddr), 0, N); \88 }89 90NSAN_SET_VALUE_UNKNOWN_N(4)91NSAN_SET_VALUE_UNKNOWN_N(8)92NSAN_SET_VALUE_UNKNOWN_N(16)93 94const char *FTInfo<float>::kCppTypeName = "float";95const char *FTInfo<double>::kCppTypeName = "double";96const char *FTInfo<long double>::kCppTypeName = "long double";97const char *FTInfo<__float128>::kCppTypeName = "__float128";98 99const char FTInfo<float>::kTypePattern[sizeof(float)];100const char FTInfo<double>::kTypePattern[sizeof(double)];101const char FTInfo<long double>::kTypePattern[sizeof(long double)];102 103// Helper for __nsan_dump_shadow_mem: Reads the value at address `ptr`,104// identified by its type id.105template <typename ShadowFT>106static __float128 ReadShadowInternal(const u8 *ptr) {107 ShadowFT Shadow;108 __builtin_memcpy(&Shadow, ptr, sizeof(Shadow));109 return Shadow;110}111 112static __float128 ReadShadow(const u8 *ptr, const char ShadowTypeId) {113 switch (ShadowTypeId) {114 case 'd':115 return ReadShadowInternal<double>(ptr);116 case 'l':117 return ReadShadowInternal<long double>(ptr);118 case 'q':119 return ReadShadowInternal<__float128>(ptr);120 default:121 return 0.0;122 }123}124 125namespace {126class Decorator : public __sanitizer::SanitizerCommonDecorator {127public:128 Decorator() : SanitizerCommonDecorator() {}129 const char *Warning() { return Red(); }130 const char *Name() { return Green(); }131 const char *End() { return Default(); }132};133 134// Workaround for the fact that Printf() does not support floats.135struct PrintBuffer {136 char Buffer[64];137};138template <typename FT> struct FTPrinter {};139 140template <> struct FTPrinter<double> {141 static PrintBuffer dec(double value) {142 PrintBuffer result;143 snprintf(result.Buffer, sizeof(result.Buffer) - 1, "%.20f", value);144 return result;145 }146 static PrintBuffer hex(double value) {147 PrintBuffer result;148 snprintf(result.Buffer, sizeof(result.Buffer) - 1, "%.20a", value);149 return result;150 }151};152 153template <> struct FTPrinter<float> : FTPrinter<double> {};154 155template <> struct FTPrinter<long double> {156 static PrintBuffer dec(long double value) {157 PrintBuffer result;158 snprintf(result.Buffer, sizeof(result.Buffer) - 1, "%.20Lf", value);159 return result;160 }161 static PrintBuffer hex(long double value) {162 PrintBuffer result;163 snprintf(result.Buffer, sizeof(result.Buffer) - 1, "%.20La", value);164 return result;165 }166};167 168// FIXME: print with full precision.169template <> struct FTPrinter<__float128> : FTPrinter<long double> {};170 171// This is a template so that there are no implicit conversions.172template <typename FT> inline FT ftAbs(FT v);173 174template <> inline long double ftAbs(long double v) { return fabsl(v); }175template <> inline double ftAbs(double v) { return fabs(v); }176 177// We don't care about nans.178// std::abs(__float128) code is suboptimal and generates a function call to179// __getf2().180template <typename FT> inline FT ftAbs(FT v) { return v >= FT{0} ? v : -v; }181 182template <typename FT1, typename FT2, bool Enable> struct LargestFTImpl {183 using type = FT2;184};185 186template <typename FT1, typename FT2> struct LargestFTImpl<FT1, FT2, true> {187 using type = FT1;188};189 190template <typename FT1, typename FT2>191using LargestFT =192 typename LargestFTImpl<FT1, FT2, (sizeof(FT1) > sizeof(FT2))>::type;193 194template <typename T> T max(T a, T b) { return a < b ? b : a; }195 196} // end anonymous namespace197 198void __sanitizer::BufferedStackTrace::UnwindImpl(uptr pc, uptr bp,199 void *context,200 bool request_fast,201 u32 max_depth) {202 using namespace __nsan;203 NsanThread *t = GetCurrentThread();204 if (!t || !StackTrace::WillUseFastUnwind(request_fast))205 return Unwind(max_depth, pc, bp, context, t ? t->stack_top() : 0,206 t ? t->stack_bottom() : 0, false);207 if (StackTrace::WillUseFastUnwind(request_fast))208 Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), true);209 else210 Unwind(max_depth, pc, 0, context, 0, 0, false);211}212 213extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_print_accumulated_stats() {214 if (nsan_stats)215 nsan_stats->Print();216}217 218static void NsanAtexit() {219 Printf("Numerical Sanitizer exit stats:\n");220 __nsan_print_accumulated_stats();221 nsan_stats = nullptr;222}223 224// The next three functions return a pointer for storing a shadow value for `n`225// values, after setting the shadow types. We return the pointer instead of226// storing ourselves because it avoids having to rely on the calling convention227// around long double being the same for nsan and the target application.228// We have to have 3 versions because we need to know which type we are storing229// since we are setting the type shadow memory.230template <typename FT> static u8 *getShadowPtrForStore(u8 *store_addr, uptr n) {231 unsigned char *shadow_type = GetShadowTypeAddrFor(store_addr);232 for (uptr i = 0; i < n; ++i) {233 __builtin_memcpy(shadow_type + i * sizeof(FT), FTInfo<FT>::kTypePattern,234 sizeof(FTInfo<FT>::kTypePattern));235 }236 return GetShadowAddrFor(store_addr);237}238 239extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *240__nsan_get_shadow_ptr_for_float_store(u8 *store_addr, uptr n) {241 return getShadowPtrForStore<float>(store_addr, n);242}243 244extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *245__nsan_get_shadow_ptr_for_double_store(u8 *store_addr, uptr n) {246 return getShadowPtrForStore<double>(store_addr, n);247}248 249extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *250__nsan_get_shadow_ptr_for_longdouble_store(u8 *store_addr, uptr n) {251 return getShadowPtrForStore<long double>(store_addr, n);252}253 254template <typename FT> static bool IsValidShadowType(const u8 *shadow_type) {255 return __builtin_memcmp(shadow_type, FTInfo<FT>::kTypePattern, sizeof(FT)) ==256 0;257}258 259template <int kSize, typename T> static bool IsZero(const T *ptr) {260 constexpr const char kZeros[kSize] = {}; // Zero initialized.261 return __builtin_memcmp(ptr, kZeros, kSize) == 0;262}263 264template <typename FT> static bool IsUnknownShadowType(const u8 *shadow_type) {265 return IsZero<sizeof(FTInfo<FT>::kTypePattern)>(shadow_type);266}267 268// The three folowing functions check that the address stores a complete269// shadow value of the given type and return a pointer for loading.270// They return nullptr if the type of the value is unknown or incomplete.271template <typename FT>272static const u8 *getShadowPtrForLoad(const u8 *load_addr, uptr n) {273 const u8 *const shadow_type = GetShadowTypeAddrFor(load_addr);274 for (uptr i = 0; i < n; ++i) {275 if (!IsValidShadowType<FT>(shadow_type + i * sizeof(FT))) {276 // If loadtracking stats are enabled, log loads with invalid types277 // (tampered with through type punning).278 if (flags().enable_loadtracking_stats) {279 if (IsUnknownShadowType<FT>(shadow_type + i * sizeof(FT))) {280 // Warn only if the value is non-zero. Zero is special because281 // applications typically initialize large buffers to zero in an282 // untyped way.283 if (!IsZero<sizeof(FT)>(load_addr)) {284 GET_CALLER_PC_BP;285 nsan_stats->AddUnknownLoadTrackingEvent(pc, bp);286 }287 } else {288 GET_CALLER_PC_BP;289 nsan_stats->AddInvalidLoadTrackingEvent(pc, bp);290 }291 }292 return nullptr;293 }294 }295 return GetShadowAddrFor(load_addr);296}297 298extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *299__nsan_get_shadow_ptr_for_float_load(const u8 *load_addr, uptr n) {300 return getShadowPtrForLoad<float>(load_addr, n);301}302 303extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *304__nsan_get_shadow_ptr_for_double_load(const u8 *load_addr, uptr n) {305 return getShadowPtrForLoad<double>(load_addr, n);306}307 308extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *309__nsan_get_shadow_ptr_for_longdouble_load(const u8 *load_addr, uptr n) {310 return getShadowPtrForLoad<long double>(load_addr, n);311}312 313// Returns the raw shadow pointer. The returned pointer should be considered314// opaque.315extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *316__nsan_internal_get_raw_shadow_ptr(const u8 *addr) {317 return GetShadowAddrFor(addr);318}319 320// Returns the raw shadow type pointer. The returned pointer should be321// considered opaque.322extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *323__nsan_internal_get_raw_shadow_type_ptr(const u8 *addr) {324 return reinterpret_cast<u8 *>(GetShadowTypeAddrFor(addr));325}326 327static ValueType getValueType(u8 c) { return static_cast<ValueType>(c & 0x3); }328 329static int getValuePos(u8 c) { return c >> kValueSizeSizeBits; }330 331// Checks the consistency of the value types at the given type pointer.332// If the value is inconsistent, returns ValueType::kUnknown. Else, return the333// consistent type.334template <typename FT>335static bool checkValueConsistency(const u8 *shadow_type) {336 const int pos = getValuePos(*shadow_type);337 // Check that all bytes from the start of the value are ordered.338 for (uptr i = 0; i < sizeof(FT); ++i) {339 const u8 T = *(shadow_type - pos + i);340 if (!(getValueType(T) == FTInfo<FT>::kValueType && getValuePos(T) == i))341 return false;342 }343 return true;344}345 346// The instrumentation automatically appends `shadow_value_type_ids`, see347// maybeAddSuffixForNsanInterface.348extern "C" SANITIZER_INTERFACE_ATTRIBUTE void349__nsan_dump_shadow_mem(const u8 *addr, size_t size_bytes, size_t bytes_per_line,350 size_t shadow_value_type_ids) {351 const u8 *const shadow_type = GetShadowTypeAddrFor(addr);352 const u8 *const shadow = GetShadowAddrFor(addr);353 354 constexpr int kMaxNumDecodedValues = 16;355 __float128 decoded_values[kMaxNumDecodedValues];356 int num_decoded_values = 0;357 if (bytes_per_line > 4 * kMaxNumDecodedValues)358 bytes_per_line = 4 * kMaxNumDecodedValues;359 360 // We keep track of the current type and position as we go.361 ValueType LastValueTy = kUnknownValueType;362 int LastPos = -1;363 size_t Offset = 0;364 for (size_t R = 0; R < (size_bytes + bytes_per_line - 1) / bytes_per_line;365 ++R) {366 printf("%p: ", (void *)(addr + R * bytes_per_line));367 for (size_t C = 0; C < bytes_per_line && Offset < size_bytes; ++C) {368 const ValueType ValueTy = getValueType(shadow_type[Offset]);369 const int pos = getValuePos(shadow_type[Offset]);370 if (ValueTy == LastValueTy && pos == LastPos + 1) {371 ++LastPos;372 } else {373 LastValueTy = ValueTy;374 LastPos = pos == 0 ? 0 : -1;375 }376 377 switch (ValueTy) {378 case kUnknownValueType:379 printf("__ ");380 break;381 case kFloatValueType:382 printf("f%x ", pos);383 if (LastPos == sizeof(float) - 1) {384 decoded_values[num_decoded_values] =385 ReadShadow(shadow + kShadowScale * (Offset + 1 - sizeof(float)),386 static_cast<char>(shadow_value_type_ids & 0xff));387 ++num_decoded_values;388 }389 break;390 case kDoubleValueType:391 printf("d%x ", pos);392 if (LastPos == sizeof(double) - 1) {393 decoded_values[num_decoded_values] = ReadShadow(394 shadow + kShadowScale * (Offset + 1 - sizeof(double)),395 static_cast<char>((shadow_value_type_ids >> 8) & 0xff));396 ++num_decoded_values;397 }398 break;399 case kFp80ValueType:400 printf("l%x ", pos);401 if (LastPos == sizeof(long double) - 1) {402 decoded_values[num_decoded_values] = ReadShadow(403 shadow + kShadowScale * (Offset + 1 - sizeof(long double)),404 static_cast<char>((shadow_value_type_ids >> 16) & 0xff));405 ++num_decoded_values;406 }407 break;408 }409 ++Offset;410 }411 for (int i = 0; i < num_decoded_values; ++i) {412 printf(" (%s)", FTPrinter<__float128>::dec(decoded_values[i]).Buffer);413 }414 num_decoded_values = 0;415 printf("\n");416 }417}418 419alignas(64) SANITIZER_INTERFACE_ATTRIBUTE420 thread_local uptr __nsan_shadow_ret_tag = 0;421 422alignas(64) SANITIZER_INTERFACE_ATTRIBUTE423 thread_local char __nsan_shadow_ret_ptr[kMaxVectorWidth *424 sizeof(__float128)];425 426alignas(64) SANITIZER_INTERFACE_ATTRIBUTE427 thread_local uptr __nsan_shadow_args_tag = 0;428 429// Maximum number of args. This should be enough for anyone (tm). An alternate430// scheme is to have the generated code create an alloca and make431// __nsan_shadow_args_ptr point ot the alloca.432constexpr const int kMaxNumArgs = 128;433alignas(64) SANITIZER_INTERFACE_ATTRIBUTE434 thread_local char __nsan_shadow_args_ptr[kMaxVectorWidth * kMaxNumArgs *435 sizeof(__float128)];436 437enum ContinuationType { // Keep in sync with instrumentation pass.438 kContinueWithShadow = 0,439 kResumeFromValue = 1,440};441 442// Checks the consistency between application and shadow value. Returns true443// when the instrumented code should resume computations from the original value444// rather than the shadow value. This prevents one error to propagate to all445// subsequent operations. This behaviour is tunable with flags.446template <typename FT, typename ShadowFT>447int32_t checkFT(const FT value, ShadowFT Shadow, CheckTypeT CheckType,448 uptr CheckArg) {449 // We do all comparisons in the InternalFT domain, which is the largest FT450 // type.451 using InternalFT = LargestFT<FT, ShadowFT>;452 const InternalFT check_value = value;453 const InternalFT check_shadow = Shadow;454 455 // We only check for NaNs in the value, not the shadow.456 if (flags().check_nan && isnan(value)) {457 GET_CALLER_PC_BP;458 BufferedStackTrace stack;459 stack.Unwind(pc, bp, nullptr, false);460 if (GetSuppressionForStack(&stack, CheckKind::Consistency)) {461 // FIXME: optionally print.462 return flags().resume_after_suppression ? kResumeFromValue463 : kContinueWithShadow;464 }465 Decorator D;466 Printf("%s", D.Warning());467 Printf("WARNING: NumericalStabilitySanitizer: NaN detected\n");468 Printf("%s", D.Default());469 stack.Print();470 if (flags().halt_on_error) {471 if (common_flags()->abort_on_error)472 Printf("ABORTING\n");473 else474 Printf("Exiting\n");475 Die();476 }477 // Performing other tests for NaN values is meaningless when dealing with numbers.478 return kResumeFromValue;479 }480 481 // See this article for an interesting discussion of how to compare floats:482 // https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/483 static constexpr const FT Eps = FTInfo<FT>::kEpsilon;484 485 const InternalFT abs_err = ftAbs(check_value - check_shadow);486 487 if (flags().enable_check_stats) {488 GET_CALLER_PC_BP;489 // We are re-computing `largest` here because this is a cold branch, and we490 // want to avoid having to move the computation of `largest` before the491 // absolute value check when this branch is not taken.492 const InternalFT largest = max(ftAbs(check_value), ftAbs(check_shadow));493 nsan_stats->AddCheck(CheckType, pc, bp, abs_err / largest);494 }495 496 // Note: writing the comparison that way ensures that when `abs_err` is Nan497 // (value and shadow are inf or -inf), we pass the test.498 if (!(abs_err >= flags().cached_absolute_error_threshold))499 return kContinueWithShadow;500 501 const InternalFT largest = max(ftAbs(check_value), ftAbs(check_shadow));502 if (abs_err * (1ull << flags().log2_max_relative_error) <= largest)503 return kContinueWithShadow; // No problem here.504 505 if (!flags().disable_warnings) {506 GET_CALLER_PC_BP;507 UNINITIALIZED BufferedStackTrace stack;508 stack.Unwind(pc, bp, nullptr, false);509 if (GetSuppressionForStack(&stack, CheckKind::Consistency)) {510 // FIXME: optionally print.511 return flags().resume_after_suppression ? kResumeFromValue512 : kContinueWithShadow;513 }514 515 Decorator D;516 Printf("%s", D.Warning());517 // Printf does not support float formatting.518 char RelErrBuf[64] = "inf";519 if (largest > Eps) {520 snprintf(RelErrBuf, sizeof(RelErrBuf) - 1, "%.20Lf%% (2^%.0Lf epsilons)",521 static_cast<long double>(100.0 * abs_err / largest),522 log2l(static_cast<long double>(abs_err / largest / Eps)));523 }524 char ulp_err_buf[128] = "";525 const double shadow_ulp_diff = GetULPDiff(check_value, check_shadow);526 if (shadow_ulp_diff != kMaxULPDiff) {527 // This is the ULP diff in the internal domain. The user actually cares528 // about that in the original domain.529 const double ulp_diff =530 shadow_ulp_diff / (u64{1} << (FTInfo<InternalFT>::kMantissaBits -531 FTInfo<FT>::kMantissaBits));532 snprintf(ulp_err_buf, sizeof(ulp_err_buf) - 1,533 "(%.0f ULPs == %.1f digits == %.1f bits)", ulp_diff,534 log10(ulp_diff), log2(ulp_diff));535 }536 Printf("WARNING: NumericalStabilitySanitizer: inconsistent shadow results");537 switch (CheckType) {538 case CheckTypeT::kUnknown:539 case CheckTypeT::kFcmp:540 case CheckTypeT::kMaxCheckType:541 break;542 case CheckTypeT::kRet:543 Printf(" while checking return value");544 break;545 case CheckTypeT::kArg:546 Printf(" while checking call argument #%d", static_cast<int>(CheckArg));547 break;548 case CheckTypeT::kLoad:549 Printf(550 " while checking load from address 0x%lx. This is due to incorrect "551 "shadow memory tracking, typically due to uninstrumented code "552 "writing to memory.",553 CheckArg);554 break;555 case CheckTypeT::kStore:556 Printf(" while checking store to address 0x%lx", CheckArg);557 break;558 case CheckTypeT::kInsert:559 Printf(" while checking vector insert");560 break;561 case CheckTypeT::kUser:562 Printf(" in user-initiated check");563 break;564 }565 using ValuePrinter = FTPrinter<FT>;566 using ShadowPrinter = FTPrinter<ShadowFT>;567 Printf("%s", D.Default());568 569 Printf("\n"570 "%-12s precision (native): dec: %s hex: %s\n"571 "%-12s precision (shadow): dec: %s hex: %s\n"572 "shadow truncated to %-12s: dec: %s hex: %s\n"573 "Relative error: %s\n"574 "Absolute error: %s\n"575 "%s\n",576 FTInfo<FT>::kCppTypeName, ValuePrinter::dec(value).Buffer,577 ValuePrinter::hex(value).Buffer, FTInfo<ShadowFT>::kCppTypeName,578 ShadowPrinter::dec(Shadow).Buffer, ShadowPrinter::hex(Shadow).Buffer,579 FTInfo<FT>::kCppTypeName, ValuePrinter::dec(Shadow).Buffer,580 ValuePrinter::hex(Shadow).Buffer, RelErrBuf,581 ValuePrinter::hex(abs_err).Buffer, ulp_err_buf);582 stack.Print();583 }584 585 if (flags().enable_warning_stats) {586 GET_CALLER_PC_BP;587 nsan_stats->AddWarning(CheckType, pc, bp, abs_err / largest);588 }589 590 if (flags().halt_on_error) {591 if (common_flags()->abort_on_error)592 Printf("ABORTING\n");593 else594 Printf("Exiting\n");595 Die();596 }597 return flags().resume_after_warning ? kResumeFromValue : kContinueWithShadow;598}599 600extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_float_d(601 float value, double shadow, int32_t check_type, uptr check_arg) {602 return checkFT(value, shadow, static_cast<CheckTypeT>(check_type), check_arg);603}604 605extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_double_l(606 double value, long double shadow, int32_t check_type, uptr check_arg) {607 return checkFT(value, shadow, static_cast<CheckTypeT>(check_type), check_arg);608}609 610extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_double_q(611 double value, __float128 shadow, int32_t check_type, uptr check_arg) {612 return checkFT(value, shadow, static_cast<CheckTypeT>(check_type), check_arg);613}614 615extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t616__nsan_internal_check_longdouble_q(long double value, __float128 shadow,617 int32_t check_type, uptr check_arg) {618 return checkFT(value, shadow, static_cast<CheckTypeT>(check_type), check_arg);619}620 621static const char *GetTruthValueName(bool v) { return v ? "true" : "false"; }622 623// This uses the same values as CmpInst::Predicate.624static const char *GetPredicateName(int v) {625 switch (v) {626 case 0:627 return "(false)";628 case 1:629 return "==";630 case 2:631 return ">";632 case 3:633 return ">=";634 case 4:635 return "<";636 case 5:637 return "<=";638 case 6:639 return "!=";640 case 7:641 return "(ordered)";642 case 8:643 return "(unordered)";644 case 9:645 return "==";646 case 10:647 return ">";648 case 11:649 return ">=";650 case 12:651 return "<";652 case 13:653 return "<=";654 case 14:655 return "!=";656 case 15:657 return "(true)";658 }659 return "??";660}661 662template <typename FT, typename ShadowFT>663void fCmpFailFT(const FT Lhs, const FT Rhs, ShadowFT LhsShadow,664 ShadowFT RhsShadow, int Predicate, bool result,665 bool ShadowResult) {666 if (result == ShadowResult) {667 // When a vector comparison fails, we fail each element of the comparison668 // to simplify instrumented code. Skip elements where the shadow comparison669 // gave the same result as the original one.670 return;671 }672 673 GET_CALLER_PC_BP;674 UNINITIALIZED BufferedStackTrace stack;675 stack.Unwind(pc, bp, nullptr, false);676 677 if (GetSuppressionForStack(&stack, CheckKind::Fcmp)) {678 // FIXME: optionally print.679 return;680 }681 682 if (flags().enable_warning_stats)683 nsan_stats->AddWarning(CheckTypeT::kFcmp, pc, bp, 0.0);684 685 if (flags().disable_warnings || !flags().check_cmp)686 return;687 688 // FIXME: ideally we would print the shadow value as FP128. Right now because689 // we truncate to long double we can sometimes see stuff like:690 // shadow <value> == <value> (false)691 using ValuePrinter = FTPrinter<FT>;692 using ShadowPrinter = FTPrinter<ShadowFT>;693 Decorator D;694 const char *const PredicateName = GetPredicateName(Predicate);695 Printf("%s", D.Warning());696 Printf("WARNING: NumericalStabilitySanitizer: floating-point comparison "697 "results depend on precision\n");698 Printf("%s", D.Default());699 Printf("%-12s precision dec (native): %s %s %s (%s)\n"700 "%-12s precision dec (shadow): %s %s %s (%s)\n"701 "%-12s precision hex (native): %s %s %s (%s)\n"702 "%-12s precision hex (shadow): %s %s %s (%s)\n"703 "%s",704 // Native, decimal.705 FTInfo<FT>::kCppTypeName, ValuePrinter::dec(Lhs).Buffer, PredicateName,706 ValuePrinter::dec(Rhs).Buffer, GetTruthValueName(result),707 // Shadow, decimal708 FTInfo<ShadowFT>::kCppTypeName, ShadowPrinter::dec(LhsShadow).Buffer,709 PredicateName, ShadowPrinter::dec(RhsShadow).Buffer,710 GetTruthValueName(ShadowResult),711 // Native, hex.712 FTInfo<FT>::kCppTypeName, ValuePrinter::hex(Lhs).Buffer, PredicateName,713 ValuePrinter::hex(Rhs).Buffer, GetTruthValueName(result),714 // Shadow, hex715 FTInfo<ShadowFT>::kCppTypeName, ShadowPrinter::hex(LhsShadow).Buffer,716 PredicateName, ShadowPrinter::hex(RhsShadow).Buffer,717 GetTruthValueName(ShadowResult), D.End());718 stack.Print();719 if (flags().halt_on_error) {720 Printf("Exiting\n");721 Die();722 }723}724 725extern "C" SANITIZER_INTERFACE_ATTRIBUTE void726__nsan_fcmp_fail_float_d(float lhs, float rhs, double lhs_shadow,727 double rhs_shadow, int predicate, bool result,728 bool shadow_result) {729 fCmpFailFT(lhs, rhs, lhs_shadow, rhs_shadow, predicate, result,730 shadow_result);731}732 733extern "C" SANITIZER_INTERFACE_ATTRIBUTE void734__nsan_fcmp_fail_double_q(double lhs, double rhs, __float128 lhs_shadow,735 __float128 rhs_shadow, int predicate, bool result,736 bool shadow_result) {737 fCmpFailFT(lhs, rhs, lhs_shadow, rhs_shadow, predicate, result,738 shadow_result);739}740 741extern "C" SANITIZER_INTERFACE_ATTRIBUTE void742__nsan_fcmp_fail_double_l(double lhs, double rhs, long double lhs_shadow,743 long double rhs_shadow, int predicate, bool result,744 bool shadow_result) {745 fCmpFailFT(lhs, rhs, lhs_shadow, rhs_shadow, predicate, result,746 shadow_result);747}748 749extern "C" SANITIZER_INTERFACE_ATTRIBUTE void750__nsan_fcmp_fail_longdouble_q(long double lhs, long double rhs,751 __float128 lhs_shadow, __float128 rhs_shadow,752 int predicate, bool result, bool shadow_result) {753 fCmpFailFT(lhs, rhs, lhs_shadow, rhs_shadow, predicate, result,754 shadow_result);755}756 757template <typename FT> void checkFTFromShadowStack(const FT value) {758 // Get the shadow 2FT value from the shadow stack. Note that759 // __nsan_check_{float,double,long double} is a function like any other, so760 // the instrumentation will have placed the shadow value on the shadow stack.761 using ShadowFT = typename FTInfo<FT>::shadow_type;762 ShadowFT Shadow;763 __builtin_memcpy(&Shadow, __nsan_shadow_args_ptr, sizeof(ShadowFT));764 checkFT(value, Shadow, CheckTypeT::kUser, 0);765}766 767// FIXME: Add suffixes and let the instrumentation pass automatically add768// suffixes.769extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_check_float(float value) {770 assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_float &&771 "__nsan_check_float called from non-instrumented function");772 checkFTFromShadowStack(value);773}774 775extern "C" SANITIZER_INTERFACE_ATTRIBUTE void776__nsan_check_double(double value) {777 assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_double &&778 "__nsan_check_double called from non-instrumented function");779 checkFTFromShadowStack(value);780}781 782extern "C" SANITIZER_INTERFACE_ATTRIBUTE void783__nsan_check_longdouble(long double value) {784 assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_longdouble &&785 "__nsan_check_longdouble called from non-instrumented function");786 checkFTFromShadowStack(value);787}788 789template <typename FT> static void dumpFTFromShadowStack(const FT value) {790 // Get the shadow 2FT value from the shadow stack. Note that791 // __nsan_dump_{float,double,long double} is a function like any other, so792 // the instrumentation will have placed the shadow value on the shadow stack.793 using ShadowFT = typename FTInfo<FT>::shadow_type;794 ShadowFT shadow;795 __builtin_memcpy(&shadow, __nsan_shadow_args_ptr, sizeof(ShadowFT));796 using ValuePrinter = FTPrinter<FT>;797 using ShadowPrinter = FTPrinter<typename FTInfo<FT>::shadow_type>;798 printf("value dec:%s hex:%s\n"799 "shadow dec:%s hex:%s\n",800 ValuePrinter::dec(value).Buffer, ValuePrinter::hex(value).Buffer,801 ShadowPrinter::dec(shadow).Buffer, ShadowPrinter::hex(shadow).Buffer);802}803 804extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_float(float value) {805 assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_float &&806 "__nsan_dump_float called from non-instrumented function");807 dumpFTFromShadowStack(value);808}809 810extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_double(double value) {811 assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_double &&812 "__nsan_dump_double called from non-instrumented function");813 dumpFTFromShadowStack(value);814}815 816extern "C" SANITIZER_INTERFACE_ATTRIBUTE void817__nsan_dump_longdouble(long double value) {818 assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_longdouble &&819 "__nsan_dump_longdouble called from non-instrumented function");820 dumpFTFromShadowStack(value);821}822 823extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_shadow_ret() {824 printf("ret tag: %lx\n", __nsan_shadow_ret_tag);825 double v;826 __builtin_memcpy(&v, __nsan_shadow_ret_ptr, sizeof(double));827 printf("double value: %f\n", v);828 // FIXME: float128 value.829}830 831extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_shadow_args() {832 printf("args tag: %lx\n", __nsan_shadow_args_tag);833}834 835bool __nsan::nsan_initialized;836bool __nsan::nsan_init_is_running;837 838extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_init() {839 CHECK(!nsan_init_is_running);840 if (nsan_initialized)841 return;842 nsan_init_is_running = true;843 SanitizerToolName = "NumericalStabilitySanitizer";844 845 InitializeFlags();846 InitializeSuppressions();847 InitializePlatformEarly();848 849 DisableCoreDumperIfNecessary();850 851 if (!MmapFixedNoReserve(TypesAddr(), AllocatorAddr() - TypesAddr()))852 Die();853 854 InitializeInterceptors();855 NsanTSDInit(NsanTSDDtor);856 NsanAllocatorInit();857 858 NsanThread *main_thread = NsanThread::Create(nullptr, nullptr);859 SetCurrentThread(main_thread);860 main_thread->Init();861 862 InitializeStats();863 if (flags().print_stats_on_exit)864 Atexit(NsanAtexit);865 866 nsan_init_is_running = false;867 nsan_initialized = true;868}869