335 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * KMSAN compiler API.4 *5 * This file implements __msan_XXX hooks that Clang inserts into the code6 * compiled with -fsanitize=kernel-memory.7 * See Documentation/dev-tools/kmsan.rst for more information on how KMSAN8 * instrumentation works.9 *10 * Copyright (C) 2017-2022 Google LLC11 * Author: Alexander Potapenko <glider@google.com>12 *13 */14 15#include "kmsan.h"16#include <linux/gfp.h>17#include <linux/kmsan.h>18#include <linux/kmsan_string.h>19#include <linux/mm.h>20#include <linux/uaccess.h>21 22static inline bool is_bad_asm_addr(void *addr, uintptr_t size, bool is_store)23{24 if (IS_ENABLED(CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE) &&25 (u64)addr < TASK_SIZE)26 return true;27 if (!kmsan_get_metadata(addr, KMSAN_META_SHADOW))28 return true;29 return false;30}31 32static inline struct shadow_origin_ptr33get_shadow_origin_ptr(void *addr, u64 size, bool store)34{35 unsigned long ua_flags = user_access_save();36 struct shadow_origin_ptr ret;37 38 ret = kmsan_get_shadow_origin_ptr(addr, size, store);39 user_access_restore(ua_flags);40 return ret;41}42 43/*44 * KMSAN instrumentation functions follow. They are not declared elsewhere in45 * the kernel code, so they are preceded by prototypes, to silence46 * -Wmissing-prototypes warnings.47 */48 49/* Get shadow and origin pointers for a memory load with non-standard size. */50struct shadow_origin_ptr __msan_metadata_ptr_for_load_n(void *addr,51 uintptr_t size);52struct shadow_origin_ptr __msan_metadata_ptr_for_load_n(void *addr,53 uintptr_t size)54{55 return get_shadow_origin_ptr(addr, size, /*store*/ false);56}57EXPORT_SYMBOL(__msan_metadata_ptr_for_load_n);58 59/* Get shadow and origin pointers for a memory store with non-standard size. */60struct shadow_origin_ptr __msan_metadata_ptr_for_store_n(void *addr,61 uintptr_t size);62struct shadow_origin_ptr __msan_metadata_ptr_for_store_n(void *addr,63 uintptr_t size)64{65 return get_shadow_origin_ptr(addr, size, /*store*/ true);66}67EXPORT_SYMBOL(__msan_metadata_ptr_for_store_n);68 69/*70 * Declare functions that obtain shadow/origin pointers for loads and stores71 * with fixed size.72 */73#define DECLARE_METADATA_PTR_GETTER(size) \74 struct shadow_origin_ptr __msan_metadata_ptr_for_load_##size( \75 void *addr); \76 struct shadow_origin_ptr __msan_metadata_ptr_for_load_##size( \77 void *addr) \78 { \79 return get_shadow_origin_ptr(addr, size, /*store*/ false); \80 } \81 EXPORT_SYMBOL(__msan_metadata_ptr_for_load_##size); \82 struct shadow_origin_ptr __msan_metadata_ptr_for_store_##size( \83 void *addr); \84 struct shadow_origin_ptr __msan_metadata_ptr_for_store_##size( \85 void *addr) \86 { \87 return get_shadow_origin_ptr(addr, size, /*store*/ true); \88 } \89 EXPORT_SYMBOL(__msan_metadata_ptr_for_store_##size)90 91DECLARE_METADATA_PTR_GETTER(1);92DECLARE_METADATA_PTR_GETTER(2);93DECLARE_METADATA_PTR_GETTER(4);94DECLARE_METADATA_PTR_GETTER(8);95 96/*97 * Handle a memory store performed by inline assembly. KMSAN conservatively98 * attempts to unpoison the outputs of asm() directives to prevent false99 * positives caused by missed stores.100 *101 * __msan_instrument_asm_store() may be called for inline assembly code when102 * entering or leaving IRQ. We omit the check for kmsan_in_runtime() to ensure103 * the memory written to in these cases is also marked as initialized.104 */105void __msan_instrument_asm_store(void *addr, uintptr_t size);106void __msan_instrument_asm_store(void *addr, uintptr_t size)107{108 unsigned long ua_flags;109 110 if (!kmsan_enabled)111 return;112 113 ua_flags = user_access_save();114 /*115 * Most of the accesses are below 32 bytes. The exceptions so far are116 * clwb() (64 bytes), FPU state (512 bytes) and chsc() (4096 bytes).117 */118 if (size > 4096) {119 WARN_ONCE(1, "assembly store size too big: %ld\n", size);120 size = 8;121 }122 if (is_bad_asm_addr(addr, size, /*is_store*/ true)) {123 user_access_restore(ua_flags);124 return;125 }126 /* Unpoisoning the memory on best effort. */127 kmsan_internal_unpoison_memory(addr, size, /*checked*/ false);128 user_access_restore(ua_flags);129}130EXPORT_SYMBOL(__msan_instrument_asm_store);131 132/*133 * KMSAN instrumentation pass replaces LLVM memcpy, memmove and memset134 * intrinsics with calls to respective __msan_ functions. We use135 * get_param0_metadata() and set_retval_metadata() to store the shadow/origin136 * values for the destination argument of these functions and use them for the137 * functions' return values.138 */139static inline void get_param0_metadata(u64 *shadow,140 depot_stack_handle_t *origin)141{142 struct kmsan_ctx *ctx = kmsan_get_context();143 144 *shadow = *(u64 *)(ctx->cstate.param_tls);145 *origin = ctx->cstate.param_origin_tls[0];146}147 148static inline void set_retval_metadata(u64 shadow, depot_stack_handle_t origin)149{150 struct kmsan_ctx *ctx = kmsan_get_context();151 152 *(u64 *)(ctx->cstate.retval_tls) = shadow;153 ctx->cstate.retval_origin_tls = origin;154}155 156/* Handle llvm.memmove intrinsic. */157void *__msan_memmove(void *dst, const void *src, uintptr_t n);158void *__msan_memmove(void *dst, const void *src, uintptr_t n)159{160 depot_stack_handle_t origin;161 void *result;162 u64 shadow;163 164 get_param0_metadata(&shadow, &origin);165 result = __memmove(dst, src, n);166 if (!n)167 /* Some people call memmove() with zero length. */168 return result;169 if (!kmsan_enabled || kmsan_in_runtime())170 return result;171 172 kmsan_enter_runtime();173 kmsan_internal_memmove_metadata(dst, (void *)src, n);174 kmsan_leave_runtime();175 176 set_retval_metadata(shadow, origin);177 return result;178}179EXPORT_SYMBOL(__msan_memmove);180 181/* Handle llvm.memcpy intrinsic. */182void *__msan_memcpy(void *dst, const void *src, uintptr_t n);183void *__msan_memcpy(void *dst, const void *src, uintptr_t n)184{185 depot_stack_handle_t origin;186 void *result;187 u64 shadow;188 189 get_param0_metadata(&shadow, &origin);190 result = __memcpy(dst, src, n);191 if (!n)192 /* Some people call memcpy() with zero length. */193 return result;194 195 if (!kmsan_enabled || kmsan_in_runtime())196 return result;197 198 kmsan_enter_runtime();199 /* Using memmove instead of memcpy doesn't affect correctness. */200 kmsan_internal_memmove_metadata(dst, (void *)src, n);201 kmsan_leave_runtime();202 203 set_retval_metadata(shadow, origin);204 return result;205}206EXPORT_SYMBOL(__msan_memcpy);207 208/* Handle llvm.memset intrinsic. */209void *__msan_memset(void *dst, int c, uintptr_t n);210void *__msan_memset(void *dst, int c, uintptr_t n)211{212 depot_stack_handle_t origin;213 void *result;214 u64 shadow;215 216 get_param0_metadata(&shadow, &origin);217 result = __memset(dst, c, n);218 if (!kmsan_enabled || kmsan_in_runtime())219 return result;220 221 kmsan_enter_runtime();222 /*223 * Clang doesn't pass parameter metadata here, so it is impossible to224 * use shadow of @c to set up the shadow for @dst.225 */226 kmsan_internal_unpoison_memory(dst, n, /*checked*/ false);227 kmsan_leave_runtime();228 229 set_retval_metadata(shadow, origin);230 return result;231}232EXPORT_SYMBOL(__msan_memset);233 234/*235 * Create a new origin from an old one. This is done when storing an236 * uninitialized value to memory. When reporting an error, KMSAN unrolls and237 * prints the whole chain of stores that preceded the use of this value.238 */239depot_stack_handle_t __msan_chain_origin(depot_stack_handle_t origin);240depot_stack_handle_t __msan_chain_origin(depot_stack_handle_t origin)241{242 depot_stack_handle_t ret = 0;243 unsigned long ua_flags;244 245 if (!kmsan_enabled || kmsan_in_runtime())246 return ret;247 248 ua_flags = user_access_save();249 250 /* Creating new origins may allocate memory. */251 kmsan_enter_runtime();252 ret = kmsan_internal_chain_origin(origin);253 kmsan_leave_runtime();254 user_access_restore(ua_flags);255 return ret;256}257EXPORT_SYMBOL(__msan_chain_origin);258 259/* Poison a local variable when entering a function. */260void __msan_poison_alloca(void *address, uintptr_t size, char *descr);261void __msan_poison_alloca(void *address, uintptr_t size, char *descr)262{263 depot_stack_handle_t handle;264 unsigned long entries[4];265 unsigned long ua_flags;266 267 if (!kmsan_enabled || kmsan_in_runtime())268 return;269 270 ua_flags = user_access_save();271 entries[0] = KMSAN_ALLOCA_MAGIC_ORIGIN;272 entries[1] = (u64)descr;273 entries[2] = (u64)__builtin_return_address(0);274 /*275 * With frame pointers enabled, it is possible to quickly fetch the276 * second frame of the caller stack without calling the unwinder.277 * Without them, simply do not bother.278 */279 if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER))280 entries[3] = (u64)__builtin_return_address(1);281 else282 entries[3] = 0;283 284 /* stack_depot_save() may allocate memory. */285 kmsan_enter_runtime();286 handle = stack_depot_save(entries, ARRAY_SIZE(entries), __GFP_HIGH);287 kmsan_leave_runtime();288 289 kmsan_internal_set_shadow_origin(address, size, -1, handle,290 /*checked*/ true);291 user_access_restore(ua_flags);292}293EXPORT_SYMBOL(__msan_poison_alloca);294 295/* Unpoison a local variable. */296void __msan_unpoison_alloca(void *address, uintptr_t size);297void __msan_unpoison_alloca(void *address, uintptr_t size)298{299 if (!kmsan_enabled || kmsan_in_runtime())300 return;301 302 kmsan_enter_runtime();303 kmsan_internal_unpoison_memory(address, size, /*checked*/ true);304 kmsan_leave_runtime();305}306EXPORT_SYMBOL(__msan_unpoison_alloca);307 308/*309 * Report that an uninitialized value with the given origin was used in a way310 * that constituted undefined behavior.311 */312void __msan_warning(u32 origin);313void __msan_warning(u32 origin)314{315 if (!kmsan_enabled || kmsan_in_runtime())316 return;317 kmsan_enter_runtime();318 kmsan_report(origin, /*address*/ NULL, /*size*/ 0,319 /*off_first*/ 0, /*off_last*/ 0, /*user_addr*/ NULL,320 REASON_ANY);321 kmsan_leave_runtime();322}323EXPORT_SYMBOL(__msan_warning);324 325/*326 * At the beginning of an instrumented function, obtain the pointer to327 * `struct kmsan_context_state` holding the metadata for function parameters.328 */329struct kmsan_context_state *__msan_get_context_state(void);330struct kmsan_context_state *__msan_get_context_state(void)331{332 return &kmsan_get_context()->cstate;333}334EXPORT_SYMBOL(__msan_get_context_state);335