332 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * KFENCE reporting.4 *5 * Copyright (C) 2020, Google LLC.6 */7 8#include <linux/stdarg.h>9 10#include <linux/kernel.h>11#include <linux/lockdep.h>12#include <linux/math.h>13#include <linux/printk.h>14#include <linux/sched/debug.h>15#include <linux/seq_file.h>16#include <linux/sprintf.h>17#include <linux/stacktrace.h>18#include <linux/string.h>19#include <linux/sched/clock.h>20#include <trace/events/error_report.h>21 22#include <asm/kfence.h>23 24#include "kfence.h"25 26/* May be overridden by <asm/kfence.h>. */27#ifndef ARCH_FUNC_PREFIX28#define ARCH_FUNC_PREFIX ""29#endif30 31/* Helper function to either print to a seq_file or to console. */32__printf(2, 3)33static void seq_con_printf(struct seq_file *seq, const char *fmt, ...)34{35 va_list args;36 37 va_start(args, fmt);38 if (seq)39 seq_vprintf(seq, fmt, args);40 else41 vprintk(fmt, args);42 va_end(args);43}44 45/*46 * Get the number of stack entries to skip to get out of MM internals. @type is47 * optional, and if set to NULL, assumes an allocation or free stack.48 */49static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries,50 const enum kfence_error_type *type)51{52 char buf[64];53 int skipnr, fallback = 0;54 55 if (type) {56 /* Depending on error type, find different stack entries. */57 switch (*type) {58 case KFENCE_ERROR_UAF:59 case KFENCE_ERROR_OOB:60 case KFENCE_ERROR_INVALID:61 /*62 * kfence_handle_page_fault() may be called with pt_regs63 * set to NULL; in that case we'll simply show the full64 * stack trace.65 */66 return 0;67 case KFENCE_ERROR_CORRUPTION:68 case KFENCE_ERROR_INVALID_FREE:69 break;70 }71 }72 73 for (skipnr = 0; skipnr < num_entries; skipnr++) {74 int len = scnprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skipnr]);75 76 if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfence_") ||77 str_has_prefix(buf, ARCH_FUNC_PREFIX "__kfence_") ||78 str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmem_cache_free") ||79 !strncmp(buf, ARCH_FUNC_PREFIX "__slab_free", len)) {80 /*81 * In case of tail calls from any of the below to any of82 * the above, optimized by the compiler such that the83 * stack trace would omit the initial entry point below.84 */85 fallback = skipnr + 1;86 }87 88 /*89 * The below list should only include the initial entry points90 * into the slab allocators. Includes the *_bulk() variants by91 * checking prefixes.92 */93 if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfree") ||94 str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_free") ||95 str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmalloc") ||96 str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_alloc"))97 goto found;98 }99 if (fallback < num_entries)100 return fallback;101found:102 skipnr++;103 return skipnr < num_entries ? skipnr : 0;104}105 106static void kfence_print_stack(struct seq_file *seq, const struct kfence_metadata *meta,107 bool show_alloc)108{109 const struct kfence_track *track = show_alloc ? &meta->alloc_track : &meta->free_track;110 u64 ts_sec = track->ts_nsec;111 unsigned long rem_nsec = do_div(ts_sec, NSEC_PER_SEC);112 u64 interval_nsec = local_clock() - track->ts_nsec;113 unsigned long rem_interval_nsec = do_div(interval_nsec, NSEC_PER_SEC);114 115 /* Timestamp matches printk timestamp format. */116 seq_con_printf(seq, "%s by task %d on cpu %d at %lu.%06lus (%lu.%06lus ago):\n",117 show_alloc ? "allocated" : meta->state == KFENCE_OBJECT_RCU_FREEING ?118 "rcu freeing" : "freed", track->pid,119 track->cpu, (unsigned long)ts_sec, rem_nsec / 1000,120 (unsigned long)interval_nsec, rem_interval_nsec / 1000);121 122 if (track->num_stack_entries) {123 /* Skip allocation/free internals stack. */124 int i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL);125 126 /* stack_trace_seq_print() does not exist; open code our own. */127 for (; i < track->num_stack_entries; i++)128 seq_con_printf(seq, " %pS\n", (void *)track->stack_entries[i]);129 } else {130 seq_con_printf(seq, " no %s stack\n", show_alloc ? "allocation" : "deallocation");131 }132}133 134void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta)135{136 const int size = abs(meta->size);137 const unsigned long start = meta->addr;138 const struct kmem_cache *const cache = meta->cache;139 140 lockdep_assert_held(&meta->lock);141 142 if (meta->state == KFENCE_OBJECT_UNUSED) {143 seq_con_printf(seq, "kfence-#%td unused\n", meta - kfence_metadata);144 return;145 }146 147 seq_con_printf(seq, "kfence-#%td: 0x%p-0x%p, size=%d, cache=%s\n\n",148 meta - kfence_metadata, (void *)start, (void *)(start + size - 1),149 size, (cache && cache->name) ? cache->name : "<destroyed>");150 151 kfence_print_stack(seq, meta, true);152 153 if (meta->state == KFENCE_OBJECT_FREED || meta->state == KFENCE_OBJECT_RCU_FREEING) {154 seq_con_printf(seq, "\n");155 kfence_print_stack(seq, meta, false);156 }157}158 159/*160 * Show bytes at @addr that are different from the expected canary values, up to161 * @max_bytes.162 */163static void print_diff_canary(unsigned long address, size_t bytes_to_show,164 const struct kfence_metadata *meta)165{166 const unsigned long show_until_addr = address + bytes_to_show;167 const u8 *cur, *end;168 169 /* Do not show contents of object nor read into following guard page. */170 end = (const u8 *)(address < meta->addr ? min(show_until_addr, meta->addr)171 : min(show_until_addr, PAGE_ALIGN(address)));172 173 pr_cont("[");174 for (cur = (const u8 *)address; cur < end; cur++) {175 if (*cur == KFENCE_CANARY_PATTERN_U8(cur))176 pr_cont(" .");177 else if (no_hash_pointers)178 pr_cont(" 0x%02x", *cur);179 else /* Do not leak kernel memory in non-debug builds. */180 pr_cont(" !");181 }182 pr_cont(" ]");183}184 185static const char *get_access_type(bool is_write)186{187 return is_write ? "write" : "read";188}189 190void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs,191 const struct kfence_metadata *meta, enum kfence_error_type type)192{193 unsigned long stack_entries[KFENCE_STACK_DEPTH] = { 0 };194 const ptrdiff_t object_index = meta ? meta - kfence_metadata : -1;195 int num_stack_entries;196 int skipnr = 0;197 198 if (regs) {199 num_stack_entries = stack_trace_save_regs(regs, stack_entries, KFENCE_STACK_DEPTH, 0);200 } else {201 num_stack_entries = stack_trace_save(stack_entries, KFENCE_STACK_DEPTH, 1);202 skipnr = get_stack_skipnr(stack_entries, num_stack_entries, &type);203 }204 205 /* Require non-NULL meta, except if KFENCE_ERROR_INVALID. */206 if (WARN_ON(type != KFENCE_ERROR_INVALID && !meta))207 return;208 209 if (meta)210 lockdep_assert_held(&meta->lock);211 /*212 * Because we may generate reports in printk-unfriendly parts of the213 * kernel, such as scheduler code, the use of printk() could deadlock.214 * Until such time that all printing code here is safe in all parts of215 * the kernel, accept the risk, and just get our message out (given the216 * system might already behave unpredictably due to the memory error).217 * As such, also disable lockdep to hide warnings, and avoid disabling218 * lockdep for the rest of the kernel.219 */220 lockdep_off();221 222 pr_err("==================================================================\n");223 /* Print report header. */224 switch (type) {225 case KFENCE_ERROR_OOB: {226 const bool left_of_object = address < meta->addr;227 228 pr_err("BUG: KFENCE: out-of-bounds %s in %pS\n\n", get_access_type(is_write),229 (void *)stack_entries[skipnr]);230 pr_err("Out-of-bounds %s at 0x%p (%luB %s of kfence-#%td):\n",231 get_access_type(is_write), (void *)address,232 left_of_object ? meta->addr - address : address - meta->addr,233 left_of_object ? "left" : "right", object_index);234 break;235 }236 case KFENCE_ERROR_UAF:237 pr_err("BUG: KFENCE: use-after-free %s in %pS\n\n", get_access_type(is_write),238 (void *)stack_entries[skipnr]);239 pr_err("Use-after-free %s at 0x%p (in kfence-#%td):\n",240 get_access_type(is_write), (void *)address, object_index);241 break;242 case KFENCE_ERROR_CORRUPTION:243 pr_err("BUG: KFENCE: memory corruption in %pS\n\n", (void *)stack_entries[skipnr]);244 pr_err("Corrupted memory at 0x%p ", (void *)address);245 print_diff_canary(address, 16, meta);246 pr_cont(" (in kfence-#%td):\n", object_index);247 break;248 case KFENCE_ERROR_INVALID:249 pr_err("BUG: KFENCE: invalid %s in %pS\n\n", get_access_type(is_write),250 (void *)stack_entries[skipnr]);251 pr_err("Invalid %s at 0x%p:\n", get_access_type(is_write),252 (void *)address);253 break;254 case KFENCE_ERROR_INVALID_FREE:255 pr_err("BUG: KFENCE: invalid free in %pS\n\n", (void *)stack_entries[skipnr]);256 pr_err("Invalid free of 0x%p (in kfence-#%td):\n", (void *)address,257 object_index);258 break;259 }260 261 /* Print stack trace and object info. */262 stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr, 0);263 264 if (meta) {265 pr_err("\n");266 kfence_print_object(NULL, meta);267 }268 269 /* Print report footer. */270 pr_err("\n");271 if (no_hash_pointers && regs)272 show_regs(regs);273 else274 dump_stack_print_info(KERN_ERR);275 trace_error_report_end(ERROR_DETECTOR_KFENCE, address);276 pr_err("==================================================================\n");277 278 lockdep_on();279 280 check_panic_on_warn("KFENCE");281 282 /* We encountered a memory safety error, taint the kernel! */283 add_taint(TAINT_BAD_PAGE, LOCKDEP_STILL_OK);284}285 286#ifdef CONFIG_PRINTK287static void kfence_to_kp_stack(const struct kfence_track *track, void **kp_stack)288{289 int i, j;290 291 i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL);292 for (j = 0; i < track->num_stack_entries && j < KS_ADDRS_COUNT; ++i, ++j)293 kp_stack[j] = (void *)track->stack_entries[i];294 if (j < KS_ADDRS_COUNT)295 kp_stack[j] = NULL;296}297 298bool __kfence_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)299{300 struct kfence_metadata *meta = addr_to_metadata((unsigned long)object);301 unsigned long flags;302 303 if (!meta)304 return false;305 306 /*307 * If state is UNUSED at least show the pointer requested; the rest308 * would be garbage data.309 */310 kpp->kp_ptr = object;311 312 /* Requesting info an a never-used object is almost certainly a bug. */313 if (WARN_ON(meta->state == KFENCE_OBJECT_UNUSED))314 return true;315 316 raw_spin_lock_irqsave(&meta->lock, flags);317 318 kpp->kp_slab = slab;319 kpp->kp_slab_cache = meta->cache;320 kpp->kp_objp = (void *)meta->addr;321 kfence_to_kp_stack(&meta->alloc_track, kpp->kp_stack);322 if (meta->state == KFENCE_OBJECT_FREED || meta->state == KFENCE_OBJECT_RCU_FREEING)323 kfence_to_kp_stack(&meta->free_track, kpp->kp_free_stack);324 /* get_stack_skipnr() ensures the first entry is outside allocator. */325 kpp->kp_ret = kpp->kp_stack[0];326 327 raw_spin_unlock_irqrestore(&meta->lock, flags);328 329 return true;330}331#endif332