440 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * KMSAN hooks for kernel subsystems.4 *5 * These functions handle creation of KMSAN metadata for memory allocations.6 *7 * Copyright (C) 2018-2022 Google LLC8 * Author: Alexander Potapenko <glider@google.com>9 *10 */11 12#include <linux/cacheflush.h>13#include <linux/dma-direction.h>14#include <linux/gfp.h>15#include <linux/kmsan.h>16#include <linux/mm.h>17#include <linux/mm_types.h>18#include <linux/scatterlist.h>19#include <linux/slab.h>20#include <linux/uaccess.h>21#include <linux/usb.h>22 23#include "../internal.h"24#include "../slab.h"25#include "kmsan.h"26 27/*28 * Instrumented functions shouldn't be called under29 * kmsan_enter_runtime()/kmsan_leave_runtime(), because this will lead to30 * skipping effects of functions like memset() inside instrumented code.31 */32 33void kmsan_task_create(struct task_struct *task)34{35 kmsan_enter_runtime();36 kmsan_internal_task_create(task);37 kmsan_leave_runtime();38}39 40void kmsan_task_exit(struct task_struct *task)41{42 if (!kmsan_enabled || kmsan_in_runtime())43 return;44 45 kmsan_disable_current();46}47 48void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags)49{50 if (unlikely(object == NULL))51 return;52 if (!kmsan_enabled || kmsan_in_runtime())53 return;54 /*55 * There's a ctor or this is an RCU cache - do nothing. The memory56 * status hasn't changed since last use.57 */58 if (s->ctor || (s->flags & SLAB_TYPESAFE_BY_RCU))59 return;60 61 kmsan_enter_runtime();62 if (flags & __GFP_ZERO)63 kmsan_internal_unpoison_memory(object, s->object_size,64 KMSAN_POISON_CHECK);65 else66 kmsan_internal_poison_memory(object, s->object_size, flags,67 KMSAN_POISON_CHECK);68 kmsan_leave_runtime();69}70 71void kmsan_slab_free(struct kmem_cache *s, void *object)72{73 if (!kmsan_enabled || kmsan_in_runtime())74 return;75 76 /* RCU slabs could be legally used after free within the RCU period */77 if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU))78 return;79 /*80 * If there's a constructor, freed memory must remain in the same state81 * until the next allocation. We cannot save its state to detect82 * use-after-free bugs, instead we just keep it unpoisoned.83 */84 if (s->ctor)85 return;86 kmsan_enter_runtime();87 kmsan_internal_poison_memory(object, s->object_size, GFP_KERNEL,88 KMSAN_POISON_CHECK | KMSAN_POISON_FREE);89 kmsan_leave_runtime();90}91 92void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)93{94 if (unlikely(ptr == NULL))95 return;96 if (!kmsan_enabled || kmsan_in_runtime())97 return;98 kmsan_enter_runtime();99 if (flags & __GFP_ZERO)100 kmsan_internal_unpoison_memory((void *)ptr, size,101 /*checked*/ true);102 else103 kmsan_internal_poison_memory((void *)ptr, size, flags,104 KMSAN_POISON_CHECK);105 kmsan_leave_runtime();106}107 108void kmsan_kfree_large(const void *ptr)109{110 struct page *page;111 112 if (!kmsan_enabled || kmsan_in_runtime())113 return;114 kmsan_enter_runtime();115 page = virt_to_head_page((void *)ptr);116 KMSAN_WARN_ON(ptr != page_address(page));117 kmsan_internal_poison_memory((void *)ptr,118 page_size(page),119 GFP_KERNEL,120 KMSAN_POISON_CHECK | KMSAN_POISON_FREE);121 kmsan_leave_runtime();122}123 124static unsigned long vmalloc_shadow(unsigned long addr)125{126 return (unsigned long)kmsan_get_metadata((void *)addr,127 KMSAN_META_SHADOW);128}129 130static unsigned long vmalloc_origin(unsigned long addr)131{132 return (unsigned long)kmsan_get_metadata((void *)addr,133 KMSAN_META_ORIGIN);134}135 136void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end)137{138 __vunmap_range_noflush(vmalloc_shadow(start), vmalloc_shadow(end));139 __vunmap_range_noflush(vmalloc_origin(start), vmalloc_origin(end));140 flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));141 flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));142}143 144/*145 * This function creates new shadow/origin pages for the physical pages mapped146 * into the virtual memory. If those physical pages already had shadow/origin,147 * those are ignored.148 */149int kmsan_ioremap_page_range(unsigned long start, unsigned long end,150 phys_addr_t phys_addr, pgprot_t prot,151 unsigned int page_shift)152{153 gfp_t gfp_mask = GFP_KERNEL | __GFP_ZERO;154 struct page *shadow, *origin;155 unsigned long off = 0;156 int nr, err = 0, clean = 0, mapped;157 158 if (!kmsan_enabled || kmsan_in_runtime())159 return 0;160 161 nr = (end - start) / PAGE_SIZE;162 kmsan_enter_runtime();163 for (int i = 0; i < nr; i++, off += PAGE_SIZE, clean = i) {164 shadow = alloc_pages(gfp_mask, 1);165 origin = alloc_pages(gfp_mask, 1);166 if (!shadow || !origin) {167 err = -ENOMEM;168 goto ret;169 }170 mapped = __vmap_pages_range_noflush(171 vmalloc_shadow(start + off),172 vmalloc_shadow(start + off + PAGE_SIZE), prot, &shadow,173 PAGE_SHIFT);174 if (mapped) {175 err = mapped;176 goto ret;177 }178 shadow = NULL;179 mapped = __vmap_pages_range_noflush(180 vmalloc_origin(start + off),181 vmalloc_origin(start + off + PAGE_SIZE), prot, &origin,182 PAGE_SHIFT);183 if (mapped) {184 __vunmap_range_noflush(185 vmalloc_shadow(start + off),186 vmalloc_shadow(start + off + PAGE_SIZE));187 err = mapped;188 goto ret;189 }190 origin = NULL;191 }192 /* Page mapping loop finished normally, nothing to clean up. */193 clean = 0;194 195ret:196 if (clean > 0) {197 /*198 * Something went wrong. Clean up shadow/origin pages allocated199 * on the last loop iteration, then delete mappings created200 * during the previous iterations.201 */202 if (shadow)203 __free_pages(shadow, 1);204 if (origin)205 __free_pages(origin, 1);206 __vunmap_range_noflush(207 vmalloc_shadow(start),208 vmalloc_shadow(start + clean * PAGE_SIZE));209 __vunmap_range_noflush(210 vmalloc_origin(start),211 vmalloc_origin(start + clean * PAGE_SIZE));212 }213 flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));214 flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));215 kmsan_leave_runtime();216 return err;217}218 219void kmsan_iounmap_page_range(unsigned long start, unsigned long end)220{221 unsigned long v_shadow, v_origin;222 struct page *shadow, *origin;223 int nr;224 225 if (!kmsan_enabled || kmsan_in_runtime())226 return;227 228 nr = (end - start) / PAGE_SIZE;229 kmsan_enter_runtime();230 v_shadow = (unsigned long)vmalloc_shadow(start);231 v_origin = (unsigned long)vmalloc_origin(start);232 for (int i = 0; i < nr;233 i++, v_shadow += PAGE_SIZE, v_origin += PAGE_SIZE) {234 shadow = kmsan_vmalloc_to_page_or_null((void *)v_shadow);235 origin = kmsan_vmalloc_to_page_or_null((void *)v_origin);236 __vunmap_range_noflush(v_shadow, vmalloc_shadow(end));237 __vunmap_range_noflush(v_origin, vmalloc_origin(end));238 if (shadow)239 __free_pages(shadow, 1);240 if (origin)241 __free_pages(origin, 1);242 }243 flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));244 flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));245 kmsan_leave_runtime();246}247 248void kmsan_copy_to_user(void __user *to, const void *from, size_t to_copy,249 size_t left)250{251 unsigned long ua_flags;252 253 if (!kmsan_enabled || kmsan_in_runtime())254 return;255 /*256 * At this point we've copied the memory already. It's hard to check it257 * before copying, as the size of actually copied buffer is unknown.258 */259 260 /* copy_to_user() may copy zero bytes. No need to check. */261 if (!to_copy)262 return;263 /* Or maybe copy_to_user() failed to copy anything. */264 if (to_copy <= left)265 return;266 267 ua_flags = user_access_save();268 if (!IS_ENABLED(CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE) ||269 (u64)to < TASK_SIZE) {270 /* This is a user memory access, check it. */271 kmsan_internal_check_memory((void *)from, to_copy - left, to,272 REASON_COPY_TO_USER);273 } else {274 /* Otherwise this is a kernel memory access. This happens when a275 * compat syscall passes an argument allocated on the kernel276 * stack to a real syscall.277 * Don't check anything, just copy the shadow of the copied278 * bytes.279 */280 kmsan_internal_memmove_metadata((void *)to, (void *)from,281 to_copy - left);282 }283 user_access_restore(ua_flags);284}285EXPORT_SYMBOL(kmsan_copy_to_user);286 287void kmsan_memmove(void *to, const void *from, size_t size)288{289 if (!kmsan_enabled || kmsan_in_runtime())290 return;291 292 kmsan_enter_runtime();293 kmsan_internal_memmove_metadata(to, (void *)from, size);294 kmsan_leave_runtime();295}296EXPORT_SYMBOL(kmsan_memmove);297 298/* Helper function to check an URB. */299void kmsan_handle_urb(const struct urb *urb, bool is_out)300{301 if (!urb)302 return;303 if (is_out)304 kmsan_internal_check_memory(urb->transfer_buffer,305 urb->transfer_buffer_length,306 /*user_addr*/ NULL,307 REASON_SUBMIT_URB);308 else309 kmsan_internal_unpoison_memory(urb->transfer_buffer,310 urb->transfer_buffer_length,311 /*checked*/ false);312}313EXPORT_SYMBOL_GPL(kmsan_handle_urb);314 315static void kmsan_handle_dma_page(const void *addr, size_t size,316 enum dma_data_direction dir)317{318 switch (dir) {319 case DMA_BIDIRECTIONAL:320 kmsan_internal_check_memory((void *)addr, size,321 /*user_addr*/ NULL, REASON_ANY);322 kmsan_internal_unpoison_memory((void *)addr, size,323 /*checked*/ false);324 break;325 case DMA_TO_DEVICE:326 kmsan_internal_check_memory((void *)addr, size,327 /*user_addr*/ NULL, REASON_ANY);328 break;329 case DMA_FROM_DEVICE:330 kmsan_internal_unpoison_memory((void *)addr, size,331 /*checked*/ false);332 break;333 case DMA_NONE:334 break;335 }336}337 338/* Helper function to handle DMA data transfers. */339void kmsan_handle_dma(struct page *page, size_t offset, size_t size,340 enum dma_data_direction dir)341{342 u64 page_offset, to_go, addr;343 344 if (PageHighMem(page))345 return;346 addr = (u64)page_address(page) + offset;347 /*348 * The kernel may occasionally give us adjacent DMA pages not belonging349 * to the same allocation. Process them separately to avoid triggering350 * internal KMSAN checks.351 */352 while (size > 0) {353 page_offset = offset_in_page(addr);354 to_go = min(PAGE_SIZE - page_offset, (u64)size);355 kmsan_handle_dma_page((void *)addr, to_go, dir);356 addr += to_go;357 size -= to_go;358 }359}360 361void kmsan_handle_dma_sg(struct scatterlist *sg, int nents,362 enum dma_data_direction dir)363{364 struct scatterlist *item;365 int i;366 367 for_each_sg(sg, item, nents, i)368 kmsan_handle_dma(sg_page(item), item->offset, item->length,369 dir);370}371 372/* Functions from kmsan-checks.h follow. */373 374/*375 * To create an origin, kmsan_poison_memory() unwinds the stacks and stores it376 * into the stack depot. This may cause deadlocks if done from within KMSAN377 * runtime, therefore we bail out if kmsan_in_runtime().378 */379void kmsan_poison_memory(const void *address, size_t size, gfp_t flags)380{381 if (!kmsan_enabled || kmsan_in_runtime())382 return;383 kmsan_enter_runtime();384 /* The users may want to poison/unpoison random memory. */385 kmsan_internal_poison_memory((void *)address, size, flags,386 KMSAN_POISON_NOCHECK);387 kmsan_leave_runtime();388}389EXPORT_SYMBOL(kmsan_poison_memory);390 391/*392 * Unlike kmsan_poison_memory(), this function can be used from within KMSAN393 * runtime, because it does not trigger allocations or call instrumented code.394 */395void kmsan_unpoison_memory(const void *address, size_t size)396{397 unsigned long ua_flags;398 399 if (!kmsan_enabled)400 return;401 402 ua_flags = user_access_save();403 /* The users may want to poison/unpoison random memory. */404 kmsan_internal_unpoison_memory((void *)address, size,405 KMSAN_POISON_NOCHECK);406 user_access_restore(ua_flags);407}408EXPORT_SYMBOL(kmsan_unpoison_memory);409 410/*411 * Version of kmsan_unpoison_memory() called from IRQ entry functions.412 */413void kmsan_unpoison_entry_regs(const struct pt_regs *regs)414{415 kmsan_unpoison_memory((void *)regs, sizeof(*regs));416}417 418void kmsan_check_memory(const void *addr, size_t size)419{420 if (!kmsan_enabled)421 return;422 return kmsan_internal_check_memory((void *)addr, size,423 /*user_addr*/ NULL, REASON_ANY);424}425EXPORT_SYMBOL(kmsan_check_memory);426 427void kmsan_enable_current(void)428{429 KMSAN_WARN_ON(current->kmsan_ctx.depth == 0);430 current->kmsan_ctx.depth--;431}432EXPORT_SYMBOL(kmsan_enable_current);433 434void kmsan_disable_current(void)435{436 current->kmsan_ctx.depth++;437 KMSAN_WARN_ON(current->kmsan_ctx.depth == 0);438}439EXPORT_SYMBOL(kmsan_disable_current);440