231 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Access kernel or user memory without faulting.4 */5#include <linux/export.h>6#include <linux/mm.h>7#include <linux/uaccess.h>8#include <asm/tlb.h>9 10bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,11 size_t size)12{13 return true;14}15 16#define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \17 while (len >= sizeof(type)) { \18 __get_kernel_nofault(dst, src, type, err_label); \19 dst += sizeof(type); \20 src += sizeof(type); \21 len -= sizeof(type); \22 }23 24long copy_from_kernel_nofault(void *dst, const void *src, size_t size)25{26 unsigned long align = 0;27 28 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))29 align = (unsigned long)dst | (unsigned long)src;30 31 if (!copy_from_kernel_nofault_allowed(src, size))32 return -ERANGE;33 34 pagefault_disable();35 if (!(align & 7))36 copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);37 if (!(align & 3))38 copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);39 if (!(align & 1))40 copy_from_kernel_nofault_loop(dst, src, size, u16, Efault);41 copy_from_kernel_nofault_loop(dst, src, size, u8, Efault);42 pagefault_enable();43 return 0;44Efault:45 pagefault_enable();46 return -EFAULT;47}48EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);49 50#define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \51 while (len >= sizeof(type)) { \52 __put_kernel_nofault(dst, src, type, err_label); \53 dst += sizeof(type); \54 src += sizeof(type); \55 len -= sizeof(type); \56 }57 58long copy_to_kernel_nofault(void *dst, const void *src, size_t size)59{60 unsigned long align = 0;61 62 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))63 align = (unsigned long)dst | (unsigned long)src;64 65 pagefault_disable();66 if (!(align & 7))67 copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);68 if (!(align & 3))69 copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);70 if (!(align & 1))71 copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);72 copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);73 pagefault_enable();74 return 0;75Efault:76 pagefault_enable();77 return -EFAULT;78}79 80long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)81{82 const void *src = unsafe_addr;83 84 if (unlikely(count <= 0))85 return 0;86 if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))87 return -ERANGE;88 89 pagefault_disable();90 do {91 __get_kernel_nofault(dst, src, u8, Efault);92 dst++;93 src++;94 } while (dst[-1] && src - unsafe_addr < count);95 pagefault_enable();96 97 dst[-1] = '\0';98 return src - unsafe_addr;99Efault:100 pagefault_enable();101 dst[0] = '\0';102 return -EFAULT;103}104 105/**106 * copy_from_user_nofault(): safely attempt to read from a user-space location107 * @dst: pointer to the buffer that shall take the data108 * @src: address to read from. This must be a user address.109 * @size: size of the data chunk110 *111 * Safely read from user address @src to the buffer at @dst. If a kernel fault112 * happens, handle that and return -EFAULT.113 */114long copy_from_user_nofault(void *dst, const void __user *src, size_t size)115{116 long ret = -EFAULT;117 118 if (!__access_ok(src, size))119 return ret;120 121 if (!nmi_uaccess_okay())122 return ret;123 124 pagefault_disable();125 ret = __copy_from_user_inatomic(dst, src, size);126 pagefault_enable();127 128 if (ret)129 return -EFAULT;130 return 0;131}132EXPORT_SYMBOL_GPL(copy_from_user_nofault);133 134/**135 * copy_to_user_nofault(): safely attempt to write to a user-space location136 * @dst: address to write to137 * @src: pointer to the data that shall be written138 * @size: size of the data chunk139 *140 * Safely write to address @dst from the buffer at @src. If a kernel fault141 * happens, handle that and return -EFAULT.142 */143long copy_to_user_nofault(void __user *dst, const void *src, size_t size)144{145 long ret = -EFAULT;146 147 if (access_ok(dst, size)) {148 pagefault_disable();149 ret = __copy_to_user_inatomic(dst, src, size);150 pagefault_enable();151 }152 153 if (ret)154 return -EFAULT;155 return 0;156}157EXPORT_SYMBOL_GPL(copy_to_user_nofault);158 159/**160 * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user161 * address.162 * @dst: Destination address, in kernel space. This buffer must be at163 * least @count bytes long.164 * @unsafe_addr: Unsafe user address.165 * @count: Maximum number of bytes to copy, including the trailing NUL.166 *167 * Copies a NUL-terminated string from unsafe user address to kernel buffer.168 *169 * On success, returns the length of the string INCLUDING the trailing NUL.170 *171 * If access fails, returns -EFAULT (some data may have been copied172 * and the trailing NUL added).173 *174 * If @count is smaller than the length of the string, copies @count-1 bytes,175 * sets the last byte of @dst buffer to NUL and returns @count.176 */177long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,178 long count)179{180 long ret;181 182 if (unlikely(count <= 0))183 return 0;184 185 pagefault_disable();186 ret = strncpy_from_user(dst, unsafe_addr, count);187 pagefault_enable();188 189 if (ret >= count) {190 ret = count;191 dst[ret - 1] = '\0';192 } else if (ret > 0) {193 ret++;194 }195 196 return ret;197}198 199/**200 * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.201 * @unsafe_addr: The string to measure.202 * @count: Maximum count (including NUL)203 *204 * Get the size of a NUL-terminated string in user space without pagefault.205 *206 * Returns the size of the string INCLUDING the terminating NUL.207 *208 * If the string is too long, returns a number larger than @count. User209 * has to check the return value against "> count".210 * On exception (or invalid count), returns 0.211 *212 * Unlike strnlen_user, this can be used from IRQ handler etc. because213 * it disables pagefaults.214 */215long strnlen_user_nofault(const void __user *unsafe_addr, long count)216{217 int ret;218 219 pagefault_disable();220 ret = strnlen_user(unsafe_addr, count);221 pagefault_enable();222 223 return ret;224}225 226void __copy_overflow(int size, unsigned long count)227{228 WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);229}230EXPORT_SYMBOL(__copy_overflow);231