396 lines · c
1#include <linux/kernel.h>2#include <linux/mm.h>3#include <linux/slab.h>4#include <linux/uaccess.h>5#include <linux/ktime.h>6#include <linux/debugfs.h>7#include <linux/highmem.h>8#include "gup_test.h"9 10static void put_back_pages(unsigned int cmd, struct page **pages,11 unsigned long nr_pages, unsigned int gup_test_flags)12{13 unsigned long i;14 15 switch (cmd) {16 case GUP_FAST_BENCHMARK:17 case GUP_BASIC_TEST:18 for (i = 0; i < nr_pages; i++)19 put_page(pages[i]);20 break;21 22 case PIN_FAST_BENCHMARK:23 case PIN_BASIC_TEST:24 case PIN_LONGTERM_BENCHMARK:25 unpin_user_pages(pages, nr_pages);26 break;27 case DUMP_USER_PAGES_TEST:28 if (gup_test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) {29 unpin_user_pages(pages, nr_pages);30 } else {31 for (i = 0; i < nr_pages; i++)32 put_page(pages[i]);33 34 }35 break;36 }37}38 39static void verify_dma_pinned(unsigned int cmd, struct page **pages,40 unsigned long nr_pages)41{42 unsigned long i;43 struct folio *folio;44 45 switch (cmd) {46 case PIN_FAST_BENCHMARK:47 case PIN_BASIC_TEST:48 case PIN_LONGTERM_BENCHMARK:49 for (i = 0; i < nr_pages; i++) {50 folio = page_folio(pages[i]);51 52 if (WARN(!folio_maybe_dma_pinned(folio),53 "pages[%lu] is NOT dma-pinned\n", i)) {54 55 dump_page(&folio->page, "gup_test failure");56 break;57 } else if (cmd == PIN_LONGTERM_BENCHMARK &&58 WARN(!folio_is_longterm_pinnable(folio),59 "pages[%lu] is NOT pinnable but pinned\n",60 i)) {61 dump_page(&folio->page, "gup_test failure");62 break;63 }64 }65 break;66 }67}68 69static void dump_pages_test(struct gup_test *gup, struct page **pages,70 unsigned long nr_pages)71{72 unsigned int index_to_dump;73 unsigned int i;74 75 /*76 * Zero out any user-supplied page index that is out of range. Remember:77 * .which_pages[] contains a 1-based set of page indices.78 */79 for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {80 if (gup->which_pages[i] > nr_pages) {81 pr_warn("ZEROING due to out of range: .which_pages[%u]: %u\n",82 i, gup->which_pages[i]);83 gup->which_pages[i] = 0;84 }85 }86 87 for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {88 index_to_dump = gup->which_pages[i];89 90 if (index_to_dump) {91 index_to_dump--; // Decode from 1-based, to 0-based92 pr_info("---- page #%u, starting from user virt addr: 0x%llx\n",93 index_to_dump, gup->addr);94 dump_page(pages[index_to_dump],95 "gup_test: dump_pages() test");96 }97 }98}99 100static int __gup_test_ioctl(unsigned int cmd,101 struct gup_test *gup)102{103 ktime_t start_time, end_time;104 unsigned long i, nr_pages, addr, next;105 long nr;106 struct page **pages;107 int ret = 0;108 bool needs_mmap_lock =109 cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;110 111 if (gup->size > ULONG_MAX)112 return -EINVAL;113 114 nr_pages = gup->size / PAGE_SIZE;115 pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);116 if (!pages)117 return -ENOMEM;118 119 if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {120 ret = -EINTR;121 goto free_pages;122 }123 124 i = 0;125 nr = gup->nr_pages_per_call;126 start_time = ktime_get();127 for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {128 if (nr != gup->nr_pages_per_call)129 break;130 131 next = addr + nr * PAGE_SIZE;132 if (next > gup->addr + gup->size) {133 next = gup->addr + gup->size;134 nr = (next - addr) / PAGE_SIZE;135 }136 137 switch (cmd) {138 case GUP_FAST_BENCHMARK:139 nr = get_user_pages_fast(addr, nr, gup->gup_flags,140 pages + i);141 break;142 case GUP_BASIC_TEST:143 nr = get_user_pages(addr, nr, gup->gup_flags, pages + i);144 break;145 case PIN_FAST_BENCHMARK:146 nr = pin_user_pages_fast(addr, nr, gup->gup_flags,147 pages + i);148 break;149 case PIN_BASIC_TEST:150 nr = pin_user_pages(addr, nr, gup->gup_flags, pages + i);151 break;152 case PIN_LONGTERM_BENCHMARK:153 nr = pin_user_pages(addr, nr,154 gup->gup_flags | FOLL_LONGTERM,155 pages + i);156 break;157 case DUMP_USER_PAGES_TEST:158 if (gup->test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN)159 nr = pin_user_pages(addr, nr, gup->gup_flags,160 pages + i);161 else162 nr = get_user_pages(addr, nr, gup->gup_flags,163 pages + i);164 break;165 default:166 ret = -EINVAL;167 goto unlock;168 }169 170 if (nr <= 0)171 break;172 i += nr;173 }174 end_time = ktime_get();175 176 /* Shifting the meaning of nr_pages: now it is actual number pinned: */177 nr_pages = i;178 179 gup->get_delta_usec = ktime_us_delta(end_time, start_time);180 gup->size = addr - gup->addr;181 182 /*183 * Take an un-benchmark-timed moment to verify DMA pinned184 * state: print a warning if any non-dma-pinned pages are found:185 */186 verify_dma_pinned(cmd, pages, nr_pages);187 188 if (cmd == DUMP_USER_PAGES_TEST)189 dump_pages_test(gup, pages, nr_pages);190 191 start_time = ktime_get();192 193 put_back_pages(cmd, pages, nr_pages, gup->test_flags);194 195 end_time = ktime_get();196 gup->put_delta_usec = ktime_us_delta(end_time, start_time);197 198unlock:199 if (needs_mmap_lock)200 mmap_read_unlock(current->mm);201free_pages:202 kvfree(pages);203 return ret;204}205 206static DEFINE_MUTEX(pin_longterm_test_mutex);207static struct page **pin_longterm_test_pages;208static unsigned long pin_longterm_test_nr_pages;209 210static inline void pin_longterm_test_stop(void)211{212 if (pin_longterm_test_pages) {213 if (pin_longterm_test_nr_pages)214 unpin_user_pages(pin_longterm_test_pages,215 pin_longterm_test_nr_pages);216 kvfree(pin_longterm_test_pages);217 pin_longterm_test_pages = NULL;218 pin_longterm_test_nr_pages = 0;219 }220}221 222static inline int pin_longterm_test_start(unsigned long arg)223{224 long nr_pages, cur_pages, addr, remaining_pages;225 int gup_flags = FOLL_LONGTERM;226 struct pin_longterm_test args;227 struct page **pages;228 int ret = 0;229 bool fast;230 231 if (pin_longterm_test_pages)232 return -EINVAL;233 234 if (copy_from_user(&args, (void __user *)arg, sizeof(args)))235 return -EFAULT;236 237 if (args.flags &238 ~(PIN_LONGTERM_TEST_FLAG_USE_WRITE|PIN_LONGTERM_TEST_FLAG_USE_FAST))239 return -EINVAL;240 if (!IS_ALIGNED(args.addr | args.size, PAGE_SIZE))241 return -EINVAL;242 if (args.size > LONG_MAX)243 return -EINVAL;244 nr_pages = args.size / PAGE_SIZE;245 if (!nr_pages)246 return -EINVAL;247 248 pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);249 if (!pages)250 return -ENOMEM;251 252 if (args.flags & PIN_LONGTERM_TEST_FLAG_USE_WRITE)253 gup_flags |= FOLL_WRITE;254 fast = !!(args.flags & PIN_LONGTERM_TEST_FLAG_USE_FAST);255 256 if (!fast && mmap_read_lock_killable(current->mm)) {257 kvfree(pages);258 return -EINTR;259 }260 261 pin_longterm_test_pages = pages;262 pin_longterm_test_nr_pages = 0;263 264 while (nr_pages - pin_longterm_test_nr_pages) {265 remaining_pages = nr_pages - pin_longterm_test_nr_pages;266 addr = args.addr + pin_longterm_test_nr_pages * PAGE_SIZE;267 268 if (fast)269 cur_pages = pin_user_pages_fast(addr, remaining_pages,270 gup_flags, pages);271 else272 cur_pages = pin_user_pages(addr, remaining_pages,273 gup_flags, pages);274 if (cur_pages < 0) {275 pin_longterm_test_stop();276 ret = cur_pages;277 break;278 }279 pin_longterm_test_nr_pages += cur_pages;280 pages += cur_pages;281 }282 283 if (!fast)284 mmap_read_unlock(current->mm);285 return ret;286}287 288static inline int pin_longterm_test_read(unsigned long arg)289{290 __u64 user_addr;291 unsigned long i;292 293 if (!pin_longterm_test_pages)294 return -EINVAL;295 296 if (copy_from_user(&user_addr, (void __user *)arg, sizeof(user_addr)))297 return -EFAULT;298 299 for (i = 0; i < pin_longterm_test_nr_pages; i++) {300 void *addr = kmap_local_page(pin_longterm_test_pages[i]);301 unsigned long ret;302 303 ret = copy_to_user((void __user *)(unsigned long)user_addr, addr,304 PAGE_SIZE);305 kunmap_local(addr);306 if (ret)307 return -EFAULT;308 user_addr += PAGE_SIZE;309 }310 return 0;311}312 313static long pin_longterm_test_ioctl(struct file *filep, unsigned int cmd,314 unsigned long arg)315{316 int ret = -EINVAL;317 318 if (mutex_lock_killable(&pin_longterm_test_mutex))319 return -EINTR;320 321 switch (cmd) {322 case PIN_LONGTERM_TEST_START:323 ret = pin_longterm_test_start(arg);324 break;325 case PIN_LONGTERM_TEST_STOP:326 pin_longterm_test_stop();327 ret = 0;328 break;329 case PIN_LONGTERM_TEST_READ:330 ret = pin_longterm_test_read(arg);331 break;332 }333 334 mutex_unlock(&pin_longterm_test_mutex);335 return ret;336}337 338static long gup_test_ioctl(struct file *filep, unsigned int cmd,339 unsigned long arg)340{341 struct gup_test gup;342 int ret;343 344 switch (cmd) {345 case GUP_FAST_BENCHMARK:346 case PIN_FAST_BENCHMARK:347 case PIN_LONGTERM_BENCHMARK:348 case GUP_BASIC_TEST:349 case PIN_BASIC_TEST:350 case DUMP_USER_PAGES_TEST:351 break;352 case PIN_LONGTERM_TEST_START:353 case PIN_LONGTERM_TEST_STOP:354 case PIN_LONGTERM_TEST_READ:355 return pin_longterm_test_ioctl(filep, cmd, arg);356 default:357 return -EINVAL;358 }359 360 if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))361 return -EFAULT;362 363 ret = __gup_test_ioctl(cmd, &gup);364 if (ret)365 return ret;366 367 if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))368 return -EFAULT;369 370 return 0;371}372 373static int gup_test_release(struct inode *inode, struct file *file)374{375 pin_longterm_test_stop();376 377 return 0;378}379 380static const struct file_operations gup_test_fops = {381 .open = nonseekable_open,382 .unlocked_ioctl = gup_test_ioctl,383 .compat_ioctl = compat_ptr_ioctl,384 .release = gup_test_release,385};386 387static int __init gup_test_init(void)388{389 debugfs_create_file_unsafe("gup_test", 0600, NULL, NULL,390 &gup_test_fops);391 392 return 0;393}394 395late_initcall(gup_test_init);396