1561 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Userfaultfd unit tests.4 *5 * Copyright (C) 2015-2023 Red Hat, Inc.6 */7 8#include <asm-generic/unistd.h>9#include "uffd-common.h"10 11#include "../../../../mm/gup_test.h"12 13/* The unit test doesn't need a large or random size, make it 32MB for now */14#define UFFD_TEST_MEM_SIZE (32UL << 20)15 16#define MEM_ANON BIT_ULL(0)17#define MEM_SHMEM BIT_ULL(1)18#define MEM_SHMEM_PRIVATE BIT_ULL(2)19#define MEM_HUGETLB BIT_ULL(3)20#define MEM_HUGETLB_PRIVATE BIT_ULL(4)21 22#define MEM_ALL (MEM_ANON | MEM_SHMEM | MEM_SHMEM_PRIVATE | \23 MEM_HUGETLB | MEM_HUGETLB_PRIVATE)24 25#define ALIGN_UP(x, align_to) \26 ((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1)))27 28struct mem_type {29 const char *name;30 unsigned int mem_flag;31 uffd_test_ops_t *mem_ops;32 bool shared;33};34typedef struct mem_type mem_type_t;35 36mem_type_t mem_types[] = {37 {38 .name = "anon",39 .mem_flag = MEM_ANON,40 .mem_ops = &anon_uffd_test_ops,41 .shared = false,42 },43 {44 .name = "shmem",45 .mem_flag = MEM_SHMEM,46 .mem_ops = &shmem_uffd_test_ops,47 .shared = true,48 },49 {50 .name = "shmem-private",51 .mem_flag = MEM_SHMEM_PRIVATE,52 .mem_ops = &shmem_uffd_test_ops,53 .shared = false,54 },55 {56 .name = "hugetlb",57 .mem_flag = MEM_HUGETLB,58 .mem_ops = &hugetlb_uffd_test_ops,59 .shared = true,60 },61 {62 .name = "hugetlb-private",63 .mem_flag = MEM_HUGETLB_PRIVATE,64 .mem_ops = &hugetlb_uffd_test_ops,65 .shared = false,66 },67};68 69/* Arguments to be passed over to each uffd unit test */70struct uffd_test_args {71 mem_type_t *mem_type;72};73typedef struct uffd_test_args uffd_test_args_t;74 75/* Returns: UFFD_TEST_* */76typedef void (*uffd_test_fn)(uffd_test_args_t *);77 78typedef struct {79 const char *name;80 uffd_test_fn uffd_fn;81 unsigned int mem_targets;82 uint64_t uffd_feature_required;83 uffd_test_case_ops_t *test_case_ops;84} uffd_test_case_t;85 86static void uffd_test_report(void)87{88 printf("Userfaults unit tests: pass=%u, skip=%u, fail=%u (total=%u)\n",89 ksft_get_pass_cnt(),90 ksft_get_xskip_cnt(),91 ksft_get_fail_cnt(),92 ksft_test_num());93}94 95static void uffd_test_pass(void)96{97 printf("done\n");98 ksft_inc_pass_cnt();99}100 101#define uffd_test_start(...) do { \102 printf("Testing "); \103 printf(__VA_ARGS__); \104 printf("... "); \105 fflush(stdout); \106 } while (0)107 108#define uffd_test_fail(...) do { \109 printf("failed [reason: "); \110 printf(__VA_ARGS__); \111 printf("]\n"); \112 ksft_inc_fail_cnt(); \113 } while (0)114 115static void uffd_test_skip(const char *message)116{117 printf("skipped [reason: %s]\n", message);118 ksft_inc_xskip_cnt();119}120 121/*122 * Returns 1 if specific userfaultfd supported, 0 otherwise. Note, we'll123 * return 1 even if some test failed as long as uffd supported, because in124 * that case we still want to proceed with the rest uffd unit tests.125 */126static int test_uffd_api(bool use_dev)127{128 struct uffdio_api uffdio_api;129 int uffd;130 131 uffd_test_start("UFFDIO_API (with %s)",132 use_dev ? "/dev/userfaultfd" : "syscall");133 134 if (use_dev)135 uffd = uffd_open_dev(UFFD_FLAGS);136 else137 uffd = uffd_open_sys(UFFD_FLAGS);138 if (uffd < 0) {139 uffd_test_skip("cannot open userfaultfd handle");140 return 0;141 }142 143 /* Test wrong UFFD_API */144 uffdio_api.api = 0xab;145 uffdio_api.features = 0;146 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {147 uffd_test_fail("UFFDIO_API should fail with wrong api but didn't");148 goto out;149 }150 151 /* Test wrong feature bit */152 uffdio_api.api = UFFD_API;153 uffdio_api.features = BIT_ULL(63);154 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {155 uffd_test_fail("UFFDIO_API should fail with wrong feature but didn't");156 goto out;157 }158 159 /* Test normal UFFDIO_API */160 uffdio_api.api = UFFD_API;161 uffdio_api.features = 0;162 if (ioctl(uffd, UFFDIO_API, &uffdio_api)) {163 uffd_test_fail("UFFDIO_API should succeed but failed");164 goto out;165 }166 167 /* Test double requests of UFFDIO_API with a random feature set */168 uffdio_api.features = BIT_ULL(0);169 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {170 uffd_test_fail("UFFDIO_API should reject initialized uffd");171 goto out;172 }173 174 uffd_test_pass();175out:176 close(uffd);177 /* We have a valid uffd handle */178 return 1;179}180 181/*182 * This function initializes the global variables. TODO: remove global183 * vars and then remove this.184 */185static int186uffd_setup_environment(uffd_test_args_t *args, uffd_test_case_t *test,187 mem_type_t *mem_type, const char **errmsg)188{189 map_shared = mem_type->shared;190 uffd_test_ops = mem_type->mem_ops;191 uffd_test_case_ops = test->test_case_ops;192 193 if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB))194 page_size = default_huge_page_size();195 else196 page_size = psize();197 198 nr_pages = UFFD_TEST_MEM_SIZE / page_size;199 /* TODO: remove this global var.. it's so ugly */200 nr_cpus = 1;201 202 /* Initialize test arguments */203 args->mem_type = mem_type;204 205 return uffd_test_ctx_init(test->uffd_feature_required, errmsg);206}207 208static bool uffd_feature_supported(uffd_test_case_t *test)209{210 uint64_t features;211 212 if (uffd_get_features(&features))213 return false;214 215 return (features & test->uffd_feature_required) ==216 test->uffd_feature_required;217}218 219static int pagemap_open(void)220{221 int fd = open("/proc/self/pagemap", O_RDONLY);222 223 if (fd < 0)224 err("open pagemap");225 226 return fd;227}228 229/* This macro let __LINE__ works in err() */230#define pagemap_check_wp(value, wp) do { \231 if (!!(value & PM_UFFD_WP) != wp) \232 err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \233 } while (0)234 235typedef struct {236 int parent_uffd, child_uffd;237} fork_event_args;238 239static void *fork_event_consumer(void *data)240{241 fork_event_args *args = data;242 struct uffd_msg msg = { 0 };243 244 ready_for_fork = true;245 246 /* Read until a full msg received */247 while (uffd_read_msg(args->parent_uffd, &msg));248 249 if (msg.event != UFFD_EVENT_FORK)250 err("wrong message: %u\n", msg.event);251 252 /* Just to be properly freed later */253 args->child_uffd = msg.arg.fork.ufd;254 return NULL;255}256 257typedef struct {258 int gup_fd;259 bool pinned;260} pin_args;261 262/*263 * Returns 0 if succeed, <0 for errors. pin_pages() needs to be paired264 * with unpin_pages(). Currently it needs to be RO longterm pin to satisfy265 * all needs of the test cases (e.g., trigger unshare, trigger fork() early266 * CoW, etc.).267 */268static int pin_pages(pin_args *args, void *buffer, size_t size)269{270 struct pin_longterm_test test = {271 .addr = (uintptr_t)buffer,272 .size = size,273 /* Read-only pins */274 .flags = 0,275 };276 277 if (args->pinned)278 err("already pinned");279 280 args->gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);281 if (args->gup_fd < 0)282 return -errno;283 284 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) {285 /* Even if gup_test existed, can be an old gup_test / kernel */286 close(args->gup_fd);287 return -errno;288 }289 args->pinned = true;290 return 0;291}292 293static void unpin_pages(pin_args *args)294{295 if (!args->pinned)296 err("unpin without pin first");297 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP))298 err("PIN_LONGTERM_TEST_STOP");299 close(args->gup_fd);300 args->pinned = false;301}302 303static int pagemap_test_fork(int uffd, bool with_event, bool test_pin)304{305 fork_event_args args = { .parent_uffd = uffd, .child_uffd = -1 };306 pthread_t thread;307 pid_t child;308 uint64_t value;309 int fd, result;310 311 /* Prepare a thread to resolve EVENT_FORK */312 if (with_event) {313 ready_for_fork = false;314 if (pthread_create(&thread, NULL, fork_event_consumer, &args))315 err("pthread_create()");316 while (!ready_for_fork)317 ; /* Wait for the poll_thread to start executing before forking */318 }319 320 child = fork();321 if (!child) {322 /* Open the pagemap fd of the child itself */323 pin_args args = {};324 325 fd = pagemap_open();326 327 if (test_pin && pin_pages(&args, area_dst, page_size))328 /*329 * Normally when reach here we have pinned in330 * previous tests, so shouldn't fail anymore331 */332 err("pin page failed in child");333 334 value = pagemap_get_entry(fd, area_dst);335 /*336 * After fork(), we should handle uffd-wp bit differently:337 *338 * (1) when with EVENT_FORK, it should persist339 * (2) when without EVENT_FORK, it should be dropped340 */341 pagemap_check_wp(value, with_event);342 if (test_pin)343 unpin_pages(&args);344 /* Succeed */345 exit(0);346 }347 waitpid(child, &result, 0);348 349 if (with_event) {350 if (pthread_join(thread, NULL))351 err("pthread_join()");352 if (args.child_uffd < 0)353 err("Didn't receive child uffd");354 close(args.child_uffd);355 }356 357 return result;358}359 360static void uffd_wp_unpopulated_test(uffd_test_args_t *args)361{362 uint64_t value;363 int pagemap_fd;364 365 if (uffd_register(uffd, area_dst, nr_pages * page_size,366 false, true, false))367 err("register failed");368 369 pagemap_fd = pagemap_open();370 371 /* Test applying pte marker to anon unpopulated */372 wp_range(uffd, (uint64_t)area_dst, page_size, true);373 value = pagemap_get_entry(pagemap_fd, area_dst);374 pagemap_check_wp(value, true);375 376 /* Test unprotect on anon pte marker */377 wp_range(uffd, (uint64_t)area_dst, page_size, false);378 value = pagemap_get_entry(pagemap_fd, area_dst);379 pagemap_check_wp(value, false);380 381 /* Test zap on anon marker */382 wp_range(uffd, (uint64_t)area_dst, page_size, true);383 if (madvise(area_dst, page_size, MADV_DONTNEED))384 err("madvise(MADV_DONTNEED) failed");385 value = pagemap_get_entry(pagemap_fd, area_dst);386 pagemap_check_wp(value, false);387 388 /* Test fault in after marker removed */389 *area_dst = 1;390 value = pagemap_get_entry(pagemap_fd, area_dst);391 pagemap_check_wp(value, false);392 /* Drop it to make pte none again */393 if (madvise(area_dst, page_size, MADV_DONTNEED))394 err("madvise(MADV_DONTNEED) failed");395 396 /* Test read-zero-page upon pte marker */397 wp_range(uffd, (uint64_t)area_dst, page_size, true);398 *(volatile char *)area_dst;399 /* Drop it to make pte none again */400 if (madvise(area_dst, page_size, MADV_DONTNEED))401 err("madvise(MADV_DONTNEED) failed");402 403 uffd_test_pass();404}405 406static void uffd_wp_fork_test_common(uffd_test_args_t *args,407 bool with_event)408{409 int pagemap_fd;410 uint64_t value;411 412 if (uffd_register(uffd, area_dst, nr_pages * page_size,413 false, true, false))414 err("register failed");415 416 pagemap_fd = pagemap_open();417 418 /* Touch the page */419 *area_dst = 1;420 wp_range(uffd, (uint64_t)area_dst, page_size, true);421 value = pagemap_get_entry(pagemap_fd, area_dst);422 pagemap_check_wp(value, true);423 if (pagemap_test_fork(uffd, with_event, false)) {424 uffd_test_fail("Detected %s uffd-wp bit in child in present pte",425 with_event ? "missing" : "stall");426 goto out;427 }428 429 /*430 * This is an attempt for zapping the pgtable so as to test the431 * markers.432 *433 * For private mappings, PAGEOUT will only work on exclusive ptes434 * (PM_MMAP_EXCLUSIVE) which we should satisfy.435 *436 * For shared, PAGEOUT may not work. Use DONTNEED instead which437 * plays a similar role of zapping (rather than freeing the page)438 * to expose pte markers.439 */440 if (args->mem_type->shared) {441 if (madvise(area_dst, page_size, MADV_DONTNEED))442 err("MADV_DONTNEED");443 } else {444 /*445 * NOTE: ignore retval because private-hugetlb doesn't yet446 * support swapping, so it could fail.447 */448 madvise(area_dst, page_size, MADV_PAGEOUT);449 }450 451 /* Uffd-wp should persist even swapped out */452 value = pagemap_get_entry(pagemap_fd, area_dst);453 pagemap_check_wp(value, true);454 if (pagemap_test_fork(uffd, with_event, false)) {455 uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte",456 with_event ? "missing" : "stall");457 goto out;458 }459 460 /* Unprotect; this tests swap pte modifications */461 wp_range(uffd, (uint64_t)area_dst, page_size, false);462 value = pagemap_get_entry(pagemap_fd, area_dst);463 pagemap_check_wp(value, false);464 465 /* Fault in the page from disk */466 *area_dst = 2;467 value = pagemap_get_entry(pagemap_fd, area_dst);468 pagemap_check_wp(value, false);469 uffd_test_pass();470out:471 if (uffd_unregister(uffd, area_dst, nr_pages * page_size))472 err("unregister failed");473 close(pagemap_fd);474}475 476static void uffd_wp_fork_test(uffd_test_args_t *args)477{478 uffd_wp_fork_test_common(args, false);479}480 481static void uffd_wp_fork_with_event_test(uffd_test_args_t *args)482{483 uffd_wp_fork_test_common(args, true);484}485 486static void uffd_wp_fork_pin_test_common(uffd_test_args_t *args,487 bool with_event)488{489 int pagemap_fd;490 pin_args pin_args = {};491 492 if (uffd_register(uffd, area_dst, page_size, false, true, false))493 err("register failed");494 495 pagemap_fd = pagemap_open();496 497 /* Touch the page */498 *area_dst = 1;499 wp_range(uffd, (uint64_t)area_dst, page_size, true);500 501 /*502 * 1. First pin, then fork(). This tests fork() special path when503 * doing early CoW if the page is private.504 */505 if (pin_pages(&pin_args, area_dst, page_size)) {506 uffd_test_skip("Possibly CONFIG_GUP_TEST missing "507 "or unprivileged");508 close(pagemap_fd);509 uffd_unregister(uffd, area_dst, page_size);510 return;511 }512 513 if (pagemap_test_fork(uffd, with_event, false)) {514 uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()",515 with_event ? "missing" : "stall");516 unpin_pages(&pin_args);517 goto out;518 }519 520 unpin_pages(&pin_args);521 522 /*523 * 2. First fork(), then pin (in the child, where test_pin==true).524 * This tests COR, aka, page unsharing on private memories.525 */526 if (pagemap_test_fork(uffd, with_event, true)) {527 uffd_test_fail("Detected %s uffd-wp bit when RO pin",528 with_event ? "missing" : "stall");529 goto out;530 }531 uffd_test_pass();532out:533 if (uffd_unregister(uffd, area_dst, page_size))534 err("register failed");535 close(pagemap_fd);536}537 538static void uffd_wp_fork_pin_test(uffd_test_args_t *args)539{540 uffd_wp_fork_pin_test_common(args, false);541}542 543static void uffd_wp_fork_pin_with_event_test(uffd_test_args_t *args)544{545 uffd_wp_fork_pin_test_common(args, true);546}547 548static void check_memory_contents(char *p)549{550 unsigned long i, j;551 uint8_t expected_byte;552 553 for (i = 0; i < nr_pages; ++i) {554 expected_byte = ~((uint8_t)(i % ((uint8_t)-1)));555 for (j = 0; j < page_size; j++) {556 uint8_t v = *(uint8_t *)(p + (i * page_size) + j);557 if (v != expected_byte)558 err("unexpected page contents");559 }560 }561}562 563static void uffd_minor_test_common(bool test_collapse, bool test_wp)564{565 unsigned long p;566 pthread_t uffd_mon;567 char c;568 struct uffd_args args = { 0 };569 570 /*571 * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing572 * both do not make much sense.573 */574 assert(!(test_collapse && test_wp));575 576 if (uffd_register(uffd, area_dst_alias, nr_pages * page_size,577 /* NOTE! MADV_COLLAPSE may not work with uffd-wp */578 false, test_wp, true))579 err("register failure");580 581 /*582 * After registering with UFFD, populate the non-UFFD-registered side of583 * the shared mapping. This should *not* trigger any UFFD minor faults.584 */585 for (p = 0; p < nr_pages; ++p)586 memset(area_dst + (p * page_size), p % ((uint8_t)-1),587 page_size);588 589 args.apply_wp = test_wp;590 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))591 err("uffd_poll_thread create");592 593 /*594 * Read each of the pages back using the UFFD-registered mapping. We595 * expect that the first time we touch a page, it will result in a minor596 * fault. uffd_poll_thread will resolve the fault by bit-flipping the597 * page's contents, and then issuing a CONTINUE ioctl.598 */599 check_memory_contents(area_dst_alias);600 601 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))602 err("pipe write");603 if (pthread_join(uffd_mon, NULL))604 err("join() failed");605 606 if (test_collapse) {607 if (madvise(area_dst_alias, nr_pages * page_size,608 MADV_COLLAPSE)) {609 /* It's fine to fail for this one... */610 uffd_test_skip("MADV_COLLAPSE failed");611 return;612 }613 614 uffd_test_ops->check_pmd_mapping(area_dst,615 nr_pages * page_size /616 read_pmd_pagesize());617 /*618 * This won't cause uffd-fault - it purely just makes sure there619 * was no corruption.620 */621 check_memory_contents(area_dst_alias);622 }623 624 if (args.missing_faults != 0 || args.minor_faults != nr_pages)625 uffd_test_fail("stats check error");626 else627 uffd_test_pass();628}629 630void uffd_minor_test(uffd_test_args_t *args)631{632 uffd_minor_test_common(false, false);633}634 635void uffd_minor_wp_test(uffd_test_args_t *args)636{637 uffd_minor_test_common(false, true);638}639 640void uffd_minor_collapse_test(uffd_test_args_t *args)641{642 uffd_minor_test_common(true, false);643}644 645static sigjmp_buf jbuf, *sigbuf;646 647static void sighndl(int sig, siginfo_t *siginfo, void *ptr)648{649 if (sig == SIGBUS) {650 if (sigbuf)651 siglongjmp(*sigbuf, 1);652 abort();653 }654}655 656/*657 * For non-cooperative userfaultfd test we fork() a process that will658 * generate pagefaults, will mremap the area monitored by the659 * userfaultfd and at last this process will release the monitored660 * area.661 * For the anonymous and shared memory the area is divided into two662 * parts, the first part is accessed before mremap, and the second663 * part is accessed after mremap. Since hugetlbfs does not support664 * mremap, the entire monitored area is accessed in a single pass for665 * HUGETLB_TEST.666 * The release of the pages currently generates event for shmem and667 * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked668 * for hugetlb.669 * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register670 * monitored area, generate pagefaults and test that signal is delivered.671 * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2672 * test robustness use case - we release monitored area, fork a process673 * that will generate pagefaults and verify signal is generated.674 * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal675 * feature. Using monitor thread, verify no userfault events are generated.676 */677static int faulting_process(int signal_test, bool wp)678{679 unsigned long nr, i;680 unsigned long long count;681 unsigned long split_nr_pages;682 unsigned long lastnr;683 struct sigaction act;684 volatile unsigned long signalled = 0;685 686 split_nr_pages = (nr_pages + 1) / 2;687 688 if (signal_test) {689 sigbuf = &jbuf;690 memset(&act, 0, sizeof(act));691 act.sa_sigaction = sighndl;692 act.sa_flags = SA_SIGINFO;693 if (sigaction(SIGBUS, &act, 0))694 err("sigaction");695 lastnr = (unsigned long)-1;696 }697 698 for (nr = 0; nr < split_nr_pages; nr++) {699 volatile int steps = 1;700 unsigned long offset = nr * page_size;701 702 if (signal_test) {703 if (sigsetjmp(*sigbuf, 1) != 0) {704 if (steps == 1 && nr == lastnr)705 err("Signal repeated");706 707 lastnr = nr;708 if (signal_test == 1) {709 if (steps == 1) {710 /* This is a MISSING request */711 steps++;712 if (copy_page(uffd, offset, wp))713 signalled++;714 } else {715 /* This is a WP request */716 assert(steps == 2);717 wp_range(uffd,718 (__u64)area_dst +719 offset,720 page_size, false);721 }722 } else {723 signalled++;724 continue;725 }726 }727 }728 729 count = *area_count(area_dst, nr);730 if (count != count_verify[nr])731 err("nr %lu memory corruption %llu %llu\n",732 nr, count, count_verify[nr]);733 /*734 * Trigger write protection if there is by writing735 * the same value back.736 */737 *area_count(area_dst, nr) = count;738 }739 740 if (signal_test)741 return signalled != split_nr_pages;742 743 area_dst = mremap(area_dst, nr_pages * page_size, nr_pages * page_size,744 MREMAP_MAYMOVE | MREMAP_FIXED, area_src);745 if (area_dst == MAP_FAILED)746 err("mremap");747 /* Reset area_src since we just clobbered it */748 area_src = NULL;749 750 for (; nr < nr_pages; nr++) {751 count = *area_count(area_dst, nr);752 if (count != count_verify[nr]) {753 err("nr %lu memory corruption %llu %llu\n",754 nr, count, count_verify[nr]);755 }756 /*757 * Trigger write protection if there is by writing758 * the same value back.759 */760 *area_count(area_dst, nr) = count;761 }762 763 uffd_test_ops->release_pages(area_dst);764 765 for (nr = 0; nr < nr_pages; nr++)766 for (i = 0; i < page_size; i++)767 if (*(area_dst + nr * page_size + i) != 0)768 err("page %lu offset %lu is not zero", nr, i);769 770 return 0;771}772 773static void uffd_sigbus_test_common(bool wp)774{775 unsigned long userfaults;776 pthread_t uffd_mon;777 pid_t pid;778 int err;779 char c;780 struct uffd_args args = { 0 };781 782 ready_for_fork = false;783 784 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);785 786 if (uffd_register(uffd, area_dst, nr_pages * page_size,787 true, wp, false))788 err("register failure");789 790 if (faulting_process(1, wp))791 err("faulting process failed");792 793 uffd_test_ops->release_pages(area_dst);794 795 args.apply_wp = wp;796 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))797 err("uffd_poll_thread create");798 799 while (!ready_for_fork)800 ; /* Wait for the poll_thread to start executing before forking */801 802 pid = fork();803 if (pid < 0)804 err("fork");805 806 if (!pid)807 exit(faulting_process(2, wp));808 809 waitpid(pid, &err, 0);810 if (err)811 err("faulting process failed");812 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))813 err("pipe write");814 if (pthread_join(uffd_mon, (void **)&userfaults))815 err("pthread_join()");816 817 if (userfaults)818 uffd_test_fail("Signal test failed, userfaults: %ld", userfaults);819 else820 uffd_test_pass();821}822 823static void uffd_sigbus_test(uffd_test_args_t *args)824{825 uffd_sigbus_test_common(false);826}827 828static void uffd_sigbus_wp_test(uffd_test_args_t *args)829{830 uffd_sigbus_test_common(true);831}832 833static void uffd_events_test_common(bool wp)834{835 pthread_t uffd_mon;836 pid_t pid;837 int err;838 char c;839 struct uffd_args args = { 0 };840 841 ready_for_fork = false;842 843 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);844 if (uffd_register(uffd, area_dst, nr_pages * page_size,845 true, wp, false))846 err("register failure");847 848 args.apply_wp = wp;849 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))850 err("uffd_poll_thread create");851 852 while (!ready_for_fork)853 ; /* Wait for the poll_thread to start executing before forking */854 855 pid = fork();856 if (pid < 0)857 err("fork");858 859 if (!pid)860 exit(faulting_process(0, wp));861 862 waitpid(pid, &err, 0);863 if (err)864 err("faulting process failed");865 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))866 err("pipe write");867 if (pthread_join(uffd_mon, NULL))868 err("pthread_join()");869 870 if (args.missing_faults != nr_pages)871 uffd_test_fail("Fault counts wrong");872 else873 uffd_test_pass();874}875 876static void uffd_events_test(uffd_test_args_t *args)877{878 uffd_events_test_common(false);879}880 881static void uffd_events_wp_test(uffd_test_args_t *args)882{883 uffd_events_test_common(true);884}885 886static void retry_uffdio_zeropage(int ufd,887 struct uffdio_zeropage *uffdio_zeropage)888{889 uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,890 uffdio_zeropage->range.len,891 0);892 if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {893 if (uffdio_zeropage->zeropage != -EEXIST)894 err("UFFDIO_ZEROPAGE error: %"PRId64,895 (int64_t)uffdio_zeropage->zeropage);896 } else {897 err("UFFDIO_ZEROPAGE error: %"PRId64,898 (int64_t)uffdio_zeropage->zeropage);899 }900}901 902static bool do_uffdio_zeropage(int ufd, bool has_zeropage)903{904 struct uffdio_zeropage uffdio_zeropage = { 0 };905 int ret;906 __s64 res;907 908 uffdio_zeropage.range.start = (unsigned long) area_dst;909 uffdio_zeropage.range.len = page_size;910 uffdio_zeropage.mode = 0;911 ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage);912 res = uffdio_zeropage.zeropage;913 if (ret) {914 /* real retval in ufdio_zeropage.zeropage */915 if (has_zeropage)916 err("UFFDIO_ZEROPAGE error: %"PRId64, (int64_t)res);917 else if (res != -EINVAL)918 err("UFFDIO_ZEROPAGE not -EINVAL");919 } else if (has_zeropage) {920 if (res != page_size)921 err("UFFDIO_ZEROPAGE unexpected size");922 else923 retry_uffdio_zeropage(ufd, &uffdio_zeropage);924 return true;925 } else926 err("UFFDIO_ZEROPAGE succeeded");927 928 return false;929}930 931/*932 * Registers a range with MISSING mode only for zeropage test. Return true933 * if UFFDIO_ZEROPAGE supported, false otherwise. Can't use uffd_register()934 * because we want to detect .ioctls along the way.935 */936static bool937uffd_register_detect_zeropage(int uffd, void *addr, uint64_t len)938{939 uint64_t ioctls = 0;940 941 if (uffd_register_with_ioctls(uffd, addr, len, true,942 false, false, &ioctls))943 err("zeropage register fail");944 945 return ioctls & (1 << _UFFDIO_ZEROPAGE);946}947 948/* exercise UFFDIO_ZEROPAGE */949static void uffd_zeropage_test(uffd_test_args_t *args)950{951 bool has_zeropage;952 int i;953 954 has_zeropage = uffd_register_detect_zeropage(uffd, area_dst, page_size);955 if (area_dst_alias)956 /* Ignore the retval; we already have it */957 uffd_register_detect_zeropage(uffd, area_dst_alias, page_size);958 959 if (do_uffdio_zeropage(uffd, has_zeropage))960 for (i = 0; i < page_size; i++)961 if (area_dst[i] != 0)962 err("data non-zero at offset %d\n", i);963 964 if (uffd_unregister(uffd, area_dst, page_size))965 err("unregister");966 967 if (area_dst_alias && uffd_unregister(uffd, area_dst_alias, page_size))968 err("unregister");969 970 uffd_test_pass();971}972 973static void uffd_register_poison(int uffd, void *addr, uint64_t len)974{975 uint64_t ioctls = 0;976 uint64_t expected = (1 << _UFFDIO_COPY) | (1 << _UFFDIO_POISON);977 978 if (uffd_register_with_ioctls(uffd, addr, len, true,979 false, false, &ioctls))980 err("poison register fail");981 982 if ((ioctls & expected) != expected)983 err("registered area doesn't support COPY and POISON ioctls");984}985 986static void do_uffdio_poison(int uffd, unsigned long offset)987{988 struct uffdio_poison uffdio_poison = { 0 };989 int ret;990 __s64 res;991 992 uffdio_poison.range.start = (unsigned long) area_dst + offset;993 uffdio_poison.range.len = page_size;994 uffdio_poison.mode = 0;995 ret = ioctl(uffd, UFFDIO_POISON, &uffdio_poison);996 res = uffdio_poison.updated;997 998 if (ret)999 err("UFFDIO_POISON error: %"PRId64, (int64_t)res);1000 else if (res != page_size)1001 err("UFFDIO_POISON unexpected size: %"PRId64, (int64_t)res);1002}1003 1004static void uffd_poison_handle_fault(1005 struct uffd_msg *msg, struct uffd_args *args)1006{1007 unsigned long offset;1008 1009 if (msg->event != UFFD_EVENT_PAGEFAULT)1010 err("unexpected msg event %u", msg->event);1011 1012 if (msg->arg.pagefault.flags &1013 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR))1014 err("unexpected fault type %llu", msg->arg.pagefault.flags);1015 1016 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;1017 offset &= ~(page_size-1);1018 1019 /* Odd pages -> copy zeroed page; even pages -> poison. */1020 if (offset & page_size)1021 copy_page(uffd, offset, false);1022 else1023 do_uffdio_poison(uffd, offset);1024}1025 1026static void uffd_poison_test(uffd_test_args_t *targs)1027{1028 pthread_t uffd_mon;1029 char c;1030 struct uffd_args args = { 0 };1031 struct sigaction act = { 0 };1032 unsigned long nr_sigbus = 0;1033 unsigned long nr;1034 1035 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);1036 1037 uffd_register_poison(uffd, area_dst, nr_pages * page_size);1038 memset(area_src, 0, nr_pages * page_size);1039 1040 args.handle_fault = uffd_poison_handle_fault;1041 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))1042 err("uffd_poll_thread create");1043 1044 sigbuf = &jbuf;1045 act.sa_sigaction = sighndl;1046 act.sa_flags = SA_SIGINFO;1047 if (sigaction(SIGBUS, &act, 0))1048 err("sigaction");1049 1050 for (nr = 0; nr < nr_pages; ++nr) {1051 unsigned long offset = nr * page_size;1052 const char *bytes = (const char *) area_dst + offset;1053 const char *i;1054 1055 if (sigsetjmp(*sigbuf, 1)) {1056 /*1057 * Access below triggered a SIGBUS, which was caught by1058 * sighndl, which then jumped here. Count this SIGBUS,1059 * and move on to next page.1060 */1061 ++nr_sigbus;1062 continue;1063 }1064 1065 for (i = bytes; i < bytes + page_size; ++i) {1066 if (*i)1067 err("nonzero byte in area_dst (%p) at %p: %u",1068 area_dst, i, *i);1069 }1070 }1071 1072 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))1073 err("pipe write");1074 if (pthread_join(uffd_mon, NULL))1075 err("pthread_join()");1076 1077 if (nr_sigbus != nr_pages / 2)1078 err("expected to receive %lu SIGBUS, actually received %lu",1079 nr_pages / 2, nr_sigbus);1080 1081 uffd_test_pass();1082}1083 1084static void1085uffd_move_handle_fault_common(struct uffd_msg *msg, struct uffd_args *args,1086 unsigned long len)1087{1088 unsigned long offset;1089 1090 if (msg->event != UFFD_EVENT_PAGEFAULT)1091 err("unexpected msg event %u", msg->event);1092 1093 if (msg->arg.pagefault.flags &1094 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR | UFFD_PAGEFAULT_FLAG_WRITE))1095 err("unexpected fault type %llu", msg->arg.pagefault.flags);1096 1097 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;1098 offset &= ~(len-1);1099 1100 if (move_page(uffd, offset, len))1101 args->missing_faults++;1102}1103 1104static void uffd_move_handle_fault(struct uffd_msg *msg,1105 struct uffd_args *args)1106{1107 uffd_move_handle_fault_common(msg, args, page_size);1108}1109 1110static void uffd_move_pmd_handle_fault(struct uffd_msg *msg,1111 struct uffd_args *args)1112{1113 uffd_move_handle_fault_common(msg, args, read_pmd_pagesize());1114}1115 1116static void1117uffd_move_test_common(uffd_test_args_t *targs, unsigned long chunk_size,1118 void (*handle_fault)(struct uffd_msg *msg, struct uffd_args *args))1119{1120 unsigned long nr;1121 pthread_t uffd_mon;1122 char c;1123 unsigned long long count;1124 struct uffd_args args = { 0 };1125 char *orig_area_src, *orig_area_dst;1126 unsigned long step_size, step_count;1127 unsigned long src_offs = 0;1128 unsigned long dst_offs = 0;1129 1130 /* Prevent source pages from being mapped more than once */1131 if (madvise(area_src, nr_pages * page_size, MADV_DONTFORK))1132 err("madvise(MADV_DONTFORK) failure");1133 1134 if (uffd_register(uffd, area_dst, nr_pages * page_size,1135 true, false, false))1136 err("register failure");1137 1138 args.handle_fault = handle_fault;1139 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))1140 err("uffd_poll_thread create");1141 1142 step_size = chunk_size / page_size;1143 step_count = nr_pages / step_size;1144 1145 if (chunk_size > page_size) {1146 char *aligned_src = ALIGN_UP(area_src, chunk_size);1147 char *aligned_dst = ALIGN_UP(area_dst, chunk_size);1148 1149 if (aligned_src != area_src || aligned_dst != area_dst) {1150 src_offs = (aligned_src - area_src) / page_size;1151 dst_offs = (aligned_dst - area_dst) / page_size;1152 step_count--;1153 }1154 orig_area_src = area_src;1155 orig_area_dst = area_dst;1156 area_src = aligned_src;1157 area_dst = aligned_dst;1158 }1159 1160 /*1161 * Read each of the pages back using the UFFD-registered mapping. We1162 * expect that the first time we touch a page, it will result in a missing1163 * fault. uffd_poll_thread will resolve the fault by moving source1164 * page to destination.1165 */1166 for (nr = 0; nr < step_count * step_size; nr += step_size) {1167 unsigned long i;1168 1169 /* Check area_src content */1170 for (i = 0; i < step_size; i++) {1171 count = *area_count(area_src, nr + i);1172 if (count != count_verify[src_offs + nr + i])1173 err("nr %lu source memory invalid %llu %llu\n",1174 nr + i, count, count_verify[src_offs + nr + i]);1175 }1176 1177 /* Faulting into area_dst should move the page or the huge page */1178 for (i = 0; i < step_size; i++) {1179 count = *area_count(area_dst, nr + i);1180 if (count != count_verify[dst_offs + nr + i])1181 err("nr %lu memory corruption %llu %llu\n",1182 nr, count, count_verify[dst_offs + nr + i]);1183 }1184 1185 /* Re-check area_src content which should be empty */1186 for (i = 0; i < step_size; i++) {1187 count = *area_count(area_src, nr + i);1188 if (count != 0)1189 err("nr %lu move failed %llu %llu\n",1190 nr, count, count_verify[src_offs + nr + i]);1191 }1192 }1193 if (step_size > page_size) {1194 area_src = orig_area_src;1195 area_dst = orig_area_dst;1196 }1197 1198 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))1199 err("pipe write");1200 if (pthread_join(uffd_mon, NULL))1201 err("join() failed");1202 1203 if (args.missing_faults != step_count || args.minor_faults != 0)1204 uffd_test_fail("stats check error");1205 else1206 uffd_test_pass();1207}1208 1209static void uffd_move_test(uffd_test_args_t *targs)1210{1211 uffd_move_test_common(targs, page_size, uffd_move_handle_fault);1212}1213 1214static void uffd_move_pmd_test(uffd_test_args_t *targs)1215{1216 if (madvise(area_dst, nr_pages * page_size, MADV_HUGEPAGE))1217 err("madvise(MADV_HUGEPAGE) failure");1218 uffd_move_test_common(targs, read_pmd_pagesize(),1219 uffd_move_pmd_handle_fault);1220}1221 1222static void uffd_move_pmd_split_test(uffd_test_args_t *targs)1223{1224 if (madvise(area_dst, nr_pages * page_size, MADV_NOHUGEPAGE))1225 err("madvise(MADV_NOHUGEPAGE) failure");1226 uffd_move_test_common(targs, read_pmd_pagesize(),1227 uffd_move_pmd_handle_fault);1228}1229 1230static int prevent_hugepages(const char **errmsg)1231{1232 /* This should be done before source area is populated */1233 if (madvise(area_src, nr_pages * page_size, MADV_NOHUGEPAGE)) {1234 /* Ignore only if CONFIG_TRANSPARENT_HUGEPAGE=n */1235 if (errno != EINVAL) {1236 if (errmsg)1237 *errmsg = "madvise(MADV_NOHUGEPAGE) failed";1238 return -errno;1239 }1240 }1241 return 0;1242}1243 1244static int request_hugepages(const char **errmsg)1245{1246 /* This should be done before source area is populated */1247 if (madvise(area_src, nr_pages * page_size, MADV_HUGEPAGE)) {1248 if (errmsg) {1249 *errmsg = (errno == EINVAL) ?1250 "CONFIG_TRANSPARENT_HUGEPAGE is not set" :1251 "madvise(MADV_HUGEPAGE) failed";1252 }1253 return -errno;1254 }1255 return 0;1256}1257 1258struct uffd_test_case_ops uffd_move_test_case_ops = {1259 .post_alloc = prevent_hugepages,1260};1261 1262struct uffd_test_case_ops uffd_move_test_pmd_case_ops = {1263 .post_alloc = request_hugepages,1264};1265 1266/*1267 * Test the returned uffdio_register.ioctls with different register modes.1268 * Note that _UFFDIO_ZEROPAGE is tested separately in the zeropage test.1269 */1270static void1271do_register_ioctls_test(uffd_test_args_t *args, bool miss, bool wp, bool minor)1272{1273 uint64_t ioctls = 0, expected = BIT_ULL(_UFFDIO_WAKE);1274 mem_type_t *mem_type = args->mem_type;1275 int ret;1276 1277 ret = uffd_register_with_ioctls(uffd, area_dst, page_size,1278 miss, wp, minor, &ioctls);1279 1280 /*1281 * Handle special cases of UFFDIO_REGISTER here where it should1282 * just fail with -EINVAL first..1283 *1284 * Case 1: register MINOR on anon1285 * Case 2: register with no mode selected1286 */1287 if ((minor && (mem_type->mem_flag == MEM_ANON)) ||1288 (!miss && !wp && !minor)) {1289 if (ret != -EINVAL)1290 err("register (miss=%d, wp=%d, minor=%d) failed "1291 "with wrong errno=%d", miss, wp, minor, ret);1292 return;1293 }1294 1295 /* UFFDIO_REGISTER should succeed, then check ioctls returned */1296 if (miss)1297 expected |= BIT_ULL(_UFFDIO_COPY);1298 if (wp)1299 expected |= BIT_ULL(_UFFDIO_WRITEPROTECT);1300 if (minor)1301 expected |= BIT_ULL(_UFFDIO_CONTINUE);1302 1303 if ((ioctls & expected) != expected)1304 err("unexpected uffdio_register.ioctls "1305 "(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", "1306 "returned=0x%"PRIx64, miss, wp, minor, expected, ioctls);1307 1308 if (uffd_unregister(uffd, area_dst, page_size))1309 err("unregister");1310}1311 1312static void uffd_register_ioctls_test(uffd_test_args_t *args)1313{1314 int miss, wp, minor;1315 1316 for (miss = 0; miss <= 1; miss++)1317 for (wp = 0; wp <= 1; wp++)1318 for (minor = 0; minor <= 1; minor++)1319 do_register_ioctls_test(args, miss, wp, minor);1320 1321 uffd_test_pass();1322}1323 1324uffd_test_case_t uffd_tests[] = {1325 {1326 /* Test returned uffdio_register.ioctls. */1327 .name = "register-ioctls",1328 .uffd_fn = uffd_register_ioctls_test,1329 .mem_targets = MEM_ALL,1330 .uffd_feature_required = UFFD_FEATURE_MISSING_HUGETLBFS |1331 UFFD_FEATURE_MISSING_SHMEM |1332 UFFD_FEATURE_PAGEFAULT_FLAG_WP |1333 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |1334 UFFD_FEATURE_MINOR_HUGETLBFS |1335 UFFD_FEATURE_MINOR_SHMEM,1336 },1337 {1338 .name = "zeropage",1339 .uffd_fn = uffd_zeropage_test,1340 .mem_targets = MEM_ALL,1341 .uffd_feature_required = 0,1342 },1343 {1344 .name = "move",1345 .uffd_fn = uffd_move_test,1346 .mem_targets = MEM_ANON,1347 .uffd_feature_required = UFFD_FEATURE_MOVE,1348 .test_case_ops = &uffd_move_test_case_ops,1349 },1350 {1351 .name = "move-pmd",1352 .uffd_fn = uffd_move_pmd_test,1353 .mem_targets = MEM_ANON,1354 .uffd_feature_required = UFFD_FEATURE_MOVE,1355 .test_case_ops = &uffd_move_test_pmd_case_ops,1356 },1357 {1358 .name = "move-pmd-split",1359 .uffd_fn = uffd_move_pmd_split_test,1360 .mem_targets = MEM_ANON,1361 .uffd_feature_required = UFFD_FEATURE_MOVE,1362 .test_case_ops = &uffd_move_test_pmd_case_ops,1363 },1364 {1365 .name = "wp-fork",1366 .uffd_fn = uffd_wp_fork_test,1367 .mem_targets = MEM_ALL,1368 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |1369 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,1370 },1371 {1372 .name = "wp-fork-with-event",1373 .uffd_fn = uffd_wp_fork_with_event_test,1374 .mem_targets = MEM_ALL,1375 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |1376 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |1377 /* when set, child process should inherit uffd-wp bits */1378 UFFD_FEATURE_EVENT_FORK,1379 },1380 {1381 .name = "wp-fork-pin",1382 .uffd_fn = uffd_wp_fork_pin_test,1383 .mem_targets = MEM_ALL,1384 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |1385 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,1386 },1387 {1388 .name = "wp-fork-pin-with-event",1389 .uffd_fn = uffd_wp_fork_pin_with_event_test,1390 .mem_targets = MEM_ALL,1391 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |1392 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |1393 /* when set, child process should inherit uffd-wp bits */1394 UFFD_FEATURE_EVENT_FORK,1395 },1396 {1397 .name = "wp-unpopulated",1398 .uffd_fn = uffd_wp_unpopulated_test,1399 .mem_targets = MEM_ANON,1400 .uffd_feature_required =1401 UFFD_FEATURE_PAGEFAULT_FLAG_WP | UFFD_FEATURE_WP_UNPOPULATED,1402 },1403 {1404 .name = "minor",1405 .uffd_fn = uffd_minor_test,1406 .mem_targets = MEM_SHMEM | MEM_HUGETLB,1407 .uffd_feature_required =1408 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM,1409 },1410 {1411 .name = "minor-wp",1412 .uffd_fn = uffd_minor_wp_test,1413 .mem_targets = MEM_SHMEM | MEM_HUGETLB,1414 .uffd_feature_required =1415 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM |1416 UFFD_FEATURE_PAGEFAULT_FLAG_WP |1417 /*1418 * HACK: here we leveraged WP_UNPOPULATED to detect whether1419 * minor mode supports wr-protect. There's no feature flag1420 * for it so this is the best we can test against.1421 */1422 UFFD_FEATURE_WP_UNPOPULATED,1423 },1424 {1425 .name = "minor-collapse",1426 .uffd_fn = uffd_minor_collapse_test,1427 /* MADV_COLLAPSE only works with shmem */1428 .mem_targets = MEM_SHMEM,1429 /* We can't test MADV_COLLAPSE, so try our luck */1430 .uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM,1431 },1432 {1433 .name = "sigbus",1434 .uffd_fn = uffd_sigbus_test,1435 .mem_targets = MEM_ALL,1436 .uffd_feature_required = UFFD_FEATURE_SIGBUS |1437 UFFD_FEATURE_EVENT_FORK,1438 },1439 {1440 .name = "sigbus-wp",1441 .uffd_fn = uffd_sigbus_wp_test,1442 .mem_targets = MEM_ALL,1443 .uffd_feature_required = UFFD_FEATURE_SIGBUS |1444 UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_PAGEFAULT_FLAG_WP |1445 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,1446 },1447 {1448 .name = "events",1449 .uffd_fn = uffd_events_test,1450 .mem_targets = MEM_ALL,1451 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |1452 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE,1453 },1454 {1455 .name = "events-wp",1456 .uffd_fn = uffd_events_wp_test,1457 .mem_targets = MEM_ALL,1458 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |1459 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE |1460 UFFD_FEATURE_PAGEFAULT_FLAG_WP |1461 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,1462 },1463 {1464 .name = "poison",1465 .uffd_fn = uffd_poison_test,1466 .mem_targets = MEM_ALL,1467 .uffd_feature_required = UFFD_FEATURE_POISON,1468 },1469};1470 1471static void usage(const char *prog)1472{1473 printf("usage: %s [-f TESTNAME]\n", prog);1474 puts("");1475 puts(" -f: test name to filter (e.g., event)");1476 puts(" -h: show the help msg");1477 puts(" -l: list tests only");1478 puts("");1479 exit(KSFT_FAIL);1480}1481 1482int main(int argc, char *argv[])1483{1484 int n_tests = sizeof(uffd_tests) / sizeof(uffd_test_case_t);1485 int n_mems = sizeof(mem_types) / sizeof(mem_type_t);1486 const char *test_filter = NULL;1487 bool list_only = false;1488 uffd_test_case_t *test;1489 mem_type_t *mem_type;1490 uffd_test_args_t args;1491 const char *errmsg;1492 int has_uffd, opt;1493 int i, j;1494 1495 while ((opt = getopt(argc, argv, "f:hl")) != -1) {1496 switch (opt) {1497 case 'f':1498 test_filter = optarg;1499 break;1500 case 'l':1501 list_only = true;1502 break;1503 case 'h':1504 default:1505 /* Unknown */1506 usage(argv[0]);1507 break;1508 }1509 }1510 1511 if (!test_filter && !list_only) {1512 has_uffd = test_uffd_api(false);1513 has_uffd |= test_uffd_api(true);1514 1515 if (!has_uffd) {1516 printf("Userfaultfd not supported or unprivileged, skip all tests\n");1517 exit(KSFT_SKIP);1518 }1519 }1520 1521 for (i = 0; i < n_tests; i++) {1522 test = &uffd_tests[i];1523 if (test_filter && !strstr(test->name, test_filter))1524 continue;1525 if (list_only) {1526 printf("%s\n", test->name);1527 continue;1528 }1529 for (j = 0; j < n_mems; j++) {1530 mem_type = &mem_types[j];1531 if (!(test->mem_targets & mem_type->mem_flag))1532 continue;1533 1534 uffd_test_start("%s on %s", test->name, mem_type->name);1535 if ((mem_type->mem_flag == MEM_HUGETLB ||1536 mem_type->mem_flag == MEM_HUGETLB_PRIVATE) &&1537 (default_huge_page_size() == 0)) {1538 uffd_test_skip("huge page size is 0, feature missing?");1539 continue;1540 }1541 if (!uffd_feature_supported(test)) {1542 uffd_test_skip("feature missing");1543 continue;1544 }1545 if (uffd_setup_environment(&args, test, mem_type,1546 &errmsg)) {1547 uffd_test_skip(errmsg);1548 continue;1549 }1550 test->uffd_fn(&args);1551 uffd_test_ctx_clear();1552 }1553 }1554 1555 if (!list_only)1556 uffd_test_report();1557 1558 return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS;1559}1560 1561