1136 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * page_fault_test.c - Test stage 2 faults.4 *5 * This test tries different combinations of guest accesses (e.g., write,6 * S1PTW), backing source type (e.g., anon) and types of faults (e.g., read on7 * hugetlbfs with a hole). It checks that the expected handling method is8 * called (e.g., uffd faults with the right address and write/read flag).9 */10#include <linux/bitmap.h>11#include <fcntl.h>12#include <test_util.h>13#include <kvm_util.h>14#include <processor.h>15#include <asm/sysreg.h>16#include <linux/bitfield.h>17#include "guest_modes.h"18#include "userfaultfd_util.h"19 20/* Guest virtual addresses that point to the test page and its PTE. */21#define TEST_GVA 0xc000000022#define TEST_EXEC_GVA (TEST_GVA + 0x8)23#define TEST_PTE_GVA 0xb000000024#define TEST_DATA 0x0123456789ABCDEF25 26static uint64_t *guest_test_memory = (uint64_t *)TEST_GVA;27 28#define CMD_NONE (0)29#define CMD_SKIP_TEST (1ULL << 1)30#define CMD_HOLE_PT (1ULL << 2)31#define CMD_HOLE_DATA (1ULL << 3)32#define CMD_CHECK_WRITE_IN_DIRTY_LOG (1ULL << 4)33#define CMD_CHECK_S1PTW_WR_IN_DIRTY_LOG (1ULL << 5)34#define CMD_CHECK_NO_WRITE_IN_DIRTY_LOG (1ULL << 6)35#define CMD_CHECK_NO_S1PTW_WR_IN_DIRTY_LOG (1ULL << 7)36#define CMD_SET_PTE_AF (1ULL << 8)37 38#define PREPARE_FN_NR 1039#define CHECK_FN_NR 1040 41static struct event_cnt {42 int mmio_exits;43 int fail_vcpu_runs;44 int uffd_faults;45 /* uffd_faults is incremented from multiple threads. */46 pthread_mutex_t uffd_faults_mutex;47} events;48 49struct test_desc {50 const char *name;51 uint64_t mem_mark_cmd;52 /* Skip the test if any prepare function returns false */53 bool (*guest_prepare[PREPARE_FN_NR])(void);54 void (*guest_test)(void);55 void (*guest_test_check[CHECK_FN_NR])(void);56 uffd_handler_t uffd_pt_handler;57 uffd_handler_t uffd_data_handler;58 void (*dabt_handler)(struct ex_regs *regs);59 void (*iabt_handler)(struct ex_regs *regs);60 void (*mmio_handler)(struct kvm_vm *vm, struct kvm_run *run);61 void (*fail_vcpu_run_handler)(int ret);62 uint32_t pt_memslot_flags;63 uint32_t data_memslot_flags;64 bool skip;65 struct event_cnt expected_events;66};67 68struct test_params {69 enum vm_mem_backing_src_type src_type;70 struct test_desc *test_desc;71};72 73static inline void flush_tlb_page(uint64_t vaddr)74{75 uint64_t page = vaddr >> 12;76 77 dsb(ishst);78 asm volatile("tlbi vaae1is, %0" :: "r" (page));79 dsb(ish);80 isb();81}82 83static void guest_write64(void)84{85 uint64_t val;86 87 WRITE_ONCE(*guest_test_memory, TEST_DATA);88 val = READ_ONCE(*guest_test_memory);89 GUEST_ASSERT_EQ(val, TEST_DATA);90}91 92/* Check the system for atomic instructions. */93static bool guest_check_lse(void)94{95 uint64_t isar0 = read_sysreg(id_aa64isar0_el1);96 uint64_t atomic;97 98 atomic = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_ATOMIC), isar0);99 return atomic >= 2;100}101 102static bool guest_check_dc_zva(void)103{104 uint64_t dczid = read_sysreg(dczid_el0);105 uint64_t dzp = FIELD_GET(ARM64_FEATURE_MASK(DCZID_EL0_DZP), dczid);106 107 return dzp == 0;108}109 110/* Compare and swap instruction. */111static void guest_cas(void)112{113 uint64_t val;114 115 GUEST_ASSERT(guest_check_lse());116 asm volatile(".arch_extension lse\n"117 "casal %0, %1, [%2]\n"118 :: "r" (0ul), "r" (TEST_DATA), "r" (guest_test_memory));119 val = READ_ONCE(*guest_test_memory);120 GUEST_ASSERT_EQ(val, TEST_DATA);121}122 123static void guest_read64(void)124{125 uint64_t val;126 127 val = READ_ONCE(*guest_test_memory);128 GUEST_ASSERT_EQ(val, 0);129}130 131/* Address translation instruction */132static void guest_at(void)133{134 uint64_t par;135 136 asm volatile("at s1e1r, %0" :: "r" (guest_test_memory));137 isb();138 par = read_sysreg(par_el1);139 140 /* Bit 1 indicates whether the AT was successful */141 GUEST_ASSERT_EQ(par & 1, 0);142}143 144/*145 * The size of the block written by "dc zva" is guaranteed to be between (2 <<146 * 0) and (2 << 9), which is safe in our case as we need the write to happen147 * for at least a word, and not more than a page.148 */149static void guest_dc_zva(void)150{151 uint16_t val;152 153 asm volatile("dc zva, %0" :: "r" (guest_test_memory));154 dsb(ish);155 val = READ_ONCE(*guest_test_memory);156 GUEST_ASSERT_EQ(val, 0);157}158 159/*160 * Pre-indexing loads and stores don't have a valid syndrome (ESR_EL2.ISV==0).161 * And that's special because KVM must take special care with those: they162 * should still count as accesses for dirty logging or user-faulting, but163 * should be handled differently on mmio.164 */165static void guest_ld_preidx(void)166{167 uint64_t val;168 uint64_t addr = TEST_GVA - 8;169 170 /*171 * This ends up accessing "TEST_GVA + 8 - 8", where "TEST_GVA - 8" is172 * in a gap between memslots not backing by anything.173 */174 asm volatile("ldr %0, [%1, #8]!"175 : "=r" (val), "+r" (addr));176 GUEST_ASSERT_EQ(val, 0);177 GUEST_ASSERT_EQ(addr, TEST_GVA);178}179 180static void guest_st_preidx(void)181{182 uint64_t val = TEST_DATA;183 uint64_t addr = TEST_GVA - 8;184 185 asm volatile("str %0, [%1, #8]!"186 : "+r" (val), "+r" (addr));187 188 GUEST_ASSERT_EQ(addr, TEST_GVA);189 val = READ_ONCE(*guest_test_memory);190}191 192static bool guest_set_ha(void)193{194 uint64_t mmfr1 = read_sysreg(id_aa64mmfr1_el1);195 uint64_t hadbs, tcr;196 197 /* Skip if HA is not supported. */198 hadbs = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_HAFDBS), mmfr1);199 if (hadbs == 0)200 return false;201 202 tcr = read_sysreg(tcr_el1) | TCR_EL1_HA;203 write_sysreg(tcr, tcr_el1);204 isb();205 206 return true;207}208 209static bool guest_clear_pte_af(void)210{211 *((uint64_t *)TEST_PTE_GVA) &= ~PTE_AF;212 flush_tlb_page(TEST_GVA);213 214 return true;215}216 217static void guest_check_pte_af(void)218{219 dsb(ish);220 GUEST_ASSERT_EQ(*((uint64_t *)TEST_PTE_GVA) & PTE_AF, PTE_AF);221}222 223static void guest_check_write_in_dirty_log(void)224{225 GUEST_SYNC(CMD_CHECK_WRITE_IN_DIRTY_LOG);226}227 228static void guest_check_no_write_in_dirty_log(void)229{230 GUEST_SYNC(CMD_CHECK_NO_WRITE_IN_DIRTY_LOG);231}232 233static void guest_check_s1ptw_wr_in_dirty_log(void)234{235 GUEST_SYNC(CMD_CHECK_S1PTW_WR_IN_DIRTY_LOG);236}237 238static void guest_check_no_s1ptw_wr_in_dirty_log(void)239{240 GUEST_SYNC(CMD_CHECK_NO_S1PTW_WR_IN_DIRTY_LOG);241}242 243static void guest_exec(void)244{245 int (*code)(void) = (int (*)(void))TEST_EXEC_GVA;246 int ret;247 248 ret = code();249 GUEST_ASSERT_EQ(ret, 0x77);250}251 252static bool guest_prepare(struct test_desc *test)253{254 bool (*prepare_fn)(void);255 int i;256 257 for (i = 0; i < PREPARE_FN_NR; i++) {258 prepare_fn = test->guest_prepare[i];259 if (prepare_fn && !prepare_fn())260 return false;261 }262 263 return true;264}265 266static void guest_test_check(struct test_desc *test)267{268 void (*check_fn)(void);269 int i;270 271 for (i = 0; i < CHECK_FN_NR; i++) {272 check_fn = test->guest_test_check[i];273 if (check_fn)274 check_fn();275 }276}277 278static void guest_code(struct test_desc *test)279{280 if (!guest_prepare(test))281 GUEST_SYNC(CMD_SKIP_TEST);282 283 GUEST_SYNC(test->mem_mark_cmd);284 285 if (test->guest_test)286 test->guest_test();287 288 guest_test_check(test);289 GUEST_DONE();290}291 292static void no_dabt_handler(struct ex_regs *regs)293{294 GUEST_FAIL("Unexpected dabt, far_el1 = 0x%lx", read_sysreg(far_el1));295}296 297static void no_iabt_handler(struct ex_regs *regs)298{299 GUEST_FAIL("Unexpected iabt, pc = 0x%lx", regs->pc);300}301 302static struct uffd_args {303 char *copy;304 void *hva;305 uint64_t paging_size;306} pt_args, data_args;307 308/* Returns true to continue the test, and false if it should be skipped. */309static int uffd_generic_handler(int uffd_mode, int uffd, struct uffd_msg *msg,310 struct uffd_args *args)311{312 uint64_t addr = msg->arg.pagefault.address;313 uint64_t flags = msg->arg.pagefault.flags;314 struct uffdio_copy copy;315 int ret;316 317 TEST_ASSERT(uffd_mode == UFFDIO_REGISTER_MODE_MISSING,318 "The only expected UFFD mode is MISSING");319 TEST_ASSERT_EQ(addr, (uint64_t)args->hva);320 321 pr_debug("uffd fault: addr=%p write=%d\n",322 (void *)addr, !!(flags & UFFD_PAGEFAULT_FLAG_WRITE));323 324 copy.src = (uint64_t)args->copy;325 copy.dst = addr;326 copy.len = args->paging_size;327 copy.mode = 0;328 329 ret = ioctl(uffd, UFFDIO_COPY, ©);330 if (ret == -1) {331 pr_info("Failed UFFDIO_COPY in 0x%lx with errno: %d\n",332 addr, errno);333 return ret;334 }335 336 pthread_mutex_lock(&events.uffd_faults_mutex);337 events.uffd_faults += 1;338 pthread_mutex_unlock(&events.uffd_faults_mutex);339 return 0;340}341 342static int uffd_pt_handler(int mode, int uffd, struct uffd_msg *msg)343{344 return uffd_generic_handler(mode, uffd, msg, &pt_args);345}346 347static int uffd_data_handler(int mode, int uffd, struct uffd_msg *msg)348{349 return uffd_generic_handler(mode, uffd, msg, &data_args);350}351 352static void setup_uffd_args(struct userspace_mem_region *region,353 struct uffd_args *args)354{355 args->hva = (void *)region->region.userspace_addr;356 args->paging_size = region->region.memory_size;357 358 args->copy = malloc(args->paging_size);359 TEST_ASSERT(args->copy, "Failed to allocate data copy.");360 memcpy(args->copy, args->hva, args->paging_size);361}362 363static void setup_uffd(struct kvm_vm *vm, struct test_params *p,364 struct uffd_desc **pt_uffd, struct uffd_desc **data_uffd)365{366 struct test_desc *test = p->test_desc;367 int uffd_mode = UFFDIO_REGISTER_MODE_MISSING;368 369 setup_uffd_args(vm_get_mem_region(vm, MEM_REGION_PT), &pt_args);370 setup_uffd_args(vm_get_mem_region(vm, MEM_REGION_TEST_DATA), &data_args);371 372 *pt_uffd = NULL;373 if (test->uffd_pt_handler)374 *pt_uffd = uffd_setup_demand_paging(uffd_mode, 0,375 pt_args.hva,376 pt_args.paging_size,377 1, test->uffd_pt_handler);378 379 *data_uffd = NULL;380 if (test->uffd_data_handler)381 *data_uffd = uffd_setup_demand_paging(uffd_mode, 0,382 data_args.hva,383 data_args.paging_size,384 1, test->uffd_data_handler);385}386 387static void free_uffd(struct test_desc *test, struct uffd_desc *pt_uffd,388 struct uffd_desc *data_uffd)389{390 if (test->uffd_pt_handler)391 uffd_stop_demand_paging(pt_uffd);392 if (test->uffd_data_handler)393 uffd_stop_demand_paging(data_uffd);394 395 free(pt_args.copy);396 free(data_args.copy);397}398 399static int uffd_no_handler(int mode, int uffd, struct uffd_msg *msg)400{401 TEST_FAIL("There was no UFFD fault expected.");402 return -1;403}404 405/* Returns false if the test should be skipped. */406static bool punch_hole_in_backing_store(struct kvm_vm *vm,407 struct userspace_mem_region *region)408{409 void *hva = (void *)region->region.userspace_addr;410 uint64_t paging_size = region->region.memory_size;411 int ret, fd = region->fd;412 413 if (fd != -1) {414 ret = fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,415 0, paging_size);416 TEST_ASSERT(ret == 0, "fallocate failed");417 } else {418 ret = madvise(hva, paging_size, MADV_DONTNEED);419 TEST_ASSERT(ret == 0, "madvise failed");420 }421 422 return true;423}424 425static void mmio_on_test_gpa_handler(struct kvm_vm *vm, struct kvm_run *run)426{427 struct userspace_mem_region *region;428 void *hva;429 430 region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);431 hva = (void *)region->region.userspace_addr;432 433 TEST_ASSERT_EQ(run->mmio.phys_addr, region->region.guest_phys_addr);434 435 memcpy(hva, run->mmio.data, run->mmio.len);436 events.mmio_exits += 1;437}438 439static void mmio_no_handler(struct kvm_vm *vm, struct kvm_run *run)440{441 uint64_t data;442 443 memcpy(&data, run->mmio.data, sizeof(data));444 pr_debug("addr=%lld len=%d w=%d data=%lx\n",445 run->mmio.phys_addr, run->mmio.len,446 run->mmio.is_write, data);447 TEST_FAIL("There was no MMIO exit expected.");448}449 450static bool check_write_in_dirty_log(struct kvm_vm *vm,451 struct userspace_mem_region *region,452 uint64_t host_pg_nr)453{454 unsigned long *bmap;455 bool first_page_dirty;456 uint64_t size = region->region.memory_size;457 458 /* getpage_size() is not always equal to vm->page_size */459 bmap = bitmap_zalloc(size / getpagesize());460 kvm_vm_get_dirty_log(vm, region->region.slot, bmap);461 first_page_dirty = test_bit(host_pg_nr, bmap);462 free(bmap);463 return first_page_dirty;464}465 466/* Returns true to continue the test, and false if it should be skipped. */467static bool handle_cmd(struct kvm_vm *vm, int cmd)468{469 struct userspace_mem_region *data_region, *pt_region;470 bool continue_test = true;471 uint64_t pte_gpa, pte_pg;472 473 data_region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);474 pt_region = vm_get_mem_region(vm, MEM_REGION_PT);475 pte_gpa = addr_hva2gpa(vm, virt_get_pte_hva(vm, TEST_GVA));476 pte_pg = (pte_gpa - pt_region->region.guest_phys_addr) / getpagesize();477 478 if (cmd == CMD_SKIP_TEST)479 continue_test = false;480 481 if (cmd & CMD_HOLE_PT)482 continue_test = punch_hole_in_backing_store(vm, pt_region);483 if (cmd & CMD_HOLE_DATA)484 continue_test = punch_hole_in_backing_store(vm, data_region);485 if (cmd & CMD_CHECK_WRITE_IN_DIRTY_LOG)486 TEST_ASSERT(check_write_in_dirty_log(vm, data_region, 0),487 "Missing write in dirty log");488 if (cmd & CMD_CHECK_S1PTW_WR_IN_DIRTY_LOG)489 TEST_ASSERT(check_write_in_dirty_log(vm, pt_region, pte_pg),490 "Missing s1ptw write in dirty log");491 if (cmd & CMD_CHECK_NO_WRITE_IN_DIRTY_LOG)492 TEST_ASSERT(!check_write_in_dirty_log(vm, data_region, 0),493 "Unexpected write in dirty log");494 if (cmd & CMD_CHECK_NO_S1PTW_WR_IN_DIRTY_LOG)495 TEST_ASSERT(!check_write_in_dirty_log(vm, pt_region, pte_pg),496 "Unexpected s1ptw write in dirty log");497 498 return continue_test;499}500 501void fail_vcpu_run_no_handler(int ret)502{503 TEST_FAIL("Unexpected vcpu run failure");504}505 506void fail_vcpu_run_mmio_no_syndrome_handler(int ret)507{508 TEST_ASSERT(errno == ENOSYS,509 "The mmio handler should have returned not implemented.");510 events.fail_vcpu_runs += 1;511}512 513typedef uint32_t aarch64_insn_t;514extern aarch64_insn_t __exec_test[2];515 516noinline void __return_0x77(void)517{518 asm volatile("__exec_test: mov x0, #0x77\n"519 "ret\n");520}521 522/*523 * Note that this function runs on the host before the test VM starts: there's524 * no need to sync the D$ and I$ caches.525 */526static void load_exec_code_for_test(struct kvm_vm *vm)527{528 uint64_t *code;529 struct userspace_mem_region *region;530 void *hva;531 532 region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);533 hva = (void *)region->region.userspace_addr;534 535 assert(TEST_EXEC_GVA > TEST_GVA);536 code = hva + TEST_EXEC_GVA - TEST_GVA;537 memcpy(code, __exec_test, sizeof(__exec_test));538}539 540static void setup_abort_handlers(struct kvm_vm *vm, struct kvm_vcpu *vcpu,541 struct test_desc *test)542{543 vm_init_descriptor_tables(vm);544 vcpu_init_descriptor_tables(vcpu);545 546 vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,547 ESR_EC_DABT, no_dabt_handler);548 vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,549 ESR_EC_IABT, no_iabt_handler);550}551 552static void setup_gva_maps(struct kvm_vm *vm)553{554 struct userspace_mem_region *region;555 uint64_t pte_gpa;556 557 region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);558 /* Map TEST_GVA first. This will install a new PTE. */559 virt_pg_map(vm, TEST_GVA, region->region.guest_phys_addr);560 /* Then map TEST_PTE_GVA to the above PTE. */561 pte_gpa = addr_hva2gpa(vm, virt_get_pte_hva(vm, TEST_GVA));562 virt_pg_map(vm, TEST_PTE_GVA, pte_gpa);563}564 565enum pf_test_memslots {566 CODE_AND_DATA_MEMSLOT,567 PAGE_TABLE_MEMSLOT,568 TEST_DATA_MEMSLOT,569};570 571/*572 * Create a memslot for code and data at pfn=0, and test-data and PT ones573 * at max_gfn.574 */575static void setup_memslots(struct kvm_vm *vm, struct test_params *p)576{577 uint64_t backing_src_pagesz = get_backing_src_pagesz(p->src_type);578 uint64_t guest_page_size = vm->page_size;579 uint64_t max_gfn = vm_compute_max_gfn(vm);580 /* Enough for 2M of code when using 4K guest pages. */581 uint64_t code_npages = 512;582 uint64_t pt_size, data_size, data_gpa;583 584 /*585 * This test requires 1 pgd, 2 pud, 4 pmd, and 6 pte pages when using586 * VM_MODE_P48V48_4K. Note that the .text takes ~1.6MBs. That's 13587 * pages. VM_MODE_P48V48_4K is the mode with most PT pages; let's use588 * twice that just in case.589 */590 pt_size = 26 * guest_page_size;591 592 /* memslot sizes and gpa's must be aligned to the backing page size */593 pt_size = align_up(pt_size, backing_src_pagesz);594 data_size = align_up(guest_page_size, backing_src_pagesz);595 data_gpa = (max_gfn * guest_page_size) - data_size;596 data_gpa = align_down(data_gpa, backing_src_pagesz);597 598 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0,599 CODE_AND_DATA_MEMSLOT, code_npages, 0);600 vm->memslots[MEM_REGION_CODE] = CODE_AND_DATA_MEMSLOT;601 vm->memslots[MEM_REGION_DATA] = CODE_AND_DATA_MEMSLOT;602 603 vm_userspace_mem_region_add(vm, p->src_type, data_gpa - pt_size,604 PAGE_TABLE_MEMSLOT, pt_size / guest_page_size,605 p->test_desc->pt_memslot_flags);606 vm->memslots[MEM_REGION_PT] = PAGE_TABLE_MEMSLOT;607 608 vm_userspace_mem_region_add(vm, p->src_type, data_gpa, TEST_DATA_MEMSLOT,609 data_size / guest_page_size,610 p->test_desc->data_memslot_flags);611 vm->memslots[MEM_REGION_TEST_DATA] = TEST_DATA_MEMSLOT;612}613 614static void setup_ucall(struct kvm_vm *vm)615{616 struct userspace_mem_region *region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);617 618 ucall_init(vm, region->region.guest_phys_addr + region->region.memory_size);619}620 621static void setup_default_handlers(struct test_desc *test)622{623 if (!test->mmio_handler)624 test->mmio_handler = mmio_no_handler;625 626 if (!test->fail_vcpu_run_handler)627 test->fail_vcpu_run_handler = fail_vcpu_run_no_handler;628}629 630static void check_event_counts(struct test_desc *test)631{632 TEST_ASSERT_EQ(test->expected_events.uffd_faults, events.uffd_faults);633 TEST_ASSERT_EQ(test->expected_events.mmio_exits, events.mmio_exits);634 TEST_ASSERT_EQ(test->expected_events.fail_vcpu_runs, events.fail_vcpu_runs);635}636 637static void print_test_banner(enum vm_guest_mode mode, struct test_params *p)638{639 struct test_desc *test = p->test_desc;640 641 pr_debug("Test: %s\n", test->name);642 pr_debug("Testing guest mode: %s\n", vm_guest_mode_string(mode));643 pr_debug("Testing memory backing src type: %s\n",644 vm_mem_backing_src_alias(p->src_type)->name);645}646 647static void reset_event_counts(void)648{649 memset(&events, 0, sizeof(events));650}651 652/*653 * This function either succeeds, skips the test (after setting test->skip), or654 * fails with a TEST_FAIL that aborts all tests.655 */656static void vcpu_run_loop(struct kvm_vm *vm, struct kvm_vcpu *vcpu,657 struct test_desc *test)658{659 struct kvm_run *run;660 struct ucall uc;661 int ret;662 663 run = vcpu->run;664 665 for (;;) {666 ret = _vcpu_run(vcpu);667 if (ret) {668 test->fail_vcpu_run_handler(ret);669 goto done;670 }671 672 switch (get_ucall(vcpu, &uc)) {673 case UCALL_SYNC:674 if (!handle_cmd(vm, uc.args[1])) {675 test->skip = true;676 goto done;677 }678 break;679 case UCALL_ABORT:680 REPORT_GUEST_ASSERT(uc);681 break;682 case UCALL_DONE:683 goto done;684 case UCALL_NONE:685 if (run->exit_reason == KVM_EXIT_MMIO)686 test->mmio_handler(vm, run);687 break;688 default:689 TEST_FAIL("Unknown ucall %lu", uc.cmd);690 }691 }692 693done:694 pr_debug(test->skip ? "Skipped.\n" : "Done.\n");695}696 697static void run_test(enum vm_guest_mode mode, void *arg)698{699 struct test_params *p = (struct test_params *)arg;700 struct test_desc *test = p->test_desc;701 struct kvm_vm *vm;702 struct kvm_vcpu *vcpu;703 struct uffd_desc *pt_uffd, *data_uffd;704 705 print_test_banner(mode, p);706 707 vm = ____vm_create(VM_SHAPE(mode));708 setup_memslots(vm, p);709 kvm_vm_elf_load(vm, program_invocation_name);710 setup_ucall(vm);711 vcpu = vm_vcpu_add(vm, 0, guest_code);712 713 setup_gva_maps(vm);714 715 reset_event_counts();716 717 /*718 * Set some code in the data memslot for the guest to execute (only719 * applicable to the EXEC tests). This has to be done before720 * setup_uffd() as that function copies the memslot data for the uffd721 * handler.722 */723 load_exec_code_for_test(vm);724 setup_uffd(vm, p, &pt_uffd, &data_uffd);725 setup_abort_handlers(vm, vcpu, test);726 setup_default_handlers(test);727 vcpu_args_set(vcpu, 1, test);728 729 vcpu_run_loop(vm, vcpu, test);730 731 kvm_vm_free(vm);732 free_uffd(test, pt_uffd, data_uffd);733 734 /*735 * Make sure we check the events after the uffd threads have exited,736 * which means they updated their respective event counters.737 */738 if (!test->skip)739 check_event_counts(test);740}741 742static void help(char *name)743{744 puts("");745 printf("usage: %s [-h] [-s mem-type]\n", name);746 puts("");747 guest_modes_help();748 backing_src_help("-s");749 puts("");750}751 752#define SNAME(s) #s753#define SCAT2(a, b) SNAME(a ## _ ## b)754#define SCAT3(a, b, c) SCAT2(a, SCAT2(b, c))755#define SCAT4(a, b, c, d) SCAT2(a, SCAT3(b, c, d))756 757#define _CHECK(_test) _CHECK_##_test758#define _PREPARE(_test) _PREPARE_##_test759#define _PREPARE_guest_read64 NULL760#define _PREPARE_guest_ld_preidx NULL761#define _PREPARE_guest_write64 NULL762#define _PREPARE_guest_st_preidx NULL763#define _PREPARE_guest_exec NULL764#define _PREPARE_guest_at NULL765#define _PREPARE_guest_dc_zva guest_check_dc_zva766#define _PREPARE_guest_cas guest_check_lse767 768/* With or without access flag checks */769#define _PREPARE_with_af guest_set_ha, guest_clear_pte_af770#define _PREPARE_no_af NULL771#define _CHECK_with_af guest_check_pte_af772#define _CHECK_no_af NULL773 774/* Performs an access and checks that no faults were triggered. */775#define TEST_ACCESS(_access, _with_af, _mark_cmd) \776{ \777 .name = SCAT3(_access, _with_af, #_mark_cmd), \778 .guest_prepare = { _PREPARE(_with_af), \779 _PREPARE(_access) }, \780 .mem_mark_cmd = _mark_cmd, \781 .guest_test = _access, \782 .guest_test_check = { _CHECK(_with_af) }, \783 .expected_events = { 0 }, \784}785 786#define TEST_UFFD(_access, _with_af, _mark_cmd, \787 _uffd_data_handler, _uffd_pt_handler, _uffd_faults) \788{ \789 .name = SCAT4(uffd, _access, _with_af, #_mark_cmd), \790 .guest_prepare = { _PREPARE(_with_af), \791 _PREPARE(_access) }, \792 .guest_test = _access, \793 .mem_mark_cmd = _mark_cmd, \794 .guest_test_check = { _CHECK(_with_af) }, \795 .uffd_data_handler = _uffd_data_handler, \796 .uffd_pt_handler = _uffd_pt_handler, \797 .expected_events = { .uffd_faults = _uffd_faults, }, \798}799 800#define TEST_DIRTY_LOG(_access, _with_af, _test_check, _pt_check) \801{ \802 .name = SCAT3(dirty_log, _access, _with_af), \803 .data_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \804 .pt_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \805 .guest_prepare = { _PREPARE(_with_af), \806 _PREPARE(_access) }, \807 .guest_test = _access, \808 .guest_test_check = { _CHECK(_with_af), _test_check, _pt_check }, \809 .expected_events = { 0 }, \810}811 812#define TEST_UFFD_AND_DIRTY_LOG(_access, _with_af, _uffd_data_handler, \813 _uffd_faults, _test_check, _pt_check) \814{ \815 .name = SCAT3(uffd_and_dirty_log, _access, _with_af), \816 .data_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \817 .pt_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \818 .guest_prepare = { _PREPARE(_with_af), \819 _PREPARE(_access) }, \820 .guest_test = _access, \821 .mem_mark_cmd = CMD_HOLE_DATA | CMD_HOLE_PT, \822 .guest_test_check = { _CHECK(_with_af), _test_check, _pt_check }, \823 .uffd_data_handler = _uffd_data_handler, \824 .uffd_pt_handler = uffd_pt_handler, \825 .expected_events = { .uffd_faults = _uffd_faults, }, \826}827 828#define TEST_RO_MEMSLOT(_access, _mmio_handler, _mmio_exits) \829{ \830 .name = SCAT2(ro_memslot, _access), \831 .data_memslot_flags = KVM_MEM_READONLY, \832 .pt_memslot_flags = KVM_MEM_READONLY, \833 .guest_prepare = { _PREPARE(_access) }, \834 .guest_test = _access, \835 .mmio_handler = _mmio_handler, \836 .expected_events = { .mmio_exits = _mmio_exits }, \837}838 839#define TEST_RO_MEMSLOT_NO_SYNDROME(_access) \840{ \841 .name = SCAT2(ro_memslot_no_syndrome, _access), \842 .data_memslot_flags = KVM_MEM_READONLY, \843 .pt_memslot_flags = KVM_MEM_READONLY, \844 .guest_prepare = { _PREPARE(_access) }, \845 .guest_test = _access, \846 .fail_vcpu_run_handler = fail_vcpu_run_mmio_no_syndrome_handler, \847 .expected_events = { .fail_vcpu_runs = 1 }, \848}849 850#define TEST_RO_MEMSLOT_AND_DIRTY_LOG(_access, _mmio_handler, _mmio_exits, \851 _test_check) \852{ \853 .name = SCAT2(ro_memslot, _access), \854 .data_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \855 .pt_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \856 .guest_prepare = { _PREPARE(_access) }, \857 .guest_test = _access, \858 .guest_test_check = { _test_check }, \859 .mmio_handler = _mmio_handler, \860 .expected_events = { .mmio_exits = _mmio_exits}, \861}862 863#define TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(_access, _test_check) \864{ \865 .name = SCAT2(ro_memslot_no_syn_and_dlog, _access), \866 .data_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \867 .pt_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \868 .guest_prepare = { _PREPARE(_access) }, \869 .guest_test = _access, \870 .guest_test_check = { _test_check }, \871 .fail_vcpu_run_handler = fail_vcpu_run_mmio_no_syndrome_handler, \872 .expected_events = { .fail_vcpu_runs = 1 }, \873}874 875#define TEST_RO_MEMSLOT_AND_UFFD(_access, _mmio_handler, _mmio_exits, \876 _uffd_data_handler, _uffd_faults) \877{ \878 .name = SCAT2(ro_memslot_uffd, _access), \879 .data_memslot_flags = KVM_MEM_READONLY, \880 .pt_memslot_flags = KVM_MEM_READONLY, \881 .mem_mark_cmd = CMD_HOLE_DATA | CMD_HOLE_PT, \882 .guest_prepare = { _PREPARE(_access) }, \883 .guest_test = _access, \884 .uffd_data_handler = _uffd_data_handler, \885 .uffd_pt_handler = uffd_pt_handler, \886 .mmio_handler = _mmio_handler, \887 .expected_events = { .mmio_exits = _mmio_exits, \888 .uffd_faults = _uffd_faults }, \889}890 891#define TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(_access, _uffd_data_handler, \892 _uffd_faults) \893{ \894 .name = SCAT2(ro_memslot_no_syndrome, _access), \895 .data_memslot_flags = KVM_MEM_READONLY, \896 .pt_memslot_flags = KVM_MEM_READONLY, \897 .mem_mark_cmd = CMD_HOLE_DATA | CMD_HOLE_PT, \898 .guest_prepare = { _PREPARE(_access) }, \899 .guest_test = _access, \900 .uffd_data_handler = _uffd_data_handler, \901 .uffd_pt_handler = uffd_pt_handler, \902 .fail_vcpu_run_handler = fail_vcpu_run_mmio_no_syndrome_handler, \903 .expected_events = { .fail_vcpu_runs = 1, \904 .uffd_faults = _uffd_faults }, \905}906 907static struct test_desc tests[] = {908 909 /* Check that HW is setting the Access Flag (AF) (sanity checks). */910 TEST_ACCESS(guest_read64, with_af, CMD_NONE),911 TEST_ACCESS(guest_ld_preidx, with_af, CMD_NONE),912 TEST_ACCESS(guest_cas, with_af, CMD_NONE),913 TEST_ACCESS(guest_write64, with_af, CMD_NONE),914 TEST_ACCESS(guest_st_preidx, with_af, CMD_NONE),915 TEST_ACCESS(guest_dc_zva, with_af, CMD_NONE),916 TEST_ACCESS(guest_exec, with_af, CMD_NONE),917 918 /*919 * Punch a hole in the data backing store, and then try multiple920 * accesses: reads should rturn zeroes, and writes should921 * re-populate the page. Moreover, the test also check that no922 * exception was generated in the guest. Note that this923 * reading/writing behavior is the same as reading/writing a924 * punched page (with fallocate(FALLOC_FL_PUNCH_HOLE)) from925 * userspace.926 */927 TEST_ACCESS(guest_read64, no_af, CMD_HOLE_DATA),928 TEST_ACCESS(guest_cas, no_af, CMD_HOLE_DATA),929 TEST_ACCESS(guest_ld_preidx, no_af, CMD_HOLE_DATA),930 TEST_ACCESS(guest_write64, no_af, CMD_HOLE_DATA),931 TEST_ACCESS(guest_st_preidx, no_af, CMD_HOLE_DATA),932 TEST_ACCESS(guest_at, no_af, CMD_HOLE_DATA),933 TEST_ACCESS(guest_dc_zva, no_af, CMD_HOLE_DATA),934 935 /*936 * Punch holes in the data and PT backing stores and mark them for937 * userfaultfd handling. This should result in 2 faults: the access938 * on the data backing store, and its respective S1 page table walk939 * (S1PTW).940 */941 TEST_UFFD(guest_read64, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,942 uffd_data_handler, uffd_pt_handler, 2),943 TEST_UFFD(guest_read64, no_af, CMD_HOLE_DATA | CMD_HOLE_PT,944 uffd_data_handler, uffd_pt_handler, 2),945 TEST_UFFD(guest_cas, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,946 uffd_data_handler, uffd_pt_handler, 2),947 /*948 * Can't test guest_at with_af as it's IMPDEF whether the AF is set.949 * The S1PTW fault should still be marked as a write.950 */951 TEST_UFFD(guest_at, no_af, CMD_HOLE_DATA | CMD_HOLE_PT,952 uffd_no_handler, uffd_pt_handler, 1),953 TEST_UFFD(guest_ld_preidx, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,954 uffd_data_handler, uffd_pt_handler, 2),955 TEST_UFFD(guest_write64, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,956 uffd_data_handler, uffd_pt_handler, 2),957 TEST_UFFD(guest_dc_zva, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,958 uffd_data_handler, uffd_pt_handler, 2),959 TEST_UFFD(guest_st_preidx, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,960 uffd_data_handler, uffd_pt_handler, 2),961 TEST_UFFD(guest_exec, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,962 uffd_data_handler, uffd_pt_handler, 2),963 964 /*965 * Try accesses when the data and PT memory regions are both966 * tracked for dirty logging.967 */968 TEST_DIRTY_LOG(guest_read64, with_af, guest_check_no_write_in_dirty_log,969 guest_check_s1ptw_wr_in_dirty_log),970 TEST_DIRTY_LOG(guest_read64, no_af, guest_check_no_write_in_dirty_log,971 guest_check_no_s1ptw_wr_in_dirty_log),972 TEST_DIRTY_LOG(guest_ld_preidx, with_af,973 guest_check_no_write_in_dirty_log,974 guest_check_s1ptw_wr_in_dirty_log),975 TEST_DIRTY_LOG(guest_at, no_af, guest_check_no_write_in_dirty_log,976 guest_check_no_s1ptw_wr_in_dirty_log),977 TEST_DIRTY_LOG(guest_exec, with_af, guest_check_no_write_in_dirty_log,978 guest_check_s1ptw_wr_in_dirty_log),979 TEST_DIRTY_LOG(guest_write64, with_af, guest_check_write_in_dirty_log,980 guest_check_s1ptw_wr_in_dirty_log),981 TEST_DIRTY_LOG(guest_cas, with_af, guest_check_write_in_dirty_log,982 guest_check_s1ptw_wr_in_dirty_log),983 TEST_DIRTY_LOG(guest_dc_zva, with_af, guest_check_write_in_dirty_log,984 guest_check_s1ptw_wr_in_dirty_log),985 TEST_DIRTY_LOG(guest_st_preidx, with_af, guest_check_write_in_dirty_log,986 guest_check_s1ptw_wr_in_dirty_log),987 988 /*989 * Access when the data and PT memory regions are both marked for990 * dirty logging and UFFD at the same time. The expected result is991 * that writes should mark the dirty log and trigger a userfaultfd992 * write fault. Reads/execs should result in a read userfaultfd993 * fault, and nothing in the dirty log. Any S1PTW should result in994 * a write in the dirty log and a userfaultfd write.995 */996 TEST_UFFD_AND_DIRTY_LOG(guest_read64, with_af,997 uffd_data_handler, 2,998 guest_check_no_write_in_dirty_log,999 guest_check_s1ptw_wr_in_dirty_log),1000 TEST_UFFD_AND_DIRTY_LOG(guest_read64, no_af,1001 uffd_data_handler, 2,1002 guest_check_no_write_in_dirty_log,1003 guest_check_no_s1ptw_wr_in_dirty_log),1004 TEST_UFFD_AND_DIRTY_LOG(guest_ld_preidx, with_af,1005 uffd_data_handler,1006 2, guest_check_no_write_in_dirty_log,1007 guest_check_s1ptw_wr_in_dirty_log),1008 TEST_UFFD_AND_DIRTY_LOG(guest_at, with_af, uffd_no_handler, 1,1009 guest_check_no_write_in_dirty_log,1010 guest_check_s1ptw_wr_in_dirty_log),1011 TEST_UFFD_AND_DIRTY_LOG(guest_exec, with_af,1012 uffd_data_handler, 2,1013 guest_check_no_write_in_dirty_log,1014 guest_check_s1ptw_wr_in_dirty_log),1015 TEST_UFFD_AND_DIRTY_LOG(guest_write64, with_af,1016 uffd_data_handler,1017 2, guest_check_write_in_dirty_log,1018 guest_check_s1ptw_wr_in_dirty_log),1019 TEST_UFFD_AND_DIRTY_LOG(guest_cas, with_af,1020 uffd_data_handler, 2,1021 guest_check_write_in_dirty_log,1022 guest_check_s1ptw_wr_in_dirty_log),1023 TEST_UFFD_AND_DIRTY_LOG(guest_dc_zva, with_af,1024 uffd_data_handler,1025 2, guest_check_write_in_dirty_log,1026 guest_check_s1ptw_wr_in_dirty_log),1027 TEST_UFFD_AND_DIRTY_LOG(guest_st_preidx, with_af,1028 uffd_data_handler, 2,1029 guest_check_write_in_dirty_log,1030 guest_check_s1ptw_wr_in_dirty_log),1031 /*1032 * Access when both the PT and data regions are marked read-only1033 * (with KVM_MEM_READONLY). Writes with a syndrome result in an1034 * MMIO exit, writes with no syndrome (e.g., CAS) result in a1035 * failed vcpu run, and reads/execs with and without syndroms do1036 * not fault.1037 */1038 TEST_RO_MEMSLOT(guest_read64, 0, 0),1039 TEST_RO_MEMSLOT(guest_ld_preidx, 0, 0),1040 TEST_RO_MEMSLOT(guest_at, 0, 0),1041 TEST_RO_MEMSLOT(guest_exec, 0, 0),1042 TEST_RO_MEMSLOT(guest_write64, mmio_on_test_gpa_handler, 1),1043 TEST_RO_MEMSLOT_NO_SYNDROME(guest_dc_zva),1044 TEST_RO_MEMSLOT_NO_SYNDROME(guest_cas),1045 TEST_RO_MEMSLOT_NO_SYNDROME(guest_st_preidx),1046 1047 /*1048 * The PT and data regions are both read-only and marked1049 * for dirty logging at the same time. The expected result is that1050 * for writes there should be no write in the dirty log. The1051 * readonly handling is the same as if the memslot was not marked1052 * for dirty logging: writes with a syndrome result in an MMIO1053 * exit, and writes with no syndrome result in a failed vcpu run.1054 */1055 TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_read64, 0, 0,1056 guest_check_no_write_in_dirty_log),1057 TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_ld_preidx, 0, 0,1058 guest_check_no_write_in_dirty_log),1059 TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_at, 0, 0,1060 guest_check_no_write_in_dirty_log),1061 TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_exec, 0, 0,1062 guest_check_no_write_in_dirty_log),1063 TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_write64, mmio_on_test_gpa_handler,1064 1, guest_check_no_write_in_dirty_log),1065 TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(guest_dc_zva,1066 guest_check_no_write_in_dirty_log),1067 TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(guest_cas,1068 guest_check_no_write_in_dirty_log),1069 TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(guest_st_preidx,1070 guest_check_no_write_in_dirty_log),1071 1072 /*1073 * The PT and data regions are both read-only and punched with1074 * holes tracked with userfaultfd. The expected result is the1075 * union of both userfaultfd and read-only behaviors. For example,1076 * write accesses result in a userfaultfd write fault and an MMIO1077 * exit. Writes with no syndrome result in a failed vcpu run and1078 * no userfaultfd write fault. Reads result in userfaultfd getting1079 * triggered.1080 */1081 TEST_RO_MEMSLOT_AND_UFFD(guest_read64, 0, 0, uffd_data_handler, 2),1082 TEST_RO_MEMSLOT_AND_UFFD(guest_ld_preidx, 0, 0, uffd_data_handler, 2),1083 TEST_RO_MEMSLOT_AND_UFFD(guest_at, 0, 0, uffd_no_handler, 1),1084 TEST_RO_MEMSLOT_AND_UFFD(guest_exec, 0, 0, uffd_data_handler, 2),1085 TEST_RO_MEMSLOT_AND_UFFD(guest_write64, mmio_on_test_gpa_handler, 1,1086 uffd_data_handler, 2),1087 TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_cas, uffd_data_handler, 2),1088 TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_dc_zva, uffd_no_handler, 1),1089 TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_st_preidx, uffd_no_handler, 1),1090 1091 { 0 }1092};1093 1094static void for_each_test_and_guest_mode(enum vm_mem_backing_src_type src_type)1095{1096 struct test_desc *t;1097 1098 for (t = &tests[0]; t->name; t++) {1099 if (t->skip)1100 continue;1101 1102 struct test_params p = {1103 .src_type = src_type,1104 .test_desc = t,1105 };1106 1107 for_each_guest_mode(run_test, &p);1108 }1109}1110 1111int main(int argc, char *argv[])1112{1113 enum vm_mem_backing_src_type src_type;1114 int opt;1115 1116 src_type = DEFAULT_VM_MEM_SRC;1117 1118 while ((opt = getopt(argc, argv, "hm:s:")) != -1) {1119 switch (opt) {1120 case 'm':1121 guest_modes_cmdline(optarg);1122 break;1123 case 's':1124 src_type = parse_backing_src_type(optarg);1125 break;1126 case 'h':1127 default:1128 help(argv[0]);1129 exit(0);1130 }1131 }1132 1133 for_each_test_and_guest_mode(src_type);1134 return 0;1135}1136