606 lines · c
1// SPDX-License-Identifier: GPL-2.02#include <fcntl.h>3#include <pthread.h>4#include <sched.h>5#include <semaphore.h>6#include <signal.h>7#include <stdio.h>8#include <stdlib.h>9#include <string.h>10#include <sys/ioctl.h>11#include <sys/mman.h>12 13#include <linux/compiler.h>14 15#include <test_util.h>16#include <kvm_util.h>17#include <processor.h>18 19/*20 * s390x needs at least 1MB alignment, and the x86_64 MOVE/DELETE tests need a21 * 2MB sized and aligned region so that the initial region corresponds to22 * exactly one large page.23 */24#define MEM_REGION_SIZE 0x20000025 26#ifdef __x86_64__27/*28 * Somewhat arbitrary location and slot, intended to not overlap anything.29 */30#define MEM_REGION_GPA 0xc000000031#define MEM_REGION_SLOT 1032 33static const uint64_t MMIO_VAL = 0xbeefull;34 35extern const uint64_t final_rip_start;36extern const uint64_t final_rip_end;37 38static sem_t vcpu_ready;39 40static inline uint64_t guest_spin_on_val(uint64_t spin_val)41{42 uint64_t val;43 44 do {45 val = READ_ONCE(*((uint64_t *)MEM_REGION_GPA));46 } while (val == spin_val);47 48 GUEST_SYNC(0);49 return val;50}51 52static void *vcpu_worker(void *data)53{54 struct kvm_vcpu *vcpu = data;55 struct kvm_run *run = vcpu->run;56 struct ucall uc;57 uint64_t cmd;58 59 /*60 * Loop until the guest is done. Re-enter the guest on all MMIO exits,61 * which will occur if the guest attempts to access a memslot after it62 * has been deleted or while it is being moved .63 */64 while (1) {65 vcpu_run(vcpu);66 67 if (run->exit_reason == KVM_EXIT_IO) {68 cmd = get_ucall(vcpu, &uc);69 if (cmd != UCALL_SYNC)70 break;71 72 sem_post(&vcpu_ready);73 continue;74 }75 76 if (run->exit_reason != KVM_EXIT_MMIO)77 break;78 79 TEST_ASSERT(!run->mmio.is_write, "Unexpected exit mmio write");80 TEST_ASSERT(run->mmio.len == 8,81 "Unexpected exit mmio size = %u", run->mmio.len);82 83 TEST_ASSERT(run->mmio.phys_addr == MEM_REGION_GPA,84 "Unexpected exit mmio address = 0x%llx",85 run->mmio.phys_addr);86 memcpy(run->mmio.data, &MMIO_VAL, 8);87 }88 89 if (run->exit_reason == KVM_EXIT_IO && cmd == UCALL_ABORT)90 REPORT_GUEST_ASSERT(uc);91 92 return NULL;93}94 95static void wait_for_vcpu(void)96{97 struct timespec ts;98 99 TEST_ASSERT(!clock_gettime(CLOCK_REALTIME, &ts),100 "clock_gettime() failed: %d", errno);101 102 ts.tv_sec += 2;103 TEST_ASSERT(!sem_timedwait(&vcpu_ready, &ts),104 "sem_timedwait() failed: %d", errno);105 106 /* Wait for the vCPU thread to reenter the guest. */107 usleep(100000);108}109 110static struct kvm_vm *spawn_vm(struct kvm_vcpu **vcpu, pthread_t *vcpu_thread,111 void *guest_code)112{113 struct kvm_vm *vm;114 uint64_t *hva;115 uint64_t gpa;116 117 vm = vm_create_with_one_vcpu(vcpu, guest_code);118 119 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP,120 MEM_REGION_GPA, MEM_REGION_SLOT,121 MEM_REGION_SIZE / getpagesize(), 0);122 123 /*124 * Allocate and map two pages so that the GPA accessed by guest_code()125 * stays valid across the memslot move.126 */127 gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_GPA, MEM_REGION_SLOT);128 TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc\n");129 130 virt_map(vm, MEM_REGION_GPA, MEM_REGION_GPA, 2);131 132 /* Ditto for the host mapping so that both pages can be zeroed. */133 hva = addr_gpa2hva(vm, MEM_REGION_GPA);134 memset(hva, 0, 2 * 4096);135 136 pthread_create(vcpu_thread, NULL, vcpu_worker, *vcpu);137 138 /* Ensure the guest thread is spun up. */139 wait_for_vcpu();140 141 return vm;142}143 144 145static void guest_code_move_memory_region(void)146{147 uint64_t val;148 149 GUEST_SYNC(0);150 151 /*152 * Spin until the memory region starts getting moved to a153 * misaligned address.154 * Every region move may or may not trigger MMIO, as the155 * window where the memslot is invalid is usually quite small.156 */157 val = guest_spin_on_val(0);158 __GUEST_ASSERT(val == 1 || val == MMIO_VAL,159 "Expected '1' or MMIO ('%lx'), got '%lx'", MMIO_VAL, val);160 161 /* Spin until the misaligning memory region move completes. */162 val = guest_spin_on_val(MMIO_VAL);163 __GUEST_ASSERT(val == 1 || val == 0,164 "Expected '0' or '1' (no MMIO), got '%lx'", val);165 166 /* Spin until the memory region starts to get re-aligned. */167 val = guest_spin_on_val(0);168 __GUEST_ASSERT(val == 1 || val == MMIO_VAL,169 "Expected '1' or MMIO ('%lx'), got '%lx'", MMIO_VAL, val);170 171 /* Spin until the re-aligning memory region move completes. */172 val = guest_spin_on_val(MMIO_VAL);173 GUEST_ASSERT_EQ(val, 1);174 175 GUEST_DONE();176}177 178static void test_move_memory_region(bool disable_slot_zap_quirk)179{180 pthread_t vcpu_thread;181 struct kvm_vcpu *vcpu;182 struct kvm_vm *vm;183 uint64_t *hva;184 185 vm = spawn_vm(&vcpu, &vcpu_thread, guest_code_move_memory_region);186 187 if (disable_slot_zap_quirk)188 vm_enable_cap(vm, KVM_CAP_DISABLE_QUIRKS2, KVM_X86_QUIRK_SLOT_ZAP_ALL);189 190 hva = addr_gpa2hva(vm, MEM_REGION_GPA);191 192 /*193 * Shift the region's base GPA. The guest should not see "2" as the194 * hva->gpa translation is misaligned, i.e. the guest is accessing a195 * different host pfn.196 */197 vm_mem_region_move(vm, MEM_REGION_SLOT, MEM_REGION_GPA - 4096);198 WRITE_ONCE(*hva, 2);199 200 /*201 * The guest _might_ see an invalid memslot and trigger MMIO, but it's202 * a tiny window. Spin and defer the sync until the memslot is203 * restored and guest behavior is once again deterministic.204 */205 usleep(100000);206 207 /*208 * Note, value in memory needs to be changed *before* restoring the209 * memslot, else the guest could race the update and see "2".210 */211 WRITE_ONCE(*hva, 1);212 213 /* Restore the original base, the guest should see "1". */214 vm_mem_region_move(vm, MEM_REGION_SLOT, MEM_REGION_GPA);215 wait_for_vcpu();216 /* Defered sync from when the memslot was misaligned (above). */217 wait_for_vcpu();218 219 pthread_join(vcpu_thread, NULL);220 221 kvm_vm_free(vm);222}223 224static void guest_code_delete_memory_region(void)225{226 struct desc_ptr idt;227 uint64_t val;228 229 /*230 * Clobber the IDT so that a #PF due to the memory region being deleted231 * escalates to triple-fault shutdown. Because the memory region is232 * deleted, there will be no valid mappings. As a result, KVM will233 * repeatedly intercepts the state-2 page fault that occurs when trying234 * to vector the guest's #PF. I.e. trying to actually handle the #PF235 * in the guest will never succeed, and so isn't an option.236 */237 memset(&idt, 0, sizeof(idt));238 __asm__ __volatile__("lidt %0" :: "m"(idt));239 240 GUEST_SYNC(0);241 242 /* Spin until the memory region is deleted. */243 val = guest_spin_on_val(0);244 GUEST_ASSERT_EQ(val, MMIO_VAL);245 246 /* Spin until the memory region is recreated. */247 val = guest_spin_on_val(MMIO_VAL);248 GUEST_ASSERT_EQ(val, 0);249 250 /* Spin until the memory region is deleted. */251 val = guest_spin_on_val(0);252 GUEST_ASSERT_EQ(val, MMIO_VAL);253 254 asm("1:\n\t"255 ".pushsection .rodata\n\t"256 ".global final_rip_start\n\t"257 "final_rip_start: .quad 1b\n\t"258 ".popsection");259 260 /* Spin indefinitely (until the code memslot is deleted). */261 guest_spin_on_val(MMIO_VAL);262 263 asm("1:\n\t"264 ".pushsection .rodata\n\t"265 ".global final_rip_end\n\t"266 "final_rip_end: .quad 1b\n\t"267 ".popsection");268 269 GUEST_ASSERT(0);270}271 272static void test_delete_memory_region(bool disable_slot_zap_quirk)273{274 pthread_t vcpu_thread;275 struct kvm_vcpu *vcpu;276 struct kvm_regs regs;277 struct kvm_run *run;278 struct kvm_vm *vm;279 280 vm = spawn_vm(&vcpu, &vcpu_thread, guest_code_delete_memory_region);281 282 if (disable_slot_zap_quirk)283 vm_enable_cap(vm, KVM_CAP_DISABLE_QUIRKS2, KVM_X86_QUIRK_SLOT_ZAP_ALL);284 285 /* Delete the memory region, the guest should not die. */286 vm_mem_region_delete(vm, MEM_REGION_SLOT);287 wait_for_vcpu();288 289 /* Recreate the memory region. The guest should see "0". */290 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP,291 MEM_REGION_GPA, MEM_REGION_SLOT,292 MEM_REGION_SIZE / getpagesize(), 0);293 wait_for_vcpu();294 295 /* Delete the region again so that there's only one memslot left. */296 vm_mem_region_delete(vm, MEM_REGION_SLOT);297 wait_for_vcpu();298 299 /*300 * Delete the primary memslot. This should cause an emulation error or301 * shutdown due to the page tables getting nuked.302 */303 vm_mem_region_delete(vm, 0);304 305 pthread_join(vcpu_thread, NULL);306 307 run = vcpu->run;308 309 TEST_ASSERT(run->exit_reason == KVM_EXIT_SHUTDOWN ||310 run->exit_reason == KVM_EXIT_INTERNAL_ERROR,311 "Unexpected exit reason = %d", run->exit_reason);312 313 vcpu_regs_get(vcpu, ®s);314 315 /*316 * On AMD, after KVM_EXIT_SHUTDOWN the VMCB has been reinitialized already,317 * so the instruction pointer would point to the reset vector.318 */319 if (run->exit_reason == KVM_EXIT_INTERNAL_ERROR)320 TEST_ASSERT(regs.rip >= final_rip_start &&321 regs.rip < final_rip_end,322 "Bad rip, expected 0x%lx - 0x%lx, got 0x%llx",323 final_rip_start, final_rip_end, regs.rip);324 325 kvm_vm_free(vm);326}327 328static void test_zero_memory_regions(void)329{330 struct kvm_vcpu *vcpu;331 struct kvm_vm *vm;332 333 pr_info("Testing KVM_RUN with zero added memory regions\n");334 335 vm = vm_create_barebones();336 vcpu = __vm_vcpu_add(vm, 0);337 338 vm_ioctl(vm, KVM_SET_NR_MMU_PAGES, (void *)64ul);339 vcpu_run(vcpu);340 TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_INTERNAL_ERROR);341 342 kvm_vm_free(vm);343}344#endif /* __x86_64__ */345 346static void test_invalid_memory_region_flags(void)347{348 uint32_t supported_flags = KVM_MEM_LOG_DIRTY_PAGES;349 const uint32_t v2_only_flags = KVM_MEM_GUEST_MEMFD;350 struct kvm_vm *vm;351 int r, i;352 353#if defined __aarch64__ || defined __riscv || defined __x86_64__354 supported_flags |= KVM_MEM_READONLY;355#endif356 357#ifdef __x86_64__358 if (kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM))359 vm = vm_create_barebones_type(KVM_X86_SW_PROTECTED_VM);360 else361#endif362 vm = vm_create_barebones();363 364 if (kvm_check_cap(KVM_CAP_MEMORY_ATTRIBUTES) & KVM_MEMORY_ATTRIBUTE_PRIVATE)365 supported_flags |= KVM_MEM_GUEST_MEMFD;366 367 for (i = 0; i < 32; i++) {368 if ((supported_flags & BIT(i)) && !(v2_only_flags & BIT(i)))369 continue;370 371 r = __vm_set_user_memory_region(vm, 0, BIT(i),372 0, MEM_REGION_SIZE, NULL);373 374 TEST_ASSERT(r && errno == EINVAL,375 "KVM_SET_USER_MEMORY_REGION should have failed on v2 only flag 0x%lx", BIT(i));376 377 if (supported_flags & BIT(i))378 continue;379 380 r = __vm_set_user_memory_region2(vm, 0, BIT(i),381 0, MEM_REGION_SIZE, NULL, 0, 0);382 TEST_ASSERT(r && errno == EINVAL,383 "KVM_SET_USER_MEMORY_REGION2 should have failed on unsupported flag 0x%lx", BIT(i));384 }385 386 if (supported_flags & KVM_MEM_GUEST_MEMFD) {387 int guest_memfd = vm_create_guest_memfd(vm, MEM_REGION_SIZE, 0);388 389 r = __vm_set_user_memory_region2(vm, 0,390 KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_GUEST_MEMFD,391 0, MEM_REGION_SIZE, NULL, guest_memfd, 0);392 TEST_ASSERT(r && errno == EINVAL,393 "KVM_SET_USER_MEMORY_REGION2 should have failed, dirty logging private memory is unsupported");394 395 r = __vm_set_user_memory_region2(vm, 0,396 KVM_MEM_READONLY | KVM_MEM_GUEST_MEMFD,397 0, MEM_REGION_SIZE, NULL, guest_memfd, 0);398 TEST_ASSERT(r && errno == EINVAL,399 "KVM_SET_USER_MEMORY_REGION2 should have failed, read-only GUEST_MEMFD memslots are unsupported");400 401 close(guest_memfd);402 }403}404 405/*406 * Test it can be added memory slots up to KVM_CAP_NR_MEMSLOTS, then any407 * tentative to add further slots should fail.408 */409static void test_add_max_memory_regions(void)410{411 int ret;412 struct kvm_vm *vm;413 uint32_t max_mem_slots;414 uint32_t slot;415 void *mem, *mem_aligned, *mem_extra;416 size_t alignment;417 418#ifdef __s390x__419 /* On s390x, the host address must be aligned to 1M (due to PGSTEs) */420 alignment = 0x100000;421#else422 alignment = 1;423#endif424 425 max_mem_slots = kvm_check_cap(KVM_CAP_NR_MEMSLOTS);426 TEST_ASSERT(max_mem_slots > 0,427 "KVM_CAP_NR_MEMSLOTS should be greater than 0");428 pr_info("Allowed number of memory slots: %i\n", max_mem_slots);429 430 vm = vm_create_barebones();431 432 /* Check it can be added memory slots up to the maximum allowed */433 pr_info("Adding slots 0..%i, each memory region with %dK size\n",434 (max_mem_slots - 1), MEM_REGION_SIZE >> 10);435 436 mem = mmap(NULL, (size_t)max_mem_slots * MEM_REGION_SIZE + alignment,437 PROT_READ | PROT_WRITE,438 MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);439 TEST_ASSERT(mem != MAP_FAILED, "Failed to mmap() host");440 mem_aligned = (void *)(((size_t) mem + alignment - 1) & ~(alignment - 1));441 442 for (slot = 0; slot < max_mem_slots; slot++)443 vm_set_user_memory_region(vm, slot, 0,444 ((uint64_t)slot * MEM_REGION_SIZE),445 MEM_REGION_SIZE,446 mem_aligned + (uint64_t)slot * MEM_REGION_SIZE);447 448 /* Check it cannot be added memory slots beyond the limit */449 mem_extra = mmap(NULL, MEM_REGION_SIZE, PROT_READ | PROT_WRITE,450 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);451 TEST_ASSERT(mem_extra != MAP_FAILED, "Failed to mmap() host");452 453 ret = __vm_set_user_memory_region(vm, max_mem_slots, 0,454 (uint64_t)max_mem_slots * MEM_REGION_SIZE,455 MEM_REGION_SIZE, mem_extra);456 TEST_ASSERT(ret == -1 && errno == EINVAL,457 "Adding one more memory slot should fail with EINVAL");458 459 munmap(mem, (size_t)max_mem_slots * MEM_REGION_SIZE + alignment);460 munmap(mem_extra, MEM_REGION_SIZE);461 kvm_vm_free(vm);462}463 464 465#ifdef __x86_64__466static void test_invalid_guest_memfd(struct kvm_vm *vm, int memfd,467 size_t offset, const char *msg)468{469 int r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD,470 MEM_REGION_GPA, MEM_REGION_SIZE,471 0, memfd, offset);472 TEST_ASSERT(r == -1 && errno == EINVAL, "%s", msg);473}474 475static void test_add_private_memory_region(void)476{477 struct kvm_vm *vm, *vm2;478 int memfd, i;479 480 pr_info("Testing ADD of KVM_MEM_GUEST_MEMFD memory regions\n");481 482 vm = vm_create_barebones_type(KVM_X86_SW_PROTECTED_VM);483 484 test_invalid_guest_memfd(vm, vm->kvm_fd, 0, "KVM fd should fail");485 test_invalid_guest_memfd(vm, vm->fd, 0, "VM's fd should fail");486 487 memfd = kvm_memfd_alloc(MEM_REGION_SIZE, false);488 test_invalid_guest_memfd(vm, memfd, 0, "Regular memfd() should fail");489 close(memfd);490 491 vm2 = vm_create_barebones_type(KVM_X86_SW_PROTECTED_VM);492 memfd = vm_create_guest_memfd(vm2, MEM_REGION_SIZE, 0);493 test_invalid_guest_memfd(vm, memfd, 0, "Other VM's guest_memfd() should fail");494 495 vm_set_user_memory_region2(vm2, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD,496 MEM_REGION_GPA, MEM_REGION_SIZE, 0, memfd, 0);497 close(memfd);498 kvm_vm_free(vm2);499 500 memfd = vm_create_guest_memfd(vm, MEM_REGION_SIZE, 0);501 for (i = 1; i < PAGE_SIZE; i++)502 test_invalid_guest_memfd(vm, memfd, i, "Unaligned offset should fail");503 504 vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD,505 MEM_REGION_GPA, MEM_REGION_SIZE, 0, memfd, 0);506 close(memfd);507 508 kvm_vm_free(vm);509}510 511static void test_add_overlapping_private_memory_regions(void)512{513 struct kvm_vm *vm;514 int memfd;515 int r;516 517 pr_info("Testing ADD of overlapping KVM_MEM_GUEST_MEMFD memory regions\n");518 519 vm = vm_create_barebones_type(KVM_X86_SW_PROTECTED_VM);520 521 memfd = vm_create_guest_memfd(vm, MEM_REGION_SIZE * 4, 0);522 523 vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD,524 MEM_REGION_GPA, MEM_REGION_SIZE * 2, 0, memfd, 0);525 526 vm_set_user_memory_region2(vm, MEM_REGION_SLOT + 1, KVM_MEM_GUEST_MEMFD,527 MEM_REGION_GPA * 2, MEM_REGION_SIZE * 2,528 0, memfd, MEM_REGION_SIZE * 2);529 530 /*531 * Delete the first memslot, and then attempt to recreate it except532 * with a "bad" offset that results in overlap in the guest_memfd().533 */534 vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD,535 MEM_REGION_GPA, 0, NULL, -1, 0);536 537 /* Overlap the front half of the other slot. */538 r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD,539 MEM_REGION_GPA * 2 - MEM_REGION_SIZE,540 MEM_REGION_SIZE * 2,541 0, memfd, 0);542 TEST_ASSERT(r == -1 && errno == EEXIST, "%s",543 "Overlapping guest_memfd() bindings should fail with EEXIST");544 545 /* And now the back half of the other slot. */546 r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD,547 MEM_REGION_GPA * 2 + MEM_REGION_SIZE,548 MEM_REGION_SIZE * 2,549 0, memfd, 0);550 TEST_ASSERT(r == -1 && errno == EEXIST, "%s",551 "Overlapping guest_memfd() bindings should fail with EEXIST");552 553 close(memfd);554 kvm_vm_free(vm);555}556#endif557 558int main(int argc, char *argv[])559{560#ifdef __x86_64__561 int i, loops;562 int j, disable_slot_zap_quirk = 0;563 564 if (kvm_check_cap(KVM_CAP_DISABLE_QUIRKS2) & KVM_X86_QUIRK_SLOT_ZAP_ALL)565 disable_slot_zap_quirk = 1;566 /*567 * FIXME: the zero-memslot test fails on aarch64 and s390x because568 * KVM_RUN fails with ENOEXEC or EFAULT.569 */570 test_zero_memory_regions();571#endif572 573 test_invalid_memory_region_flags();574 575 test_add_max_memory_regions();576 577#ifdef __x86_64__578 if (kvm_has_cap(KVM_CAP_GUEST_MEMFD) &&579 (kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM))) {580 test_add_private_memory_region();581 test_add_overlapping_private_memory_regions();582 } else {583 pr_info("Skipping tests for KVM_MEM_GUEST_MEMFD memory regions\n");584 }585 586 if (argc > 1)587 loops = atoi_positive("Number of iterations", argv[1]);588 else589 loops = 10;590 591 for (j = 0; j <= disable_slot_zap_quirk; j++) {592 pr_info("Testing MOVE of in-use region, %d loops, slot zap quirk %s\n",593 loops, j ? "disabled" : "enabled");594 for (i = 0; i < loops; i++)595 test_move_memory_region(!!j);596 597 pr_info("Testing DELETE of in-use region, %d loops, slot zap quirk %s\n",598 loops, j ? "disabled" : "enabled");599 for (i = 0; i < loops; i++)600 test_delete_memory_region(!!j);601 }602#endif603 604 return 0;605}606