2273 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * tools/testing/selftests/kvm/lib/kvm_util.c4 *5 * Copyright (C) 2018, Google LLC.6 */7#include "test_util.h"8#include "kvm_util.h"9#include "processor.h"10#include "ucall_common.h"11 12#include <assert.h>13#include <sched.h>14#include <sys/mman.h>15#include <sys/types.h>16#include <sys/stat.h>17#include <unistd.h>18#include <linux/kernel.h>19 20#define KVM_UTIL_MIN_PFN 221 22uint32_t guest_random_seed;23struct guest_random_state guest_rng;24static uint32_t last_guest_seed;25 26static int vcpu_mmap_sz(void);27 28int open_path_or_exit(const char *path, int flags)29{30 int fd;31 32 fd = open(path, flags);33 __TEST_REQUIRE(fd >= 0 || errno != ENOENT, "Cannot open %s: %s", path, strerror(errno));34 TEST_ASSERT(fd >= 0, "Failed to open '%s'", path);35 36 return fd;37}38 39/*40 * Open KVM_DEV_PATH if available, otherwise exit the entire program.41 *42 * Input Args:43 * flags - The flags to pass when opening KVM_DEV_PATH.44 *45 * Return:46 * The opened file descriptor of /dev/kvm.47 */48static int _open_kvm_dev_path_or_exit(int flags)49{50 return open_path_or_exit(KVM_DEV_PATH, flags);51}52 53int open_kvm_dev_path_or_exit(void)54{55 return _open_kvm_dev_path_or_exit(O_RDONLY);56}57 58static ssize_t get_module_param(const char *module_name, const char *param,59 void *buffer, size_t buffer_size)60{61 const int path_size = 128;62 char path[path_size];63 ssize_t bytes_read;64 int fd, r;65 66 r = snprintf(path, path_size, "/sys/module/%s/parameters/%s",67 module_name, param);68 TEST_ASSERT(r < path_size,69 "Failed to construct sysfs path in %d bytes.", path_size);70 71 fd = open_path_or_exit(path, O_RDONLY);72 73 bytes_read = read(fd, buffer, buffer_size);74 TEST_ASSERT(bytes_read > 0, "read(%s) returned %ld, wanted %ld bytes",75 path, bytes_read, buffer_size);76 77 r = close(fd);78 TEST_ASSERT(!r, "close(%s) failed", path);79 return bytes_read;80}81 82static int get_module_param_integer(const char *module_name, const char *param)83{84 /*85 * 16 bytes to hold a 64-bit value (1 byte per char), 1 byte for the86 * NUL char, and 1 byte because the kernel sucks and inserts a newline87 * at the end.88 */89 char value[16 + 1 + 1];90 ssize_t r;91 92 memset(value, '\0', sizeof(value));93 94 r = get_module_param(module_name, param, value, sizeof(value));95 TEST_ASSERT(value[r - 1] == '\n',96 "Expected trailing newline, got char '%c'", value[r - 1]);97 98 /*99 * Squash the newline, otherwise atoi_paranoid() will complain about100 * trailing non-NUL characters in the string.101 */102 value[r - 1] = '\0';103 return atoi_paranoid(value);104}105 106static bool get_module_param_bool(const char *module_name, const char *param)107{108 char value;109 ssize_t r;110 111 r = get_module_param(module_name, param, &value, sizeof(value));112 TEST_ASSERT_EQ(r, 1);113 114 if (value == 'Y')115 return true;116 else if (value == 'N')117 return false;118 119 TEST_FAIL("Unrecognized value '%c' for boolean module param", value);120}121 122bool get_kvm_param_bool(const char *param)123{124 return get_module_param_bool("kvm", param);125}126 127bool get_kvm_intel_param_bool(const char *param)128{129 return get_module_param_bool("kvm_intel", param);130}131 132bool get_kvm_amd_param_bool(const char *param)133{134 return get_module_param_bool("kvm_amd", param);135}136 137int get_kvm_param_integer(const char *param)138{139 return get_module_param_integer("kvm", param);140}141 142int get_kvm_intel_param_integer(const char *param)143{144 return get_module_param_integer("kvm_intel", param);145}146 147int get_kvm_amd_param_integer(const char *param)148{149 return get_module_param_integer("kvm_amd", param);150}151 152/*153 * Capability154 *155 * Input Args:156 * cap - Capability157 *158 * Output Args: None159 *160 * Return:161 * On success, the Value corresponding to the capability (KVM_CAP_*)162 * specified by the value of cap. On failure a TEST_ASSERT failure163 * is produced.164 *165 * Looks up and returns the value corresponding to the capability166 * (KVM_CAP_*) given by cap.167 */168unsigned int kvm_check_cap(long cap)169{170 int ret;171 int kvm_fd;172 173 kvm_fd = open_kvm_dev_path_or_exit();174 ret = __kvm_ioctl(kvm_fd, KVM_CHECK_EXTENSION, (void *)cap);175 TEST_ASSERT(ret >= 0, KVM_IOCTL_ERROR(KVM_CHECK_EXTENSION, ret));176 177 close(kvm_fd);178 179 return (unsigned int)ret;180}181 182void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size)183{184 if (vm_check_cap(vm, KVM_CAP_DIRTY_LOG_RING_ACQ_REL))185 vm_enable_cap(vm, KVM_CAP_DIRTY_LOG_RING_ACQ_REL, ring_size);186 else187 vm_enable_cap(vm, KVM_CAP_DIRTY_LOG_RING, ring_size);188 vm->dirty_ring_size = ring_size;189}190 191static void vm_open(struct kvm_vm *vm)192{193 vm->kvm_fd = _open_kvm_dev_path_or_exit(O_RDWR);194 195 TEST_REQUIRE(kvm_has_cap(KVM_CAP_IMMEDIATE_EXIT));196 197 vm->fd = __kvm_ioctl(vm->kvm_fd, KVM_CREATE_VM, (void *)vm->type);198 TEST_ASSERT(vm->fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_VM, vm->fd));199}200 201const char *vm_guest_mode_string(uint32_t i)202{203 static const char * const strings[] = {204 [VM_MODE_P52V48_4K] = "PA-bits:52, VA-bits:48, 4K pages",205 [VM_MODE_P52V48_16K] = "PA-bits:52, VA-bits:48, 16K pages",206 [VM_MODE_P52V48_64K] = "PA-bits:52, VA-bits:48, 64K pages",207 [VM_MODE_P48V48_4K] = "PA-bits:48, VA-bits:48, 4K pages",208 [VM_MODE_P48V48_16K] = "PA-bits:48, VA-bits:48, 16K pages",209 [VM_MODE_P48V48_64K] = "PA-bits:48, VA-bits:48, 64K pages",210 [VM_MODE_P40V48_4K] = "PA-bits:40, VA-bits:48, 4K pages",211 [VM_MODE_P40V48_16K] = "PA-bits:40, VA-bits:48, 16K pages",212 [VM_MODE_P40V48_64K] = "PA-bits:40, VA-bits:48, 64K pages",213 [VM_MODE_PXXV48_4K] = "PA-bits:ANY, VA-bits:48, 4K pages",214 [VM_MODE_P47V64_4K] = "PA-bits:47, VA-bits:64, 4K pages",215 [VM_MODE_P44V64_4K] = "PA-bits:44, VA-bits:64, 4K pages",216 [VM_MODE_P36V48_4K] = "PA-bits:36, VA-bits:48, 4K pages",217 [VM_MODE_P36V48_16K] = "PA-bits:36, VA-bits:48, 16K pages",218 [VM_MODE_P36V48_64K] = "PA-bits:36, VA-bits:48, 64K pages",219 [VM_MODE_P36V47_16K] = "PA-bits:36, VA-bits:47, 16K pages",220 };221 _Static_assert(sizeof(strings)/sizeof(char *) == NUM_VM_MODES,222 "Missing new mode strings?");223 224 TEST_ASSERT(i < NUM_VM_MODES, "Guest mode ID %d too big", i);225 226 return strings[i];227}228 229const struct vm_guest_mode_params vm_guest_mode_params[] = {230 [VM_MODE_P52V48_4K] = { 52, 48, 0x1000, 12 },231 [VM_MODE_P52V48_16K] = { 52, 48, 0x4000, 14 },232 [VM_MODE_P52V48_64K] = { 52, 48, 0x10000, 16 },233 [VM_MODE_P48V48_4K] = { 48, 48, 0x1000, 12 },234 [VM_MODE_P48V48_16K] = { 48, 48, 0x4000, 14 },235 [VM_MODE_P48V48_64K] = { 48, 48, 0x10000, 16 },236 [VM_MODE_P40V48_4K] = { 40, 48, 0x1000, 12 },237 [VM_MODE_P40V48_16K] = { 40, 48, 0x4000, 14 },238 [VM_MODE_P40V48_64K] = { 40, 48, 0x10000, 16 },239 [VM_MODE_PXXV48_4K] = { 0, 0, 0x1000, 12 },240 [VM_MODE_P47V64_4K] = { 47, 64, 0x1000, 12 },241 [VM_MODE_P44V64_4K] = { 44, 64, 0x1000, 12 },242 [VM_MODE_P36V48_4K] = { 36, 48, 0x1000, 12 },243 [VM_MODE_P36V48_16K] = { 36, 48, 0x4000, 14 },244 [VM_MODE_P36V48_64K] = { 36, 48, 0x10000, 16 },245 [VM_MODE_P36V47_16K] = { 36, 47, 0x4000, 14 },246};247_Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES,248 "Missing new mode params?");249 250/*251 * Initializes vm->vpages_valid to match the canonical VA space of the252 * architecture.253 *254 * The default implementation is valid for architectures which split the255 * range addressed by a single page table into a low and high region256 * based on the MSB of the VA. On architectures with this behavior257 * the VA region spans [0, 2^(va_bits - 1)), [-(2^(va_bits - 1), -1].258 */259__weak void vm_vaddr_populate_bitmap(struct kvm_vm *vm)260{261 sparsebit_set_num(vm->vpages_valid,262 0, (1ULL << (vm->va_bits - 1)) >> vm->page_shift);263 sparsebit_set_num(vm->vpages_valid,264 (~((1ULL << (vm->va_bits - 1)) - 1)) >> vm->page_shift,265 (1ULL << (vm->va_bits - 1)) >> vm->page_shift);266}267 268struct kvm_vm *____vm_create(struct vm_shape shape)269{270 struct kvm_vm *vm;271 272 vm = calloc(1, sizeof(*vm));273 TEST_ASSERT(vm != NULL, "Insufficient Memory");274 275 INIT_LIST_HEAD(&vm->vcpus);276 vm->regions.gpa_tree = RB_ROOT;277 vm->regions.hva_tree = RB_ROOT;278 hash_init(vm->regions.slot_hash);279 280 vm->mode = shape.mode;281 vm->type = shape.type;282 283 vm->pa_bits = vm_guest_mode_params[vm->mode].pa_bits;284 vm->va_bits = vm_guest_mode_params[vm->mode].va_bits;285 vm->page_size = vm_guest_mode_params[vm->mode].page_size;286 vm->page_shift = vm_guest_mode_params[vm->mode].page_shift;287 288 /* Setup mode specific traits. */289 switch (vm->mode) {290 case VM_MODE_P52V48_4K:291 vm->pgtable_levels = 4;292 break;293 case VM_MODE_P52V48_64K:294 vm->pgtable_levels = 3;295 break;296 case VM_MODE_P48V48_4K:297 vm->pgtable_levels = 4;298 break;299 case VM_MODE_P48V48_64K:300 vm->pgtable_levels = 3;301 break;302 case VM_MODE_P40V48_4K:303 case VM_MODE_P36V48_4K:304 vm->pgtable_levels = 4;305 break;306 case VM_MODE_P40V48_64K:307 case VM_MODE_P36V48_64K:308 vm->pgtable_levels = 3;309 break;310 case VM_MODE_P52V48_16K:311 case VM_MODE_P48V48_16K:312 case VM_MODE_P40V48_16K:313 case VM_MODE_P36V48_16K:314 vm->pgtable_levels = 4;315 break;316 case VM_MODE_P36V47_16K:317 vm->pgtable_levels = 3;318 break;319 case VM_MODE_PXXV48_4K:320#ifdef __x86_64__321 kvm_get_cpu_address_width(&vm->pa_bits, &vm->va_bits);322 kvm_init_vm_address_properties(vm);323 /*324 * Ignore KVM support for 5-level paging (vm->va_bits == 57),325 * it doesn't take effect unless a CR4.LA57 is set, which it326 * isn't for this mode (48-bit virtual address space).327 */328 TEST_ASSERT(vm->va_bits == 48 || vm->va_bits == 57,329 "Linear address width (%d bits) not supported",330 vm->va_bits);331 pr_debug("Guest physical address width detected: %d\n",332 vm->pa_bits);333 vm->pgtable_levels = 4;334 vm->va_bits = 48;335#else336 TEST_FAIL("VM_MODE_PXXV48_4K not supported on non-x86 platforms");337#endif338 break;339 case VM_MODE_P47V64_4K:340 vm->pgtable_levels = 5;341 break;342 case VM_MODE_P44V64_4K:343 vm->pgtable_levels = 5;344 break;345 default:346 TEST_FAIL("Unknown guest mode: 0x%x", vm->mode);347 }348 349#ifdef __aarch64__350 TEST_ASSERT(!vm->type, "ARM doesn't support test-provided types");351 if (vm->pa_bits != 40)352 vm->type = KVM_VM_TYPE_ARM_IPA_SIZE(vm->pa_bits);353#endif354 355 vm_open(vm);356 357 /* Limit to VA-bit canonical virtual addresses. */358 vm->vpages_valid = sparsebit_alloc();359 vm_vaddr_populate_bitmap(vm);360 361 /* Limit physical addresses to PA-bits. */362 vm->max_gfn = vm_compute_max_gfn(vm);363 364 /* Allocate and setup memory for guest. */365 vm->vpages_mapped = sparsebit_alloc();366 367 return vm;368}369 370static uint64_t vm_nr_pages_required(enum vm_guest_mode mode,371 uint32_t nr_runnable_vcpus,372 uint64_t extra_mem_pages)373{374 uint64_t page_size = vm_guest_mode_params[mode].page_size;375 uint64_t nr_pages;376 377 TEST_ASSERT(nr_runnable_vcpus,378 "Use vm_create_barebones() for VMs that _never_ have vCPUs");379 380 TEST_ASSERT(nr_runnable_vcpus <= kvm_check_cap(KVM_CAP_MAX_VCPUS),381 "nr_vcpus = %d too large for host, max-vcpus = %d",382 nr_runnable_vcpus, kvm_check_cap(KVM_CAP_MAX_VCPUS));383 384 /*385 * Arbitrarily allocate 512 pages (2mb when page size is 4kb) for the386 * test code and other per-VM assets that will be loaded into memslot0.387 */388 nr_pages = 512;389 390 /* Account for the per-vCPU stacks on behalf of the test. */391 nr_pages += nr_runnable_vcpus * DEFAULT_STACK_PGS;392 393 /*394 * Account for the number of pages needed for the page tables. The395 * maximum page table size for a memory region will be when the396 * smallest page size is used. Considering each page contains x page397 * table descriptors, the total extra size for page tables (for extra398 * N pages) will be: N/x+N/x^2+N/x^3+... which is definitely smaller399 * than N/x*2.400 */401 nr_pages += (nr_pages + extra_mem_pages) / PTES_PER_MIN_PAGE * 2;402 403 /* Account for the number of pages needed by ucall. */404 nr_pages += ucall_nr_pages_required(page_size);405 406 return vm_adjust_num_guest_pages(mode, nr_pages);407}408 409struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus,410 uint64_t nr_extra_pages)411{412 uint64_t nr_pages = vm_nr_pages_required(shape.mode, nr_runnable_vcpus,413 nr_extra_pages);414 struct userspace_mem_region *slot0;415 struct kvm_vm *vm;416 int i;417 418 pr_debug("%s: mode='%s' type='%d', pages='%ld'\n", __func__,419 vm_guest_mode_string(shape.mode), shape.type, nr_pages);420 421 vm = ____vm_create(shape);422 423 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, 0);424 for (i = 0; i < NR_MEM_REGIONS; i++)425 vm->memslots[i] = 0;426 427 kvm_vm_elf_load(vm, program_invocation_name);428 429 /*430 * TODO: Add proper defines to protect the library's memslots, and then431 * carve out memslot1 for the ucall MMIO address. KVM treats writes to432 * read-only memslots as MMIO, and creating a read-only memslot for the433 * MMIO region would prevent silently clobbering the MMIO region.434 */435 slot0 = memslot2region(vm, 0);436 ucall_init(vm, slot0->region.guest_phys_addr + slot0->region.memory_size);437 438 if (guest_random_seed != last_guest_seed) {439 pr_info("Random seed: 0x%x\n", guest_random_seed);440 last_guest_seed = guest_random_seed;441 }442 guest_rng = new_guest_random_state(guest_random_seed);443 sync_global_to_guest(vm, guest_rng);444 445 kvm_arch_vm_post_create(vm);446 447 return vm;448}449 450/*451 * VM Create with customized parameters452 *453 * Input Args:454 * mode - VM Mode (e.g. VM_MODE_P52V48_4K)455 * nr_vcpus - VCPU count456 * extra_mem_pages - Non-slot0 physical memory total size457 * guest_code - Guest entry point458 * vcpuids - VCPU IDs459 *460 * Output Args: None461 *462 * Return:463 * Pointer to opaque structure that describes the created VM.464 *465 * Creates a VM with the mode specified by mode (e.g. VM_MODE_P52V48_4K).466 * extra_mem_pages is only used to calculate the maximum page table size,467 * no real memory allocation for non-slot0 memory in this function.468 */469struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus,470 uint64_t extra_mem_pages,471 void *guest_code, struct kvm_vcpu *vcpus[])472{473 struct kvm_vm *vm;474 int i;475 476 TEST_ASSERT(!nr_vcpus || vcpus, "Must provide vCPU array");477 478 vm = __vm_create(shape, nr_vcpus, extra_mem_pages);479 480 for (i = 0; i < nr_vcpus; ++i)481 vcpus[i] = vm_vcpu_add(vm, i, guest_code);482 483 return vm;484}485 486struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape,487 struct kvm_vcpu **vcpu,488 uint64_t extra_mem_pages,489 void *guest_code)490{491 struct kvm_vcpu *vcpus[1];492 struct kvm_vm *vm;493 494 vm = __vm_create_with_vcpus(shape, 1, extra_mem_pages, guest_code, vcpus);495 496 *vcpu = vcpus[0];497 return vm;498}499 500/*501 * VM Restart502 *503 * Input Args:504 * vm - VM that has been released before505 *506 * Output Args: None507 *508 * Reopens the file descriptors associated to the VM and reinstates the509 * global state, such as the irqchip and the memory regions that are mapped510 * into the guest.511 */512void kvm_vm_restart(struct kvm_vm *vmp)513{514 int ctr;515 struct userspace_mem_region *region;516 517 vm_open(vmp);518 if (vmp->has_irqchip)519 vm_create_irqchip(vmp);520 521 hash_for_each(vmp->regions.slot_hash, ctr, region, slot_node) {522 int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION2, ®ion->region);523 524 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"525 " rc: %i errno: %i\n"526 " slot: %u flags: 0x%x\n"527 " guest_phys_addr: 0x%llx size: 0x%llx",528 ret, errno, region->region.slot,529 region->region.flags,530 region->region.guest_phys_addr,531 region->region.memory_size);532 }533}534 535__weak struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm,536 uint32_t vcpu_id)537{538 return __vm_vcpu_add(vm, vcpu_id);539}540 541struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm)542{543 kvm_vm_restart(vm);544 545 return vm_vcpu_recreate(vm, 0);546}547 548void kvm_pin_this_task_to_pcpu(uint32_t pcpu)549{550 cpu_set_t mask;551 int r;552 553 CPU_ZERO(&mask);554 CPU_SET(pcpu, &mask);555 r = sched_setaffinity(0, sizeof(mask), &mask);556 TEST_ASSERT(!r, "sched_setaffinity() failed for pCPU '%u'.", pcpu);557}558 559static uint32_t parse_pcpu(const char *cpu_str, const cpu_set_t *allowed_mask)560{561 uint32_t pcpu = atoi_non_negative("CPU number", cpu_str);562 563 TEST_ASSERT(CPU_ISSET(pcpu, allowed_mask),564 "Not allowed to run on pCPU '%d', check cgroups?", pcpu);565 return pcpu;566}567 568void kvm_print_vcpu_pinning_help(void)569{570 const char *name = program_invocation_name;571 572 printf(" -c: Pin tasks to physical CPUs. Takes a list of comma separated\n"573 " values (target pCPU), one for each vCPU, plus an optional\n"574 " entry for the main application task (specified via entry\n"575 " <nr_vcpus + 1>). If used, entries must be provided for all\n"576 " vCPUs, i.e. pinning vCPUs is all or nothing.\n\n"577 " E.g. to create 3 vCPUs, pin vCPU0=>pCPU22, vCPU1=>pCPU23,\n"578 " vCPU2=>pCPU24, and pin the application task to pCPU50:\n\n"579 " %s -v 3 -c 22,23,24,50\n\n"580 " To leave the application task unpinned, drop the final entry:\n\n"581 " %s -v 3 -c 22,23,24\n\n"582 " (default: no pinning)\n", name, name);583}584 585void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[],586 int nr_vcpus)587{588 cpu_set_t allowed_mask;589 char *cpu, *cpu_list;590 char delim[2] = ",";591 int i, r;592 593 cpu_list = strdup(pcpus_string);594 TEST_ASSERT(cpu_list, "strdup() allocation failed.");595 596 r = sched_getaffinity(0, sizeof(allowed_mask), &allowed_mask);597 TEST_ASSERT(!r, "sched_getaffinity() failed");598 599 cpu = strtok(cpu_list, delim);600 601 /* 1. Get all pcpus for vcpus. */602 for (i = 0; i < nr_vcpus; i++) {603 TEST_ASSERT(cpu, "pCPU not provided for vCPU '%d'", i);604 vcpu_to_pcpu[i] = parse_pcpu(cpu, &allowed_mask);605 cpu = strtok(NULL, delim);606 }607 608 /* 2. Check if the main worker needs to be pinned. */609 if (cpu) {610 kvm_pin_this_task_to_pcpu(parse_pcpu(cpu, &allowed_mask));611 cpu = strtok(NULL, delim);612 }613 614 TEST_ASSERT(!cpu, "pCPU list contains trailing garbage characters '%s'", cpu);615 free(cpu_list);616}617 618/*619 * Userspace Memory Region Find620 *621 * Input Args:622 * vm - Virtual Machine623 * start - Starting VM physical address624 * end - Ending VM physical address, inclusive.625 *626 * Output Args: None627 *628 * Return:629 * Pointer to overlapping region, NULL if no such region.630 *631 * Searches for a region with any physical memory that overlaps with632 * any portion of the guest physical addresses from start to end633 * inclusive. If multiple overlapping regions exist, a pointer to any634 * of the regions is returned. Null is returned only when no overlapping635 * region exists.636 */637static struct userspace_mem_region *638userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end)639{640 struct rb_node *node;641 642 for (node = vm->regions.gpa_tree.rb_node; node; ) {643 struct userspace_mem_region *region =644 container_of(node, struct userspace_mem_region, gpa_node);645 uint64_t existing_start = region->region.guest_phys_addr;646 uint64_t existing_end = region->region.guest_phys_addr647 + region->region.memory_size - 1;648 if (start <= existing_end && end >= existing_start)649 return region;650 651 if (start < existing_start)652 node = node->rb_left;653 else654 node = node->rb_right;655 }656 657 return NULL;658}659 660__weak void vcpu_arch_free(struct kvm_vcpu *vcpu)661{662 663}664 665/*666 * VM VCPU Remove667 *668 * Input Args:669 * vcpu - VCPU to remove670 *671 * Output Args: None672 *673 * Return: None, TEST_ASSERT failures for all error conditions674 *675 * Removes a vCPU from a VM and frees its resources.676 */677static void vm_vcpu_rm(struct kvm_vm *vm, struct kvm_vcpu *vcpu)678{679 int ret;680 681 if (vcpu->dirty_gfns) {682 ret = munmap(vcpu->dirty_gfns, vm->dirty_ring_size);683 TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));684 vcpu->dirty_gfns = NULL;685 }686 687 ret = munmap(vcpu->run, vcpu_mmap_sz());688 TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));689 690 ret = close(vcpu->fd);691 TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("close()", ret));692 693 list_del(&vcpu->list);694 695 vcpu_arch_free(vcpu);696 free(vcpu);697}698 699void kvm_vm_release(struct kvm_vm *vmp)700{701 struct kvm_vcpu *vcpu, *tmp;702 int ret;703 704 list_for_each_entry_safe(vcpu, tmp, &vmp->vcpus, list)705 vm_vcpu_rm(vmp, vcpu);706 707 ret = close(vmp->fd);708 TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("close()", ret));709 710 ret = close(vmp->kvm_fd);711 TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("close()", ret));712}713 714static void __vm_mem_region_delete(struct kvm_vm *vm,715 struct userspace_mem_region *region)716{717 int ret;718 719 rb_erase(®ion->gpa_node, &vm->regions.gpa_tree);720 rb_erase(®ion->hva_node, &vm->regions.hva_tree);721 hash_del(®ion->slot_node);722 723 region->region.memory_size = 0;724 vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, ®ion->region);725 726 sparsebit_free(®ion->unused_phy_pages);727 sparsebit_free(®ion->protected_phy_pages);728 ret = munmap(region->mmap_start, region->mmap_size);729 TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));730 if (region->fd >= 0) {731 /* There's an extra map when using shared memory. */732 ret = munmap(region->mmap_alias, region->mmap_size);733 TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));734 close(region->fd);735 }736 if (region->region.guest_memfd >= 0)737 close(region->region.guest_memfd);738 739 free(region);740}741 742/*743 * Destroys and frees the VM pointed to by vmp.744 */745void kvm_vm_free(struct kvm_vm *vmp)746{747 int ctr;748 struct hlist_node *node;749 struct userspace_mem_region *region;750 751 if (vmp == NULL)752 return;753 754 /* Free cached stats metadata and close FD */755 if (vmp->stats_fd) {756 free(vmp->stats_desc);757 close(vmp->stats_fd);758 }759 760 /* Free userspace_mem_regions. */761 hash_for_each_safe(vmp->regions.slot_hash, ctr, node, region, slot_node)762 __vm_mem_region_delete(vmp, region);763 764 /* Free sparsebit arrays. */765 sparsebit_free(&vmp->vpages_valid);766 sparsebit_free(&vmp->vpages_mapped);767 768 kvm_vm_release(vmp);769 770 /* Free the structure describing the VM. */771 free(vmp);772}773 774int kvm_memfd_alloc(size_t size, bool hugepages)775{776 int memfd_flags = MFD_CLOEXEC;777 int fd, r;778 779 if (hugepages)780 memfd_flags |= MFD_HUGETLB;781 782 fd = memfd_create("kvm_selftest", memfd_flags);783 TEST_ASSERT(fd != -1, __KVM_SYSCALL_ERROR("memfd_create()", fd));784 785 r = ftruncate(fd, size);786 TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("ftruncate()", r));787 788 r = fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 0, size);789 TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("fallocate()", r));790 791 return fd;792}793 794static void vm_userspace_mem_region_gpa_insert(struct rb_root *gpa_tree,795 struct userspace_mem_region *region)796{797 struct rb_node **cur, *parent;798 799 for (cur = &gpa_tree->rb_node, parent = NULL; *cur; ) {800 struct userspace_mem_region *cregion;801 802 cregion = container_of(*cur, typeof(*cregion), gpa_node);803 parent = *cur;804 if (region->region.guest_phys_addr <805 cregion->region.guest_phys_addr)806 cur = &(*cur)->rb_left;807 else {808 TEST_ASSERT(region->region.guest_phys_addr !=809 cregion->region.guest_phys_addr,810 "Duplicate GPA in region tree");811 812 cur = &(*cur)->rb_right;813 }814 }815 816 rb_link_node(®ion->gpa_node, parent, cur);817 rb_insert_color(®ion->gpa_node, gpa_tree);818}819 820static void vm_userspace_mem_region_hva_insert(struct rb_root *hva_tree,821 struct userspace_mem_region *region)822{823 struct rb_node **cur, *parent;824 825 for (cur = &hva_tree->rb_node, parent = NULL; *cur; ) {826 struct userspace_mem_region *cregion;827 828 cregion = container_of(*cur, typeof(*cregion), hva_node);829 parent = *cur;830 if (region->host_mem < cregion->host_mem)831 cur = &(*cur)->rb_left;832 else {833 TEST_ASSERT(region->host_mem !=834 cregion->host_mem,835 "Duplicate HVA in region tree");836 837 cur = &(*cur)->rb_right;838 }839 }840 841 rb_link_node(®ion->hva_node, parent, cur);842 rb_insert_color(®ion->hva_node, hva_tree);843}844 845 846int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,847 uint64_t gpa, uint64_t size, void *hva)848{849 struct kvm_userspace_memory_region region = {850 .slot = slot,851 .flags = flags,852 .guest_phys_addr = gpa,853 .memory_size = size,854 .userspace_addr = (uintptr_t)hva,855 };856 857 return ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion);858}859 860void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,861 uint64_t gpa, uint64_t size, void *hva)862{863 int ret = __vm_set_user_memory_region(vm, slot, flags, gpa, size, hva);864 865 TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION failed, errno = %d (%s)",866 errno, strerror(errno));867}868 869#define TEST_REQUIRE_SET_USER_MEMORY_REGION2() \870 __TEST_REQUIRE(kvm_has_cap(KVM_CAP_USER_MEMORY2), \871 "KVM selftests now require KVM_SET_USER_MEMORY_REGION2 (introduced in v6.8)")872 873int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,874 uint64_t gpa, uint64_t size, void *hva,875 uint32_t guest_memfd, uint64_t guest_memfd_offset)876{877 struct kvm_userspace_memory_region2 region = {878 .slot = slot,879 .flags = flags,880 .guest_phys_addr = gpa,881 .memory_size = size,882 .userspace_addr = (uintptr_t)hva,883 .guest_memfd = guest_memfd,884 .guest_memfd_offset = guest_memfd_offset,885 };886 887 TEST_REQUIRE_SET_USER_MEMORY_REGION2();888 889 return ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION2, ®ion);890}891 892void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,893 uint64_t gpa, uint64_t size, void *hva,894 uint32_t guest_memfd, uint64_t guest_memfd_offset)895{896 int ret = __vm_set_user_memory_region2(vm, slot, flags, gpa, size, hva,897 guest_memfd, guest_memfd_offset);898 899 TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed, errno = %d (%s)",900 errno, strerror(errno));901}902 903 904/* FIXME: This thing needs to be ripped apart and rewritten. */905void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,906 uint64_t guest_paddr, uint32_t slot, uint64_t npages,907 uint32_t flags, int guest_memfd, uint64_t guest_memfd_offset)908{909 int ret;910 struct userspace_mem_region *region;911 size_t backing_src_pagesz = get_backing_src_pagesz(src_type);912 size_t mem_size = npages * vm->page_size;913 size_t alignment;914 915 TEST_REQUIRE_SET_USER_MEMORY_REGION2();916 917 TEST_ASSERT(vm_adjust_num_guest_pages(vm->mode, npages) == npages,918 "Number of guest pages is not compatible with the host. "919 "Try npages=%d", vm_adjust_num_guest_pages(vm->mode, npages));920 921 TEST_ASSERT((guest_paddr % vm->page_size) == 0, "Guest physical "922 "address not on a page boundary.\n"923 " guest_paddr: 0x%lx vm->page_size: 0x%x",924 guest_paddr, vm->page_size);925 TEST_ASSERT((((guest_paddr >> vm->page_shift) + npages) - 1)926 <= vm->max_gfn, "Physical range beyond maximum "927 "supported physical address,\n"928 " guest_paddr: 0x%lx npages: 0x%lx\n"929 " vm->max_gfn: 0x%lx vm->page_size: 0x%x",930 guest_paddr, npages, vm->max_gfn, vm->page_size);931 932 /*933 * Confirm a mem region with an overlapping address doesn't934 * already exist.935 */936 region = (struct userspace_mem_region *) userspace_mem_region_find(937 vm, guest_paddr, (guest_paddr + npages * vm->page_size) - 1);938 if (region != NULL)939 TEST_FAIL("overlapping userspace_mem_region already "940 "exists\n"941 " requested guest_paddr: 0x%lx npages: 0x%lx "942 "page_size: 0x%x\n"943 " existing guest_paddr: 0x%lx size: 0x%lx",944 guest_paddr, npages, vm->page_size,945 (uint64_t) region->region.guest_phys_addr,946 (uint64_t) region->region.memory_size);947 948 /* Confirm no region with the requested slot already exists. */949 hash_for_each_possible(vm->regions.slot_hash, region, slot_node,950 slot) {951 if (region->region.slot != slot)952 continue;953 954 TEST_FAIL("A mem region with the requested slot "955 "already exists.\n"956 " requested slot: %u paddr: 0x%lx npages: 0x%lx\n"957 " existing slot: %u paddr: 0x%lx size: 0x%lx",958 slot, guest_paddr, npages,959 region->region.slot,960 (uint64_t) region->region.guest_phys_addr,961 (uint64_t) region->region.memory_size);962 }963 964 /* Allocate and initialize new mem region structure. */965 region = calloc(1, sizeof(*region));966 TEST_ASSERT(region != NULL, "Insufficient Memory");967 region->mmap_size = mem_size;968 969#ifdef __s390x__970 /* On s390x, the host address must be aligned to 1M (due to PGSTEs) */971 alignment = 0x100000;972#else973 alignment = 1;974#endif975 976 /*977 * When using THP mmap is not guaranteed to returned a hugepage aligned978 * address so we have to pad the mmap. Padding is not needed for HugeTLB979 * because mmap will always return an address aligned to the HugeTLB980 * page size.981 */982 if (src_type == VM_MEM_SRC_ANONYMOUS_THP)983 alignment = max(backing_src_pagesz, alignment);984 985 TEST_ASSERT_EQ(guest_paddr, align_up(guest_paddr, backing_src_pagesz));986 987 /* Add enough memory to align up if necessary */988 if (alignment > 1)989 region->mmap_size += alignment;990 991 region->fd = -1;992 if (backing_src_is_shared(src_type))993 region->fd = kvm_memfd_alloc(region->mmap_size,994 src_type == VM_MEM_SRC_SHARED_HUGETLB);995 996 region->mmap_start = mmap(NULL, region->mmap_size,997 PROT_READ | PROT_WRITE,998 vm_mem_backing_src_alias(src_type)->flag,999 region->fd, 0);1000 TEST_ASSERT(region->mmap_start != MAP_FAILED,1001 __KVM_SYSCALL_ERROR("mmap()", (int)(unsigned long)MAP_FAILED));1002 1003 TEST_ASSERT(!is_backing_src_hugetlb(src_type) ||1004 region->mmap_start == align_ptr_up(region->mmap_start, backing_src_pagesz),1005 "mmap_start %p is not aligned to HugeTLB page size 0x%lx",1006 region->mmap_start, backing_src_pagesz);1007 1008 /* Align host address */1009 region->host_mem = align_ptr_up(region->mmap_start, alignment);1010 1011 /* As needed perform madvise */1012 if ((src_type == VM_MEM_SRC_ANONYMOUS ||1013 src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) {1014 ret = madvise(region->host_mem, mem_size,1015 src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE);1016 TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s",1017 region->host_mem, mem_size,1018 vm_mem_backing_src_alias(src_type)->name);1019 }1020 1021 region->backing_src_type = src_type;1022 1023 if (flags & KVM_MEM_GUEST_MEMFD) {1024 if (guest_memfd < 0) {1025 uint32_t guest_memfd_flags = 0;1026 TEST_ASSERT(!guest_memfd_offset,1027 "Offset must be zero when creating new guest_memfd");1028 guest_memfd = vm_create_guest_memfd(vm, mem_size, guest_memfd_flags);1029 } else {1030 /*1031 * Install a unique fd for each memslot so that the fd1032 * can be closed when the region is deleted without1033 * needing to track if the fd is owned by the framework1034 * or by the caller.1035 */1036 guest_memfd = dup(guest_memfd);1037 TEST_ASSERT(guest_memfd >= 0, __KVM_SYSCALL_ERROR("dup()", guest_memfd));1038 }1039 1040 region->region.guest_memfd = guest_memfd;1041 region->region.guest_memfd_offset = guest_memfd_offset;1042 } else {1043 region->region.guest_memfd = -1;1044 }1045 1046 region->unused_phy_pages = sparsebit_alloc();1047 if (vm_arch_has_protected_memory(vm))1048 region->protected_phy_pages = sparsebit_alloc();1049 sparsebit_set_num(region->unused_phy_pages,1050 guest_paddr >> vm->page_shift, npages);1051 region->region.slot = slot;1052 region->region.flags = flags;1053 region->region.guest_phys_addr = guest_paddr;1054 region->region.memory_size = npages * vm->page_size;1055 region->region.userspace_addr = (uintptr_t) region->host_mem;1056 ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, ®ion->region);1057 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"1058 " rc: %i errno: %i\n"1059 " slot: %u flags: 0x%x\n"1060 " guest_phys_addr: 0x%lx size: 0x%lx guest_memfd: %d",1061 ret, errno, slot, flags,1062 guest_paddr, (uint64_t) region->region.memory_size,1063 region->region.guest_memfd);1064 1065 /* Add to quick lookup data structures */1066 vm_userspace_mem_region_gpa_insert(&vm->regions.gpa_tree, region);1067 vm_userspace_mem_region_hva_insert(&vm->regions.hva_tree, region);1068 hash_add(vm->regions.slot_hash, ®ion->slot_node, slot);1069 1070 /* If shared memory, create an alias. */1071 if (region->fd >= 0) {1072 region->mmap_alias = mmap(NULL, region->mmap_size,1073 PROT_READ | PROT_WRITE,1074 vm_mem_backing_src_alias(src_type)->flag,1075 region->fd, 0);1076 TEST_ASSERT(region->mmap_alias != MAP_FAILED,1077 __KVM_SYSCALL_ERROR("mmap()", (int)(unsigned long)MAP_FAILED));1078 1079 /* Align host alias address */1080 region->host_alias = align_ptr_up(region->mmap_alias, alignment);1081 }1082}1083 1084void vm_userspace_mem_region_add(struct kvm_vm *vm,1085 enum vm_mem_backing_src_type src_type,1086 uint64_t guest_paddr, uint32_t slot,1087 uint64_t npages, uint32_t flags)1088{1089 vm_mem_add(vm, src_type, guest_paddr, slot, npages, flags, -1, 0);1090}1091 1092/*1093 * Memslot to region1094 *1095 * Input Args:1096 * vm - Virtual Machine1097 * memslot - KVM memory slot ID1098 *1099 * Output Args: None1100 *1101 * Return:1102 * Pointer to memory region structure that describe memory region1103 * using kvm memory slot ID given by memslot. TEST_ASSERT failure1104 * on error (e.g. currently no memory region using memslot as a KVM1105 * memory slot ID).1106 */1107struct userspace_mem_region *1108memslot2region(struct kvm_vm *vm, uint32_t memslot)1109{1110 struct userspace_mem_region *region;1111 1112 hash_for_each_possible(vm->regions.slot_hash, region, slot_node,1113 memslot)1114 if (region->region.slot == memslot)1115 return region;1116 1117 fprintf(stderr, "No mem region with the requested slot found,\n"1118 " requested slot: %u\n", memslot);1119 fputs("---- vm dump ----\n", stderr);1120 vm_dump(stderr, vm, 2);1121 TEST_FAIL("Mem region not found");1122 return NULL;1123}1124 1125/*1126 * VM Memory Region Flags Set1127 *1128 * Input Args:1129 * vm - Virtual Machine1130 * flags - Starting guest physical address1131 *1132 * Output Args: None1133 *1134 * Return: None1135 *1136 * Sets the flags of the memory region specified by the value of slot,1137 * to the values given by flags.1138 */1139void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags)1140{1141 int ret;1142 struct userspace_mem_region *region;1143 1144 region = memslot2region(vm, slot);1145 1146 region->region.flags = flags;1147 1148 ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, ®ion->region);1149 1150 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"1151 " rc: %i errno: %i slot: %u flags: 0x%x",1152 ret, errno, slot, flags);1153}1154 1155/*1156 * VM Memory Region Move1157 *1158 * Input Args:1159 * vm - Virtual Machine1160 * slot - Slot of the memory region to move1161 * new_gpa - Starting guest physical address1162 *1163 * Output Args: None1164 *1165 * Return: None1166 *1167 * Change the gpa of a memory region.1168 */1169void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa)1170{1171 struct userspace_mem_region *region;1172 int ret;1173 1174 region = memslot2region(vm, slot);1175 1176 region->region.guest_phys_addr = new_gpa;1177 1178 ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, ®ion->region);1179 1180 TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed\n"1181 "ret: %i errno: %i slot: %u new_gpa: 0x%lx",1182 ret, errno, slot, new_gpa);1183}1184 1185/*1186 * VM Memory Region Delete1187 *1188 * Input Args:1189 * vm - Virtual Machine1190 * slot - Slot of the memory region to delete1191 *1192 * Output Args: None1193 *1194 * Return: None1195 *1196 * Delete a memory region.1197 */1198void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot)1199{1200 __vm_mem_region_delete(vm, memslot2region(vm, slot));1201}1202 1203void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t base, uint64_t size,1204 bool punch_hole)1205{1206 const int mode = FALLOC_FL_KEEP_SIZE | (punch_hole ? FALLOC_FL_PUNCH_HOLE : 0);1207 struct userspace_mem_region *region;1208 uint64_t end = base + size;1209 uint64_t gpa, len;1210 off_t fd_offset;1211 int ret;1212 1213 for (gpa = base; gpa < end; gpa += len) {1214 uint64_t offset;1215 1216 region = userspace_mem_region_find(vm, gpa, gpa);1217 TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD,1218 "Private memory region not found for GPA 0x%lx", gpa);1219 1220 offset = gpa - region->region.guest_phys_addr;1221 fd_offset = region->region.guest_memfd_offset + offset;1222 len = min_t(uint64_t, end - gpa, region->region.memory_size - offset);1223 1224 ret = fallocate(region->region.guest_memfd, mode, fd_offset, len);1225 TEST_ASSERT(!ret, "fallocate() failed to %s at %lx (len = %lu), fd = %d, mode = %x, offset = %lx",1226 punch_hole ? "punch hole" : "allocate", gpa, len,1227 region->region.guest_memfd, mode, fd_offset);1228 }1229}1230 1231/* Returns the size of a vCPU's kvm_run structure. */1232static int vcpu_mmap_sz(void)1233{1234 int dev_fd, ret;1235 1236 dev_fd = open_kvm_dev_path_or_exit();1237 1238 ret = ioctl(dev_fd, KVM_GET_VCPU_MMAP_SIZE, NULL);1239 TEST_ASSERT(ret >= sizeof(struct kvm_run),1240 KVM_IOCTL_ERROR(KVM_GET_VCPU_MMAP_SIZE, ret));1241 1242 close(dev_fd);1243 1244 return ret;1245}1246 1247static bool vcpu_exists(struct kvm_vm *vm, uint32_t vcpu_id)1248{1249 struct kvm_vcpu *vcpu;1250 1251 list_for_each_entry(vcpu, &vm->vcpus, list) {1252 if (vcpu->id == vcpu_id)1253 return true;1254 }1255 1256 return false;1257}1258 1259/*1260 * Adds a virtual CPU to the VM specified by vm with the ID given by vcpu_id.1261 * No additional vCPU setup is done. Returns the vCPU.1262 */1263struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)1264{1265 struct kvm_vcpu *vcpu;1266 1267 /* Confirm a vcpu with the specified id doesn't already exist. */1268 TEST_ASSERT(!vcpu_exists(vm, vcpu_id), "vCPU%d already exists", vcpu_id);1269 1270 /* Allocate and initialize new vcpu structure. */1271 vcpu = calloc(1, sizeof(*vcpu));1272 TEST_ASSERT(vcpu != NULL, "Insufficient Memory");1273 1274 vcpu->vm = vm;1275 vcpu->id = vcpu_id;1276 vcpu->fd = __vm_ioctl(vm, KVM_CREATE_VCPU, (void *)(unsigned long)vcpu_id);1277 TEST_ASSERT_VM_VCPU_IOCTL(vcpu->fd >= 0, KVM_CREATE_VCPU, vcpu->fd, vm);1278 1279 TEST_ASSERT(vcpu_mmap_sz() >= sizeof(*vcpu->run), "vcpu mmap size "1280 "smaller than expected, vcpu_mmap_sz: %i expected_min: %zi",1281 vcpu_mmap_sz(), sizeof(*vcpu->run));1282 vcpu->run = (struct kvm_run *) mmap(NULL, vcpu_mmap_sz(),1283 PROT_READ | PROT_WRITE, MAP_SHARED, vcpu->fd, 0);1284 TEST_ASSERT(vcpu->run != MAP_FAILED,1285 __KVM_SYSCALL_ERROR("mmap()", (int)(unsigned long)MAP_FAILED));1286 1287 /* Add to linked-list of VCPUs. */1288 list_add(&vcpu->list, &vm->vcpus);1289 1290 return vcpu;1291}1292 1293/*1294 * VM Virtual Address Unused Gap1295 *1296 * Input Args:1297 * vm - Virtual Machine1298 * sz - Size (bytes)1299 * vaddr_min - Minimum Virtual Address1300 *1301 * Output Args: None1302 *1303 * Return:1304 * Lowest virtual address at or below vaddr_min, with at least1305 * sz unused bytes. TEST_ASSERT failure if no area of at least1306 * size sz is available.1307 *1308 * Within the VM specified by vm, locates the lowest starting virtual1309 * address >= vaddr_min, that has at least sz unallocated bytes. A1310 * TEST_ASSERT failure occurs for invalid input or no area of at least1311 * sz unallocated bytes >= vaddr_min is available.1312 */1313vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz,1314 vm_vaddr_t vaddr_min)1315{1316 uint64_t pages = (sz + vm->page_size - 1) >> vm->page_shift;1317 1318 /* Determine lowest permitted virtual page index. */1319 uint64_t pgidx_start = (vaddr_min + vm->page_size - 1) >> vm->page_shift;1320 if ((pgidx_start * vm->page_size) < vaddr_min)1321 goto no_va_found;1322 1323 /* Loop over section with enough valid virtual page indexes. */1324 if (!sparsebit_is_set_num(vm->vpages_valid,1325 pgidx_start, pages))1326 pgidx_start = sparsebit_next_set_num(vm->vpages_valid,1327 pgidx_start, pages);1328 do {1329 /*1330 * Are there enough unused virtual pages available at1331 * the currently proposed starting virtual page index.1332 * If not, adjust proposed starting index to next1333 * possible.1334 */1335 if (sparsebit_is_clear_num(vm->vpages_mapped,1336 pgidx_start, pages))1337 goto va_found;1338 pgidx_start = sparsebit_next_clear_num(vm->vpages_mapped,1339 pgidx_start, pages);1340 if (pgidx_start == 0)1341 goto no_va_found;1342 1343 /*1344 * If needed, adjust proposed starting virtual address,1345 * to next range of valid virtual addresses.1346 */1347 if (!sparsebit_is_set_num(vm->vpages_valid,1348 pgidx_start, pages)) {1349 pgidx_start = sparsebit_next_set_num(1350 vm->vpages_valid, pgidx_start, pages);1351 if (pgidx_start == 0)1352 goto no_va_found;1353 }1354 } while (pgidx_start != 0);1355 1356no_va_found:1357 TEST_FAIL("No vaddr of specified pages available, pages: 0x%lx", pages);1358 1359 /* NOT REACHED */1360 return -1;1361 1362va_found:1363 TEST_ASSERT(sparsebit_is_set_num(vm->vpages_valid,1364 pgidx_start, pages),1365 "Unexpected, invalid virtual page index range,\n"1366 " pgidx_start: 0x%lx\n"1367 " pages: 0x%lx",1368 pgidx_start, pages);1369 TEST_ASSERT(sparsebit_is_clear_num(vm->vpages_mapped,1370 pgidx_start, pages),1371 "Unexpected, pages already mapped,\n"1372 " pgidx_start: 0x%lx\n"1373 " pages: 0x%lx",1374 pgidx_start, pages);1375 1376 return pgidx_start * vm->page_size;1377}1378 1379static vm_vaddr_t ____vm_vaddr_alloc(struct kvm_vm *vm, size_t sz,1380 vm_vaddr_t vaddr_min,1381 enum kvm_mem_region_type type,1382 bool protected)1383{1384 uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);1385 1386 virt_pgd_alloc(vm);1387 vm_paddr_t paddr = __vm_phy_pages_alloc(vm, pages,1388 KVM_UTIL_MIN_PFN * vm->page_size,1389 vm->memslots[type], protected);1390 1391 /*1392 * Find an unused range of virtual page addresses of at least1393 * pages in length.1394 */1395 vm_vaddr_t vaddr_start = vm_vaddr_unused_gap(vm, sz, vaddr_min);1396 1397 /* Map the virtual pages. */1398 for (vm_vaddr_t vaddr = vaddr_start; pages > 0;1399 pages--, vaddr += vm->page_size, paddr += vm->page_size) {1400 1401 virt_pg_map(vm, vaddr, paddr);1402 1403 sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift);1404 }1405 1406 return vaddr_start;1407}1408 1409vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,1410 enum kvm_mem_region_type type)1411{1412 return ____vm_vaddr_alloc(vm, sz, vaddr_min, type,1413 vm_arch_has_protected_memory(vm));1414}1415 1416vm_vaddr_t vm_vaddr_alloc_shared(struct kvm_vm *vm, size_t sz,1417 vm_vaddr_t vaddr_min,1418 enum kvm_mem_region_type type)1419{1420 return ____vm_vaddr_alloc(vm, sz, vaddr_min, type, false);1421}1422 1423/*1424 * VM Virtual Address Allocate1425 *1426 * Input Args:1427 * vm - Virtual Machine1428 * sz - Size in bytes1429 * vaddr_min - Minimum starting virtual address1430 *1431 * Output Args: None1432 *1433 * Return:1434 * Starting guest virtual address1435 *1436 * Allocates at least sz bytes within the virtual address space of the vm1437 * given by vm. The allocated bytes are mapped to a virtual address >=1438 * the address given by vaddr_min. Note that each allocation uses a1439 * a unique set of pages, with the minimum real allocation being at least1440 * a page. The allocated physical space comes from the TEST_DATA memory region.1441 */1442vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min)1443{1444 return __vm_vaddr_alloc(vm, sz, vaddr_min, MEM_REGION_TEST_DATA);1445}1446 1447/*1448 * VM Virtual Address Allocate Pages1449 *1450 * Input Args:1451 * vm - Virtual Machine1452 *1453 * Output Args: None1454 *1455 * Return:1456 * Starting guest virtual address1457 *1458 * Allocates at least N system pages worth of bytes within the virtual address1459 * space of the vm.1460 */1461vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages)1462{1463 return vm_vaddr_alloc(vm, nr_pages * getpagesize(), KVM_UTIL_MIN_VADDR);1464}1465 1466vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm, enum kvm_mem_region_type type)1467{1468 return __vm_vaddr_alloc(vm, getpagesize(), KVM_UTIL_MIN_VADDR, type);1469}1470 1471/*1472 * VM Virtual Address Allocate Page1473 *1474 * Input Args:1475 * vm - Virtual Machine1476 *1477 * Output Args: None1478 *1479 * Return:1480 * Starting guest virtual address1481 *1482 * Allocates at least one system page worth of bytes within the virtual address1483 * space of the vm.1484 */1485vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm)1486{1487 return vm_vaddr_alloc_pages(vm, 1);1488}1489 1490/*1491 * Map a range of VM virtual address to the VM's physical address1492 *1493 * Input Args:1494 * vm - Virtual Machine1495 * vaddr - Virtuall address to map1496 * paddr - VM Physical Address1497 * npages - The number of pages to map1498 *1499 * Output Args: None1500 *1501 * Return: None1502 *1503 * Within the VM given by @vm, creates a virtual translation for1504 * @npages starting at @vaddr to the page range starting at @paddr.1505 */1506void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,1507 unsigned int npages)1508{1509 size_t page_size = vm->page_size;1510 size_t size = npages * page_size;1511 1512 TEST_ASSERT(vaddr + size > vaddr, "Vaddr overflow");1513 TEST_ASSERT(paddr + size > paddr, "Paddr overflow");1514 1515 while (npages--) {1516 virt_pg_map(vm, vaddr, paddr);1517 sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift);1518 1519 vaddr += page_size;1520 paddr += page_size;1521 }1522}1523 1524/*1525 * Address VM Physical to Host Virtual1526 *1527 * Input Args:1528 * vm - Virtual Machine1529 * gpa - VM physical address1530 *1531 * Output Args: None1532 *1533 * Return:1534 * Equivalent host virtual address1535 *1536 * Locates the memory region containing the VM physical address given1537 * by gpa, within the VM given by vm. When found, the host virtual1538 * address providing the memory to the vm physical address is returned.1539 * A TEST_ASSERT failure occurs if no region containing gpa exists.1540 */1541void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa)1542{1543 struct userspace_mem_region *region;1544 1545 gpa = vm_untag_gpa(vm, gpa);1546 1547 region = userspace_mem_region_find(vm, gpa, gpa);1548 if (!region) {1549 TEST_FAIL("No vm physical memory at 0x%lx", gpa);1550 return NULL;1551 }1552 1553 return (void *)((uintptr_t)region->host_mem1554 + (gpa - region->region.guest_phys_addr));1555}1556 1557/*1558 * Address Host Virtual to VM Physical1559 *1560 * Input Args:1561 * vm - Virtual Machine1562 * hva - Host virtual address1563 *1564 * Output Args: None1565 *1566 * Return:1567 * Equivalent VM physical address1568 *1569 * Locates the memory region containing the host virtual address given1570 * by hva, within the VM given by vm. When found, the equivalent1571 * VM physical address is returned. A TEST_ASSERT failure occurs if no1572 * region containing hva exists.1573 */1574vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva)1575{1576 struct rb_node *node;1577 1578 for (node = vm->regions.hva_tree.rb_node; node; ) {1579 struct userspace_mem_region *region =1580 container_of(node, struct userspace_mem_region, hva_node);1581 1582 if (hva >= region->host_mem) {1583 if (hva <= (region->host_mem1584 + region->region.memory_size - 1))1585 return (vm_paddr_t)((uintptr_t)1586 region->region.guest_phys_addr1587 + (hva - (uintptr_t)region->host_mem));1588 1589 node = node->rb_right;1590 } else1591 node = node->rb_left;1592 }1593 1594 TEST_FAIL("No mapping to a guest physical address, hva: %p", hva);1595 return -1;1596}1597 1598/*1599 * Address VM physical to Host Virtual *alias*.1600 *1601 * Input Args:1602 * vm - Virtual Machine1603 * gpa - VM physical address1604 *1605 * Output Args: None1606 *1607 * Return:1608 * Equivalent address within the host virtual *alias* area, or NULL1609 * (without failing the test) if the guest memory is not shared (so1610 * no alias exists).1611 *1612 * Create a writable, shared virtual=>physical alias for the specific GPA.1613 * The primary use case is to allow the host selftest to manipulate guest1614 * memory without mapping said memory in the guest's address space. And, for1615 * userfaultfd-based demand paging, to do so without triggering userfaults.1616 */1617void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa)1618{1619 struct userspace_mem_region *region;1620 uintptr_t offset;1621 1622 region = userspace_mem_region_find(vm, gpa, gpa);1623 if (!region)1624 return NULL;1625 1626 if (!region->host_alias)1627 return NULL;1628 1629 offset = gpa - region->region.guest_phys_addr;1630 return (void *) ((uintptr_t) region->host_alias + offset);1631}1632 1633/* Create an interrupt controller chip for the specified VM. */1634void vm_create_irqchip(struct kvm_vm *vm)1635{1636 vm_ioctl(vm, KVM_CREATE_IRQCHIP, NULL);1637 1638 vm->has_irqchip = true;1639}1640 1641int _vcpu_run(struct kvm_vcpu *vcpu)1642{1643 int rc;1644 1645 do {1646 rc = __vcpu_run(vcpu);1647 } while (rc == -1 && errno == EINTR);1648 1649 assert_on_unhandled_exception(vcpu);1650 1651 return rc;1652}1653 1654/*1655 * Invoke KVM_RUN on a vCPU until KVM returns something other than -EINTR.1656 * Assert if the KVM returns an error (other than -EINTR).1657 */1658void vcpu_run(struct kvm_vcpu *vcpu)1659{1660 int ret = _vcpu_run(vcpu);1661 1662 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_RUN, ret));1663}1664 1665void vcpu_run_complete_io(struct kvm_vcpu *vcpu)1666{1667 int ret;1668 1669 vcpu->run->immediate_exit = 1;1670 ret = __vcpu_run(vcpu);1671 vcpu->run->immediate_exit = 0;1672 1673 TEST_ASSERT(ret == -1 && errno == EINTR,1674 "KVM_RUN IOCTL didn't exit immediately, rc: %i, errno: %i",1675 ret, errno);1676}1677 1678/*1679 * Get the list of guest registers which are supported for1680 * KVM_GET_ONE_REG/KVM_SET_ONE_REG ioctls. Returns a kvm_reg_list pointer,1681 * it is the caller's responsibility to free the list.1682 */1683struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu)1684{1685 struct kvm_reg_list reg_list_n = { .n = 0 }, *reg_list;1686 int ret;1687 1688 ret = __vcpu_ioctl(vcpu, KVM_GET_REG_LIST, ®_list_n);1689 TEST_ASSERT(ret == -1 && errno == E2BIG, "KVM_GET_REG_LIST n=0");1690 1691 reg_list = calloc(1, sizeof(*reg_list) + reg_list_n.n * sizeof(__u64));1692 reg_list->n = reg_list_n.n;1693 vcpu_ioctl(vcpu, KVM_GET_REG_LIST, reg_list);1694 return reg_list;1695}1696 1697void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu)1698{1699 uint32_t page_size = getpagesize();1700 uint32_t size = vcpu->vm->dirty_ring_size;1701 1702 TEST_ASSERT(size > 0, "Should enable dirty ring first");1703 1704 if (!vcpu->dirty_gfns) {1705 void *addr;1706 1707 addr = mmap(NULL, size, PROT_READ, MAP_PRIVATE, vcpu->fd,1708 page_size * KVM_DIRTY_LOG_PAGE_OFFSET);1709 TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped private");1710 1711 addr = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_PRIVATE, vcpu->fd,1712 page_size * KVM_DIRTY_LOG_PAGE_OFFSET);1713 TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped exec");1714 1715 addr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, vcpu->fd,1716 page_size * KVM_DIRTY_LOG_PAGE_OFFSET);1717 TEST_ASSERT(addr != MAP_FAILED, "Dirty ring map failed");1718 1719 vcpu->dirty_gfns = addr;1720 vcpu->dirty_gfns_count = size / sizeof(struct kvm_dirty_gfn);1721 }1722 1723 return vcpu->dirty_gfns;1724}1725 1726/*1727 * Device Ioctl1728 */1729 1730int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr)1731{1732 struct kvm_device_attr attribute = {1733 .group = group,1734 .attr = attr,1735 .flags = 0,1736 };1737 1738 return ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute);1739}1740 1741int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type)1742{1743 struct kvm_create_device create_dev = {1744 .type = type,1745 .flags = KVM_CREATE_DEVICE_TEST,1746 };1747 1748 return __vm_ioctl(vm, KVM_CREATE_DEVICE, &create_dev);1749}1750 1751int __kvm_create_device(struct kvm_vm *vm, uint64_t type)1752{1753 struct kvm_create_device create_dev = {1754 .type = type,1755 .fd = -1,1756 .flags = 0,1757 };1758 int err;1759 1760 err = __vm_ioctl(vm, KVM_CREATE_DEVICE, &create_dev);1761 TEST_ASSERT(err <= 0, "KVM_CREATE_DEVICE shouldn't return a positive value");1762 return err ? : create_dev.fd;1763}1764 1765int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val)1766{1767 struct kvm_device_attr kvmattr = {1768 .group = group,1769 .attr = attr,1770 .flags = 0,1771 .addr = (uintptr_t)val,1772 };1773 1774 return __kvm_ioctl(dev_fd, KVM_GET_DEVICE_ATTR, &kvmattr);1775}1776 1777int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val)1778{1779 struct kvm_device_attr kvmattr = {1780 .group = group,1781 .attr = attr,1782 .flags = 0,1783 .addr = (uintptr_t)val,1784 };1785 1786 return __kvm_ioctl(dev_fd, KVM_SET_DEVICE_ATTR, &kvmattr);1787}1788 1789/*1790 * IRQ related functions.1791 */1792 1793int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)1794{1795 struct kvm_irq_level irq_level = {1796 .irq = irq,1797 .level = level,1798 };1799 1800 return __vm_ioctl(vm, KVM_IRQ_LINE, &irq_level);1801}1802 1803void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)1804{1805 int ret = _kvm_irq_line(vm, irq, level);1806 1807 TEST_ASSERT(ret >= 0, KVM_IOCTL_ERROR(KVM_IRQ_LINE, ret));1808}1809 1810struct kvm_irq_routing *kvm_gsi_routing_create(void)1811{1812 struct kvm_irq_routing *routing;1813 size_t size;1814 1815 size = sizeof(struct kvm_irq_routing);1816 /* Allocate space for the max number of entries: this wastes 196 KBs. */1817 size += KVM_MAX_IRQ_ROUTES * sizeof(struct kvm_irq_routing_entry);1818 routing = calloc(1, size);1819 assert(routing);1820 1821 return routing;1822}1823 1824void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,1825 uint32_t gsi, uint32_t pin)1826{1827 int i;1828 1829 assert(routing);1830 assert(routing->nr < KVM_MAX_IRQ_ROUTES);1831 1832 i = routing->nr;1833 routing->entries[i].gsi = gsi;1834 routing->entries[i].type = KVM_IRQ_ROUTING_IRQCHIP;1835 routing->entries[i].flags = 0;1836 routing->entries[i].u.irqchip.irqchip = 0;1837 routing->entries[i].u.irqchip.pin = pin;1838 routing->nr++;1839}1840 1841int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing)1842{1843 int ret;1844 1845 assert(routing);1846 ret = __vm_ioctl(vm, KVM_SET_GSI_ROUTING, routing);1847 free(routing);1848 1849 return ret;1850}1851 1852void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing)1853{1854 int ret;1855 1856 ret = _kvm_gsi_routing_write(vm, routing);1857 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_GSI_ROUTING, ret));1858}1859 1860/*1861 * VM Dump1862 *1863 * Input Args:1864 * vm - Virtual Machine1865 * indent - Left margin indent amount1866 *1867 * Output Args:1868 * stream - Output FILE stream1869 *1870 * Return: None1871 *1872 * Dumps the current state of the VM given by vm, to the FILE stream1873 * given by stream.1874 */1875void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)1876{1877 int ctr;1878 struct userspace_mem_region *region;1879 struct kvm_vcpu *vcpu;1880 1881 fprintf(stream, "%*smode: 0x%x\n", indent, "", vm->mode);1882 fprintf(stream, "%*sfd: %i\n", indent, "", vm->fd);1883 fprintf(stream, "%*spage_size: 0x%x\n", indent, "", vm->page_size);1884 fprintf(stream, "%*sMem Regions:\n", indent, "");1885 hash_for_each(vm->regions.slot_hash, ctr, region, slot_node) {1886 fprintf(stream, "%*sguest_phys: 0x%lx size: 0x%lx "1887 "host_virt: %p\n", indent + 2, "",1888 (uint64_t) region->region.guest_phys_addr,1889 (uint64_t) region->region.memory_size,1890 region->host_mem);1891 fprintf(stream, "%*sunused_phy_pages: ", indent + 2, "");1892 sparsebit_dump(stream, region->unused_phy_pages, 0);1893 if (region->protected_phy_pages) {1894 fprintf(stream, "%*sprotected_phy_pages: ", indent + 2, "");1895 sparsebit_dump(stream, region->protected_phy_pages, 0);1896 }1897 }1898 fprintf(stream, "%*sMapped Virtual Pages:\n", indent, "");1899 sparsebit_dump(stream, vm->vpages_mapped, indent + 2);1900 fprintf(stream, "%*spgd_created: %u\n", indent, "",1901 vm->pgd_created);1902 if (vm->pgd_created) {1903 fprintf(stream, "%*sVirtual Translation Tables:\n",1904 indent + 2, "");1905 virt_dump(stream, vm, indent + 4);1906 }1907 fprintf(stream, "%*sVCPUs:\n", indent, "");1908 1909 list_for_each_entry(vcpu, &vm->vcpus, list)1910 vcpu_dump(stream, vcpu, indent + 2);1911}1912 1913#define KVM_EXIT_STRING(x) {KVM_EXIT_##x, #x}1914 1915/* Known KVM exit reasons */1916static struct exit_reason {1917 unsigned int reason;1918 const char *name;1919} exit_reasons_known[] = {1920 KVM_EXIT_STRING(UNKNOWN),1921 KVM_EXIT_STRING(EXCEPTION),1922 KVM_EXIT_STRING(IO),1923 KVM_EXIT_STRING(HYPERCALL),1924 KVM_EXIT_STRING(DEBUG),1925 KVM_EXIT_STRING(HLT),1926 KVM_EXIT_STRING(MMIO),1927 KVM_EXIT_STRING(IRQ_WINDOW_OPEN),1928 KVM_EXIT_STRING(SHUTDOWN),1929 KVM_EXIT_STRING(FAIL_ENTRY),1930 KVM_EXIT_STRING(INTR),1931 KVM_EXIT_STRING(SET_TPR),1932 KVM_EXIT_STRING(TPR_ACCESS),1933 KVM_EXIT_STRING(S390_SIEIC),1934 KVM_EXIT_STRING(S390_RESET),1935 KVM_EXIT_STRING(DCR),1936 KVM_EXIT_STRING(NMI),1937 KVM_EXIT_STRING(INTERNAL_ERROR),1938 KVM_EXIT_STRING(OSI),1939 KVM_EXIT_STRING(PAPR_HCALL),1940 KVM_EXIT_STRING(S390_UCONTROL),1941 KVM_EXIT_STRING(WATCHDOG),1942 KVM_EXIT_STRING(S390_TSCH),1943 KVM_EXIT_STRING(EPR),1944 KVM_EXIT_STRING(SYSTEM_EVENT),1945 KVM_EXIT_STRING(S390_STSI),1946 KVM_EXIT_STRING(IOAPIC_EOI),1947 KVM_EXIT_STRING(HYPERV),1948 KVM_EXIT_STRING(ARM_NISV),1949 KVM_EXIT_STRING(X86_RDMSR),1950 KVM_EXIT_STRING(X86_WRMSR),1951 KVM_EXIT_STRING(DIRTY_RING_FULL),1952 KVM_EXIT_STRING(AP_RESET_HOLD),1953 KVM_EXIT_STRING(X86_BUS_LOCK),1954 KVM_EXIT_STRING(XEN),1955 KVM_EXIT_STRING(RISCV_SBI),1956 KVM_EXIT_STRING(RISCV_CSR),1957 KVM_EXIT_STRING(NOTIFY),1958#ifdef KVM_EXIT_MEMORY_NOT_PRESENT1959 KVM_EXIT_STRING(MEMORY_NOT_PRESENT),1960#endif1961};1962 1963/*1964 * Exit Reason String1965 *1966 * Input Args:1967 * exit_reason - Exit reason1968 *1969 * Output Args: None1970 *1971 * Return:1972 * Constant string pointer describing the exit reason.1973 *1974 * Locates and returns a constant string that describes the KVM exit1975 * reason given by exit_reason. If no such string is found, a constant1976 * string of "Unknown" is returned.1977 */1978const char *exit_reason_str(unsigned int exit_reason)1979{1980 unsigned int n1;1981 1982 for (n1 = 0; n1 < ARRAY_SIZE(exit_reasons_known); n1++) {1983 if (exit_reason == exit_reasons_known[n1].reason)1984 return exit_reasons_known[n1].name;1985 }1986 1987 return "Unknown";1988}1989 1990/*1991 * Physical Contiguous Page Allocator1992 *1993 * Input Args:1994 * vm - Virtual Machine1995 * num - number of pages1996 * paddr_min - Physical address minimum1997 * memslot - Memory region to allocate page from1998 * protected - True if the pages will be used as protected/private memory1999 *2000 * Output Args: None2001 *2002 * Return:2003 * Starting physical address2004 *2005 * Within the VM specified by vm, locates a range of available physical2006 * pages at or above paddr_min. If found, the pages are marked as in use2007 * and their base address is returned. A TEST_ASSERT failure occurs if2008 * not enough pages are available at or above paddr_min.2009 */2010vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,2011 vm_paddr_t paddr_min, uint32_t memslot,2012 bool protected)2013{2014 struct userspace_mem_region *region;2015 sparsebit_idx_t pg, base;2016 2017 TEST_ASSERT(num > 0, "Must allocate at least one page");2018 2019 TEST_ASSERT((paddr_min % vm->page_size) == 0, "Min physical address "2020 "not divisible by page size.\n"2021 " paddr_min: 0x%lx page_size: 0x%x",2022 paddr_min, vm->page_size);2023 2024 region = memslot2region(vm, memslot);2025 TEST_ASSERT(!protected || region->protected_phy_pages,2026 "Region doesn't support protected memory");2027 2028 base = pg = paddr_min >> vm->page_shift;2029 do {2030 for (; pg < base + num; ++pg) {2031 if (!sparsebit_is_set(region->unused_phy_pages, pg)) {2032 base = pg = sparsebit_next_set(region->unused_phy_pages, pg);2033 break;2034 }2035 }2036 } while (pg && pg != base + num);2037 2038 if (pg == 0) {2039 fprintf(stderr, "No guest physical page available, "2040 "paddr_min: 0x%lx page_size: 0x%x memslot: %u\n",2041 paddr_min, vm->page_size, memslot);2042 fputs("---- vm dump ----\n", stderr);2043 vm_dump(stderr, vm, 2);2044 abort();2045 }2046 2047 for (pg = base; pg < base + num; ++pg) {2048 sparsebit_clear(region->unused_phy_pages, pg);2049 if (protected)2050 sparsebit_set(region->protected_phy_pages, pg);2051 }2052 2053 return base * vm->page_size;2054}2055 2056vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,2057 uint32_t memslot)2058{2059 return vm_phy_pages_alloc(vm, 1, paddr_min, memslot);2060}2061 2062vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm)2063{2064 return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR,2065 vm->memslots[MEM_REGION_PT]);2066}2067 2068/*2069 * Address Guest Virtual to Host Virtual2070 *2071 * Input Args:2072 * vm - Virtual Machine2073 * gva - VM virtual address2074 *2075 * Output Args: None2076 *2077 * Return:2078 * Equivalent host virtual address2079 */2080void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva)2081{2082 return addr_gpa2hva(vm, addr_gva2gpa(vm, gva));2083}2084 2085unsigned long __weak vm_compute_max_gfn(struct kvm_vm *vm)2086{2087 return ((1ULL << vm->pa_bits) >> vm->page_shift) - 1;2088}2089 2090static unsigned int vm_calc_num_pages(unsigned int num_pages,2091 unsigned int page_shift,2092 unsigned int new_page_shift,2093 bool ceil)2094{2095 unsigned int n = 1 << (new_page_shift - page_shift);2096 2097 if (page_shift >= new_page_shift)2098 return num_pages * (1 << (page_shift - new_page_shift));2099 2100 return num_pages / n + !!(ceil && num_pages % n);2101}2102 2103static inline int getpageshift(void)2104{2105 return __builtin_ffs(getpagesize()) - 1;2106}2107 2108unsigned int2109vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)2110{2111 return vm_calc_num_pages(num_guest_pages,2112 vm_guest_mode_params[mode].page_shift,2113 getpageshift(), true);2114}2115 2116unsigned int2117vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages)2118{2119 return vm_calc_num_pages(num_host_pages, getpageshift(),2120 vm_guest_mode_params[mode].page_shift, false);2121}2122 2123unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size)2124{2125 unsigned int n;2126 n = DIV_ROUND_UP(size, vm_guest_mode_params[mode].page_size);2127 return vm_adjust_num_guest_pages(mode, n);2128}2129 2130/*2131 * Read binary stats descriptors2132 *2133 * Input Args:2134 * stats_fd - the file descriptor for the binary stats file from which to read2135 * header - the binary stats metadata header corresponding to the given FD2136 *2137 * Output Args: None2138 *2139 * Return:2140 * A pointer to a newly allocated series of stat descriptors.2141 * Caller is responsible for freeing the returned kvm_stats_desc.2142 *2143 * Read the stats descriptors from the binary stats interface.2144 */2145struct kvm_stats_desc *read_stats_descriptors(int stats_fd,2146 struct kvm_stats_header *header)2147{2148 struct kvm_stats_desc *stats_desc;2149 ssize_t desc_size, total_size, ret;2150 2151 desc_size = get_stats_descriptor_size(header);2152 total_size = header->num_desc * desc_size;2153 2154 stats_desc = calloc(header->num_desc, desc_size);2155 TEST_ASSERT(stats_desc, "Allocate memory for stats descriptors");2156 2157 ret = pread(stats_fd, stats_desc, total_size, header->desc_offset);2158 TEST_ASSERT(ret == total_size, "Read KVM stats descriptors");2159 2160 return stats_desc;2161}2162 2163/*2164 * Read stat data for a particular stat2165 *2166 * Input Args:2167 * stats_fd - the file descriptor for the binary stats file from which to read2168 * header - the binary stats metadata header corresponding to the given FD2169 * desc - the binary stat metadata for the particular stat to be read2170 * max_elements - the maximum number of 8-byte values to read into data2171 *2172 * Output Args:2173 * data - the buffer into which stat data should be read2174 *2175 * Read the data values of a specified stat from the binary stats interface.2176 */2177void read_stat_data(int stats_fd, struct kvm_stats_header *header,2178 struct kvm_stats_desc *desc, uint64_t *data,2179 size_t max_elements)2180{2181 size_t nr_elements = min_t(ssize_t, desc->size, max_elements);2182 size_t size = nr_elements * sizeof(*data);2183 ssize_t ret;2184 2185 TEST_ASSERT(desc->size, "No elements in stat '%s'", desc->name);2186 TEST_ASSERT(max_elements, "Zero elements requested for stat '%s'", desc->name);2187 2188 ret = pread(stats_fd, data, size,2189 header->data_offset + desc->offset);2190 2191 TEST_ASSERT(ret >= 0, "pread() failed on stat '%s', errno: %i (%s)",2192 desc->name, errno, strerror(errno));2193 TEST_ASSERT(ret == size,2194 "pread() on stat '%s' read %ld bytes, wanted %lu bytes",2195 desc->name, size, ret);2196}2197 2198/*2199 * Read the data of the named stat2200 *2201 * Input Args:2202 * vm - the VM for which the stat should be read2203 * stat_name - the name of the stat to read2204 * max_elements - the maximum number of 8-byte values to read into data2205 *2206 * Output Args:2207 * data - the buffer into which stat data should be read2208 *2209 * Read the data values of a specified stat from the binary stats interface.2210 */2211void __vm_get_stat(struct kvm_vm *vm, const char *stat_name, uint64_t *data,2212 size_t max_elements)2213{2214 struct kvm_stats_desc *desc;2215 size_t size_desc;2216 int i;2217 2218 if (!vm->stats_fd) {2219 vm->stats_fd = vm_get_stats_fd(vm);2220 read_stats_header(vm->stats_fd, &vm->stats_header);2221 vm->stats_desc = read_stats_descriptors(vm->stats_fd,2222 &vm->stats_header);2223 }2224 2225 size_desc = get_stats_descriptor_size(&vm->stats_header);2226 2227 for (i = 0; i < vm->stats_header.num_desc; ++i) {2228 desc = (void *)vm->stats_desc + (i * size_desc);2229 2230 if (strcmp(desc->name, stat_name))2231 continue;2232 2233 read_stat_data(vm->stats_fd, &vm->stats_header, desc,2234 data, max_elements);2235 2236 break;2237 }2238}2239 2240__weak void kvm_arch_vm_post_create(struct kvm_vm *vm)2241{2242}2243 2244__weak void kvm_selftest_arch_init(void)2245{2246}2247 2248void __attribute((constructor)) kvm_selftest_init(void)2249{2250 /* Tell stdout not to buffer its content. */2251 setbuf(stdout, NULL);2252 2253 guest_random_seed = last_guest_seed = random();2254 pr_info("Random seed: 0x%x\n", guest_random_seed);2255 2256 kvm_selftest_arch_init();2257}2258 2259bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr)2260{2261 sparsebit_idx_t pg = 0;2262 struct userspace_mem_region *region;2263 2264 if (!vm_arch_has_protected_memory(vm))2265 return false;2266 2267 region = userspace_mem_region_find(vm, paddr, paddr);2268 TEST_ASSERT(region, "No vm physical memory at 0x%lx", paddr);2269 2270 pg = paddr >> vm->page_shift;2271 return sparsebit_is_set(region->protected_phy_pages, pg);2272}2273