1141 lines · c
1/* SPDX-License-Identifier: GPL-2.0-only */2/*3 * Copyright (C) 2018, Google LLC.4 */5#ifndef SELFTEST_KVM_UTIL_H6#define SELFTEST_KVM_UTIL_H7 8#include "test_util.h"9 10#include <linux/compiler.h>11#include "linux/hashtable.h"12#include "linux/list.h"13#include <linux/kernel.h>14#include <linux/kvm.h>15#include "linux/rbtree.h"16#include <linux/types.h>17 18#include <asm/atomic.h>19#include <asm/kvm.h>20 21#include <sys/ioctl.h>22 23#include "kvm_util_arch.h"24#include "kvm_util_types.h"25#include "sparsebit.h"26 27#define KVM_DEV_PATH "/dev/kvm"28#define KVM_MAX_VCPUS 51229 30#define NSEC_PER_SEC 1000000000L31 32struct userspace_mem_region {33 struct kvm_userspace_memory_region2 region;34 struct sparsebit *unused_phy_pages;35 struct sparsebit *protected_phy_pages;36 int fd;37 off_t offset;38 enum vm_mem_backing_src_type backing_src_type;39 void *host_mem;40 void *host_alias;41 void *mmap_start;42 void *mmap_alias;43 size_t mmap_size;44 struct rb_node gpa_node;45 struct rb_node hva_node;46 struct hlist_node slot_node;47};48 49struct kvm_vcpu {50 struct list_head list;51 uint32_t id;52 int fd;53 struct kvm_vm *vm;54 struct kvm_run *run;55#ifdef __x86_64__56 struct kvm_cpuid2 *cpuid;57#endif58 struct kvm_dirty_gfn *dirty_gfns;59 uint32_t fetch_index;60 uint32_t dirty_gfns_count;61};62 63struct userspace_mem_regions {64 struct rb_root gpa_tree;65 struct rb_root hva_tree;66 DECLARE_HASHTABLE(slot_hash, 9);67};68 69enum kvm_mem_region_type {70 MEM_REGION_CODE,71 MEM_REGION_DATA,72 MEM_REGION_PT,73 MEM_REGION_TEST_DATA,74 NR_MEM_REGIONS,75};76 77struct kvm_vm {78 int mode;79 unsigned long type;80 int kvm_fd;81 int fd;82 unsigned int pgtable_levels;83 unsigned int page_size;84 unsigned int page_shift;85 unsigned int pa_bits;86 unsigned int va_bits;87 uint64_t max_gfn;88 struct list_head vcpus;89 struct userspace_mem_regions regions;90 struct sparsebit *vpages_valid;91 struct sparsebit *vpages_mapped;92 bool has_irqchip;93 bool pgd_created;94 vm_paddr_t ucall_mmio_addr;95 vm_paddr_t pgd;96 vm_vaddr_t handlers;97 uint32_t dirty_ring_size;98 uint64_t gpa_tag_mask;99 100 struct kvm_vm_arch arch;101 102 /* Cache of information for binary stats interface */103 int stats_fd;104 struct kvm_stats_header stats_header;105 struct kvm_stats_desc *stats_desc;106 107 /*108 * KVM region slots. These are the default memslots used by page109 * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE]110 * memslot.111 */112 uint32_t memslots[NR_MEM_REGIONS];113};114 115struct vcpu_reg_sublist {116 const char *name;117 long capability;118 int feature;119 int feature_type;120 bool finalize;121 __u64 *regs;122 __u64 regs_n;123 __u64 *rejects_set;124 __u64 rejects_set_n;125 __u64 *skips_set;126 __u64 skips_set_n;127};128 129struct vcpu_reg_list {130 char *name;131 struct vcpu_reg_sublist sublists[];132};133 134#define for_each_sublist(c, s) \135 for ((s) = &(c)->sublists[0]; (s)->regs; ++(s))136 137#define kvm_for_each_vcpu(vm, i, vcpu) \138 for ((i) = 0; (i) <= (vm)->last_vcpu_id; (i)++) \139 if (!((vcpu) = vm->vcpus[i])) \140 continue; \141 else142 143struct userspace_mem_region *144memslot2region(struct kvm_vm *vm, uint32_t memslot);145 146static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm,147 enum kvm_mem_region_type type)148{149 assert(type < NR_MEM_REGIONS);150 return memslot2region(vm, vm->memslots[type]);151}152 153/* Minimum allocated guest virtual and physical addresses */154#define KVM_UTIL_MIN_VADDR 0x2000155#define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000156 157#define DEFAULT_GUEST_STACK_VADDR_MIN 0xab6000158#define DEFAULT_STACK_PGS 5159 160enum vm_guest_mode {161 VM_MODE_P52V48_4K,162 VM_MODE_P52V48_16K,163 VM_MODE_P52V48_64K,164 VM_MODE_P48V48_4K,165 VM_MODE_P48V48_16K,166 VM_MODE_P48V48_64K,167 VM_MODE_P40V48_4K,168 VM_MODE_P40V48_16K,169 VM_MODE_P40V48_64K,170 VM_MODE_PXXV48_4K, /* For 48bits VA but ANY bits PA */171 VM_MODE_P47V64_4K,172 VM_MODE_P44V64_4K,173 VM_MODE_P36V48_4K,174 VM_MODE_P36V48_16K,175 VM_MODE_P36V48_64K,176 VM_MODE_P36V47_16K,177 NUM_VM_MODES,178};179 180struct vm_shape {181 uint32_t type;182 uint8_t mode;183 uint8_t pad0;184 uint16_t pad1;185};186 187kvm_static_assert(sizeof(struct vm_shape) == sizeof(uint64_t));188 189#define VM_TYPE_DEFAULT 0190 191#define VM_SHAPE(__mode) \192({ \193 struct vm_shape shape = { \194 .mode = (__mode), \195 .type = VM_TYPE_DEFAULT \196 }; \197 \198 shape; \199})200 201#if defined(__aarch64__)202 203extern enum vm_guest_mode vm_mode_default;204 205#define VM_MODE_DEFAULT vm_mode_default206#define MIN_PAGE_SHIFT 12U207#define ptes_per_page(page_size) ((page_size) / 8)208 209#elif defined(__x86_64__)210 211#define VM_MODE_DEFAULT VM_MODE_PXXV48_4K212#define MIN_PAGE_SHIFT 12U213#define ptes_per_page(page_size) ((page_size) / 8)214 215#elif defined(__s390x__)216 217#define VM_MODE_DEFAULT VM_MODE_P44V64_4K218#define MIN_PAGE_SHIFT 12U219#define ptes_per_page(page_size) ((page_size) / 16)220 221#elif defined(__riscv)222 223#if __riscv_xlen == 32224#error "RISC-V 32-bit kvm selftests not supported"225#endif226 227#define VM_MODE_DEFAULT VM_MODE_P40V48_4K228#define MIN_PAGE_SHIFT 12U229#define ptes_per_page(page_size) ((page_size) / 8)230 231#endif232 233#define VM_SHAPE_DEFAULT VM_SHAPE(VM_MODE_DEFAULT)234 235#define MIN_PAGE_SIZE (1U << MIN_PAGE_SHIFT)236#define PTES_PER_MIN_PAGE ptes_per_page(MIN_PAGE_SIZE)237 238struct vm_guest_mode_params {239 unsigned int pa_bits;240 unsigned int va_bits;241 unsigned int page_size;242 unsigned int page_shift;243};244extern const struct vm_guest_mode_params vm_guest_mode_params[];245 246int open_path_or_exit(const char *path, int flags);247int open_kvm_dev_path_or_exit(void);248 249bool get_kvm_param_bool(const char *param);250bool get_kvm_intel_param_bool(const char *param);251bool get_kvm_amd_param_bool(const char *param);252 253int get_kvm_param_integer(const char *param);254int get_kvm_intel_param_integer(const char *param);255int get_kvm_amd_param_integer(const char *param);256 257unsigned int kvm_check_cap(long cap);258 259static inline bool kvm_has_cap(long cap)260{261 return kvm_check_cap(cap);262}263 264#define __KVM_SYSCALL_ERROR(_name, _ret) \265 "%s failed, rc: %i errno: %i (%s)", (_name), (_ret), errno, strerror(errno)266 267/*268 * Use the "inner", double-underscore macro when reporting errors from within269 * other macros so that the name of ioctl() and not its literal numeric value270 * is printed on error. The "outer" macro is strongly preferred when reporting271 * errors "directly", i.e. without an additional layer of macros, as it reduces272 * the probability of passing in the wrong string.273 */274#define __KVM_IOCTL_ERROR(_name, _ret) __KVM_SYSCALL_ERROR(_name, _ret)275#define KVM_IOCTL_ERROR(_ioctl, _ret) __KVM_IOCTL_ERROR(#_ioctl, _ret)276 277#define kvm_do_ioctl(fd, cmd, arg) \278({ \279 kvm_static_assert(!_IOC_SIZE(cmd) || sizeof(*arg) == _IOC_SIZE(cmd)); \280 ioctl(fd, cmd, arg); \281})282 283#define __kvm_ioctl(kvm_fd, cmd, arg) \284 kvm_do_ioctl(kvm_fd, cmd, arg)285 286#define kvm_ioctl(kvm_fd, cmd, arg) \287({ \288 int ret = __kvm_ioctl(kvm_fd, cmd, arg); \289 \290 TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \291})292 293static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { }294 295#define __vm_ioctl(vm, cmd, arg) \296({ \297 static_assert_is_vm(vm); \298 kvm_do_ioctl((vm)->fd, cmd, arg); \299})300 301/*302 * Assert that a VM or vCPU ioctl() succeeded, with extra magic to detect if303 * the ioctl() failed because KVM killed/bugged the VM. To detect a dead VM,304 * probe KVM_CAP_USER_MEMORY, which (a) has been supported by KVM since before305 * selftests existed and (b) should never outright fail, i.e. is supposed to306 * return 0 or 1. If KVM kills a VM, KVM returns -EIO for all ioctl()s for the307 * VM and its vCPUs, including KVM_CHECK_EXTENSION.308 */309#define __TEST_ASSERT_VM_VCPU_IOCTL(cond, name, ret, vm) \310do { \311 int __errno = errno; \312 \313 static_assert_is_vm(vm); \314 \315 if (cond) \316 break; \317 \318 if (errno == EIO && \319 __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)KVM_CAP_USER_MEMORY) < 0) { \320 TEST_ASSERT(errno == EIO, "KVM killed the VM, should return -EIO"); \321 TEST_FAIL("KVM killed/bugged the VM, check the kernel log for clues"); \322 } \323 errno = __errno; \324 TEST_ASSERT(cond, __KVM_IOCTL_ERROR(name, ret)); \325} while (0)326 327#define TEST_ASSERT_VM_VCPU_IOCTL(cond, cmd, ret, vm) \328 __TEST_ASSERT_VM_VCPU_IOCTL(cond, #cmd, ret, vm)329 330#define vm_ioctl(vm, cmd, arg) \331({ \332 int ret = __vm_ioctl(vm, cmd, arg); \333 \334 __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, vm); \335})336 337static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { }338 339#define __vcpu_ioctl(vcpu, cmd, arg) \340({ \341 static_assert_is_vcpu(vcpu); \342 kvm_do_ioctl((vcpu)->fd, cmd, arg); \343})344 345#define vcpu_ioctl(vcpu, cmd, arg) \346({ \347 int ret = __vcpu_ioctl(vcpu, cmd, arg); \348 \349 __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm); \350})351 352/*353 * Looks up and returns the value corresponding to the capability354 * (KVM_CAP_*) given by cap.355 */356static inline int vm_check_cap(struct kvm_vm *vm, long cap)357{358 int ret = __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)cap);359 360 TEST_ASSERT_VM_VCPU_IOCTL(ret >= 0, KVM_CHECK_EXTENSION, ret, vm);361 return ret;362}363 364static inline int __vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0)365{366 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };367 368 return __vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap);369}370static inline void vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0)371{372 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };373 374 vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap);375}376 377static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa,378 uint64_t size, uint64_t attributes)379{380 struct kvm_memory_attributes attr = {381 .attributes = attributes,382 .address = gpa,383 .size = size,384 .flags = 0,385 };386 387 /*388 * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes. These flows389 * need significant enhancements to support multiple attributes.390 */391 TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE,392 "Update me to support multiple attributes!");393 394 vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr);395}396 397 398static inline void vm_mem_set_private(struct kvm_vm *vm, uint64_t gpa,399 uint64_t size)400{401 vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE);402}403 404static inline void vm_mem_set_shared(struct kvm_vm *vm, uint64_t gpa,405 uint64_t size)406{407 vm_set_memory_attributes(vm, gpa, size, 0);408}409 410void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t gpa, uint64_t size,411 bool punch_hole);412 413static inline void vm_guest_mem_punch_hole(struct kvm_vm *vm, uint64_t gpa,414 uint64_t size)415{416 vm_guest_mem_fallocate(vm, gpa, size, true);417}418 419static inline void vm_guest_mem_allocate(struct kvm_vm *vm, uint64_t gpa,420 uint64_t size)421{422 vm_guest_mem_fallocate(vm, gpa, size, false);423}424 425void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size);426const char *vm_guest_mode_string(uint32_t i);427 428void kvm_vm_free(struct kvm_vm *vmp);429void kvm_vm_restart(struct kvm_vm *vmp);430void kvm_vm_release(struct kvm_vm *vmp);431void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename);432int kvm_memfd_alloc(size_t size, bool hugepages);433 434void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent);435 436static inline void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log)437{438 struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot };439 440 vm_ioctl(vm, KVM_GET_DIRTY_LOG, &args);441}442 443static inline void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log,444 uint64_t first_page, uint32_t num_pages)445{446 struct kvm_clear_dirty_log args = {447 .dirty_bitmap = log,448 .slot = slot,449 .first_page = first_page,450 .num_pages = num_pages451 };452 453 vm_ioctl(vm, KVM_CLEAR_DIRTY_LOG, &args);454}455 456static inline uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm)457{458 return __vm_ioctl(vm, KVM_RESET_DIRTY_RINGS, NULL);459}460 461static inline void kvm_vm_register_coalesced_io(struct kvm_vm *vm,462 uint64_t address,463 uint64_t size, bool pio)464{465 struct kvm_coalesced_mmio_zone zone = {466 .addr = address,467 .size = size,468 .pio = pio,469 };470 471 vm_ioctl(vm, KVM_REGISTER_COALESCED_MMIO, &zone);472}473 474static inline void kvm_vm_unregister_coalesced_io(struct kvm_vm *vm,475 uint64_t address,476 uint64_t size, bool pio)477{478 struct kvm_coalesced_mmio_zone zone = {479 .addr = address,480 .size = size,481 .pio = pio,482 };483 484 vm_ioctl(vm, KVM_UNREGISTER_COALESCED_MMIO, &zone);485}486 487static inline int vm_get_stats_fd(struct kvm_vm *vm)488{489 int fd = __vm_ioctl(vm, KVM_GET_STATS_FD, NULL);490 491 TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_GET_STATS_FD, fd, vm);492 return fd;493}494 495static inline void read_stats_header(int stats_fd, struct kvm_stats_header *header)496{497 ssize_t ret;498 499 ret = pread(stats_fd, header, sizeof(*header), 0);500 TEST_ASSERT(ret == sizeof(*header),501 "Failed to read '%lu' header bytes, ret = '%ld'",502 sizeof(*header), ret);503}504 505struct kvm_stats_desc *read_stats_descriptors(int stats_fd,506 struct kvm_stats_header *header);507 508static inline ssize_t get_stats_descriptor_size(struct kvm_stats_header *header)509{510 /*511 * The base size of the descriptor is defined by KVM's ABI, but the512 * size of the name field is variable, as far as KVM's ABI is513 * concerned. For a given instance of KVM, the name field is the same514 * size for all stats and is provided in the overall stats header.515 */516 return sizeof(struct kvm_stats_desc) + header->name_size;517}518 519static inline struct kvm_stats_desc *get_stats_descriptor(struct kvm_stats_desc *stats,520 int index,521 struct kvm_stats_header *header)522{523 /*524 * Note, size_desc includes the size of the name field, which is525 * variable. i.e. this is NOT equivalent to &stats_desc[i].526 */527 return (void *)stats + index * get_stats_descriptor_size(header);528}529 530void read_stat_data(int stats_fd, struct kvm_stats_header *header,531 struct kvm_stats_desc *desc, uint64_t *data,532 size_t max_elements);533 534void __vm_get_stat(struct kvm_vm *vm, const char *stat_name, uint64_t *data,535 size_t max_elements);536 537static inline uint64_t vm_get_stat(struct kvm_vm *vm, const char *stat_name)538{539 uint64_t data;540 541 __vm_get_stat(vm, stat_name, &data, 1);542 return data;543}544 545void vm_create_irqchip(struct kvm_vm *vm);546 547static inline int __vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size,548 uint64_t flags)549{550 struct kvm_create_guest_memfd guest_memfd = {551 .size = size,552 .flags = flags,553 };554 555 return __vm_ioctl(vm, KVM_CREATE_GUEST_MEMFD, &guest_memfd);556}557 558static inline int vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size,559 uint64_t flags)560{561 int fd = __vm_create_guest_memfd(vm, size, flags);562 563 TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_GUEST_MEMFD, fd));564 return fd;565}566 567void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,568 uint64_t gpa, uint64_t size, void *hva);569int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,570 uint64_t gpa, uint64_t size, void *hva);571void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,572 uint64_t gpa, uint64_t size, void *hva,573 uint32_t guest_memfd, uint64_t guest_memfd_offset);574int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,575 uint64_t gpa, uint64_t size, void *hva,576 uint32_t guest_memfd, uint64_t guest_memfd_offset);577 578void vm_userspace_mem_region_add(struct kvm_vm *vm,579 enum vm_mem_backing_src_type src_type,580 uint64_t guest_paddr, uint32_t slot, uint64_t npages,581 uint32_t flags);582void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,583 uint64_t guest_paddr, uint32_t slot, uint64_t npages,584 uint32_t flags, int guest_memfd_fd, uint64_t guest_memfd_offset);585 586#ifndef vm_arch_has_protected_memory587static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm)588{589 return false;590}591#endif592 593void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags);594void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa);595void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot);596struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id);597void vm_populate_vaddr_bitmap(struct kvm_vm *vm);598vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);599vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);600vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,601 enum kvm_mem_region_type type);602vm_vaddr_t vm_vaddr_alloc_shared(struct kvm_vm *vm, size_t sz,603 vm_vaddr_t vaddr_min,604 enum kvm_mem_region_type type);605vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages);606vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm,607 enum kvm_mem_region_type type);608vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm);609 610void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,611 unsigned int npages);612void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa);613void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva);614vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva);615void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa);616 617#ifndef vcpu_arch_put_guest618#define vcpu_arch_put_guest(mem, val) do { (mem) = (val); } while (0)619#endif620 621static inline vm_paddr_t vm_untag_gpa(struct kvm_vm *vm, vm_paddr_t gpa)622{623 return gpa & ~vm->gpa_tag_mask;624}625 626void vcpu_run(struct kvm_vcpu *vcpu);627int _vcpu_run(struct kvm_vcpu *vcpu);628 629static inline int __vcpu_run(struct kvm_vcpu *vcpu)630{631 return __vcpu_ioctl(vcpu, KVM_RUN, NULL);632}633 634void vcpu_run_complete_io(struct kvm_vcpu *vcpu);635struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu);636 637static inline void vcpu_enable_cap(struct kvm_vcpu *vcpu, uint32_t cap,638 uint64_t arg0)639{640 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };641 642 vcpu_ioctl(vcpu, KVM_ENABLE_CAP, &enable_cap);643}644 645static inline void vcpu_guest_debug_set(struct kvm_vcpu *vcpu,646 struct kvm_guest_debug *debug)647{648 vcpu_ioctl(vcpu, KVM_SET_GUEST_DEBUG, debug);649}650 651static inline void vcpu_mp_state_get(struct kvm_vcpu *vcpu,652 struct kvm_mp_state *mp_state)653{654 vcpu_ioctl(vcpu, KVM_GET_MP_STATE, mp_state);655}656static inline void vcpu_mp_state_set(struct kvm_vcpu *vcpu,657 struct kvm_mp_state *mp_state)658{659 vcpu_ioctl(vcpu, KVM_SET_MP_STATE, mp_state);660}661 662static inline void vcpu_regs_get(struct kvm_vcpu *vcpu, struct kvm_regs *regs)663{664 vcpu_ioctl(vcpu, KVM_GET_REGS, regs);665}666 667static inline void vcpu_regs_set(struct kvm_vcpu *vcpu, struct kvm_regs *regs)668{669 vcpu_ioctl(vcpu, KVM_SET_REGS, regs);670}671static inline void vcpu_sregs_get(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)672{673 vcpu_ioctl(vcpu, KVM_GET_SREGS, sregs);674 675}676static inline void vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)677{678 vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs);679}680static inline int _vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)681{682 return __vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs);683}684static inline void vcpu_fpu_get(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)685{686 vcpu_ioctl(vcpu, KVM_GET_FPU, fpu);687}688static inline void vcpu_fpu_set(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)689{690 vcpu_ioctl(vcpu, KVM_SET_FPU, fpu);691}692 693static inline int __vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr)694{695 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr };696 697 return __vcpu_ioctl(vcpu, KVM_GET_ONE_REG, ®);698}699static inline int __vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)700{701 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val };702 703 return __vcpu_ioctl(vcpu, KVM_SET_ONE_REG, ®);704}705static inline void vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr)706{707 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr };708 709 vcpu_ioctl(vcpu, KVM_GET_ONE_REG, ®);710}711static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)712{713 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val };714 715 vcpu_ioctl(vcpu, KVM_SET_ONE_REG, ®);716}717 718#ifdef __KVM_HAVE_VCPU_EVENTS719static inline void vcpu_events_get(struct kvm_vcpu *vcpu,720 struct kvm_vcpu_events *events)721{722 vcpu_ioctl(vcpu, KVM_GET_VCPU_EVENTS, events);723}724static inline void vcpu_events_set(struct kvm_vcpu *vcpu,725 struct kvm_vcpu_events *events)726{727 vcpu_ioctl(vcpu, KVM_SET_VCPU_EVENTS, events);728}729#endif730#ifdef __x86_64__731static inline void vcpu_nested_state_get(struct kvm_vcpu *vcpu,732 struct kvm_nested_state *state)733{734 vcpu_ioctl(vcpu, KVM_GET_NESTED_STATE, state);735}736static inline int __vcpu_nested_state_set(struct kvm_vcpu *vcpu,737 struct kvm_nested_state *state)738{739 return __vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state);740}741 742static inline void vcpu_nested_state_set(struct kvm_vcpu *vcpu,743 struct kvm_nested_state *state)744{745 vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state);746}747#endif748static inline int vcpu_get_stats_fd(struct kvm_vcpu *vcpu)749{750 int fd = __vcpu_ioctl(vcpu, KVM_GET_STATS_FD, NULL);751 752 TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_CHECK_EXTENSION, fd, vcpu->vm);753 return fd;754}755 756int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr);757 758static inline void kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr)759{760 int ret = __kvm_has_device_attr(dev_fd, group, attr);761 762 TEST_ASSERT(!ret, "KVM_HAS_DEVICE_ATTR failed, rc: %i errno: %i", ret, errno);763}764 765int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val);766 767static inline void kvm_device_attr_get(int dev_fd, uint32_t group,768 uint64_t attr, void *val)769{770 int ret = __kvm_device_attr_get(dev_fd, group, attr, val);771 772 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_GET_DEVICE_ATTR, ret));773}774 775int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val);776 777static inline void kvm_device_attr_set(int dev_fd, uint32_t group,778 uint64_t attr, void *val)779{780 int ret = __kvm_device_attr_set(dev_fd, group, attr, val);781 782 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_DEVICE_ATTR, ret));783}784 785static inline int __vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group,786 uint64_t attr)787{788 return __kvm_has_device_attr(vcpu->fd, group, attr);789}790 791static inline void vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group,792 uint64_t attr)793{794 kvm_has_device_attr(vcpu->fd, group, attr);795}796 797static inline int __vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group,798 uint64_t attr, void *val)799{800 return __kvm_device_attr_get(vcpu->fd, group, attr, val);801}802 803static inline void vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group,804 uint64_t attr, void *val)805{806 kvm_device_attr_get(vcpu->fd, group, attr, val);807}808 809static inline int __vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group,810 uint64_t attr, void *val)811{812 return __kvm_device_attr_set(vcpu->fd, group, attr, val);813}814 815static inline void vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group,816 uint64_t attr, void *val)817{818 kvm_device_attr_set(vcpu->fd, group, attr, val);819}820 821int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type);822int __kvm_create_device(struct kvm_vm *vm, uint64_t type);823 824static inline int kvm_create_device(struct kvm_vm *vm, uint64_t type)825{826 int fd = __kvm_create_device(vm, type);827 828 TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_DEVICE, fd));829 return fd;830}831 832void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu);833 834/*835 * VM VCPU Args Set836 *837 * Input Args:838 * vm - Virtual Machine839 * num - number of arguments840 * ... - arguments, each of type uint64_t841 *842 * Output Args: None843 *844 * Return: None845 *846 * Sets the first @num input parameters for the function at @vcpu's entry point,847 * per the C calling convention of the architecture, to the values given as848 * variable args. Each of the variable args is expected to be of type uint64_t.849 * The maximum @num can be is specific to the architecture.850 */851void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...);852 853void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);854int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);855 856#define KVM_MAX_IRQ_ROUTES 4096857 858struct kvm_irq_routing *kvm_gsi_routing_create(void);859void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,860 uint32_t gsi, uint32_t pin);861int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);862void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);863 864const char *exit_reason_str(unsigned int exit_reason);865 866vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,867 uint32_t memslot);868vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,869 vm_paddr_t paddr_min, uint32_t memslot,870 bool protected);871vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm);872 873static inline vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,874 vm_paddr_t paddr_min, uint32_t memslot)875{876 /*877 * By default, allocate memory as protected for VMs that support878 * protected memory, as the majority of memory for such VMs is879 * protected, i.e. using shared memory is effectively opt-in.880 */881 return __vm_phy_pages_alloc(vm, num, paddr_min, memslot,882 vm_arch_has_protected_memory(vm));883}884 885/*886 * ____vm_create() does KVM_CREATE_VM and little else. __vm_create() also887 * loads the test binary into guest memory and creates an IRQ chip (x86 only).888 * __vm_create() does NOT create vCPUs, @nr_runnable_vcpus is used purely to889 * calculate the amount of memory needed for per-vCPU data, e.g. stacks.890 */891struct kvm_vm *____vm_create(struct vm_shape shape);892struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus,893 uint64_t nr_extra_pages);894 895static inline struct kvm_vm *vm_create_barebones(void)896{897 return ____vm_create(VM_SHAPE_DEFAULT);898}899 900static inline struct kvm_vm *vm_create_barebones_type(unsigned long type)901{902 const struct vm_shape shape = {903 .mode = VM_MODE_DEFAULT,904 .type = type,905 };906 907 return ____vm_create(shape);908}909 910static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus)911{912 return __vm_create(VM_SHAPE_DEFAULT, nr_runnable_vcpus, 0);913}914 915struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus,916 uint64_t extra_mem_pages,917 void *guest_code, struct kvm_vcpu *vcpus[]);918 919static inline struct kvm_vm *vm_create_with_vcpus(uint32_t nr_vcpus,920 void *guest_code,921 struct kvm_vcpu *vcpus[])922{923 return __vm_create_with_vcpus(VM_SHAPE_DEFAULT, nr_vcpus, 0,924 guest_code, vcpus);925}926 927 928struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape,929 struct kvm_vcpu **vcpu,930 uint64_t extra_mem_pages,931 void *guest_code);932 933/*934 * Create a VM with a single vCPU with reasonable defaults and @extra_mem_pages935 * additional pages of guest memory. Returns the VM and vCPU (via out param).936 */937static inline struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,938 uint64_t extra_mem_pages,939 void *guest_code)940{941 return __vm_create_shape_with_one_vcpu(VM_SHAPE_DEFAULT, vcpu,942 extra_mem_pages, guest_code);943}944 945static inline struct kvm_vm *vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,946 void *guest_code)947{948 return __vm_create_with_one_vcpu(vcpu, 0, guest_code);949}950 951static inline struct kvm_vm *vm_create_shape_with_one_vcpu(struct vm_shape shape,952 struct kvm_vcpu **vcpu,953 void *guest_code)954{955 return __vm_create_shape_with_one_vcpu(shape, vcpu, 0, guest_code);956}957 958struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm);959 960void kvm_pin_this_task_to_pcpu(uint32_t pcpu);961void kvm_print_vcpu_pinning_help(void);962void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[],963 int nr_vcpus);964 965unsigned long vm_compute_max_gfn(struct kvm_vm *vm);966unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size);967unsigned int vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages);968unsigned int vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages);969static inline unsigned int970vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)971{972 unsigned int n;973 n = vm_num_guest_pages(mode, vm_num_host_pages(mode, num_guest_pages));974#ifdef __s390x__975 /* s390 requires 1M aligned guest sizes */976 n = (n + 255) & ~255;977#endif978 return n;979}980 981#define sync_global_to_guest(vm, g) ({ \982 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \983 memcpy(_p, &(g), sizeof(g)); \984})985 986#define sync_global_from_guest(vm, g) ({ \987 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \988 memcpy(&(g), _p, sizeof(g)); \989})990 991/*992 * Write a global value, but only in the VM's (guest's) domain. Primarily used993 * for "globals" that hold per-VM values (VMs always duplicate code and global994 * data into their own region of physical memory), but can be used anytime it's995 * undesirable to change the host's copy of the global.996 */997#define write_guest_global(vm, g, val) ({ \998 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \999 typeof(g) _val = val; \1000 \1001 memcpy(_p, &(_val), sizeof(g)); \1002})1003 1004void assert_on_unhandled_exception(struct kvm_vcpu *vcpu);1005 1006void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu,1007 uint8_t indent);1008 1009static inline void vcpu_dump(FILE *stream, struct kvm_vcpu *vcpu,1010 uint8_t indent)1011{1012 vcpu_arch_dump(stream, vcpu, indent);1013}1014 1015/*1016 * Adds a vCPU with reasonable defaults (e.g. a stack)1017 *1018 * Input Args:1019 * vm - Virtual Machine1020 * vcpu_id - The id of the VCPU to add to the VM.1021 */1022struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id);1023void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code);1024 1025static inline struct kvm_vcpu *vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,1026 void *guest_code)1027{1028 struct kvm_vcpu *vcpu = vm_arch_vcpu_add(vm, vcpu_id);1029 1030 vcpu_arch_set_entry_point(vcpu, guest_code);1031 1032 return vcpu;1033}1034 1035/* Re-create a vCPU after restarting a VM, e.g. for state save/restore tests. */1036struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id);1037 1038static inline struct kvm_vcpu *vm_vcpu_recreate(struct kvm_vm *vm,1039 uint32_t vcpu_id)1040{1041 return vm_arch_vcpu_recreate(vm, vcpu_id);1042}1043 1044void vcpu_arch_free(struct kvm_vcpu *vcpu);1045 1046void virt_arch_pgd_alloc(struct kvm_vm *vm);1047 1048static inline void virt_pgd_alloc(struct kvm_vm *vm)1049{1050 virt_arch_pgd_alloc(vm);1051}1052 1053/*1054 * VM Virtual Page Map1055 *1056 * Input Args:1057 * vm - Virtual Machine1058 * vaddr - VM Virtual Address1059 * paddr - VM Physical Address1060 * memslot - Memory region slot for new virtual translation tables1061 *1062 * Output Args: None1063 *1064 * Return: None1065 *1066 * Within @vm, creates a virtual translation for the page starting1067 * at @vaddr to the page starting at @paddr.1068 */1069void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr);1070 1071static inline void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)1072{1073 virt_arch_pg_map(vm, vaddr, paddr);1074}1075 1076 1077/*1078 * Address Guest Virtual to Guest Physical1079 *1080 * Input Args:1081 * vm - Virtual Machine1082 * gva - VM virtual address1083 *1084 * Output Args: None1085 *1086 * Return:1087 * Equivalent VM physical address1088 *1089 * Returns the VM physical address of the translated VM virtual1090 * address given by @gva.1091 */1092vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva);1093 1094static inline vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)1095{1096 return addr_arch_gva2gpa(vm, gva);1097}1098 1099/*1100 * Virtual Translation Tables Dump1101 *1102 * Input Args:1103 * stream - Output FILE stream1104 * vm - Virtual Machine1105 * indent - Left margin indent amount1106 *1107 * Output Args: None1108 *1109 * Return: None1110 *1111 * Dumps to the FILE stream given by @stream, the contents of all the1112 * virtual translation tables for the VM given by @vm.1113 */1114void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent);1115 1116static inline void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)1117{1118 virt_arch_dump(stream, vm, indent);1119}1120 1121 1122static inline int __vm_disable_nx_huge_pages(struct kvm_vm *vm)1123{1124 return __vm_enable_cap(vm, KVM_CAP_VM_DISABLE_NX_HUGE_PAGES, 0);1125}1126 1127/*1128 * Arch hook that is invoked via a constructor, i.e. before exeucting main(),1129 * to allow for arch-specific setup that is common to all tests, e.g. computing1130 * the default guest "mode".1131 */1132void kvm_selftest_arch_init(void);1133 1134void kvm_arch_vm_post_create(struct kvm_vm *vm);1135 1136bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr);1137 1138uint32_t guest_get_vcpuid(void);1139 1140#endif /* SELFTEST_KVM_UTIL_H */1141