1393 lines · c
1/* SPDX-License-Identifier: GPL-2.0-only */2/*3 * tools/testing/selftests/kvm/include/x86_64/processor.h4 *5 * Copyright (C) 2018, Google LLC.6 */7 8#ifndef SELFTEST_KVM_PROCESSOR_H9#define SELFTEST_KVM_PROCESSOR_H10 11#include <assert.h>12#include <stdint.h>13#include <syscall.h>14 15#include <asm/msr-index.h>16#include <asm/prctl.h>17 18#include <linux/kvm_para.h>19#include <linux/stringify.h>20 21#include "kvm_util.h"22#include "ucall_common.h"23 24extern bool host_cpu_is_intel;25extern bool host_cpu_is_amd;26extern uint64_t guest_tsc_khz;27 28#ifndef MAX_NR_CPUID_ENTRIES29#define MAX_NR_CPUID_ENTRIES 10030#endif31 32/* Forced emulation prefix, used to invoke the emulator unconditionally. */33#define KVM_FEP "ud2; .byte 'k', 'v', 'm';"34 35#define NMI_VECTOR 0x0236 37#define X86_EFLAGS_FIXED (1u << 1)38 39#define X86_CR4_VME (1ul << 0)40#define X86_CR4_PVI (1ul << 1)41#define X86_CR4_TSD (1ul << 2)42#define X86_CR4_DE (1ul << 3)43#define X86_CR4_PSE (1ul << 4)44#define X86_CR4_PAE (1ul << 5)45#define X86_CR4_MCE (1ul << 6)46#define X86_CR4_PGE (1ul << 7)47#define X86_CR4_PCE (1ul << 8)48#define X86_CR4_OSFXSR (1ul << 9)49#define X86_CR4_OSXMMEXCPT (1ul << 10)50#define X86_CR4_UMIP (1ul << 11)51#define X86_CR4_LA57 (1ul << 12)52#define X86_CR4_VMXE (1ul << 13)53#define X86_CR4_SMXE (1ul << 14)54#define X86_CR4_FSGSBASE (1ul << 16)55#define X86_CR4_PCIDE (1ul << 17)56#define X86_CR4_OSXSAVE (1ul << 18)57#define X86_CR4_SMEP (1ul << 20)58#define X86_CR4_SMAP (1ul << 21)59#define X86_CR4_PKE (1ul << 22)60 61struct xstate_header {62 u64 xstate_bv;63 u64 xcomp_bv;64 u64 reserved[6];65} __attribute__((packed));66 67struct xstate {68 u8 i387[512];69 struct xstate_header header;70 u8 extended_state_area[0];71} __attribute__ ((packed, aligned (64)));72 73#define XFEATURE_MASK_FP BIT_ULL(0)74#define XFEATURE_MASK_SSE BIT_ULL(1)75#define XFEATURE_MASK_YMM BIT_ULL(2)76#define XFEATURE_MASK_BNDREGS BIT_ULL(3)77#define XFEATURE_MASK_BNDCSR BIT_ULL(4)78#define XFEATURE_MASK_OPMASK BIT_ULL(5)79#define XFEATURE_MASK_ZMM_Hi256 BIT_ULL(6)80#define XFEATURE_MASK_Hi16_ZMM BIT_ULL(7)81#define XFEATURE_MASK_PT BIT_ULL(8)82#define XFEATURE_MASK_PKRU BIT_ULL(9)83#define XFEATURE_MASK_PASID BIT_ULL(10)84#define XFEATURE_MASK_CET_USER BIT_ULL(11)85#define XFEATURE_MASK_CET_KERNEL BIT_ULL(12)86#define XFEATURE_MASK_LBR BIT_ULL(15)87#define XFEATURE_MASK_XTILE_CFG BIT_ULL(17)88#define XFEATURE_MASK_XTILE_DATA BIT_ULL(18)89 90#define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK | \91 XFEATURE_MASK_ZMM_Hi256 | \92 XFEATURE_MASK_Hi16_ZMM)93#define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILE_DATA | \94 XFEATURE_MASK_XTILE_CFG)95 96/* Note, these are ordered alphabetically to match kvm_cpuid_entry2. Eww. */97enum cpuid_output_regs {98 KVM_CPUID_EAX,99 KVM_CPUID_EBX,100 KVM_CPUID_ECX,101 KVM_CPUID_EDX102};103 104/*105 * Pack the information into a 64-bit value so that each X86_FEATURE_XXX can be106 * passed by value with no overhead.107 */108struct kvm_x86_cpu_feature {109 u32 function;110 u16 index;111 u8 reg;112 u8 bit;113};114#define KVM_X86_CPU_FEATURE(fn, idx, gpr, __bit) \115({ \116 struct kvm_x86_cpu_feature feature = { \117 .function = fn, \118 .index = idx, \119 .reg = KVM_CPUID_##gpr, \120 .bit = __bit, \121 }; \122 \123 kvm_static_assert((fn & 0xc0000000) == 0 || \124 (fn & 0xc0000000) == 0x40000000 || \125 (fn & 0xc0000000) == 0x80000000 || \126 (fn & 0xc0000000) == 0xc0000000); \127 kvm_static_assert(idx < BIT(sizeof(feature.index) * BITS_PER_BYTE)); \128 feature; \129})130 131/*132 * Basic Leafs, a.k.a. Intel defined133 */134#define X86_FEATURE_MWAIT KVM_X86_CPU_FEATURE(0x1, 0, ECX, 3)135#define X86_FEATURE_VMX KVM_X86_CPU_FEATURE(0x1, 0, ECX, 5)136#define X86_FEATURE_SMX KVM_X86_CPU_FEATURE(0x1, 0, ECX, 6)137#define X86_FEATURE_PDCM KVM_X86_CPU_FEATURE(0x1, 0, ECX, 15)138#define X86_FEATURE_PCID KVM_X86_CPU_FEATURE(0x1, 0, ECX, 17)139#define X86_FEATURE_X2APIC KVM_X86_CPU_FEATURE(0x1, 0, ECX, 21)140#define X86_FEATURE_MOVBE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 22)141#define X86_FEATURE_TSC_DEADLINE_TIMER KVM_X86_CPU_FEATURE(0x1, 0, ECX, 24)142#define X86_FEATURE_XSAVE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 26)143#define X86_FEATURE_OSXSAVE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 27)144#define X86_FEATURE_RDRAND KVM_X86_CPU_FEATURE(0x1, 0, ECX, 30)145#define X86_FEATURE_HYPERVISOR KVM_X86_CPU_FEATURE(0x1, 0, ECX, 31)146#define X86_FEATURE_PAE KVM_X86_CPU_FEATURE(0x1, 0, EDX, 6)147#define X86_FEATURE_MCE KVM_X86_CPU_FEATURE(0x1, 0, EDX, 7)148#define X86_FEATURE_APIC KVM_X86_CPU_FEATURE(0x1, 0, EDX, 9)149#define X86_FEATURE_CLFLUSH KVM_X86_CPU_FEATURE(0x1, 0, EDX, 19)150#define X86_FEATURE_XMM KVM_X86_CPU_FEATURE(0x1, 0, EDX, 25)151#define X86_FEATURE_XMM2 KVM_X86_CPU_FEATURE(0x1, 0, EDX, 26)152#define X86_FEATURE_FSGSBASE KVM_X86_CPU_FEATURE(0x7, 0, EBX, 0)153#define X86_FEATURE_TSC_ADJUST KVM_X86_CPU_FEATURE(0x7, 0, EBX, 1)154#define X86_FEATURE_SGX KVM_X86_CPU_FEATURE(0x7, 0, EBX, 2)155#define X86_FEATURE_HLE KVM_X86_CPU_FEATURE(0x7, 0, EBX, 4)156#define X86_FEATURE_SMEP KVM_X86_CPU_FEATURE(0x7, 0, EBX, 7)157#define X86_FEATURE_INVPCID KVM_X86_CPU_FEATURE(0x7, 0, EBX, 10)158#define X86_FEATURE_RTM KVM_X86_CPU_FEATURE(0x7, 0, EBX, 11)159#define X86_FEATURE_MPX KVM_X86_CPU_FEATURE(0x7, 0, EBX, 14)160#define X86_FEATURE_SMAP KVM_X86_CPU_FEATURE(0x7, 0, EBX, 20)161#define X86_FEATURE_PCOMMIT KVM_X86_CPU_FEATURE(0x7, 0, EBX, 22)162#define X86_FEATURE_CLFLUSHOPT KVM_X86_CPU_FEATURE(0x7, 0, EBX, 23)163#define X86_FEATURE_CLWB KVM_X86_CPU_FEATURE(0x7, 0, EBX, 24)164#define X86_FEATURE_UMIP KVM_X86_CPU_FEATURE(0x7, 0, ECX, 2)165#define X86_FEATURE_PKU KVM_X86_CPU_FEATURE(0x7, 0, ECX, 3)166#define X86_FEATURE_OSPKE KVM_X86_CPU_FEATURE(0x7, 0, ECX, 4)167#define X86_FEATURE_LA57 KVM_X86_CPU_FEATURE(0x7, 0, ECX, 16)168#define X86_FEATURE_RDPID KVM_X86_CPU_FEATURE(0x7, 0, ECX, 22)169#define X86_FEATURE_SGX_LC KVM_X86_CPU_FEATURE(0x7, 0, ECX, 30)170#define X86_FEATURE_SHSTK KVM_X86_CPU_FEATURE(0x7, 0, ECX, 7)171#define X86_FEATURE_IBT KVM_X86_CPU_FEATURE(0x7, 0, EDX, 20)172#define X86_FEATURE_AMX_TILE KVM_X86_CPU_FEATURE(0x7, 0, EDX, 24)173#define X86_FEATURE_SPEC_CTRL KVM_X86_CPU_FEATURE(0x7, 0, EDX, 26)174#define X86_FEATURE_ARCH_CAPABILITIES KVM_X86_CPU_FEATURE(0x7, 0, EDX, 29)175#define X86_FEATURE_PKS KVM_X86_CPU_FEATURE(0x7, 0, ECX, 31)176#define X86_FEATURE_XTILECFG KVM_X86_CPU_FEATURE(0xD, 0, EAX, 17)177#define X86_FEATURE_XTILEDATA KVM_X86_CPU_FEATURE(0xD, 0, EAX, 18)178#define X86_FEATURE_XSAVES KVM_X86_CPU_FEATURE(0xD, 1, EAX, 3)179#define X86_FEATURE_XFD KVM_X86_CPU_FEATURE(0xD, 1, EAX, 4)180#define X86_FEATURE_XTILEDATA_XFD KVM_X86_CPU_FEATURE(0xD, 18, ECX, 2)181 182/*183 * Extended Leafs, a.k.a. AMD defined184 */185#define X86_FEATURE_SVM KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 2)186#define X86_FEATURE_NX KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 20)187#define X86_FEATURE_GBPAGES KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 26)188#define X86_FEATURE_RDTSCP KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 27)189#define X86_FEATURE_LM KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 29)190#define X86_FEATURE_INVTSC KVM_X86_CPU_FEATURE(0x80000007, 0, EDX, 8)191#define X86_FEATURE_RDPRU KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 4)192#define X86_FEATURE_AMD_IBPB KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 12)193#define X86_FEATURE_NPT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 0)194#define X86_FEATURE_LBRV KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1)195#define X86_FEATURE_NRIPS KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3)196#define X86_FEATURE_TSCRATEMSR KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 4)197#define X86_FEATURE_PAUSEFILTER KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 10)198#define X86_FEATURE_PFTHRESHOLD KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 12)199#define X86_FEATURE_VGIF KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 16)200#define X86_FEATURE_SEV KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 1)201#define X86_FEATURE_SEV_ES KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 3)202 203/*204 * KVM defined paravirt features.205 */206#define X86_FEATURE_KVM_CLOCKSOURCE KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 0)207#define X86_FEATURE_KVM_NOP_IO_DELAY KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 1)208#define X86_FEATURE_KVM_MMU_OP KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 2)209#define X86_FEATURE_KVM_CLOCKSOURCE2 KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 3)210#define X86_FEATURE_KVM_ASYNC_PF KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 4)211#define X86_FEATURE_KVM_STEAL_TIME KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 5)212#define X86_FEATURE_KVM_PV_EOI KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 6)213#define X86_FEATURE_KVM_PV_UNHALT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 7)214/* Bit 8 apparently isn't used?!?! */215#define X86_FEATURE_KVM_PV_TLB_FLUSH KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 9)216#define X86_FEATURE_KVM_ASYNC_PF_VMEXIT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 10)217#define X86_FEATURE_KVM_PV_SEND_IPI KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 11)218#define X86_FEATURE_KVM_POLL_CONTROL KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 12)219#define X86_FEATURE_KVM_PV_SCHED_YIELD KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 13)220#define X86_FEATURE_KVM_ASYNC_PF_INT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 14)221#define X86_FEATURE_KVM_MSI_EXT_DEST_ID KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 15)222#define X86_FEATURE_KVM_HC_MAP_GPA_RANGE KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 16)223#define X86_FEATURE_KVM_MIGRATION_CONTROL KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 17)224 225/*226 * Same idea as X86_FEATURE_XXX, but X86_PROPERTY_XXX retrieves a multi-bit227 * value/property as opposed to a single-bit feature. Again, pack the info228 * into a 64-bit value to pass by value with no overhead.229 */230struct kvm_x86_cpu_property {231 u32 function;232 u8 index;233 u8 reg;234 u8 lo_bit;235 u8 hi_bit;236};237#define KVM_X86_CPU_PROPERTY(fn, idx, gpr, low_bit, high_bit) \238({ \239 struct kvm_x86_cpu_property property = { \240 .function = fn, \241 .index = idx, \242 .reg = KVM_CPUID_##gpr, \243 .lo_bit = low_bit, \244 .hi_bit = high_bit, \245 }; \246 \247 kvm_static_assert(low_bit < high_bit); \248 kvm_static_assert((fn & 0xc0000000) == 0 || \249 (fn & 0xc0000000) == 0x40000000 || \250 (fn & 0xc0000000) == 0x80000000 || \251 (fn & 0xc0000000) == 0xc0000000); \252 kvm_static_assert(idx < BIT(sizeof(property.index) * BITS_PER_BYTE)); \253 property; \254})255 256#define X86_PROPERTY_MAX_BASIC_LEAF KVM_X86_CPU_PROPERTY(0, 0, EAX, 0, 31)257#define X86_PROPERTY_PMU_VERSION KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 0, 7)258#define X86_PROPERTY_PMU_NR_GP_COUNTERS KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 8, 15)259#define X86_PROPERTY_PMU_GP_COUNTERS_BIT_WIDTH KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 16, 23)260#define X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 24, 31)261#define X86_PROPERTY_PMU_EVENTS_MASK KVM_X86_CPU_PROPERTY(0xa, 0, EBX, 0, 7)262#define X86_PROPERTY_PMU_FIXED_COUNTERS_BITMASK KVM_X86_CPU_PROPERTY(0xa, 0, ECX, 0, 31)263#define X86_PROPERTY_PMU_NR_FIXED_COUNTERS KVM_X86_CPU_PROPERTY(0xa, 0, EDX, 0, 4)264#define X86_PROPERTY_PMU_FIXED_COUNTERS_BIT_WIDTH KVM_X86_CPU_PROPERTY(0xa, 0, EDX, 5, 12)265 266#define X86_PROPERTY_SUPPORTED_XCR0_LO KVM_X86_CPU_PROPERTY(0xd, 0, EAX, 0, 31)267#define X86_PROPERTY_XSTATE_MAX_SIZE_XCR0 KVM_X86_CPU_PROPERTY(0xd, 0, EBX, 0, 31)268#define X86_PROPERTY_XSTATE_MAX_SIZE KVM_X86_CPU_PROPERTY(0xd, 0, ECX, 0, 31)269#define X86_PROPERTY_SUPPORTED_XCR0_HI KVM_X86_CPU_PROPERTY(0xd, 0, EDX, 0, 31)270 271#define X86_PROPERTY_XSTATE_TILE_SIZE KVM_X86_CPU_PROPERTY(0xd, 18, EAX, 0, 31)272#define X86_PROPERTY_XSTATE_TILE_OFFSET KVM_X86_CPU_PROPERTY(0xd, 18, EBX, 0, 31)273#define X86_PROPERTY_AMX_MAX_PALETTE_TABLES KVM_X86_CPU_PROPERTY(0x1d, 0, EAX, 0, 31)274#define X86_PROPERTY_AMX_TOTAL_TILE_BYTES KVM_X86_CPU_PROPERTY(0x1d, 1, EAX, 0, 15)275#define X86_PROPERTY_AMX_BYTES_PER_TILE KVM_X86_CPU_PROPERTY(0x1d, 1, EAX, 16, 31)276#define X86_PROPERTY_AMX_BYTES_PER_ROW KVM_X86_CPU_PROPERTY(0x1d, 1, EBX, 0, 15)277#define X86_PROPERTY_AMX_NR_TILE_REGS KVM_X86_CPU_PROPERTY(0x1d, 1, EBX, 16, 31)278#define X86_PROPERTY_AMX_MAX_ROWS KVM_X86_CPU_PROPERTY(0x1d, 1, ECX, 0, 15)279 280#define X86_PROPERTY_MAX_KVM_LEAF KVM_X86_CPU_PROPERTY(0x40000000, 0, EAX, 0, 31)281 282#define X86_PROPERTY_MAX_EXT_LEAF KVM_X86_CPU_PROPERTY(0x80000000, 0, EAX, 0, 31)283#define X86_PROPERTY_MAX_PHY_ADDR KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 0, 7)284#define X86_PROPERTY_MAX_VIRT_ADDR KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 8, 15)285#define X86_PROPERTY_GUEST_MAX_PHY_ADDR KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 16, 23)286#define X86_PROPERTY_SEV_C_BIT KVM_X86_CPU_PROPERTY(0x8000001F, 0, EBX, 0, 5)287#define X86_PROPERTY_PHYS_ADDR_REDUCTION KVM_X86_CPU_PROPERTY(0x8000001F, 0, EBX, 6, 11)288 289#define X86_PROPERTY_MAX_CENTAUR_LEAF KVM_X86_CPU_PROPERTY(0xC0000000, 0, EAX, 0, 31)290 291/*292 * Intel's architectural PMU events are bizarre. They have a "feature" bit293 * that indicates the feature is _not_ supported, and a property that states294 * the length of the bit mask of unsupported features. A feature is supported295 * if the size of the bit mask is larger than the "unavailable" bit, and said296 * bit is not set. Fixed counters also bizarre enumeration, but inverted from297 * arch events for general purpose counters. Fixed counters are supported if a298 * feature flag is set **OR** the total number of fixed counters is greater299 * than index of the counter.300 *301 * Wrap the events for general purpose and fixed counters to simplify checking302 * whether or not a given architectural event is supported.303 */304struct kvm_x86_pmu_feature {305 struct kvm_x86_cpu_feature f;306};307#define KVM_X86_PMU_FEATURE(__reg, __bit) \308({ \309 struct kvm_x86_pmu_feature feature = { \310 .f = KVM_X86_CPU_FEATURE(0xa, 0, __reg, __bit), \311 }; \312 \313 kvm_static_assert(KVM_CPUID_##__reg == KVM_CPUID_EBX || \314 KVM_CPUID_##__reg == KVM_CPUID_ECX); \315 feature; \316})317 318#define X86_PMU_FEATURE_CPU_CYCLES KVM_X86_PMU_FEATURE(EBX, 0)319#define X86_PMU_FEATURE_INSNS_RETIRED KVM_X86_PMU_FEATURE(EBX, 1)320#define X86_PMU_FEATURE_REFERENCE_CYCLES KVM_X86_PMU_FEATURE(EBX, 2)321#define X86_PMU_FEATURE_LLC_REFERENCES KVM_X86_PMU_FEATURE(EBX, 3)322#define X86_PMU_FEATURE_LLC_MISSES KVM_X86_PMU_FEATURE(EBX, 4)323#define X86_PMU_FEATURE_BRANCH_INSNS_RETIRED KVM_X86_PMU_FEATURE(EBX, 5)324#define X86_PMU_FEATURE_BRANCHES_MISPREDICTED KVM_X86_PMU_FEATURE(EBX, 6)325#define X86_PMU_FEATURE_TOPDOWN_SLOTS KVM_X86_PMU_FEATURE(EBX, 7)326 327#define X86_PMU_FEATURE_INSNS_RETIRED_FIXED KVM_X86_PMU_FEATURE(ECX, 0)328#define X86_PMU_FEATURE_CPU_CYCLES_FIXED KVM_X86_PMU_FEATURE(ECX, 1)329#define X86_PMU_FEATURE_REFERENCE_TSC_CYCLES_FIXED KVM_X86_PMU_FEATURE(ECX, 2)330#define X86_PMU_FEATURE_TOPDOWN_SLOTS_FIXED KVM_X86_PMU_FEATURE(ECX, 3)331 332static inline unsigned int x86_family(unsigned int eax)333{334 unsigned int x86;335 336 x86 = (eax >> 8) & 0xf;337 338 if (x86 == 0xf)339 x86 += (eax >> 20) & 0xff;340 341 return x86;342}343 344static inline unsigned int x86_model(unsigned int eax)345{346 return ((eax >> 12) & 0xf0) | ((eax >> 4) & 0x0f);347}348 349/* Page table bitfield declarations */350#define PTE_PRESENT_MASK BIT_ULL(0)351#define PTE_WRITABLE_MASK BIT_ULL(1)352#define PTE_USER_MASK BIT_ULL(2)353#define PTE_ACCESSED_MASK BIT_ULL(5)354#define PTE_DIRTY_MASK BIT_ULL(6)355#define PTE_LARGE_MASK BIT_ULL(7)356#define PTE_GLOBAL_MASK BIT_ULL(8)357#define PTE_NX_MASK BIT_ULL(63)358 359#define PHYSICAL_PAGE_MASK GENMASK_ULL(51, 12)360 361#define PAGE_SHIFT 12362#define PAGE_SIZE (1ULL << PAGE_SHIFT)363#define PAGE_MASK (~(PAGE_SIZE-1) & PHYSICAL_PAGE_MASK)364 365#define HUGEPAGE_SHIFT(x) (PAGE_SHIFT + (((x) - 1) * 9))366#define HUGEPAGE_SIZE(x) (1UL << HUGEPAGE_SHIFT(x))367#define HUGEPAGE_MASK(x) (~(HUGEPAGE_SIZE(x) - 1) & PHYSICAL_PAGE_MASK)368 369#define PTE_GET_PA(pte) ((pte) & PHYSICAL_PAGE_MASK)370#define PTE_GET_PFN(pte) (PTE_GET_PA(pte) >> PAGE_SHIFT)371 372/* General Registers in 64-Bit Mode */373struct gpr64_regs {374 u64 rax;375 u64 rcx;376 u64 rdx;377 u64 rbx;378 u64 rsp;379 u64 rbp;380 u64 rsi;381 u64 rdi;382 u64 r8;383 u64 r9;384 u64 r10;385 u64 r11;386 u64 r12;387 u64 r13;388 u64 r14;389 u64 r15;390};391 392struct desc64 {393 uint16_t limit0;394 uint16_t base0;395 unsigned base1:8, type:4, s:1, dpl:2, p:1;396 unsigned limit1:4, avl:1, l:1, db:1, g:1, base2:8;397 uint32_t base3;398 uint32_t zero1;399} __attribute__((packed));400 401struct desc_ptr {402 uint16_t size;403 uint64_t address;404} __attribute__((packed));405 406struct kvm_x86_state {407 struct kvm_xsave *xsave;408 struct kvm_vcpu_events events;409 struct kvm_mp_state mp_state;410 struct kvm_regs regs;411 struct kvm_xcrs xcrs;412 struct kvm_sregs sregs;413 struct kvm_debugregs debugregs;414 union {415 struct kvm_nested_state nested;416 char nested_[16384];417 };418 struct kvm_msrs msrs;419};420 421static inline uint64_t get_desc64_base(const struct desc64 *desc)422{423 return ((uint64_t)desc->base3 << 32) |424 (desc->base0 | ((desc->base1) << 16) | ((desc->base2) << 24));425}426 427static inline uint64_t rdtsc(void)428{429 uint32_t eax, edx;430 uint64_t tsc_val;431 /*432 * The lfence is to wait (on Intel CPUs) until all previous433 * instructions have been executed. If software requires RDTSC to be434 * executed prior to execution of any subsequent instruction, it can435 * execute LFENCE immediately after RDTSC436 */437 __asm__ __volatile__("lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx));438 tsc_val = ((uint64_t)edx) << 32 | eax;439 return tsc_val;440}441 442static inline uint64_t rdtscp(uint32_t *aux)443{444 uint32_t eax, edx;445 446 __asm__ __volatile__("rdtscp" : "=a"(eax), "=d"(edx), "=c"(*aux));447 return ((uint64_t)edx) << 32 | eax;448}449 450static inline uint64_t rdmsr(uint32_t msr)451{452 uint32_t a, d;453 454 __asm__ __volatile__("rdmsr" : "=a"(a), "=d"(d) : "c"(msr) : "memory");455 456 return a | ((uint64_t) d << 32);457}458 459static inline void wrmsr(uint32_t msr, uint64_t value)460{461 uint32_t a = value;462 uint32_t d = value >> 32;463 464 __asm__ __volatile__("wrmsr" :: "a"(a), "d"(d), "c"(msr) : "memory");465}466 467 468static inline uint16_t inw(uint16_t port)469{470 uint16_t tmp;471 472 __asm__ __volatile__("in %%dx, %%ax"473 : /* output */ "=a" (tmp)474 : /* input */ "d" (port));475 476 return tmp;477}478 479static inline uint16_t get_es(void)480{481 uint16_t es;482 483 __asm__ __volatile__("mov %%es, %[es]"484 : /* output */ [es]"=rm"(es));485 return es;486}487 488static inline uint16_t get_cs(void)489{490 uint16_t cs;491 492 __asm__ __volatile__("mov %%cs, %[cs]"493 : /* output */ [cs]"=rm"(cs));494 return cs;495}496 497static inline uint16_t get_ss(void)498{499 uint16_t ss;500 501 __asm__ __volatile__("mov %%ss, %[ss]"502 : /* output */ [ss]"=rm"(ss));503 return ss;504}505 506static inline uint16_t get_ds(void)507{508 uint16_t ds;509 510 __asm__ __volatile__("mov %%ds, %[ds]"511 : /* output */ [ds]"=rm"(ds));512 return ds;513}514 515static inline uint16_t get_fs(void)516{517 uint16_t fs;518 519 __asm__ __volatile__("mov %%fs, %[fs]"520 : /* output */ [fs]"=rm"(fs));521 return fs;522}523 524static inline uint16_t get_gs(void)525{526 uint16_t gs;527 528 __asm__ __volatile__("mov %%gs, %[gs]"529 : /* output */ [gs]"=rm"(gs));530 return gs;531}532 533static inline uint16_t get_tr(void)534{535 uint16_t tr;536 537 __asm__ __volatile__("str %[tr]"538 : /* output */ [tr]"=rm"(tr));539 return tr;540}541 542static inline uint64_t get_cr0(void)543{544 uint64_t cr0;545 546 __asm__ __volatile__("mov %%cr0, %[cr0]"547 : /* output */ [cr0]"=r"(cr0));548 return cr0;549}550 551static inline uint64_t get_cr3(void)552{553 uint64_t cr3;554 555 __asm__ __volatile__("mov %%cr3, %[cr3]"556 : /* output */ [cr3]"=r"(cr3));557 return cr3;558}559 560static inline uint64_t get_cr4(void)561{562 uint64_t cr4;563 564 __asm__ __volatile__("mov %%cr4, %[cr4]"565 : /* output */ [cr4]"=r"(cr4));566 return cr4;567}568 569static inline void set_cr4(uint64_t val)570{571 __asm__ __volatile__("mov %0, %%cr4" : : "r" (val) : "memory");572}573 574static inline u64 xgetbv(u32 index)575{576 u32 eax, edx;577 578 __asm__ __volatile__("xgetbv;"579 : "=a" (eax), "=d" (edx)580 : "c" (index));581 return eax | ((u64)edx << 32);582}583 584static inline void xsetbv(u32 index, u64 value)585{586 u32 eax = value;587 u32 edx = value >> 32;588 589 __asm__ __volatile__("xsetbv" :: "a" (eax), "d" (edx), "c" (index));590}591 592static inline void wrpkru(u32 pkru)593{594 /* Note, ECX and EDX are architecturally required to be '0'. */595 asm volatile(".byte 0x0f,0x01,0xef\n\t"596 : : "a" (pkru), "c"(0), "d"(0));597}598 599static inline struct desc_ptr get_gdt(void)600{601 struct desc_ptr gdt;602 __asm__ __volatile__("sgdt %[gdt]"603 : /* output */ [gdt]"=m"(gdt));604 return gdt;605}606 607static inline struct desc_ptr get_idt(void)608{609 struct desc_ptr idt;610 __asm__ __volatile__("sidt %[idt]"611 : /* output */ [idt]"=m"(idt));612 return idt;613}614 615static inline void outl(uint16_t port, uint32_t value)616{617 __asm__ __volatile__("outl %%eax, %%dx" : : "d"(port), "a"(value));618}619 620static inline void __cpuid(uint32_t function, uint32_t index,621 uint32_t *eax, uint32_t *ebx,622 uint32_t *ecx, uint32_t *edx)623{624 *eax = function;625 *ecx = index;626 627 asm volatile("cpuid"628 : "=a" (*eax),629 "=b" (*ebx),630 "=c" (*ecx),631 "=d" (*edx)632 : "0" (*eax), "2" (*ecx)633 : "memory");634}635 636static inline void cpuid(uint32_t function,637 uint32_t *eax, uint32_t *ebx,638 uint32_t *ecx, uint32_t *edx)639{640 return __cpuid(function, 0, eax, ebx, ecx, edx);641}642 643static inline uint32_t this_cpu_fms(void)644{645 uint32_t eax, ebx, ecx, edx;646 647 cpuid(1, &eax, &ebx, &ecx, &edx);648 return eax;649}650 651static inline uint32_t this_cpu_family(void)652{653 return x86_family(this_cpu_fms());654}655 656static inline uint32_t this_cpu_model(void)657{658 return x86_model(this_cpu_fms());659}660 661static inline bool this_cpu_vendor_string_is(const char *vendor)662{663 const uint32_t *chunk = (const uint32_t *)vendor;664 uint32_t eax, ebx, ecx, edx;665 666 cpuid(0, &eax, &ebx, &ecx, &edx);667 return (ebx == chunk[0] && edx == chunk[1] && ecx == chunk[2]);668}669 670static inline bool this_cpu_is_intel(void)671{672 return this_cpu_vendor_string_is("GenuineIntel");673}674 675/*676 * Exclude early K5 samples with a vendor string of "AMDisbetter!"677 */678static inline bool this_cpu_is_amd(void)679{680 return this_cpu_vendor_string_is("AuthenticAMD");681}682 683static inline uint32_t __this_cpu_has(uint32_t function, uint32_t index,684 uint8_t reg, uint8_t lo, uint8_t hi)685{686 uint32_t gprs[4];687 688 __cpuid(function, index,689 &gprs[KVM_CPUID_EAX], &gprs[KVM_CPUID_EBX],690 &gprs[KVM_CPUID_ECX], &gprs[KVM_CPUID_EDX]);691 692 return (gprs[reg] & GENMASK(hi, lo)) >> lo;693}694 695static inline bool this_cpu_has(struct kvm_x86_cpu_feature feature)696{697 return __this_cpu_has(feature.function, feature.index,698 feature.reg, feature.bit, feature.bit);699}700 701static inline uint32_t this_cpu_property(struct kvm_x86_cpu_property property)702{703 return __this_cpu_has(property.function, property.index,704 property.reg, property.lo_bit, property.hi_bit);705}706 707static __always_inline bool this_cpu_has_p(struct kvm_x86_cpu_property property)708{709 uint32_t max_leaf;710 711 switch (property.function & 0xc0000000) {712 case 0:713 max_leaf = this_cpu_property(X86_PROPERTY_MAX_BASIC_LEAF);714 break;715 case 0x40000000:716 max_leaf = this_cpu_property(X86_PROPERTY_MAX_KVM_LEAF);717 break;718 case 0x80000000:719 max_leaf = this_cpu_property(X86_PROPERTY_MAX_EXT_LEAF);720 break;721 case 0xc0000000:722 max_leaf = this_cpu_property(X86_PROPERTY_MAX_CENTAUR_LEAF);723 }724 return max_leaf >= property.function;725}726 727static inline bool this_pmu_has(struct kvm_x86_pmu_feature feature)728{729 uint32_t nr_bits;730 731 if (feature.f.reg == KVM_CPUID_EBX) {732 nr_bits = this_cpu_property(X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH);733 return nr_bits > feature.f.bit && !this_cpu_has(feature.f);734 }735 736 GUEST_ASSERT(feature.f.reg == KVM_CPUID_ECX);737 nr_bits = this_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);738 return nr_bits > feature.f.bit || this_cpu_has(feature.f);739}740 741static __always_inline uint64_t this_cpu_supported_xcr0(void)742{743 if (!this_cpu_has_p(X86_PROPERTY_SUPPORTED_XCR0_LO))744 return 0;745 746 return this_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_LO) |747 ((uint64_t)this_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_HI) << 32);748}749 750typedef u32 __attribute__((vector_size(16))) sse128_t;751#define __sse128_u union { sse128_t vec; u64 as_u64[2]; u32 as_u32[4]; }752#define sse128_lo(x) ({ __sse128_u t; t.vec = x; t.as_u64[0]; })753#define sse128_hi(x) ({ __sse128_u t; t.vec = x; t.as_u64[1]; })754 755static inline void read_sse_reg(int reg, sse128_t *data)756{757 switch (reg) {758 case 0:759 asm("movdqa %%xmm0, %0" : "=m"(*data));760 break;761 case 1:762 asm("movdqa %%xmm1, %0" : "=m"(*data));763 break;764 case 2:765 asm("movdqa %%xmm2, %0" : "=m"(*data));766 break;767 case 3:768 asm("movdqa %%xmm3, %0" : "=m"(*data));769 break;770 case 4:771 asm("movdqa %%xmm4, %0" : "=m"(*data));772 break;773 case 5:774 asm("movdqa %%xmm5, %0" : "=m"(*data));775 break;776 case 6:777 asm("movdqa %%xmm6, %0" : "=m"(*data));778 break;779 case 7:780 asm("movdqa %%xmm7, %0" : "=m"(*data));781 break;782 default:783 BUG();784 }785}786 787static inline void write_sse_reg(int reg, const sse128_t *data)788{789 switch (reg) {790 case 0:791 asm("movdqa %0, %%xmm0" : : "m"(*data));792 break;793 case 1:794 asm("movdqa %0, %%xmm1" : : "m"(*data));795 break;796 case 2:797 asm("movdqa %0, %%xmm2" : : "m"(*data));798 break;799 case 3:800 asm("movdqa %0, %%xmm3" : : "m"(*data));801 break;802 case 4:803 asm("movdqa %0, %%xmm4" : : "m"(*data));804 break;805 case 5:806 asm("movdqa %0, %%xmm5" : : "m"(*data));807 break;808 case 6:809 asm("movdqa %0, %%xmm6" : : "m"(*data));810 break;811 case 7:812 asm("movdqa %0, %%xmm7" : : "m"(*data));813 break;814 default:815 BUG();816 }817}818 819static inline void cpu_relax(void)820{821 asm volatile("rep; nop" ::: "memory");822}823 824static inline void udelay(unsigned long usec)825{826 uint64_t start, now, cycles;827 828 GUEST_ASSERT(guest_tsc_khz);829 cycles = guest_tsc_khz / 1000 * usec;830 831 /*832 * Deliberately don't PAUSE, a.k.a. cpu_relax(), so that the delay is833 * as accurate as possible, e.g. doesn't trigger PAUSE-Loop VM-Exits.834 */835 start = rdtsc();836 do {837 now = rdtsc();838 } while (now - start < cycles);839}840 841#define ud2() \842 __asm__ __volatile__( \843 "ud2\n" \844 )845 846#define hlt() \847 __asm__ __volatile__( \848 "hlt\n" \849 )850 851struct kvm_x86_state *vcpu_save_state(struct kvm_vcpu *vcpu);852void vcpu_load_state(struct kvm_vcpu *vcpu, struct kvm_x86_state *state);853void kvm_x86_state_cleanup(struct kvm_x86_state *state);854 855const struct kvm_msr_list *kvm_get_msr_index_list(void);856const struct kvm_msr_list *kvm_get_feature_msr_index_list(void);857bool kvm_msr_is_in_save_restore_list(uint32_t msr_index);858uint64_t kvm_get_feature_msr(uint64_t msr_index);859 860static inline void vcpu_msrs_get(struct kvm_vcpu *vcpu,861 struct kvm_msrs *msrs)862{863 int r = __vcpu_ioctl(vcpu, KVM_GET_MSRS, msrs);864 865 TEST_ASSERT(r == msrs->nmsrs,866 "KVM_GET_MSRS failed, r: %i (failed on MSR %x)",867 r, r < 0 || r >= msrs->nmsrs ? -1 : msrs->entries[r].index);868}869static inline void vcpu_msrs_set(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs)870{871 int r = __vcpu_ioctl(vcpu, KVM_SET_MSRS, msrs);872 873 TEST_ASSERT(r == msrs->nmsrs,874 "KVM_SET_MSRS failed, r: %i (failed on MSR %x)",875 r, r < 0 || r >= msrs->nmsrs ? -1 : msrs->entries[r].index);876}877static inline void vcpu_debugregs_get(struct kvm_vcpu *vcpu,878 struct kvm_debugregs *debugregs)879{880 vcpu_ioctl(vcpu, KVM_GET_DEBUGREGS, debugregs);881}882static inline void vcpu_debugregs_set(struct kvm_vcpu *vcpu,883 struct kvm_debugregs *debugregs)884{885 vcpu_ioctl(vcpu, KVM_SET_DEBUGREGS, debugregs);886}887static inline void vcpu_xsave_get(struct kvm_vcpu *vcpu,888 struct kvm_xsave *xsave)889{890 vcpu_ioctl(vcpu, KVM_GET_XSAVE, xsave);891}892static inline void vcpu_xsave2_get(struct kvm_vcpu *vcpu,893 struct kvm_xsave *xsave)894{895 vcpu_ioctl(vcpu, KVM_GET_XSAVE2, xsave);896}897static inline void vcpu_xsave_set(struct kvm_vcpu *vcpu,898 struct kvm_xsave *xsave)899{900 vcpu_ioctl(vcpu, KVM_SET_XSAVE, xsave);901}902static inline void vcpu_xcrs_get(struct kvm_vcpu *vcpu,903 struct kvm_xcrs *xcrs)904{905 vcpu_ioctl(vcpu, KVM_GET_XCRS, xcrs);906}907static inline void vcpu_xcrs_set(struct kvm_vcpu *vcpu, struct kvm_xcrs *xcrs)908{909 vcpu_ioctl(vcpu, KVM_SET_XCRS, xcrs);910}911 912const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid,913 uint32_t function, uint32_t index);914const struct kvm_cpuid2 *kvm_get_supported_cpuid(void);915 916static inline uint32_t kvm_cpu_fms(void)917{918 return get_cpuid_entry(kvm_get_supported_cpuid(), 0x1, 0)->eax;919}920 921static inline uint32_t kvm_cpu_family(void)922{923 return x86_family(kvm_cpu_fms());924}925 926static inline uint32_t kvm_cpu_model(void)927{928 return x86_model(kvm_cpu_fms());929}930 931bool kvm_cpuid_has(const struct kvm_cpuid2 *cpuid,932 struct kvm_x86_cpu_feature feature);933 934static inline bool kvm_cpu_has(struct kvm_x86_cpu_feature feature)935{936 return kvm_cpuid_has(kvm_get_supported_cpuid(), feature);937}938 939uint32_t kvm_cpuid_property(const struct kvm_cpuid2 *cpuid,940 struct kvm_x86_cpu_property property);941 942static inline uint32_t kvm_cpu_property(struct kvm_x86_cpu_property property)943{944 return kvm_cpuid_property(kvm_get_supported_cpuid(), property);945}946 947static __always_inline bool kvm_cpu_has_p(struct kvm_x86_cpu_property property)948{949 uint32_t max_leaf;950 951 switch (property.function & 0xc0000000) {952 case 0:953 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_BASIC_LEAF);954 break;955 case 0x40000000:956 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_KVM_LEAF);957 break;958 case 0x80000000:959 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_EXT_LEAF);960 break;961 case 0xc0000000:962 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_CENTAUR_LEAF);963 }964 return max_leaf >= property.function;965}966 967static inline bool kvm_pmu_has(struct kvm_x86_pmu_feature feature)968{969 uint32_t nr_bits;970 971 if (feature.f.reg == KVM_CPUID_EBX) {972 nr_bits = kvm_cpu_property(X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH);973 return nr_bits > feature.f.bit && !kvm_cpu_has(feature.f);974 }975 976 TEST_ASSERT_EQ(feature.f.reg, KVM_CPUID_ECX);977 nr_bits = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);978 return nr_bits > feature.f.bit || kvm_cpu_has(feature.f);979}980 981static __always_inline uint64_t kvm_cpu_supported_xcr0(void)982{983 if (!kvm_cpu_has_p(X86_PROPERTY_SUPPORTED_XCR0_LO))984 return 0;985 986 return kvm_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_LO) |987 ((uint64_t)kvm_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_HI) << 32);988}989 990static inline size_t kvm_cpuid2_size(int nr_entries)991{992 return sizeof(struct kvm_cpuid2) +993 sizeof(struct kvm_cpuid_entry2) * nr_entries;994}995 996/*997 * Allocate a "struct kvm_cpuid2* instance, with the 0-length arrary of998 * entries sized to hold @nr_entries. The caller is responsible for freeing999 * the struct.1000 */1001static inline struct kvm_cpuid2 *allocate_kvm_cpuid2(int nr_entries)1002{1003 struct kvm_cpuid2 *cpuid;1004 1005 cpuid = malloc(kvm_cpuid2_size(nr_entries));1006 TEST_ASSERT(cpuid, "-ENOMEM when allocating kvm_cpuid2");1007 1008 cpuid->nent = nr_entries;1009 1010 return cpuid;1011}1012 1013void vcpu_init_cpuid(struct kvm_vcpu *vcpu, const struct kvm_cpuid2 *cpuid);1014 1015static inline struct kvm_cpuid_entry2 *__vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu,1016 uint32_t function,1017 uint32_t index)1018{1019 return (struct kvm_cpuid_entry2 *)get_cpuid_entry(vcpu->cpuid,1020 function, index);1021}1022 1023static inline struct kvm_cpuid_entry2 *vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu,1024 uint32_t function)1025{1026 return __vcpu_get_cpuid_entry(vcpu, function, 0);1027}1028 1029static inline int __vcpu_set_cpuid(struct kvm_vcpu *vcpu)1030{1031 int r;1032 1033 TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first");1034 r = __vcpu_ioctl(vcpu, KVM_SET_CPUID2, vcpu->cpuid);1035 if (r)1036 return r;1037 1038 /* On success, refresh the cache to pick up adjustments made by KVM. */1039 vcpu_ioctl(vcpu, KVM_GET_CPUID2, vcpu->cpuid);1040 return 0;1041}1042 1043static inline void vcpu_set_cpuid(struct kvm_vcpu *vcpu)1044{1045 TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first");1046 vcpu_ioctl(vcpu, KVM_SET_CPUID2, vcpu->cpuid);1047 1048 /* Refresh the cache to pick up adjustments made by KVM. */1049 vcpu_ioctl(vcpu, KVM_GET_CPUID2, vcpu->cpuid);1050}1051 1052void vcpu_set_cpuid_property(struct kvm_vcpu *vcpu,1053 struct kvm_x86_cpu_property property,1054 uint32_t value);1055void vcpu_set_cpuid_maxphyaddr(struct kvm_vcpu *vcpu, uint8_t maxphyaddr);1056 1057void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, uint32_t function);1058 1059static inline bool vcpu_cpuid_has(struct kvm_vcpu *vcpu,1060 struct kvm_x86_cpu_feature feature)1061{1062 struct kvm_cpuid_entry2 *entry;1063 1064 entry = __vcpu_get_cpuid_entry(vcpu, feature.function, feature.index);1065 return *((&entry->eax) + feature.reg) & BIT(feature.bit);1066}1067 1068void vcpu_set_or_clear_cpuid_feature(struct kvm_vcpu *vcpu,1069 struct kvm_x86_cpu_feature feature,1070 bool set);1071 1072static inline void vcpu_set_cpuid_feature(struct kvm_vcpu *vcpu,1073 struct kvm_x86_cpu_feature feature)1074{1075 vcpu_set_or_clear_cpuid_feature(vcpu, feature, true);1076 1077}1078 1079static inline void vcpu_clear_cpuid_feature(struct kvm_vcpu *vcpu,1080 struct kvm_x86_cpu_feature feature)1081{1082 vcpu_set_or_clear_cpuid_feature(vcpu, feature, false);1083}1084 1085uint64_t vcpu_get_msr(struct kvm_vcpu *vcpu, uint64_t msr_index);1086int _vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index, uint64_t msr_value);1087 1088/*1089 * Assert on an MSR access(es) and pretty print the MSR name when possible.1090 * Note, the caller provides the stringified name so that the name of macro is1091 * printed, not the value the macro resolves to (due to macro expansion).1092 */1093#define TEST_ASSERT_MSR(cond, fmt, msr, str, args...) \1094do { \1095 if (__builtin_constant_p(msr)) { \1096 TEST_ASSERT(cond, fmt, str, args); \1097 } else if (!(cond)) { \1098 char buf[16]; \1099 \1100 snprintf(buf, sizeof(buf), "MSR 0x%x", msr); \1101 TEST_ASSERT(cond, fmt, buf, args); \1102 } \1103} while (0)1104 1105/*1106 * Returns true if KVM should return the last written value when reading an MSR1107 * from userspace, e.g. the MSR isn't a command MSR, doesn't emulate state that1108 * is changing, etc. This is NOT an exhaustive list! The intent is to filter1109 * out MSRs that are not durable _and_ that a selftest wants to write.1110 */1111static inline bool is_durable_msr(uint32_t msr)1112{1113 return msr != MSR_IA32_TSC;1114}1115 1116#define vcpu_set_msr(vcpu, msr, val) \1117do { \1118 uint64_t r, v = val; \1119 \1120 TEST_ASSERT_MSR(_vcpu_set_msr(vcpu, msr, v) == 1, \1121 "KVM_SET_MSRS failed on %s, value = 0x%lx", msr, #msr, v); \1122 if (!is_durable_msr(msr)) \1123 break; \1124 r = vcpu_get_msr(vcpu, msr); \1125 TEST_ASSERT_MSR(r == v, "Set %s to '0x%lx', got back '0x%lx'", msr, #msr, v, r);\1126} while (0)1127 1128void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits);1129void kvm_init_vm_address_properties(struct kvm_vm *vm);1130bool vm_is_unrestricted_guest(struct kvm_vm *vm);1131 1132struct ex_regs {1133 uint64_t rax, rcx, rdx, rbx;1134 uint64_t rbp, rsi, rdi;1135 uint64_t r8, r9, r10, r11;1136 uint64_t r12, r13, r14, r15;1137 uint64_t vector;1138 uint64_t error_code;1139 uint64_t rip;1140 uint64_t cs;1141 uint64_t rflags;1142};1143 1144struct idt_entry {1145 uint16_t offset0;1146 uint16_t selector;1147 uint16_t ist : 3;1148 uint16_t : 5;1149 uint16_t type : 4;1150 uint16_t : 1;1151 uint16_t dpl : 2;1152 uint16_t p : 1;1153 uint16_t offset1;1154 uint32_t offset2; uint32_t reserved;1155};1156 1157void vm_install_exception_handler(struct kvm_vm *vm, int vector,1158 void (*handler)(struct ex_regs *));1159 1160/* If a toddler were to say "abracadabra". */1161#define KVM_EXCEPTION_MAGIC 0xabacadabaULL1162 1163/*1164 * KVM selftest exception fixup uses registers to coordinate with the exception1165 * handler, versus the kernel's in-memory tables and KVM-Unit-Tests's in-memory1166 * per-CPU data. Using only registers avoids having to map memory into the1167 * guest, doesn't require a valid, stable GS.base, and reduces the risk of1168 * for recursive faults when accessing memory in the handler. The downside to1169 * using registers is that it restricts what registers can be used by the actual1170 * instruction. But, selftests are 64-bit only, making register* pressure a1171 * minor concern. Use r9-r11 as they are volatile, i.e. don't need to be saved1172 * by the callee, and except for r11 are not implicit parameters to any1173 * instructions. Ideally, fixup would use r8-r10 and thus avoid implicit1174 * parameters entirely, but Hyper-V's hypercall ABI uses r8 and testing Hyper-V1175 * is higher priority than testing non-faulting SYSCALL/SYSRET.1176 *1177 * Note, the fixup handler deliberately does not handle #DE, i.e. the vector1178 * is guaranteed to be non-zero on fault.1179 *1180 * REGISTER INPUTS:1181 * r9 = MAGIC1182 * r10 = RIP1183 * r11 = new RIP on fault1184 *1185 * REGISTER OUTPUTS:1186 * r9 = exception vector (non-zero)1187 * r10 = error code1188 */1189#define __KVM_ASM_SAFE(insn, fep) \1190 "mov $" __stringify(KVM_EXCEPTION_MAGIC) ", %%r9\n\t" \1191 "lea 1f(%%rip), %%r10\n\t" \1192 "lea 2f(%%rip), %%r11\n\t" \1193 fep "1: " insn "\n\t" \1194 "xor %%r9, %%r9\n\t" \1195 "2:\n\t" \1196 "mov %%r9b, %[vector]\n\t" \1197 "mov %%r10, %[error_code]\n\t"1198 1199#define KVM_ASM_SAFE(insn) __KVM_ASM_SAFE(insn, "")1200#define KVM_ASM_SAFE_FEP(insn) __KVM_ASM_SAFE(insn, KVM_FEP)1201 1202#define KVM_ASM_SAFE_OUTPUTS(v, ec) [vector] "=qm"(v), [error_code] "=rm"(ec)1203#define KVM_ASM_SAFE_CLOBBERS "r9", "r10", "r11"1204 1205#define kvm_asm_safe(insn, inputs...) \1206({ \1207 uint64_t ign_error_code; \1208 uint8_t vector; \1209 \1210 asm volatile(KVM_ASM_SAFE(insn) \1211 : KVM_ASM_SAFE_OUTPUTS(vector, ign_error_code) \1212 : inputs \1213 : KVM_ASM_SAFE_CLOBBERS); \1214 vector; \1215})1216 1217#define kvm_asm_safe_ec(insn, error_code, inputs...) \1218({ \1219 uint8_t vector; \1220 \1221 asm volatile(KVM_ASM_SAFE(insn) \1222 : KVM_ASM_SAFE_OUTPUTS(vector, error_code) \1223 : inputs \1224 : KVM_ASM_SAFE_CLOBBERS); \1225 vector; \1226})1227 1228#define kvm_asm_safe_fep(insn, inputs...) \1229({ \1230 uint64_t ign_error_code; \1231 uint8_t vector; \1232 \1233 asm volatile(KVM_ASM_SAFE(insn) \1234 : KVM_ASM_SAFE_OUTPUTS(vector, ign_error_code) \1235 : inputs \1236 : KVM_ASM_SAFE_CLOBBERS); \1237 vector; \1238})1239 1240#define kvm_asm_safe_ec_fep(insn, error_code, inputs...) \1241({ \1242 uint8_t vector; \1243 \1244 asm volatile(KVM_ASM_SAFE_FEP(insn) \1245 : KVM_ASM_SAFE_OUTPUTS(vector, error_code) \1246 : inputs \1247 : KVM_ASM_SAFE_CLOBBERS); \1248 vector; \1249})1250 1251#define BUILD_READ_U64_SAFE_HELPER(insn, _fep, _FEP) \1252static inline uint8_t insn##_safe ##_fep(uint32_t idx, uint64_t *val) \1253{ \1254 uint64_t error_code; \1255 uint8_t vector; \1256 uint32_t a, d; \1257 \1258 asm volatile(KVM_ASM_SAFE##_FEP(#insn) \1259 : "=a"(a), "=d"(d), \1260 KVM_ASM_SAFE_OUTPUTS(vector, error_code) \1261 : "c"(idx) \1262 : KVM_ASM_SAFE_CLOBBERS); \1263 \1264 *val = (uint64_t)a | ((uint64_t)d << 32); \1265 return vector; \1266}1267 1268/*1269 * Generate {insn}_safe() and {insn}_safe_fep() helpers for instructions that1270 * use ECX as in input index, and EDX:EAX as a 64-bit output.1271 */1272#define BUILD_READ_U64_SAFE_HELPERS(insn) \1273 BUILD_READ_U64_SAFE_HELPER(insn, , ) \1274 BUILD_READ_U64_SAFE_HELPER(insn, _fep, _FEP) \1275 1276BUILD_READ_U64_SAFE_HELPERS(rdmsr)1277BUILD_READ_U64_SAFE_HELPERS(rdpmc)1278BUILD_READ_U64_SAFE_HELPERS(xgetbv)1279 1280static inline uint8_t wrmsr_safe(uint32_t msr, uint64_t val)1281{1282 return kvm_asm_safe("wrmsr", "a"(val & -1u), "d"(val >> 32), "c"(msr));1283}1284 1285static inline uint8_t xsetbv_safe(uint32_t index, uint64_t value)1286{1287 u32 eax = value;1288 u32 edx = value >> 32;1289 1290 return kvm_asm_safe("xsetbv", "a" (eax), "d" (edx), "c" (index));1291}1292 1293bool kvm_is_tdp_enabled(void);1294 1295static inline bool kvm_is_pmu_enabled(void)1296{1297 return get_kvm_param_bool("enable_pmu");1298}1299 1300static inline bool kvm_is_forced_emulation_enabled(void)1301{1302 return !!get_kvm_param_integer("force_emulation_prefix");1303}1304 1305uint64_t *__vm_get_page_table_entry(struct kvm_vm *vm, uint64_t vaddr,1306 int *level);1307uint64_t *vm_get_page_table_entry(struct kvm_vm *vm, uint64_t vaddr);1308 1309uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2,1310 uint64_t a3);1311uint64_t __xen_hypercall(uint64_t nr, uint64_t a0, void *a1);1312void xen_hypercall(uint64_t nr, uint64_t a0, void *a1);1313 1314static inline uint64_t __kvm_hypercall_map_gpa_range(uint64_t gpa,1315 uint64_t size, uint64_t flags)1316{1317 return kvm_hypercall(KVM_HC_MAP_GPA_RANGE, gpa, size >> PAGE_SHIFT, flags, 0);1318}1319 1320static inline void kvm_hypercall_map_gpa_range(uint64_t gpa, uint64_t size,1321 uint64_t flags)1322{1323 uint64_t ret = __kvm_hypercall_map_gpa_range(gpa, size, flags);1324 1325 GUEST_ASSERT(!ret);1326}1327 1328void __vm_xsave_require_permission(uint64_t xfeature, const char *name);1329 1330#define vm_xsave_require_permission(xfeature) \1331 __vm_xsave_require_permission(xfeature, #xfeature)1332 1333enum pg_level {1334 PG_LEVEL_NONE,1335 PG_LEVEL_4K,1336 PG_LEVEL_2M,1337 PG_LEVEL_1G,1338 PG_LEVEL_512G,1339 PG_LEVEL_NUM1340};1341 1342#define PG_LEVEL_SHIFT(_level) ((_level - 1) * 9 + 12)1343#define PG_LEVEL_SIZE(_level) (1ull << PG_LEVEL_SHIFT(_level))1344 1345#define PG_SIZE_4K PG_LEVEL_SIZE(PG_LEVEL_4K)1346#define PG_SIZE_2M PG_LEVEL_SIZE(PG_LEVEL_2M)1347#define PG_SIZE_1G PG_LEVEL_SIZE(PG_LEVEL_1G)1348 1349void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level);1350void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,1351 uint64_t nr_bytes, int level);1352 1353/*1354 * Basic CPU control in CR01355 */1356#define X86_CR0_PE (1UL<<0) /* Protection Enable */1357#define X86_CR0_MP (1UL<<1) /* Monitor Coprocessor */1358#define X86_CR0_EM (1UL<<2) /* Emulation */1359#define X86_CR0_TS (1UL<<3) /* Task Switched */1360#define X86_CR0_ET (1UL<<4) /* Extension Type */1361#define X86_CR0_NE (1UL<<5) /* Numeric Error */1362#define X86_CR0_WP (1UL<<16) /* Write Protect */1363#define X86_CR0_AM (1UL<<18) /* Alignment Mask */1364#define X86_CR0_NW (1UL<<29) /* Not Write-through */1365#define X86_CR0_CD (1UL<<30) /* Cache Disable */1366#define X86_CR0_PG (1UL<<31) /* Paging */1367 1368#define PFERR_PRESENT_BIT 01369#define PFERR_WRITE_BIT 11370#define PFERR_USER_BIT 21371#define PFERR_RSVD_BIT 31372#define PFERR_FETCH_BIT 41373#define PFERR_PK_BIT 51374#define PFERR_SGX_BIT 151375#define PFERR_GUEST_FINAL_BIT 321376#define PFERR_GUEST_PAGE_BIT 331377#define PFERR_IMPLICIT_ACCESS_BIT 481378 1379#define PFERR_PRESENT_MASK BIT(PFERR_PRESENT_BIT)1380#define PFERR_WRITE_MASK BIT(PFERR_WRITE_BIT)1381#define PFERR_USER_MASK BIT(PFERR_USER_BIT)1382#define PFERR_RSVD_MASK BIT(PFERR_RSVD_BIT)1383#define PFERR_FETCH_MASK BIT(PFERR_FETCH_BIT)1384#define PFERR_PK_MASK BIT(PFERR_PK_BIT)1385#define PFERR_SGX_MASK BIT(PFERR_SGX_BIT)1386#define PFERR_GUEST_FINAL_MASK BIT_ULL(PFERR_GUEST_FINAL_BIT)1387#define PFERR_GUEST_PAGE_MASK BIT_ULL(PFERR_GUEST_PAGE_BIT)1388#define PFERR_IMPLICIT_ACCESS BIT_ULL(PFERR_IMPLICIT_ACCESS_BIT)1389 1390bool sys_clocksource_is_based_on_tsc(void);1391 1392#endif /* SELFTEST_KVM_PROCESSOR_H */1393