290 lines · c
1// SPDX-License-Identifier: GPL-2.02#include <stdio.h>3#include <stdlib.h>4#include <pthread.h>5#include <semaphore.h>6#include <sys/types.h>7#include <signal.h>8#include <errno.h>9#include <linux/bitmap.h>10#include <linux/bitops.h>11#include <linux/atomic.h>12#include <linux/sizes.h>13 14#include "kvm_util.h"15#include "test_util.h"16#include "guest_modes.h"17#include "processor.h"18 19static void guest_code(uint64_t start_gpa, uint64_t end_gpa, uint64_t stride)20{21 uint64_t gpa;22 23 for (;;) {24 for (gpa = start_gpa; gpa < end_gpa; gpa += stride)25 *((volatile uint64_t *)gpa) = gpa;26 GUEST_SYNC(0);27 }28}29 30struct vcpu_info {31 struct kvm_vcpu *vcpu;32 uint64_t start_gpa;33 uint64_t end_gpa;34};35 36static int nr_vcpus;37static atomic_t rendezvous;38 39static void rendezvous_with_boss(void)40{41 int orig = atomic_read(&rendezvous);42 43 if (orig > 0) {44 atomic_dec_and_test(&rendezvous);45 while (atomic_read(&rendezvous) > 0)46 cpu_relax();47 } else {48 atomic_inc(&rendezvous);49 while (atomic_read(&rendezvous) < 0)50 cpu_relax();51 }52}53 54static void run_vcpu(struct kvm_vcpu *vcpu)55{56 vcpu_run(vcpu);57 TEST_ASSERT_EQ(get_ucall(vcpu, NULL), UCALL_SYNC);58}59 60static void *vcpu_worker(void *data)61{62 struct vcpu_info *info = data;63 struct kvm_vcpu *vcpu = info->vcpu;64 struct kvm_vm *vm = vcpu->vm;65 struct kvm_sregs sregs;66 67 vcpu_args_set(vcpu, 3, info->start_gpa, info->end_gpa, vm->page_size);68 69 rendezvous_with_boss();70 71 run_vcpu(vcpu);72 rendezvous_with_boss();73 vcpu_sregs_get(vcpu, &sregs);74#ifdef __x86_64__75 /* Toggle CR0.WP to trigger a MMU context reset. */76 sregs.cr0 ^= X86_CR0_WP;77#endif78 vcpu_sregs_set(vcpu, &sregs);79 rendezvous_with_boss();80 81 run_vcpu(vcpu);82 rendezvous_with_boss();83 84 return NULL;85}86 87static pthread_t *spawn_workers(struct kvm_vm *vm, struct kvm_vcpu **vcpus,88 uint64_t start_gpa, uint64_t end_gpa)89{90 struct vcpu_info *info;91 uint64_t gpa, nr_bytes;92 pthread_t *threads;93 int i;94 95 threads = malloc(nr_vcpus * sizeof(*threads));96 TEST_ASSERT(threads, "Failed to allocate vCPU threads");97 98 info = malloc(nr_vcpus * sizeof(*info));99 TEST_ASSERT(info, "Failed to allocate vCPU gpa ranges");100 101 nr_bytes = ((end_gpa - start_gpa) / nr_vcpus) &102 ~((uint64_t)vm->page_size - 1);103 TEST_ASSERT(nr_bytes, "C'mon, no way you have %d CPUs", nr_vcpus);104 105 for (i = 0, gpa = start_gpa; i < nr_vcpus; i++, gpa += nr_bytes) {106 info[i].vcpu = vcpus[i];107 info[i].start_gpa = gpa;108 info[i].end_gpa = gpa + nr_bytes;109 pthread_create(&threads[i], NULL, vcpu_worker, &info[i]);110 }111 return threads;112}113 114static void rendezvous_with_vcpus(struct timespec *time, const char *name)115{116 int i, rendezvoused;117 118 pr_info("Waiting for vCPUs to finish %s...\n", name);119 120 rendezvoused = atomic_read(&rendezvous);121 for (i = 0; abs(rendezvoused) != 1; i++) {122 usleep(100);123 if (!(i & 0x3f))124 pr_info("\r%d vCPUs haven't rendezvoused...",125 abs(rendezvoused) - 1);126 rendezvoused = atomic_read(&rendezvous);127 }128 129 clock_gettime(CLOCK_MONOTONIC, time);130 131 /* Release the vCPUs after getting the time of the previous action. */132 pr_info("\rAll vCPUs finished %s, releasing...\n", name);133 if (rendezvoused > 0)134 atomic_set(&rendezvous, -nr_vcpus - 1);135 else136 atomic_set(&rendezvous, nr_vcpus + 1);137}138 139static void calc_default_nr_vcpus(void)140{141 cpu_set_t possible_mask;142 int r;143 144 r = sched_getaffinity(0, sizeof(possible_mask), &possible_mask);145 TEST_ASSERT(!r, "sched_getaffinity failed, errno = %d (%s)",146 errno, strerror(errno));147 148 nr_vcpus = CPU_COUNT(&possible_mask) * 3/4;149 TEST_ASSERT(nr_vcpus > 0, "Uh, no CPUs?");150}151 152int main(int argc, char *argv[])153{154 /*155 * Skip the first 4gb and slot0. slot0 maps <1gb and is used to back156 * the guest's code, stack, and page tables. Because selftests creates157 * an IRQCHIP, a.k.a. a local APIC, KVM creates an internal memslot158 * just below the 4gb boundary. This test could create memory at159 * 1gb-3gb,but it's simpler to skip straight to 4gb.160 */161 const uint64_t start_gpa = SZ_4G;162 const int first_slot = 1;163 164 struct timespec time_start, time_run1, time_reset, time_run2;165 uint64_t max_gpa, gpa, slot_size, max_mem, i;166 int max_slots, slot, opt, fd;167 bool hugepages = false;168 struct kvm_vcpu **vcpus;169 pthread_t *threads;170 struct kvm_vm *vm;171 void *mem;172 173 /*174 * Default to 2gb so that maxing out systems with MAXPHADDR=46, which175 * are quite common for x86, requires changing only max_mem (KVM allows176 * 32k memslots, 32k * 2gb == ~64tb of guest memory).177 */178 slot_size = SZ_2G;179 180 max_slots = kvm_check_cap(KVM_CAP_NR_MEMSLOTS);181 TEST_ASSERT(max_slots > first_slot, "KVM is broken");182 183 /* All KVM MMUs should be able to survive a 128gb guest. */184 max_mem = 128ull * SZ_1G;185 186 calc_default_nr_vcpus();187 188 while ((opt = getopt(argc, argv, "c:h:m:s:H")) != -1) {189 switch (opt) {190 case 'c':191 nr_vcpus = atoi_positive("Number of vCPUs", optarg);192 break;193 case 'm':194 max_mem = 1ull * atoi_positive("Memory size", optarg) * SZ_1G;195 break;196 case 's':197 slot_size = 1ull * atoi_positive("Slot size", optarg) * SZ_1G;198 break;199 case 'H':200 hugepages = true;201 break;202 case 'h':203 default:204 printf("usage: %s [-c nr_vcpus] [-m max_mem_in_gb] [-s slot_size_in_gb] [-H]\n", argv[0]);205 exit(1);206 }207 }208 209 vcpus = malloc(nr_vcpus * sizeof(*vcpus));210 TEST_ASSERT(vcpus, "Failed to allocate vCPU array");211 212 vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus);213 214 max_gpa = vm->max_gfn << vm->page_shift;215 TEST_ASSERT(max_gpa > (4 * slot_size), "MAXPHYADDR <4gb ");216 217 fd = kvm_memfd_alloc(slot_size, hugepages);218 mem = mmap(NULL, slot_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);219 TEST_ASSERT(mem != MAP_FAILED, "mmap() failed");220 221 TEST_ASSERT(!madvise(mem, slot_size, MADV_NOHUGEPAGE), "madvise() failed");222 223 /* Pre-fault the memory to avoid taking mmap_sem on guest page faults. */224 for (i = 0; i < slot_size; i += vm->page_size)225 ((uint8_t *)mem)[i] = 0xaa;226 227 gpa = 0;228 for (slot = first_slot; slot < max_slots; slot++) {229 gpa = start_gpa + ((slot - first_slot) * slot_size);230 if (gpa + slot_size > max_gpa)231 break;232 233 if ((gpa - start_gpa) >= max_mem)234 break;235 236 vm_set_user_memory_region(vm, slot, 0, gpa, slot_size, mem);237 238#ifdef __x86_64__239 /* Identity map memory in the guest using 1gb pages. */240 for (i = 0; i < slot_size; i += SZ_1G)241 __virt_pg_map(vm, gpa + i, gpa + i, PG_LEVEL_1G);242#else243 for (i = 0; i < slot_size; i += vm->page_size)244 virt_pg_map(vm, gpa + i, gpa + i);245#endif246 }247 248 atomic_set(&rendezvous, nr_vcpus + 1);249 threads = spawn_workers(vm, vcpus, start_gpa, gpa);250 251 free(vcpus);252 vcpus = NULL;253 254 pr_info("Running with %lugb of guest memory and %u vCPUs\n",255 (gpa - start_gpa) / SZ_1G, nr_vcpus);256 257 rendezvous_with_vcpus(&time_start, "spawning");258 rendezvous_with_vcpus(&time_run1, "run 1");259 rendezvous_with_vcpus(&time_reset, "reset");260 rendezvous_with_vcpus(&time_run2, "run 2");261 262 time_run2 = timespec_sub(time_run2, time_reset);263 time_reset = timespec_sub(time_reset, time_run1);264 time_run1 = timespec_sub(time_run1, time_start);265 266 pr_info("run1 = %ld.%.9lds, reset = %ld.%.9lds, run2 = %ld.%.9lds\n",267 time_run1.tv_sec, time_run1.tv_nsec,268 time_reset.tv_sec, time_reset.tv_nsec,269 time_run2.tv_sec, time_run2.tv_nsec);270 271 /*272 * Delete even numbered slots (arbitrary) and unmap the first half of273 * the backing (also arbitrary) to verify KVM correctly drops all274 * references to the removed regions.275 */276 for (slot = (slot - 1) & ~1ull; slot >= first_slot; slot -= 2)277 vm_set_user_memory_region(vm, slot, 0, 0, 0, NULL);278 279 munmap(mem, slot_size / 2);280 281 /* Sanity check that the vCPUs actually ran. */282 for (i = 0; i < nr_vcpus; i++)283 pthread_join(threads[i], NULL);284 285 /*286 * Deliberately exit without deleting the remaining memslots or closing287 * kvm_fd to test cleanup via mmu_notifier.release.288 */289}290