478 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * KVM page table test4 *5 * Copyright (C) 2021, Huawei, Inc.6 *7 * Make sure that THP has been enabled or enough HUGETLB pages with specific8 * page size have been pre-allocated on your system, if you are planning to9 * use hugepages to back the guest memory for testing.10 */11#include <stdio.h>12#include <stdlib.h>13#include <time.h>14#include <pthread.h>15#include <semaphore.h>16 17#include "test_util.h"18#include "kvm_util.h"19#include "processor.h"20#include "guest_modes.h"21#include "ucall_common.h"22 23#define TEST_MEM_SLOT_INDEX 124 25/* Default size(1GB) of the memory for testing */26#define DEFAULT_TEST_MEM_SIZE (1 << 30)27 28/* Default guest test virtual memory offset */29#define DEFAULT_GUEST_TEST_MEM 0xc000000030 31/* Different guest memory accessing stages */32enum test_stage {33 KVM_BEFORE_MAPPINGS,34 KVM_CREATE_MAPPINGS,35 KVM_UPDATE_MAPPINGS,36 KVM_ADJUST_MAPPINGS,37 NUM_TEST_STAGES,38};39 40static const char * const test_stage_string[] = {41 "KVM_BEFORE_MAPPINGS",42 "KVM_CREATE_MAPPINGS",43 "KVM_UPDATE_MAPPINGS",44 "KVM_ADJUST_MAPPINGS",45};46 47struct test_args {48 struct kvm_vm *vm;49 uint64_t guest_test_virt_mem;50 uint64_t host_page_size;51 uint64_t host_num_pages;52 uint64_t large_page_size;53 uint64_t large_num_pages;54 uint64_t host_pages_per_lpage;55 enum vm_mem_backing_src_type src_type;56 struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];57};58 59/*60 * Guest variables. Use addr_gva2hva() if these variables need61 * to be changed in host.62 */63static enum test_stage guest_test_stage;64 65/* Host variables */66static uint32_t nr_vcpus = 1;67static struct test_args test_args;68static enum test_stage *current_stage;69static bool host_quit;70 71/* Whether the test stage is updated, or completed */72static sem_t test_stage_updated;73static sem_t test_stage_completed;74 75/*76 * Guest physical memory offset of the testing memory slot.77 * This will be set to the topmost valid physical address minus78 * the test memory size.79 */80static uint64_t guest_test_phys_mem;81 82/*83 * Guest virtual memory offset of the testing memory slot.84 * Must not conflict with identity mapped test code.85 */86static uint64_t guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM;87 88static void guest_code(bool do_write)89{90 struct test_args *p = &test_args;91 enum test_stage *current_stage = &guest_test_stage;92 uint64_t addr;93 int i, j;94 95 while (true) {96 addr = p->guest_test_virt_mem;97 98 switch (READ_ONCE(*current_stage)) {99 /*100 * All vCPU threads will be started in this stage,101 * where guest code of each vCPU will do nothing.102 */103 case KVM_BEFORE_MAPPINGS:104 break;105 106 /*107 * Before dirty logging, vCPUs concurrently access the first108 * 8 bytes of each page (host page/large page) within the same109 * memory region with different accessing types (read/write).110 * Then KVM will create normal page mappings or huge block111 * mappings for them.112 */113 case KVM_CREATE_MAPPINGS:114 for (i = 0; i < p->large_num_pages; i++) {115 if (do_write)116 *(uint64_t *)addr = 0x0123456789ABCDEF;117 else118 READ_ONCE(*(uint64_t *)addr);119 120 addr += p->large_page_size;121 }122 break;123 124 /*125 * During dirty logging, KVM will only update attributes of the126 * normal page mappings from RO to RW if memory backing src type127 * is anonymous. In other cases, KVM will split the huge block128 * mappings into normal page mappings if memory backing src type129 * is THP or HUGETLB.130 */131 case KVM_UPDATE_MAPPINGS:132 if (p->src_type == VM_MEM_SRC_ANONYMOUS) {133 for (i = 0; i < p->host_num_pages; i++) {134 *(uint64_t *)addr = 0x0123456789ABCDEF;135 addr += p->host_page_size;136 }137 break;138 }139 140 for (i = 0; i < p->large_num_pages; i++) {141 /*142 * Write to the first host page in each large143 * page region, and triger break of large pages.144 */145 *(uint64_t *)addr = 0x0123456789ABCDEF;146 147 /*148 * Access the middle host pages in each large149 * page region. Since dirty logging is enabled,150 * this will create new mappings at the smallest151 * granularity.152 */153 addr += p->large_page_size / 2;154 for (j = 0; j < p->host_pages_per_lpage / 2; j++) {155 READ_ONCE(*(uint64_t *)addr);156 addr += p->host_page_size;157 }158 }159 break;160 161 /*162 * After dirty logging is stopped, vCPUs concurrently read163 * from every single host page. Then KVM will coalesce the164 * split page mappings back to block mappings. And a TLB165 * conflict abort could occur here if TLB entries of the166 * page mappings are not fully invalidated.167 */168 case KVM_ADJUST_MAPPINGS:169 for (i = 0; i < p->host_num_pages; i++) {170 READ_ONCE(*(uint64_t *)addr);171 addr += p->host_page_size;172 }173 break;174 175 default:176 GUEST_ASSERT(0);177 }178 179 GUEST_SYNC(1);180 }181}182 183static void *vcpu_worker(void *data)184{185 struct kvm_vcpu *vcpu = data;186 bool do_write = !(vcpu->id % 2);187 struct timespec start;188 struct timespec ts_diff;189 enum test_stage stage;190 int ret;191 192 vcpu_args_set(vcpu, 1, do_write);193 194 while (!READ_ONCE(host_quit)) {195 ret = sem_wait(&test_stage_updated);196 TEST_ASSERT(ret == 0, "Error in sem_wait");197 198 if (READ_ONCE(host_quit))199 return NULL;200 201 clock_gettime(CLOCK_MONOTONIC, &start);202 ret = _vcpu_run(vcpu);203 ts_diff = timespec_elapsed(start);204 205 TEST_ASSERT(ret == 0, "vcpu_run failed: %d", ret);206 TEST_ASSERT(get_ucall(vcpu, NULL) == UCALL_SYNC,207 "Invalid guest sync status: exit_reason=%s",208 exit_reason_str(vcpu->run->exit_reason));209 210 pr_debug("Got sync event from vCPU %d\n", vcpu->id);211 stage = READ_ONCE(*current_stage);212 213 /*214 * Here we can know the execution time of every215 * single vcpu running in different test stages.216 */217 pr_debug("vCPU %d has completed stage %s\n"218 "execution time is: %ld.%.9lds\n\n",219 vcpu->id, test_stage_string[stage],220 ts_diff.tv_sec, ts_diff.tv_nsec);221 222 ret = sem_post(&test_stage_completed);223 TEST_ASSERT(ret == 0, "Error in sem_post");224 }225 226 return NULL;227}228 229struct test_params {230 uint64_t phys_offset;231 uint64_t test_mem_size;232 enum vm_mem_backing_src_type src_type;233};234 235static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg)236{237 int ret;238 struct test_params *p = arg;239 enum vm_mem_backing_src_type src_type = p->src_type;240 uint64_t large_page_size = get_backing_src_pagesz(src_type);241 uint64_t guest_page_size = vm_guest_mode_params[mode].page_size;242 uint64_t host_page_size = getpagesize();243 uint64_t test_mem_size = p->test_mem_size;244 uint64_t guest_num_pages;245 uint64_t alignment;246 void *host_test_mem;247 struct kvm_vm *vm;248 249 /* Align up the test memory size */250 alignment = max(large_page_size, guest_page_size);251 test_mem_size = (test_mem_size + alignment - 1) & ~(alignment - 1);252 253 /* Create a VM with enough guest pages */254 guest_num_pages = test_mem_size / guest_page_size;255 vm = __vm_create_with_vcpus(VM_SHAPE(mode), nr_vcpus, guest_num_pages,256 guest_code, test_args.vcpus);257 258 /* Align down GPA of the testing memslot */259 if (!p->phys_offset)260 guest_test_phys_mem = (vm->max_gfn - guest_num_pages) *261 guest_page_size;262 else263 guest_test_phys_mem = p->phys_offset;264#ifdef __s390x__265 alignment = max(0x100000UL, alignment);266#endif267 guest_test_phys_mem = align_down(guest_test_phys_mem, alignment);268 269 /* Set up the shared data structure test_args */270 test_args.vm = vm;271 test_args.guest_test_virt_mem = guest_test_virt_mem;272 test_args.host_page_size = host_page_size;273 test_args.host_num_pages = test_mem_size / host_page_size;274 test_args.large_page_size = large_page_size;275 test_args.large_num_pages = test_mem_size / large_page_size;276 test_args.host_pages_per_lpage = large_page_size / host_page_size;277 test_args.src_type = src_type;278 279 /* Add an extra memory slot with specified backing src type */280 vm_userspace_mem_region_add(vm, src_type, guest_test_phys_mem,281 TEST_MEM_SLOT_INDEX, guest_num_pages, 0);282 283 /* Do mapping(GVA->GPA) for the testing memory slot */284 virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages);285 286 /* Cache the HVA pointer of the region */287 host_test_mem = addr_gpa2hva(vm, (vm_paddr_t)guest_test_phys_mem);288 289 /* Export shared structure test_args to guest */290 sync_global_to_guest(vm, test_args);291 292 ret = sem_init(&test_stage_updated, 0, 0);293 TEST_ASSERT(ret == 0, "Error in sem_init");294 295 ret = sem_init(&test_stage_completed, 0, 0);296 TEST_ASSERT(ret == 0, "Error in sem_init");297 298 current_stage = addr_gva2hva(vm, (vm_vaddr_t)(&guest_test_stage));299 *current_stage = NUM_TEST_STAGES;300 301 pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode));302 pr_info("Testing memory backing src type: %s\n",303 vm_mem_backing_src_alias(src_type)->name);304 pr_info("Testing memory backing src granularity: 0x%lx\n",305 large_page_size);306 pr_info("Testing memory size(aligned): 0x%lx\n", test_mem_size);307 pr_info("Guest physical test memory offset: 0x%lx\n",308 guest_test_phys_mem);309 pr_info("Host virtual test memory offset: 0x%lx\n",310 (uint64_t)host_test_mem);311 pr_info("Number of testing vCPUs: %d\n", nr_vcpus);312 313 return vm;314}315 316static void vcpus_complete_new_stage(enum test_stage stage)317{318 int ret;319 int vcpus;320 321 /* Wake up all the vcpus to run new test stage */322 for (vcpus = 0; vcpus < nr_vcpus; vcpus++) {323 ret = sem_post(&test_stage_updated);324 TEST_ASSERT(ret == 0, "Error in sem_post");325 }326 pr_debug("All vcpus have been notified to continue\n");327 328 /* Wait for all the vcpus to complete new test stage */329 for (vcpus = 0; vcpus < nr_vcpus; vcpus++) {330 ret = sem_wait(&test_stage_completed);331 TEST_ASSERT(ret == 0, "Error in sem_wait");332 333 pr_debug("%d vcpus have completed stage %s\n",334 vcpus + 1, test_stage_string[stage]);335 }336 337 pr_debug("All vcpus have completed stage %s\n",338 test_stage_string[stage]);339}340 341static void run_test(enum vm_guest_mode mode, void *arg)342{343 pthread_t *vcpu_threads;344 struct kvm_vm *vm;345 struct timespec start;346 struct timespec ts_diff;347 int ret, i;348 349 /* Create VM with vCPUs and make some pre-initialization */350 vm = pre_init_before_test(mode, arg);351 352 vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads));353 TEST_ASSERT(vcpu_threads, "Memory allocation failed");354 355 host_quit = false;356 *current_stage = KVM_BEFORE_MAPPINGS;357 358 for (i = 0; i < nr_vcpus; i++)359 pthread_create(&vcpu_threads[i], NULL, vcpu_worker,360 test_args.vcpus[i]);361 362 vcpus_complete_new_stage(*current_stage);363 pr_info("Started all vCPUs successfully\n");364 365 /* Test the stage of KVM creating mappings */366 *current_stage = KVM_CREATE_MAPPINGS;367 368 clock_gettime(CLOCK_MONOTONIC, &start);369 vcpus_complete_new_stage(*current_stage);370 ts_diff = timespec_elapsed(start);371 372 pr_info("KVM_CREATE_MAPPINGS: total execution time: %ld.%.9lds\n\n",373 ts_diff.tv_sec, ts_diff.tv_nsec);374 375 /* Test the stage of KVM updating mappings */376 vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX,377 KVM_MEM_LOG_DIRTY_PAGES);378 379 *current_stage = KVM_UPDATE_MAPPINGS;380 381 clock_gettime(CLOCK_MONOTONIC, &start);382 vcpus_complete_new_stage(*current_stage);383 ts_diff = timespec_elapsed(start);384 385 pr_info("KVM_UPDATE_MAPPINGS: total execution time: %ld.%.9lds\n\n",386 ts_diff.tv_sec, ts_diff.tv_nsec);387 388 /* Test the stage of KVM adjusting mappings */389 vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX, 0);390 391 *current_stage = KVM_ADJUST_MAPPINGS;392 393 clock_gettime(CLOCK_MONOTONIC, &start);394 vcpus_complete_new_stage(*current_stage);395 ts_diff = timespec_elapsed(start);396 397 pr_info("KVM_ADJUST_MAPPINGS: total execution time: %ld.%.9lds\n\n",398 ts_diff.tv_sec, ts_diff.tv_nsec);399 400 /* Tell the vcpu thread to quit */401 host_quit = true;402 for (i = 0; i < nr_vcpus; i++) {403 ret = sem_post(&test_stage_updated);404 TEST_ASSERT(ret == 0, "Error in sem_post");405 }406 407 for (i = 0; i < nr_vcpus; i++)408 pthread_join(vcpu_threads[i], NULL);409 410 ret = sem_destroy(&test_stage_updated);411 TEST_ASSERT(ret == 0, "Error in sem_destroy");412 413 ret = sem_destroy(&test_stage_completed);414 TEST_ASSERT(ret == 0, "Error in sem_destroy");415 416 free(vcpu_threads);417 kvm_vm_free(vm);418}419 420static void help(char *name)421{422 puts("");423 printf("usage: %s [-h] [-p offset] [-m mode] "424 "[-b mem-size] [-v vcpus] [-s mem-type]\n", name);425 puts("");426 printf(" -p: specify guest physical test memory offset\n"427 " Warning: a low offset can conflict with the loaded test code.\n");428 guest_modes_help();429 printf(" -b: specify size of the memory region for testing. e.g. 10M or 3G.\n"430 " (default: 1G)\n");431 printf(" -v: specify the number of vCPUs to run\n"432 " (default: 1)\n");433 backing_src_help("-s");434 puts("");435}436 437int main(int argc, char *argv[])438{439 int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS);440 struct test_params p = {441 .test_mem_size = DEFAULT_TEST_MEM_SIZE,442 .src_type = DEFAULT_VM_MEM_SRC,443 };444 int opt;445 446 guest_modes_append_default();447 448 while ((opt = getopt(argc, argv, "hp:m:b:v:s:")) != -1) {449 switch (opt) {450 case 'p':451 p.phys_offset = strtoull(optarg, NULL, 0);452 break;453 case 'm':454 guest_modes_cmdline(optarg);455 break;456 case 'b':457 p.test_mem_size = parse_size(optarg);458 break;459 case 'v':460 nr_vcpus = atoi_positive("Number of vCPUs", optarg);461 TEST_ASSERT(nr_vcpus <= max_vcpus,462 "Invalid number of vcpus, must be between 1 and %d", max_vcpus);463 break;464 case 's':465 p.src_type = parse_backing_src_type(optarg);466 break;467 case 'h':468 default:469 help(argv[0]);470 exit(0);471 }472 }473 474 for_each_guest_mode(run_test, &p);475 476 return 0;477}478