395 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * access_tracking_perf_test4 *5 * Copyright (C) 2021, Google, Inc.6 *7 * This test measures the performance effects of KVM's access tracking.8 * Access tracking is driven by the MMU notifiers test_young, clear_young, and9 * clear_flush_young. These notifiers do not have a direct userspace API,10 * however the clear_young notifier can be triggered by marking a pages as idle11 * in /sys/kernel/mm/page_idle/bitmap. This test leverages that mechanism to12 * enable access tracking on guest memory.13 *14 * To measure performance this test runs a VM with a configurable number of15 * vCPUs that each touch every page in disjoint regions of memory. Performance16 * is measured in the time it takes all vCPUs to finish touching their17 * predefined region.18 *19 * Note that a deterministic correctness test of access tracking is not possible20 * by using page_idle as it exists today. This is for a few reasons:21 *22 * 1. page_idle only issues clear_young notifiers, which lack a TLB flush. This23 * means subsequent guest accesses are not guaranteed to see page table24 * updates made by KVM until some time in the future.25 *26 * 2. page_idle only operates on LRU pages. Newly allocated pages are not27 * immediately allocated to LRU lists. Instead they are held in a "pagevec",28 * which is drained to LRU lists some time in the future. There is no29 * userspace API to force this drain to occur.30 *31 * These limitations are worked around in this test by using a large enough32 * region of memory for each vCPU such that the number of translations cached in33 * the TLB and the number of pages held in pagevecs are a small fraction of the34 * overall workload. And if either of those conditions are not true (for example35 * in nesting, where TLB size is unlimited) this test will print a warning36 * rather than silently passing.37 */38#include <inttypes.h>39#include <limits.h>40#include <pthread.h>41#include <sys/mman.h>42#include <sys/types.h>43#include <sys/stat.h>44 45#include "kvm_util.h"46#include "test_util.h"47#include "memstress.h"48#include "guest_modes.h"49#include "processor.h"50 51/* Global variable used to synchronize all of the vCPU threads. */52static int iteration;53 54/* Defines what vCPU threads should do during a given iteration. */55static enum {56 /* Run the vCPU to access all its memory. */57 ITERATION_ACCESS_MEMORY,58 /* Mark the vCPU's memory idle in page_idle. */59 ITERATION_MARK_IDLE,60} iteration_work;61 62/* The iteration that was last completed by each vCPU. */63static int vcpu_last_completed_iteration[KVM_MAX_VCPUS];64 65/* Whether to overlap the regions of memory vCPUs access. */66static bool overlap_memory_access;67 68struct test_params {69 /* The backing source for the region of memory. */70 enum vm_mem_backing_src_type backing_src;71 72 /* The amount of memory to allocate for each vCPU. */73 uint64_t vcpu_memory_bytes;74 75 /* The number of vCPUs to create in the VM. */76 int nr_vcpus;77};78 79static uint64_t pread_uint64(int fd, const char *filename, uint64_t index)80{81 uint64_t value;82 off_t offset = index * sizeof(value);83 84 TEST_ASSERT(pread(fd, &value, sizeof(value), offset) == sizeof(value),85 "pread from %s offset 0x%" PRIx64 " failed!",86 filename, offset);87 88 return value;89 90}91 92#define PAGEMAP_PRESENT (1ULL << 63)93#define PAGEMAP_PFN_MASK ((1ULL << 55) - 1)94 95static uint64_t lookup_pfn(int pagemap_fd, struct kvm_vm *vm, uint64_t gva)96{97 uint64_t hva = (uint64_t) addr_gva2hva(vm, gva);98 uint64_t entry;99 uint64_t pfn;100 101 entry = pread_uint64(pagemap_fd, "pagemap", hva / getpagesize());102 if (!(entry & PAGEMAP_PRESENT))103 return 0;104 105 pfn = entry & PAGEMAP_PFN_MASK;106 __TEST_REQUIRE(pfn, "Looking up PFNs requires CAP_SYS_ADMIN");107 108 return pfn;109}110 111static bool is_page_idle(int page_idle_fd, uint64_t pfn)112{113 uint64_t bits = pread_uint64(page_idle_fd, "page_idle", pfn / 64);114 115 return !!((bits >> (pfn % 64)) & 1);116}117 118static void mark_page_idle(int page_idle_fd, uint64_t pfn)119{120 uint64_t bits = 1ULL << (pfn % 64);121 122 TEST_ASSERT(pwrite(page_idle_fd, &bits, 8, 8 * (pfn / 64)) == 8,123 "Set page_idle bits for PFN 0x%" PRIx64, pfn);124}125 126static void mark_vcpu_memory_idle(struct kvm_vm *vm,127 struct memstress_vcpu_args *vcpu_args)128{129 int vcpu_idx = vcpu_args->vcpu_idx;130 uint64_t base_gva = vcpu_args->gva;131 uint64_t pages = vcpu_args->pages;132 uint64_t page;133 uint64_t still_idle = 0;134 uint64_t no_pfn = 0;135 int page_idle_fd;136 int pagemap_fd;137 138 /* If vCPUs are using an overlapping region, let vCPU 0 mark it idle. */139 if (overlap_memory_access && vcpu_idx)140 return;141 142 page_idle_fd = open("/sys/kernel/mm/page_idle/bitmap", O_RDWR);143 TEST_ASSERT(page_idle_fd > 0, "Failed to open page_idle.");144 145 pagemap_fd = open("/proc/self/pagemap", O_RDONLY);146 TEST_ASSERT(pagemap_fd > 0, "Failed to open pagemap.");147 148 for (page = 0; page < pages; page++) {149 uint64_t gva = base_gva + page * memstress_args.guest_page_size;150 uint64_t pfn = lookup_pfn(pagemap_fd, vm, gva);151 152 if (!pfn) {153 no_pfn++;154 continue;155 }156 157 if (is_page_idle(page_idle_fd, pfn)) {158 still_idle++;159 continue;160 }161 162 mark_page_idle(page_idle_fd, pfn);163 }164 165 /*166 * Assumption: Less than 1% of pages are going to be swapped out from167 * under us during this test.168 */169 TEST_ASSERT(no_pfn < pages / 100,170 "vCPU %d: No PFN for %" PRIu64 " out of %" PRIu64 " pages.",171 vcpu_idx, no_pfn, pages);172 173 /*174 * Check that at least 90% of memory has been marked idle (the rest175 * might not be marked idle because the pages have not yet made it to an176 * LRU list or the translations are still cached in the TLB). 90% is177 * arbitrary; high enough that we ensure most memory access went through178 * access tracking but low enough as to not make the test too brittle179 * over time and across architectures.180 *181 * When running the guest as a nested VM, "warn" instead of asserting182 * as the TLB size is effectively unlimited and the KVM doesn't183 * explicitly flush the TLB when aging SPTEs. As a result, more pages184 * are cached and the guest won't see the "idle" bit cleared.185 */186 if (still_idle >= pages / 10) {187#ifdef __x86_64__188 TEST_ASSERT(this_cpu_has(X86_FEATURE_HYPERVISOR),189 "vCPU%d: Too many pages still idle (%lu out of %lu)",190 vcpu_idx, still_idle, pages);191#endif192 printf("WARNING: vCPU%d: Too many pages still idle (%lu out of %lu), "193 "this will affect performance results.\n",194 vcpu_idx, still_idle, pages);195 }196 197 close(page_idle_fd);198 close(pagemap_fd);199}200 201static void assert_ucall(struct kvm_vcpu *vcpu, uint64_t expected_ucall)202{203 struct ucall uc;204 uint64_t actual_ucall = get_ucall(vcpu, &uc);205 206 TEST_ASSERT(expected_ucall == actual_ucall,207 "Guest exited unexpectedly (expected ucall %" PRIu64208 ", got %" PRIu64 ")",209 expected_ucall, actual_ucall);210}211 212static bool spin_wait_for_next_iteration(int *current_iteration)213{214 int last_iteration = *current_iteration;215 216 do {217 if (READ_ONCE(memstress_args.stop_vcpus))218 return false;219 220 *current_iteration = READ_ONCE(iteration);221 } while (last_iteration == *current_iteration);222 223 return true;224}225 226static void vcpu_thread_main(struct memstress_vcpu_args *vcpu_args)227{228 struct kvm_vcpu *vcpu = vcpu_args->vcpu;229 struct kvm_vm *vm = memstress_args.vm;230 int vcpu_idx = vcpu_args->vcpu_idx;231 int current_iteration = 0;232 233 while (spin_wait_for_next_iteration(¤t_iteration)) {234 switch (READ_ONCE(iteration_work)) {235 case ITERATION_ACCESS_MEMORY:236 vcpu_run(vcpu);237 assert_ucall(vcpu, UCALL_SYNC);238 break;239 case ITERATION_MARK_IDLE:240 mark_vcpu_memory_idle(vm, vcpu_args);241 break;242 };243 244 vcpu_last_completed_iteration[vcpu_idx] = current_iteration;245 }246}247 248static void spin_wait_for_vcpu(int vcpu_idx, int target_iteration)249{250 while (READ_ONCE(vcpu_last_completed_iteration[vcpu_idx]) !=251 target_iteration) {252 continue;253 }254}255 256/* The type of memory accesses to perform in the VM. */257enum access_type {258 ACCESS_READ,259 ACCESS_WRITE,260};261 262static void run_iteration(struct kvm_vm *vm, int nr_vcpus, const char *description)263{264 struct timespec ts_start;265 struct timespec ts_elapsed;266 int next_iteration, i;267 268 /* Kick off the vCPUs by incrementing iteration. */269 next_iteration = ++iteration;270 271 clock_gettime(CLOCK_MONOTONIC, &ts_start);272 273 /* Wait for all vCPUs to finish the iteration. */274 for (i = 0; i < nr_vcpus; i++)275 spin_wait_for_vcpu(i, next_iteration);276 277 ts_elapsed = timespec_elapsed(ts_start);278 pr_info("%-30s: %ld.%09lds\n",279 description, ts_elapsed.tv_sec, ts_elapsed.tv_nsec);280}281 282static void access_memory(struct kvm_vm *vm, int nr_vcpus,283 enum access_type access, const char *description)284{285 memstress_set_write_percent(vm, (access == ACCESS_READ) ? 0 : 100);286 iteration_work = ITERATION_ACCESS_MEMORY;287 run_iteration(vm, nr_vcpus, description);288}289 290static void mark_memory_idle(struct kvm_vm *vm, int nr_vcpus)291{292 /*293 * Even though this parallelizes the work across vCPUs, this is still a294 * very slow operation because page_idle forces the test to mark one pfn295 * at a time and the clear_young notifier serializes on the KVM MMU296 * lock.297 */298 pr_debug("Marking VM memory idle (slow)...\n");299 iteration_work = ITERATION_MARK_IDLE;300 run_iteration(vm, nr_vcpus, "Mark memory idle");301}302 303static void run_test(enum vm_guest_mode mode, void *arg)304{305 struct test_params *params = arg;306 struct kvm_vm *vm;307 int nr_vcpus = params->nr_vcpus;308 309 vm = memstress_create_vm(mode, nr_vcpus, params->vcpu_memory_bytes, 1,310 params->backing_src, !overlap_memory_access);311 312 memstress_start_vcpu_threads(nr_vcpus, vcpu_thread_main);313 314 pr_info("\n");315 access_memory(vm, nr_vcpus, ACCESS_WRITE, "Populating memory");316 317 /* As a control, read and write to the populated memory first. */318 access_memory(vm, nr_vcpus, ACCESS_WRITE, "Writing to populated memory");319 access_memory(vm, nr_vcpus, ACCESS_READ, "Reading from populated memory");320 321 /* Repeat on memory that has been marked as idle. */322 mark_memory_idle(vm, nr_vcpus);323 access_memory(vm, nr_vcpus, ACCESS_WRITE, "Writing to idle memory");324 mark_memory_idle(vm, nr_vcpus);325 access_memory(vm, nr_vcpus, ACCESS_READ, "Reading from idle memory");326 327 memstress_join_vcpu_threads(nr_vcpus);328 memstress_destroy_vm(vm);329}330 331static void help(char *name)332{333 puts("");334 printf("usage: %s [-h] [-m mode] [-b vcpu_bytes] [-v vcpus] [-o] [-s mem_type]\n",335 name);336 puts("");337 printf(" -h: Display this help message.");338 guest_modes_help();339 printf(" -b: specify the size of the memory region which should be\n"340 " dirtied by each vCPU. e.g. 10M or 3G.\n"341 " (default: 1G)\n");342 printf(" -v: specify the number of vCPUs to run.\n");343 printf(" -o: Overlap guest memory accesses instead of partitioning\n"344 " them into a separate region of memory for each vCPU.\n");345 backing_src_help("-s");346 puts("");347 exit(0);348}349 350int main(int argc, char *argv[])351{352 struct test_params params = {353 .backing_src = DEFAULT_VM_MEM_SRC,354 .vcpu_memory_bytes = DEFAULT_PER_VCPU_MEM_SIZE,355 .nr_vcpus = 1,356 };357 int page_idle_fd;358 int opt;359 360 guest_modes_append_default();361 362 while ((opt = getopt(argc, argv, "hm:b:v:os:")) != -1) {363 switch (opt) {364 case 'm':365 guest_modes_cmdline(optarg);366 break;367 case 'b':368 params.vcpu_memory_bytes = parse_size(optarg);369 break;370 case 'v':371 params.nr_vcpus = atoi_positive("Number of vCPUs", optarg);372 break;373 case 'o':374 overlap_memory_access = true;375 break;376 case 's':377 params.backing_src = parse_backing_src_type(optarg);378 break;379 case 'h':380 default:381 help(argv[0]);382 break;383 }384 }385 386 page_idle_fd = open("/sys/kernel/mm/page_idle/bitmap", O_RDWR);387 __TEST_REQUIRE(page_idle_fd >= 0,388 "CONFIG_IDLE_PAGE_TRACKING is not enabled");389 close(page_idle_fd);390 391 for_each_guest_mode(run_test, ¶ms);392 393 return 0;394}395