949 lines · c
1// SPDX-License-Identifier: GPL-2.02 3#include <sys/mman.h>4#include <sys/prctl.h>5#include <sys/wait.h>6#include <stdbool.h>7#include <time.h>8#include <string.h>9#include <numa.h>10#include <unistd.h>11#include <fcntl.h>12#include <stdint.h>13#include <err.h>14 15#include "../kselftest.h"16#include <include/vdso/time64.h>17#include "vm_util.h"18 19#define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/"20#define KSM_FP(s) (KSM_SYSFS_PATH s)21#define KSM_SCAN_LIMIT_SEC_DEFAULT 12022#define KSM_PAGE_COUNT_DEFAULT 10l23#define KSM_PROT_STR_DEFAULT "rw"24#define KSM_USE_ZERO_PAGES_DEFAULT false25#define KSM_MERGE_ACROSS_NODES_DEFAULT true26#define KSM_MERGE_TYPE_DEFAULT 027#define MB (1ul << 20)28 29struct ksm_sysfs {30 unsigned long max_page_sharing;31 unsigned long merge_across_nodes;32 unsigned long pages_to_scan;33 unsigned long run;34 unsigned long sleep_millisecs;35 unsigned long stable_node_chains_prune_millisecs;36 unsigned long use_zero_pages;37};38 39enum ksm_merge_type {40 KSM_MERGE_MADVISE,41 KSM_MERGE_PRCTL,42 KSM_MERGE_LAST = KSM_MERGE_PRCTL43};44 45enum ksm_test_name {46 CHECK_KSM_MERGE,47 CHECK_KSM_UNMERGE,48 CHECK_KSM_GET_MERGE_TYPE,49 CHECK_KSM_ZERO_PAGE_MERGE,50 CHECK_KSM_NUMA_MERGE,51 KSM_MERGE_TIME,52 KSM_MERGE_TIME_HUGE_PAGES,53 KSM_UNMERGE_TIME,54 KSM_COW_TIME55};56 57int debug;58 59static int ksm_write_sysfs(const char *file_path, unsigned long val)60{61 FILE *f = fopen(file_path, "w");62 63 if (!f) {64 fprintf(stderr, "f %s\n", file_path);65 perror("fopen");66 return 1;67 }68 if (fprintf(f, "%lu", val) < 0) {69 perror("fprintf");70 fclose(f);71 return 1;72 }73 fclose(f);74 75 return 0;76}77 78static int ksm_read_sysfs(const char *file_path, unsigned long *val)79{80 FILE *f = fopen(file_path, "r");81 82 if (!f) {83 fprintf(stderr, "f %s\n", file_path);84 perror("fopen");85 return 1;86 }87 if (fscanf(f, "%lu", val) != 1) {88 perror("fscanf");89 fclose(f);90 return 1;91 }92 fclose(f);93 94 return 0;95}96 97static void ksm_print_sysfs(void)98{99 unsigned long max_page_sharing, pages_sharing, pages_shared;100 unsigned long full_scans, pages_unshared, pages_volatile;101 unsigned long stable_node_chains, stable_node_dups;102 long general_profit;103 104 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||105 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||106 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing) ||107 ksm_read_sysfs(KSM_FP("full_scans"), &full_scans) ||108 ksm_read_sysfs(KSM_FP("pages_unshared"), &pages_unshared) ||109 ksm_read_sysfs(KSM_FP("pages_volatile"), &pages_volatile) ||110 ksm_read_sysfs(KSM_FP("stable_node_chains"), &stable_node_chains) ||111 ksm_read_sysfs(KSM_FP("stable_node_dups"), &stable_node_dups) ||112 ksm_read_sysfs(KSM_FP("general_profit"), (unsigned long *)&general_profit))113 return;114 115 printf("pages_shared : %lu\n", pages_shared);116 printf("pages_sharing : %lu\n", pages_sharing);117 printf("max_page_sharing : %lu\n", max_page_sharing);118 printf("full_scans : %lu\n", full_scans);119 printf("pages_unshared : %lu\n", pages_unshared);120 printf("pages_volatile : %lu\n", pages_volatile);121 printf("stable_node_chains: %lu\n", stable_node_chains);122 printf("stable_node_dups : %lu\n", stable_node_dups);123 printf("general_profit : %ld\n", general_profit);124}125 126static void ksm_print_procfs(void)127{128 const char *file_name = "/proc/self/ksm_stat";129 char buffer[512];130 FILE *f = fopen(file_name, "r");131 132 if (!f) {133 fprintf(stderr, "f %s\n", file_name);134 perror("fopen");135 return;136 }137 138 while (fgets(buffer, sizeof(buffer), f))139 printf("%s", buffer);140 141 fclose(f);142}143 144static int str_to_prot(char *prot_str)145{146 int prot = 0;147 148 if ((strchr(prot_str, 'r')) != NULL)149 prot |= PROT_READ;150 if ((strchr(prot_str, 'w')) != NULL)151 prot |= PROT_WRITE;152 if ((strchr(prot_str, 'x')) != NULL)153 prot |= PROT_EXEC;154 155 return prot;156}157 158static void print_help(void)159{160 printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n"161 "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n");162 163 printf("Supported <test type>:\n"164 " -M (page merging)\n"165 " -Z (zero pages merging)\n"166 " -N (merging of pages in different NUMA nodes)\n"167 " -U (page unmerging)\n"168 " -P evaluate merging time and speed.\n"169 " For this test, the size of duplicated memory area (in MiB)\n"170 " must be provided using -s option\n"171 " -H evaluate merging time and speed of area allocated mostly with huge pages\n"172 " For this test, the size of duplicated memory area (in MiB)\n"173 " must be provided using -s option\n"174 " -D evaluate unmerging time and speed when disabling KSM.\n"175 " For this test, the size of duplicated memory area (in MiB)\n"176 " must be provided using -s option\n"177 " -C evaluate the time required to break COW of merged pages.\n\n");178 179 printf(" -a: specify the access protections of pages.\n"180 " <prot> must be of the form [rwx].\n"181 " Default: %s\n", KSM_PROT_STR_DEFAULT);182 printf(" -p: specify the number of pages to test.\n"183 " Default: %ld\n", KSM_PAGE_COUNT_DEFAULT);184 printf(" -l: limit the maximum running time (in seconds) for a test.\n"185 " Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT);186 printf(" -z: change use_zero_pages tunable\n"187 " Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT);188 printf(" -m: change merge_across_nodes tunable\n"189 " Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT);190 printf(" -d: turn debugging output on\n");191 printf(" -s: the size of duplicated memory area (in MiB)\n");192 printf(" -t: KSM merge type\n"193 " Default: 0\n"194 " 0: madvise merging\n"195 " 1: prctl merging\n");196 197 exit(0);198}199 200static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size)201{202 void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0);203 204 if (!map_ptr) {205 perror("mmap");206 return NULL;207 }208 memset(map_ptr, data, map_size);209 if (mprotect(map_ptr, map_size, prot)) {210 perror("mprotect");211 munmap(map_ptr, map_size);212 return NULL;213 }214 215 return map_ptr;216}217 218static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout)219{220 struct timespec cur_time;221 unsigned long cur_scan, init_scan;222 223 if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan))224 return 1;225 cur_scan = init_scan;226 227 while (cur_scan < init_scan + scan_count) {228 if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan))229 return 1;230 if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) {231 perror("clock_gettime");232 return 1;233 }234 if ((cur_time.tv_sec - start_time.tv_sec) > timeout) {235 printf("Scan time limit exceeded\n");236 return 1;237 }238 }239 240 return 0;241}242 243static int ksm_merge_pages(int merge_type, void *addr, size_t size,244 struct timespec start_time, int timeout)245{246 if (merge_type == KSM_MERGE_MADVISE) {247 if (madvise(addr, size, MADV_MERGEABLE)) {248 perror("madvise");249 return 1;250 }251 } else if (merge_type == KSM_MERGE_PRCTL) {252 if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) {253 perror("prctl");254 return 1;255 }256 }257 258 if (ksm_write_sysfs(KSM_FP("run"), 1))259 return 1;260 261 /* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */262 if (ksm_do_scan(2, start_time, timeout))263 return 1;264 265 return 0;266}267 268static int ksm_unmerge_pages(void *addr, size_t size,269 struct timespec start_time, int timeout)270{271 if (madvise(addr, size, MADV_UNMERGEABLE)) {272 perror("madvise");273 return 1;274 }275 return 0;276}277 278static bool assert_ksm_pages_count(long dupl_page_count)279{280 unsigned long max_page_sharing, pages_sharing, pages_shared;281 282 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||283 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||284 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing))285 return false;286 287 if (debug) {288 ksm_print_sysfs();289 ksm_print_procfs();290 }291 292 /*293 * Since there must be at least 2 pages for merging and 1 page can be294 * shared with the limited number of pages (max_page_sharing), sometimes295 * there are 'leftover' pages that cannot be merged. For example, if there296 * are 11 pages and max_page_sharing = 10, then only 10 pages will be297 * merged and the 11th page won't be affected. As a result, when the number298 * of duplicate pages is divided by max_page_sharing and the remainder is 1,299 * pages_shared and pages_sharing values will be equal between dupl_page_count300 * and dupl_page_count - 1.301 */302 if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) {303 if (pages_shared == dupl_page_count / max_page_sharing &&304 pages_sharing == pages_shared * (max_page_sharing - 1))305 return true;306 } else {307 if (pages_shared == (dupl_page_count / max_page_sharing + 1) &&308 pages_sharing == dupl_page_count - pages_shared)309 return true;310 }311 312 return false;313}314 315static int ksm_save_def(struct ksm_sysfs *ksm_sysfs)316{317 if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) ||318 numa_available() ? 0 :319 ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) ||320 ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) ||321 ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) ||322 ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) ||323 ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),324 &ksm_sysfs->stable_node_chains_prune_millisecs) ||325 ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages))326 return 1;327 328 return 0;329}330 331static int ksm_restore(struct ksm_sysfs *ksm_sysfs)332{333 if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) ||334 numa_available() ? 0 :335 ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) ||336 ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) ||337 ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) ||338 ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) ||339 ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),340 ksm_sysfs->stable_node_chains_prune_millisecs) ||341 ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages))342 return 1;343 344 return 0;345}346 347static int check_ksm_merge(int merge_type, int mapping, int prot,348 long page_count, int timeout, size_t page_size)349{350 void *map_ptr;351 struct timespec start_time;352 353 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {354 perror("clock_gettime");355 return KSFT_FAIL;356 }357 358 /* fill pages with the same data and merge them */359 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);360 if (!map_ptr)361 return KSFT_FAIL;362 363 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))364 goto err_out;365 366 /* verify that the right number of pages are merged */367 if (assert_ksm_pages_count(page_count)) {368 printf("OK\n");369 munmap(map_ptr, page_size * page_count);370 if (merge_type == KSM_MERGE_PRCTL)371 prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0);372 return KSFT_PASS;373 }374 375err_out:376 printf("Not OK\n");377 munmap(map_ptr, page_size * page_count);378 return KSFT_FAIL;379}380 381static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, size_t page_size)382{383 void *map_ptr;384 struct timespec start_time;385 int page_count = 2;386 387 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {388 perror("clock_gettime");389 return KSFT_FAIL;390 }391 392 /* fill pages with the same data and merge them */393 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);394 if (!map_ptr)395 return KSFT_FAIL;396 397 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))398 goto err_out;399 400 /* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */401 memset(map_ptr, '-', 1);402 memset(map_ptr + page_size, '+', 1);403 404 /* get at least 1 scan, so KSM can detect that the pages were modified */405 if (ksm_do_scan(1, start_time, timeout))406 goto err_out;407 408 /* check that unmerging was successful and 0 pages are currently merged */409 if (assert_ksm_pages_count(0)) {410 printf("OK\n");411 munmap(map_ptr, page_size * page_count);412 return KSFT_PASS;413 }414 415err_out:416 printf("Not OK\n");417 munmap(map_ptr, page_size * page_count);418 return KSFT_FAIL;419}420 421static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long page_count,422 int timeout, bool use_zero_pages, size_t page_size)423{424 void *map_ptr;425 struct timespec start_time;426 427 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {428 perror("clock_gettime");429 return KSFT_FAIL;430 }431 432 if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages))433 return KSFT_FAIL;434 435 /* fill pages with zero and try to merge them */436 map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count);437 if (!map_ptr)438 return KSFT_FAIL;439 440 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))441 goto err_out;442 443 /*444 * verify that the right number of pages are merged:445 * 1) if use_zero_pages is set to 1, empty pages are merged446 * with the kernel zero page instead of with each other;447 * 2) if use_zero_pages is set to 0, empty pages are not treated specially448 * and merged as usual.449 */450 if (use_zero_pages && !assert_ksm_pages_count(0))451 goto err_out;452 else if (!use_zero_pages && !assert_ksm_pages_count(page_count))453 goto err_out;454 455 printf("OK\n");456 munmap(map_ptr, page_size * page_count);457 return KSFT_PASS;458 459err_out:460 printf("Not OK\n");461 munmap(map_ptr, page_size * page_count);462 return KSFT_FAIL;463}464 465static int get_next_mem_node(int node)466{467 468 long node_size;469 int mem_node = 0;470 int i, max_node = numa_max_node();471 472 for (i = node + 1; i <= max_node + node; i++) {473 mem_node = i % (max_node + 1);474 node_size = numa_node_size(mem_node, NULL);475 if (node_size > 0)476 break;477 }478 return mem_node;479}480 481static int get_first_mem_node(void)482{483 return get_next_mem_node(numa_max_node());484}485 486static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeout,487 bool merge_across_nodes, size_t page_size)488{489 void *numa1_map_ptr, *numa2_map_ptr;490 struct timespec start_time;491 int page_count = 2;492 int first_node;493 494 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {495 perror("clock_gettime");496 return KSFT_FAIL;497 }498 499 if (numa_available() < 0) {500 perror("NUMA support not enabled");501 return KSFT_SKIP;502 }503 if (numa_num_configured_nodes() <= 1) {504 printf("At least 2 NUMA nodes must be available\n");505 return KSFT_SKIP;506 }507 if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes))508 return KSFT_FAIL;509 510 /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */511 first_node = get_first_mem_node();512 numa1_map_ptr = numa_alloc_onnode(page_size, first_node);513 numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node));514 if (!numa1_map_ptr || !numa2_map_ptr) {515 perror("numa_alloc_onnode");516 return KSFT_FAIL;517 }518 519 memset(numa1_map_ptr, '*', page_size);520 memset(numa2_map_ptr, '*', page_size);521 522 /* try to merge the pages */523 if (ksm_merge_pages(merge_type, numa1_map_ptr, page_size, start_time, timeout) ||524 ksm_merge_pages(merge_type, numa2_map_ptr, page_size, start_time, timeout))525 goto err_out;526 527 /*528 * verify that the right number of pages are merged:529 * 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged;530 * 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is531 * only 1 unique page in each node and they can't be shared.532 */533 if (merge_across_nodes && !assert_ksm_pages_count(page_count))534 goto err_out;535 else if (!merge_across_nodes && !assert_ksm_pages_count(0))536 goto err_out;537 538 numa_free(numa1_map_ptr, page_size);539 numa_free(numa2_map_ptr, page_size);540 printf("OK\n");541 return KSFT_PASS;542 543err_out:544 numa_free(numa1_map_ptr, page_size);545 numa_free(numa2_map_ptr, page_size);546 printf("Not OK\n");547 return KSFT_FAIL;548}549 550static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot,551 int timeout, size_t map_size)552{553 void *map_ptr, *map_ptr_orig;554 struct timespec start_time, end_time;555 unsigned long scan_time_ns;556 int pagemap_fd, n_normal_pages, n_huge_pages;557 558 map_size *= MB;559 size_t len = map_size;560 561 len -= len % HPAGE_SIZE;562 map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE,563 MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0);564 map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE;565 566 if (map_ptr_orig == MAP_FAILED)567 err(2, "initial mmap");568 569 if (madvise(map_ptr, len, MADV_HUGEPAGE))570 err(2, "MADV_HUGEPAGE");571 572 pagemap_fd = open("/proc/self/pagemap", O_RDONLY);573 if (pagemap_fd < 0)574 err(2, "open pagemap");575 576 n_normal_pages = 0;577 n_huge_pages = 0;578 for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) {579 if (allocate_transhuge(p, pagemap_fd) < 0)580 n_normal_pages++;581 else582 n_huge_pages++;583 }584 printf("Number of normal pages: %d\n", n_normal_pages);585 printf("Number of huge pages: %d\n", n_huge_pages);586 587 memset(map_ptr, '*', len);588 589 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {590 perror("clock_gettime");591 goto err_out;592 }593 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))594 goto err_out;595 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {596 perror("clock_gettime");597 goto err_out;598 }599 600 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +601 (end_time.tv_nsec - start_time.tv_nsec);602 603 printf("Total size: %lu MiB\n", map_size / MB);604 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,605 scan_time_ns % NSEC_PER_SEC);606 printf("Average speed: %.3f MiB/s\n", (map_size / MB) /607 ((double)scan_time_ns / NSEC_PER_SEC));608 609 munmap(map_ptr_orig, len + HPAGE_SIZE);610 return KSFT_PASS;611 612err_out:613 printf("Not OK\n");614 munmap(map_ptr_orig, len + HPAGE_SIZE);615 return KSFT_FAIL;616}617 618static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size)619{620 void *map_ptr;621 struct timespec start_time, end_time;622 unsigned long scan_time_ns;623 624 map_size *= MB;625 626 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);627 if (!map_ptr)628 return KSFT_FAIL;629 630 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {631 perror("clock_gettime");632 goto err_out;633 }634 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))635 goto err_out;636 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {637 perror("clock_gettime");638 goto err_out;639 }640 641 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +642 (end_time.tv_nsec - start_time.tv_nsec);643 644 printf("Total size: %lu MiB\n", map_size / MB);645 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,646 scan_time_ns % NSEC_PER_SEC);647 printf("Average speed: %.3f MiB/s\n", (map_size / MB) /648 ((double)scan_time_ns / NSEC_PER_SEC));649 650 munmap(map_ptr, map_size);651 return KSFT_PASS;652 653err_out:654 printf("Not OK\n");655 munmap(map_ptr, map_size);656 return KSFT_FAIL;657}658 659static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size)660{661 void *map_ptr;662 struct timespec start_time, end_time;663 unsigned long scan_time_ns;664 665 map_size *= MB;666 667 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);668 if (!map_ptr)669 return KSFT_FAIL;670 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {671 perror("clock_gettime");672 goto err_out;673 }674 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout))675 goto err_out;676 677 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {678 perror("clock_gettime");679 goto err_out;680 }681 if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout))682 goto err_out;683 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {684 perror("clock_gettime");685 goto err_out;686 }687 688 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +689 (end_time.tv_nsec - start_time.tv_nsec);690 691 printf("Total size: %lu MiB\n", map_size / MB);692 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,693 scan_time_ns % NSEC_PER_SEC);694 printf("Average speed: %.3f MiB/s\n", (map_size / MB) /695 ((double)scan_time_ns / NSEC_PER_SEC));696 697 munmap(map_ptr, map_size);698 return KSFT_PASS;699 700err_out:701 printf("Not OK\n");702 munmap(map_ptr, map_size);703 return KSFT_FAIL;704}705 706static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size_t page_size)707{708 void *map_ptr;709 struct timespec start_time, end_time;710 unsigned long cow_time_ns;711 712 /* page_count must be less than 2*page_size */713 size_t page_count = 4000;714 715 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);716 if (!map_ptr)717 return KSFT_FAIL;718 719 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {720 perror("clock_gettime");721 return KSFT_FAIL;722 }723 for (size_t i = 0; i < page_count - 1; i = i + 2)724 memset(map_ptr + page_size * i, '-', 1);725 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {726 perror("clock_gettime");727 return KSFT_FAIL;728 }729 730 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +731 (end_time.tv_nsec - start_time.tv_nsec);732 733 printf("Total size: %lu MiB\n\n", (page_size * page_count) / MB);734 printf("Not merged pages:\n");735 printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,736 cow_time_ns % NSEC_PER_SEC);737 printf("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) /738 ((double)cow_time_ns / NSEC_PER_SEC));739 740 /* Create 2000 pairs of duplicate pages */741 for (size_t i = 0; i < page_count - 1; i = i + 2) {742 memset(map_ptr + page_size * i, '+', i / 2 + 1);743 memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1);744 }745 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout))746 goto err_out;747 748 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {749 perror("clock_gettime");750 goto err_out;751 }752 for (size_t i = 0; i < page_count - 1; i = i + 2)753 memset(map_ptr + page_size * i, '-', 1);754 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {755 perror("clock_gettime");756 goto err_out;757 }758 759 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +760 (end_time.tv_nsec - start_time.tv_nsec);761 762 printf("Merged pages:\n");763 printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,764 cow_time_ns % NSEC_PER_SEC);765 printf("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) /766 ((double)cow_time_ns / NSEC_PER_SEC));767 768 munmap(map_ptr, page_size * page_count);769 return KSFT_PASS;770 771err_out:772 printf("Not OK\n");773 munmap(map_ptr, page_size * page_count);774 return KSFT_FAIL;775}776 777int main(int argc, char *argv[])778{779 int ret, opt;780 int prot = 0;781 int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT;782 int merge_type = KSM_MERGE_TYPE_DEFAULT;783 long page_count = KSM_PAGE_COUNT_DEFAULT;784 size_t page_size = sysconf(_SC_PAGESIZE);785 struct ksm_sysfs ksm_sysfs_old;786 int test_name = CHECK_KSM_MERGE;787 bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT;788 bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT;789 long size_MB = 0;790 791 while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) {792 switch (opt) {793 case 'a':794 prot = str_to_prot(optarg);795 break;796 case 'p':797 page_count = atol(optarg);798 if (page_count <= 0) {799 printf("The number of pages must be greater than 0\n");800 return KSFT_FAIL;801 }802 break;803 case 'l':804 ksm_scan_limit_sec = atoi(optarg);805 if (ksm_scan_limit_sec <= 0) {806 printf("Timeout value must be greater than 0\n");807 return KSFT_FAIL;808 }809 break;810 case 'h':811 print_help();812 break;813 case 'z':814 if (strcmp(optarg, "0") == 0)815 use_zero_pages = 0;816 else817 use_zero_pages = 1;818 break;819 case 'm':820 if (strcmp(optarg, "0") == 0)821 merge_across_nodes = 0;822 else823 merge_across_nodes = 1;824 break;825 case 'd':826 debug = 1;827 break;828 case 's':829 size_MB = atoi(optarg);830 if (size_MB <= 0) {831 printf("Size must be greater than 0\n");832 return KSFT_FAIL;833 }834 break;835 case 't':836 {837 int tmp = atoi(optarg);838 839 if (tmp < 0 || tmp > KSM_MERGE_LAST) {840 printf("Invalid merge type\n");841 return KSFT_FAIL;842 }843 merge_type = tmp;844 }845 break;846 case 'M':847 break;848 case 'U':849 test_name = CHECK_KSM_UNMERGE;850 break;851 case 'Z':852 test_name = CHECK_KSM_ZERO_PAGE_MERGE;853 break;854 case 'N':855 test_name = CHECK_KSM_NUMA_MERGE;856 break;857 case 'P':858 test_name = KSM_MERGE_TIME;859 break;860 case 'H':861 test_name = KSM_MERGE_TIME_HUGE_PAGES;862 break;863 case 'D':864 test_name = KSM_UNMERGE_TIME;865 break;866 case 'C':867 test_name = KSM_COW_TIME;868 break;869 default:870 return KSFT_FAIL;871 }872 }873 874 if (prot == 0)875 prot = str_to_prot(KSM_PROT_STR_DEFAULT);876 877 if (access(KSM_SYSFS_PATH, F_OK)) {878 printf("Config KSM not enabled\n");879 return KSFT_SKIP;880 }881 882 if (ksm_save_def(&ksm_sysfs_old)) {883 printf("Cannot save default tunables\n");884 return KSFT_FAIL;885 }886 887 if (ksm_write_sysfs(KSM_FP("run"), 2) ||888 ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) ||889 numa_available() ? 0 :890 ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) ||891 ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count))892 return KSFT_FAIL;893 894 switch (test_name) {895 case CHECK_KSM_MERGE:896 ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count,897 ksm_scan_limit_sec, page_size);898 break;899 case CHECK_KSM_UNMERGE:900 ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,901 ksm_scan_limit_sec, page_size);902 break;903 case CHECK_KSM_ZERO_PAGE_MERGE:904 ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,905 page_count, ksm_scan_limit_sec, use_zero_pages,906 page_size);907 break;908 case CHECK_KSM_NUMA_MERGE:909 ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,910 ksm_scan_limit_sec, merge_across_nodes, page_size);911 break;912 case KSM_MERGE_TIME:913 if (size_MB == 0) {914 printf("Option '-s' is required.\n");915 return KSFT_FAIL;916 }917 ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,918 ksm_scan_limit_sec, size_MB);919 break;920 case KSM_MERGE_TIME_HUGE_PAGES:921 if (size_MB == 0) {922 printf("Option '-s' is required.\n");923 return KSFT_FAIL;924 }925 ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,926 ksm_scan_limit_sec, size_MB);927 break;928 case KSM_UNMERGE_TIME:929 if (size_MB == 0) {930 printf("Option '-s' is required.\n");931 return KSFT_FAIL;932 }933 ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,934 ksm_scan_limit_sec, size_MB);935 break;936 case KSM_COW_TIME:937 ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot,938 ksm_scan_limit_sec, page_size);939 break;940 }941 942 if (ksm_restore(&ksm_sysfs_old)) {943 printf("Cannot restore default tunables\n");944 return KSFT_FAIL;945 }946 947 return ret;948}949