714 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * KSM functional tests4 *5 * Copyright 2022, Red Hat, Inc.6 *7 * Author(s): David Hildenbrand <david@redhat.com>8 */9#define _GNU_SOURCE10#include <stdlib.h>11#include <string.h>12#include <stdbool.h>13#include <stdint.h>14#include <asm-generic/unistd.h>15#include <errno.h>16#include <fcntl.h>17#include <sys/mman.h>18#include <sys/prctl.h>19#include <sys/syscall.h>20#include <sys/ioctl.h>21#include <sys/wait.h>22#include <linux/userfaultfd.h>23 24#include "../kselftest.h"25#include "vm_util.h"26 27#define KiB 1024u28#define MiB (1024 * KiB)29#define FORK_EXEC_CHILD_PRG_NAME "ksm_fork_exec_child"30 31#define MAP_MERGE_FAIL ((void *)-1)32#define MAP_MERGE_SKIP ((void *)-2)33 34enum ksm_merge_mode {35 KSM_MERGE_PRCTL,36 KSM_MERGE_MADVISE,37 KSM_MERGE_NONE, /* PRCTL already set */38};39 40static int mem_fd;41static int ksm_fd;42static int ksm_full_scans_fd;43static int proc_self_ksm_stat_fd;44static int proc_self_ksm_merging_pages_fd;45static int ksm_use_zero_pages_fd;46static int pagemap_fd;47static size_t pagesize;48 49static bool range_maps_duplicates(char *addr, unsigned long size)50{51 unsigned long offs_a, offs_b, pfn_a, pfn_b;52 53 /*54 * There is no easy way to check if there are KSM pages mapped into55 * this range. We only check that the range does not map the same PFN56 * twice by comparing each pair of mapped pages.57 */58 for (offs_a = 0; offs_a < size; offs_a += pagesize) {59 pfn_a = pagemap_get_pfn(pagemap_fd, addr + offs_a);60 /* Page not present or PFN not exposed by the kernel. */61 if (pfn_a == -1ul || !pfn_a)62 continue;63 64 for (offs_b = offs_a + pagesize; offs_b < size;65 offs_b += pagesize) {66 pfn_b = pagemap_get_pfn(pagemap_fd, addr + offs_b);67 if (pfn_b == -1ul || !pfn_b)68 continue;69 if (pfn_a == pfn_b)70 return true;71 }72 }73 return false;74}75 76static long get_my_ksm_zero_pages(void)77{78 char buf[200];79 char *substr_ksm_zero;80 size_t value_pos;81 ssize_t read_size;82 unsigned long my_ksm_zero_pages;83 84 if (!proc_self_ksm_stat_fd)85 return 0;86 87 read_size = pread(proc_self_ksm_stat_fd, buf, sizeof(buf) - 1, 0);88 if (read_size < 0)89 return -errno;90 91 buf[read_size] = 0;92 93 substr_ksm_zero = strstr(buf, "ksm_zero_pages");94 if (!substr_ksm_zero)95 return 0;96 97 value_pos = strcspn(substr_ksm_zero, "0123456789");98 my_ksm_zero_pages = strtol(substr_ksm_zero + value_pos, NULL, 10);99 100 return my_ksm_zero_pages;101}102 103static long get_my_merging_pages(void)104{105 char buf[10];106 ssize_t ret;107 108 if (proc_self_ksm_merging_pages_fd < 0)109 return proc_self_ksm_merging_pages_fd;110 111 ret = pread(proc_self_ksm_merging_pages_fd, buf, sizeof(buf) - 1, 0);112 if (ret <= 0)113 return -errno;114 buf[ret] = 0;115 116 return strtol(buf, NULL, 10);117}118 119static long ksm_get_full_scans(void)120{121 char buf[10];122 ssize_t ret;123 124 ret = pread(ksm_full_scans_fd, buf, sizeof(buf) - 1, 0);125 if (ret <= 0)126 return -errno;127 buf[ret] = 0;128 129 return strtol(buf, NULL, 10);130}131 132static int ksm_merge(void)133{134 long start_scans, end_scans;135 136 /* Wait for two full scans such that any possible merging happened. */137 start_scans = ksm_get_full_scans();138 if (start_scans < 0)139 return start_scans;140 if (write(ksm_fd, "1", 1) != 1)141 return -errno;142 do {143 end_scans = ksm_get_full_scans();144 if (end_scans < 0)145 return end_scans;146 } while (end_scans < start_scans + 2);147 148 return 0;149}150 151static int ksm_unmerge(void)152{153 if (write(ksm_fd, "2", 1) != 1)154 return -errno;155 return 0;156}157 158static char *__mmap_and_merge_range(char val, unsigned long size, int prot,159 enum ksm_merge_mode mode)160{161 char *map;162 char *err_map = MAP_MERGE_FAIL;163 int ret;164 165 /* Stabilize accounting by disabling KSM completely. */166 if (ksm_unmerge()) {167 ksft_print_msg("Disabling (unmerging) KSM failed\n");168 return err_map;169 }170 171 if (get_my_merging_pages() > 0) {172 ksft_print_msg("Still pages merged\n");173 return err_map;174 }175 176 map = mmap(NULL, size, PROT_READ|PROT_WRITE,177 MAP_PRIVATE|MAP_ANON, -1, 0);178 if (map == MAP_FAILED) {179 ksft_print_msg("mmap() failed\n");180 return err_map;181 }182 183 /* Don't use THP. Ignore if THP are not around on a kernel. */184 if (madvise(map, size, MADV_NOHUGEPAGE) && errno != EINVAL) {185 ksft_print_msg("MADV_NOHUGEPAGE failed\n");186 goto unmap;187 }188 189 /* Make sure each page contains the same values to merge them. */190 memset(map, val, size);191 192 if (mprotect(map, size, prot)) {193 ksft_print_msg("mprotect() failed\n");194 err_map = MAP_MERGE_SKIP;195 goto unmap;196 }197 198 switch (mode) {199 case KSM_MERGE_PRCTL:200 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0);201 if (ret < 0 && errno == EINVAL) {202 ksft_print_msg("PR_SET_MEMORY_MERGE not supported\n");203 err_map = MAP_MERGE_SKIP;204 goto unmap;205 } else if (ret) {206 ksft_print_msg("PR_SET_MEMORY_MERGE=1 failed\n");207 goto unmap;208 }209 break;210 case KSM_MERGE_MADVISE:211 if (madvise(map, size, MADV_MERGEABLE)) {212 ksft_print_msg("MADV_MERGEABLE failed\n");213 goto unmap;214 }215 break;216 case KSM_MERGE_NONE:217 break;218 }219 220 /* Run KSM to trigger merging and wait. */221 if (ksm_merge()) {222 ksft_print_msg("Running KSM failed\n");223 goto unmap;224 }225 226 /*227 * Check if anything was merged at all. Ignore the zero page that is228 * accounted differently (depending on kernel support).229 */230 if (val && !get_my_merging_pages()) {231 ksft_print_msg("No pages got merged\n");232 goto unmap;233 }234 235 return map;236unmap:237 munmap(map, size);238 return err_map;239}240 241static char *mmap_and_merge_range(char val, unsigned long size, int prot,242 enum ksm_merge_mode mode)243{244 char *map;245 char *ret = MAP_FAILED;246 247 map = __mmap_and_merge_range(val, size, prot, mode);248 if (map == MAP_MERGE_FAIL)249 ksft_test_result_fail("Merging memory failed");250 else if (map == MAP_MERGE_SKIP)251 ksft_test_result_skip("Merging memory skipped");252 else253 ret = map;254 255 return ret;256}257 258static void test_unmerge(void)259{260 const unsigned int size = 2 * MiB;261 char *map;262 263 ksft_print_msg("[RUN] %s\n", __func__);264 265 map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, KSM_MERGE_MADVISE);266 if (map == MAP_FAILED)267 return;268 269 if (madvise(map, size, MADV_UNMERGEABLE)) {270 ksft_test_result_fail("MADV_UNMERGEABLE failed\n");271 goto unmap;272 }273 274 ksft_test_result(!range_maps_duplicates(map, size),275 "Pages were unmerged\n");276unmap:277 munmap(map, size);278}279 280static void test_unmerge_zero_pages(void)281{282 const unsigned int size = 2 * MiB;283 char *map;284 unsigned int offs;285 unsigned long pages_expected;286 287 ksft_print_msg("[RUN] %s\n", __func__);288 289 if (proc_self_ksm_stat_fd < 0) {290 ksft_test_result_skip("open(\"/proc/self/ksm_stat\") failed\n");291 return;292 }293 if (ksm_use_zero_pages_fd < 0) {294 ksft_test_result_skip("open \"/sys/kernel/mm/ksm/use_zero_pages\" failed\n");295 return;296 }297 if (write(ksm_use_zero_pages_fd, "1", 1) != 1) {298 ksft_test_result_skip("write \"/sys/kernel/mm/ksm/use_zero_pages\" failed\n");299 return;300 }301 302 /* Let KSM deduplicate zero pages. */303 map = mmap_and_merge_range(0x00, size, PROT_READ | PROT_WRITE, KSM_MERGE_MADVISE);304 if (map == MAP_FAILED)305 return;306 307 /* Check if ksm_zero_pages is updated correctly after KSM merging */308 pages_expected = size / pagesize;309 if (pages_expected != get_my_ksm_zero_pages()) {310 ksft_test_result_fail("'ksm_zero_pages' updated after merging\n");311 goto unmap;312 }313 314 /* Try to unmerge half of the region */315 if (madvise(map, size / 2, MADV_UNMERGEABLE)) {316 ksft_test_result_fail("MADV_UNMERGEABLE failed\n");317 goto unmap;318 }319 320 /* Check if ksm_zero_pages is updated correctly after unmerging */321 pages_expected /= 2;322 if (pages_expected != get_my_ksm_zero_pages()) {323 ksft_test_result_fail("'ksm_zero_pages' updated after unmerging\n");324 goto unmap;325 }326 327 /* Trigger unmerging of the other half by writing to the pages. */328 for (offs = size / 2; offs < size; offs += pagesize)329 *((unsigned int *)&map[offs]) = offs;330 331 /* Now we should have no zeropages remaining. */332 if (get_my_ksm_zero_pages()) {333 ksft_test_result_fail("'ksm_zero_pages' updated after write fault\n");334 goto unmap;335 }336 337 /* Check if ksm zero pages are really unmerged */338 ksft_test_result(!range_maps_duplicates(map, size),339 "KSM zero pages were unmerged\n");340unmap:341 munmap(map, size);342}343 344static void test_unmerge_discarded(void)345{346 const unsigned int size = 2 * MiB;347 char *map;348 349 ksft_print_msg("[RUN] %s\n", __func__);350 351 map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, KSM_MERGE_MADVISE);352 if (map == MAP_FAILED)353 return;354 355 /* Discard half of all mapped pages so we have pte_none() entries. */356 if (madvise(map, size / 2, MADV_DONTNEED)) {357 ksft_test_result_fail("MADV_DONTNEED failed\n");358 goto unmap;359 }360 361 if (madvise(map, size, MADV_UNMERGEABLE)) {362 ksft_test_result_fail("MADV_UNMERGEABLE failed\n");363 goto unmap;364 }365 366 ksft_test_result(!range_maps_duplicates(map, size),367 "Pages were unmerged\n");368unmap:369 munmap(map, size);370}371 372static void test_unmerge_uffd_wp(void)373{374 struct uffdio_writeprotect uffd_writeprotect;375 const unsigned int size = 2 * MiB;376 struct uffdio_api uffdio_api;377 char *map;378 int uffd;379 380 ksft_print_msg("[RUN] %s\n", __func__);381 382 map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, KSM_MERGE_MADVISE);383 if (map == MAP_FAILED)384 return;385 386 /* See if UFFD is around. */387 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);388 if (uffd < 0) {389 ksft_test_result_skip("__NR_userfaultfd failed\n");390 goto unmap;391 }392 393 /* See if UFFD-WP is around. */394 uffdio_api.api = UFFD_API;395 uffdio_api.features = UFFD_FEATURE_PAGEFAULT_FLAG_WP;396 if (ioctl(uffd, UFFDIO_API, &uffdio_api) < 0) {397 ksft_test_result_fail("UFFDIO_API failed\n");398 goto close_uffd;399 }400 if (!(uffdio_api.features & UFFD_FEATURE_PAGEFAULT_FLAG_WP)) {401 ksft_test_result_skip("UFFD_FEATURE_PAGEFAULT_FLAG_WP not available\n");402 goto close_uffd;403 }404 405 /* Register UFFD-WP, no need for an actual handler. */406 if (uffd_register(uffd, map, size, false, true, false)) {407 ksft_test_result_fail("UFFDIO_REGISTER_MODE_WP failed\n");408 goto close_uffd;409 }410 411 /* Write-protect the range using UFFD-WP. */412 uffd_writeprotect.range.start = (unsigned long) map;413 uffd_writeprotect.range.len = size;414 uffd_writeprotect.mode = UFFDIO_WRITEPROTECT_MODE_WP;415 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &uffd_writeprotect)) {416 ksft_test_result_fail("UFFDIO_WRITEPROTECT failed\n");417 goto close_uffd;418 }419 420 if (madvise(map, size, MADV_UNMERGEABLE)) {421 ksft_test_result_fail("MADV_UNMERGEABLE failed\n");422 goto close_uffd;423 }424 425 ksft_test_result(!range_maps_duplicates(map, size),426 "Pages were unmerged\n");427close_uffd:428 close(uffd);429unmap:430 munmap(map, size);431}432 433/* Verify that KSM can be enabled / queried with prctl. */434static void test_prctl(void)435{436 int ret;437 438 ksft_print_msg("[RUN] %s\n", __func__);439 440 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0);441 if (ret < 0 && errno == EINVAL) {442 ksft_test_result_skip("PR_SET_MEMORY_MERGE not supported\n");443 return;444 } else if (ret) {445 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 failed\n");446 return;447 }448 449 ret = prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0);450 if (ret < 0) {451 ksft_test_result_fail("PR_GET_MEMORY_MERGE failed\n");452 return;453 } else if (ret != 1) {454 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 not effective\n");455 return;456 }457 458 ret = prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0);459 if (ret) {460 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n");461 return;462 }463 464 ret = prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0);465 if (ret < 0) {466 ksft_test_result_fail("PR_GET_MEMORY_MERGE failed\n");467 return;468 } else if (ret != 0) {469 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 not effective\n");470 return;471 }472 473 ksft_test_result_pass("Setting/clearing PR_SET_MEMORY_MERGE works\n");474}475 476static int test_child_ksm(void)477{478 const unsigned int size = 2 * MiB;479 char *map;480 481 /* Test if KSM is enabled for the process. */482 if (prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0) != 1)483 return -1;484 485 /* Test if merge could really happen. */486 map = __mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, KSM_MERGE_NONE);487 if (map == MAP_MERGE_FAIL)488 return -2;489 else if (map == MAP_MERGE_SKIP)490 return -3;491 492 munmap(map, size);493 return 0;494}495 496static void test_child_ksm_err(int status)497{498 if (status == -1)499 ksft_test_result_fail("unexpected PR_GET_MEMORY_MERGE result in child\n");500 else if (status == -2)501 ksft_test_result_fail("Merge in child failed\n");502 else if (status == -3)503 ksft_test_result_skip("Merge in child skipped\n");504}505 506/* Verify that prctl ksm flag is inherited. */507static void test_prctl_fork(void)508{509 int ret, status;510 pid_t child_pid;511 512 ksft_print_msg("[RUN] %s\n", __func__);513 514 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0);515 if (ret < 0 && errno == EINVAL) {516 ksft_test_result_skip("PR_SET_MEMORY_MERGE not supported\n");517 return;518 } else if (ret) {519 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 failed\n");520 return;521 }522 523 child_pid = fork();524 if (!child_pid) {525 exit(test_child_ksm());526 } else if (child_pid < 0) {527 ksft_test_result_fail("fork() failed\n");528 return;529 }530 531 if (waitpid(child_pid, &status, 0) < 0) {532 ksft_test_result_fail("waitpid() failed\n");533 return;534 }535 536 status = WEXITSTATUS(status);537 if (status) {538 test_child_ksm_err(status);539 return;540 }541 542 if (prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0)) {543 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n");544 return;545 }546 547 ksft_test_result_pass("PR_SET_MEMORY_MERGE value is inherited\n");548}549 550static void test_prctl_fork_exec(void)551{552 int ret, status;553 pid_t child_pid;554 555 ksft_print_msg("[RUN] %s\n", __func__);556 557 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0);558 if (ret < 0 && errno == EINVAL) {559 ksft_test_result_skip("PR_SET_MEMORY_MERGE not supported\n");560 return;561 } else if (ret) {562 ksft_test_result_fail("PR_SET_MEMORY_MERGE=1 failed\n");563 return;564 }565 566 child_pid = fork();567 if (child_pid == -1) {568 ksft_test_result_skip("fork() failed\n");569 return;570 } else if (child_pid == 0) {571 char *prg_name = "./ksm_functional_tests";572 char *argv_for_program[] = { prg_name, FORK_EXEC_CHILD_PRG_NAME };573 574 execv(prg_name, argv_for_program);575 return;576 }577 578 if (waitpid(child_pid, &status, 0) > 0) {579 if (WIFEXITED(status)) {580 status = WEXITSTATUS(status);581 if (status) {582 test_child_ksm_err(status);583 return;584 }585 } else {586 ksft_test_result_fail("program didn't terminate normally\n");587 return;588 }589 } else {590 ksft_test_result_fail("waitpid() failed\n");591 return;592 }593 594 if (prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0)) {595 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n");596 return;597 }598 599 ksft_test_result_pass("PR_SET_MEMORY_MERGE value is inherited\n");600}601 602static void test_prctl_unmerge(void)603{604 const unsigned int size = 2 * MiB;605 char *map;606 607 ksft_print_msg("[RUN] %s\n", __func__);608 609 map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, KSM_MERGE_PRCTL);610 if (map == MAP_FAILED)611 return;612 613 if (prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0)) {614 ksft_test_result_fail("PR_SET_MEMORY_MERGE=0 failed\n");615 goto unmap;616 }617 618 ksft_test_result(!range_maps_duplicates(map, size),619 "Pages were unmerged\n");620unmap:621 munmap(map, size);622}623 624static void test_prot_none(void)625{626 const unsigned int size = 2 * MiB;627 char *map;628 int i;629 630 ksft_print_msg("[RUN] %s\n", __func__);631 632 map = mmap_and_merge_range(0x11, size, PROT_NONE, KSM_MERGE_MADVISE);633 if (map == MAP_FAILED)634 goto unmap;635 636 /* Store a unique value in each page on one half using ptrace */637 for (i = 0; i < size / 2; i += pagesize) {638 lseek(mem_fd, (uintptr_t) map + i, SEEK_SET);639 if (write(mem_fd, &i, sizeof(i)) != sizeof(i)) {640 ksft_test_result_fail("ptrace write failed\n");641 goto unmap;642 }643 }644 645 /* Trigger unsharing on the other half. */646 if (madvise(map + size / 2, size / 2, MADV_UNMERGEABLE)) {647 ksft_test_result_fail("MADV_UNMERGEABLE failed\n");648 goto unmap;649 }650 651 ksft_test_result(!range_maps_duplicates(map, size),652 "Pages were unmerged\n");653unmap:654 munmap(map, size);655}656 657static void init_global_file_handles(void)658{659 mem_fd = open("/proc/self/mem", O_RDWR);660 if (mem_fd < 0)661 ksft_exit_fail_msg("opening /proc/self/mem failed\n");662 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR);663 if (ksm_fd < 0)664 ksft_exit_skip("open(\"/sys/kernel/mm/ksm/run\") failed\n");665 ksm_full_scans_fd = open("/sys/kernel/mm/ksm/full_scans", O_RDONLY);666 if (ksm_full_scans_fd < 0)667 ksft_exit_skip("open(\"/sys/kernel/mm/ksm/full_scans\") failed\n");668 pagemap_fd = open("/proc/self/pagemap", O_RDONLY);669 if (pagemap_fd < 0)670 ksft_exit_skip("open(\"/proc/self/pagemap\") failed\n");671 proc_self_ksm_stat_fd = open("/proc/self/ksm_stat", O_RDONLY);672 proc_self_ksm_merging_pages_fd = open("/proc/self/ksm_merging_pages",673 O_RDONLY);674 ksm_use_zero_pages_fd = open("/sys/kernel/mm/ksm/use_zero_pages", O_RDWR);675}676 677int main(int argc, char **argv)678{679 unsigned int tests = 8;680 int err;681 682 if (argc > 1 && !strcmp(argv[1], FORK_EXEC_CHILD_PRG_NAME)) {683 init_global_file_handles();684 exit(test_child_ksm());685 }686 687 tests++;688 689 ksft_print_header();690 ksft_set_plan(tests);691 692 pagesize = getpagesize();693 694 init_global_file_handles();695 696 test_unmerge();697 test_unmerge_zero_pages();698 test_unmerge_discarded();699 test_unmerge_uffd_wp();700 701 test_prot_none();702 703 test_prctl();704 test_prctl_fork();705 test_prctl_fork_exec();706 test_prctl_unmerge();707 708 err = ksft_get_fail_cnt();709 if (err)710 ksft_exit_fail_msg("%d out of %d tests failed\n",711 err, ksft_test_num());712 ksft_exit_pass();713}714