387 lines · c
1// SPDX-License-Identifier: GPL-2.02#include <string.h>3#include <fcntl.h>4#include <dirent.h>5#include <sys/ioctl.h>6#include <linux/userfaultfd.h>7#include <linux/fs.h>8#include <sys/syscall.h>9#include <unistd.h>10#include "../kselftest.h"11#include "vm_util.h"12 13#define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"14#define SMAP_FILE_PATH "/proc/self/smaps"15#define STATUS_FILE_PATH "/proc/self/status"16#define MAX_LINE_LENGTH 50017 18unsigned int __page_size;19unsigned int __page_shift;20 21uint64_t pagemap_get_entry(int fd, char *start)22{23 const unsigned long pfn = (unsigned long)start / getpagesize();24 uint64_t entry;25 int ret;26 27 ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry));28 if (ret != sizeof(entry))29 ksft_exit_fail_msg("reading pagemap failed\n");30 return entry;31}32 33static uint64_t __pagemap_scan_get_categories(int fd, char *start, struct page_region *r)34{35 struct pm_scan_arg arg;36 37 arg.start = (uintptr_t)start;38 arg.end = (uintptr_t)(start + psize());39 arg.vec = (uintptr_t)r;40 arg.vec_len = 1;41 arg.flags = 0;42 arg.size = sizeof(struct pm_scan_arg);43 arg.max_pages = 0;44 arg.category_inverted = 0;45 arg.category_mask = 0;46 arg.category_anyof_mask = PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | PAGE_IS_FILE |47 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_PFNZERO |48 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY;49 arg.return_mask = arg.category_anyof_mask;50 51 return ioctl(fd, PAGEMAP_SCAN, &arg);52}53 54static uint64_t pagemap_scan_get_categories(int fd, char *start)55{56 struct page_region r;57 long ret;58 59 ret = __pagemap_scan_get_categories(fd, start, &r);60 if (ret < 0)61 ksft_exit_fail_msg("PAGEMAP_SCAN failed: %s\n", strerror(errno));62 if (ret == 0)63 return 0;64 return r.categories;65}66 67/* `start` is any valid address. */68static bool pagemap_scan_supported(int fd, char *start)69{70 static int supported = -1;71 int ret;72 73 if (supported != -1)74 return supported;75 76 /* Provide an invalid address in order to trigger EFAULT. */77 ret = __pagemap_scan_get_categories(fd, start, (struct page_region *) ~0UL);78 if (ret == 0)79 ksft_exit_fail_msg("PAGEMAP_SCAN succeeded unexpectedly\n");80 81 supported = errno == EFAULT;82 83 return supported;84}85 86static bool page_entry_is(int fd, char *start, char *desc,87 uint64_t pagemap_flags, uint64_t pagescan_flags)88{89 bool m = pagemap_get_entry(fd, start) & pagemap_flags;90 91 if (pagemap_scan_supported(fd, start)) {92 bool s = pagemap_scan_get_categories(fd, start) & pagescan_flags;93 94 if (m == s)95 return m;96 97 ksft_exit_fail_msg(98 "read and ioctl return unmatched results for %s: %d %d", desc, m, s);99 }100 return m;101}102 103bool pagemap_is_softdirty(int fd, char *start)104{105 return page_entry_is(fd, start, "soft-dirty",106 PM_SOFT_DIRTY, PAGE_IS_SOFT_DIRTY);107}108 109bool pagemap_is_swapped(int fd, char *start)110{111 return page_entry_is(fd, start, "swap", PM_SWAP, PAGE_IS_SWAPPED);112}113 114bool pagemap_is_populated(int fd, char *start)115{116 return page_entry_is(fd, start, "populated",117 PM_PRESENT | PM_SWAP,118 PAGE_IS_PRESENT | PAGE_IS_SWAPPED);119}120 121unsigned long pagemap_get_pfn(int fd, char *start)122{123 uint64_t entry = pagemap_get_entry(fd, start);124 125 /* If present (63th bit), PFN is at bit 0 -- 54. */126 if (entry & PM_PRESENT)127 return entry & 0x007fffffffffffffull;128 return -1ul;129}130 131void clear_softdirty(void)132{133 int ret;134 const char *ctrl = "4";135 int fd = open("/proc/self/clear_refs", O_WRONLY);136 137 if (fd < 0)138 ksft_exit_fail_msg("opening clear_refs failed\n");139 ret = write(fd, ctrl, strlen(ctrl));140 close(fd);141 if (ret != strlen(ctrl))142 ksft_exit_fail_msg("writing clear_refs failed\n");143}144 145bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len)146{147 while (fgets(buf, len, fp)) {148 if (!strncmp(buf, pattern, strlen(pattern)))149 return true;150 }151 return false;152}153 154uint64_t read_pmd_pagesize(void)155{156 int fd;157 char buf[20];158 ssize_t num_read;159 160 fd = open(PMD_SIZE_FILE_PATH, O_RDONLY);161 if (fd == -1)162 return 0;163 164 num_read = read(fd, buf, 19);165 if (num_read < 1) {166 close(fd);167 return 0;168 }169 buf[num_read] = '\0';170 close(fd);171 172 return strtoul(buf, NULL, 10);173}174 175unsigned long rss_anon(void)176{177 unsigned long rss_anon = 0;178 FILE *fp;179 char buffer[MAX_LINE_LENGTH];180 181 fp = fopen(STATUS_FILE_PATH, "r");182 if (!fp)183 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH);184 185 if (!check_for_pattern(fp, "RssAnon:", buffer, sizeof(buffer)))186 goto err_out;187 188 if (sscanf(buffer, "RssAnon:%10lu kB", &rss_anon) != 1)189 ksft_exit_fail_msg("Reading status error\n");190 191err_out:192 fclose(fp);193 return rss_anon;194}195 196bool __check_huge(void *addr, char *pattern, int nr_hpages,197 uint64_t hpage_size)198{199 uint64_t thp = -1;200 int ret;201 FILE *fp;202 char buffer[MAX_LINE_LENGTH];203 char addr_pattern[MAX_LINE_LENGTH];204 205 ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",206 (unsigned long) addr);207 if (ret >= MAX_LINE_LENGTH)208 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__);209 210 fp = fopen(SMAP_FILE_PATH, "r");211 if (!fp)212 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH);213 214 if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer)))215 goto err_out;216 217 /*218 * Fetch the pattern in the same block and check the number of219 * hugepages.220 */221 if (!check_for_pattern(fp, pattern, buffer, sizeof(buffer)))222 goto err_out;223 224 snprintf(addr_pattern, MAX_LINE_LENGTH, "%s%%9ld kB", pattern);225 226 if (sscanf(buffer, addr_pattern, &thp) != 1)227 ksft_exit_fail_msg("Reading smap error\n");228 229err_out:230 fclose(fp);231 return thp == (nr_hpages * (hpage_size >> 10));232}233 234bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size)235{236 return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size);237}238 239bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size)240{241 return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size);242}243 244bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size)245{246 return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size);247}248 249int64_t allocate_transhuge(void *ptr, int pagemap_fd)250{251 uint64_t ent[2];252 253 /* drop pmd */254 if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,255 MAP_FIXED | MAP_ANONYMOUS |256 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)257 ksft_exit_fail_msg("mmap transhuge\n");258 259 if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))260 ksft_exit_fail_msg("MADV_HUGEPAGE\n");261 262 /* allocate transparent huge page */263 *(volatile void **)ptr = ptr;264 265 if (pread(pagemap_fd, ent, sizeof(ent),266 (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent))267 ksft_exit_fail_msg("read pagemap\n");268 269 if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&270 PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&271 !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1)))272 return PAGEMAP_PFN(ent[0]);273 274 return -1;275}276 277unsigned long default_huge_page_size(void)278{279 unsigned long hps = 0;280 char *line = NULL;281 size_t linelen = 0;282 FILE *f = fopen("/proc/meminfo", "r");283 284 if (!f)285 return 0;286 while (getline(&line, &linelen, f) > 0) {287 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) {288 hps <<= 10;289 break;290 }291 }292 293 free(line);294 fclose(f);295 return hps;296}297 298int detect_hugetlb_page_sizes(size_t sizes[], int max)299{300 DIR *dir = opendir("/sys/kernel/mm/hugepages/");301 int count = 0;302 303 if (!dir)304 return 0;305 306 while (count < max) {307 struct dirent *entry = readdir(dir);308 size_t kb;309 310 if (!entry)311 break;312 if (entry->d_type != DT_DIR)313 continue;314 if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1)315 continue;316 sizes[count++] = kb * 1024;317 ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n",318 kb);319 }320 closedir(dir);321 return count;322}323 324/* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */325int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len,326 bool miss, bool wp, bool minor, uint64_t *ioctls)327{328 struct uffdio_register uffdio_register = { 0 };329 uint64_t mode = 0;330 int ret = 0;331 332 if (miss)333 mode |= UFFDIO_REGISTER_MODE_MISSING;334 if (wp)335 mode |= UFFDIO_REGISTER_MODE_WP;336 if (minor)337 mode |= UFFDIO_REGISTER_MODE_MINOR;338 339 uffdio_register.range.start = (unsigned long)addr;340 uffdio_register.range.len = len;341 uffdio_register.mode = mode;342 343 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1)344 ret = -errno;345 else if (ioctls)346 *ioctls = uffdio_register.ioctls;347 348 return ret;349}350 351int uffd_register(int uffd, void *addr, uint64_t len,352 bool miss, bool wp, bool minor)353{354 return uffd_register_with_ioctls(uffd, addr, len,355 miss, wp, minor, NULL);356}357 358int uffd_unregister(int uffd, void *addr, uint64_t len)359{360 struct uffdio_range range = { .start = (uintptr_t)addr, .len = len };361 int ret = 0;362 363 if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1)364 ret = -errno;365 366 return ret;367}368 369unsigned long get_free_hugepages(void)370{371 unsigned long fhp = 0;372 char *line = NULL;373 size_t linelen = 0;374 FILE *f = fopen("/proc/meminfo", "r");375 376 if (!f)377 return fhp;378 while (getline(&line, &linelen, f) > 0) {379 if (sscanf(line, "HugePages_Free: %lu", &fhp) == 1)380 break;381 }382 383 free(line);384 fclose(f);385 return fhp;386}387