384 lines · c
1// SPDX-License-Identifier: GPL-2.02#include <fcntl.h>3#include <limits.h>4#include <stdio.h>5#include <stdlib.h>6#include <string.h>7#include <unistd.h>8 9#include "thp_settings.h"10 11#define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/"12#define MAX_SETTINGS_DEPTH 413static struct thp_settings settings_stack[MAX_SETTINGS_DEPTH];14static int settings_index;15static struct thp_settings saved_settings;16static char dev_queue_read_ahead_path[PATH_MAX];17 18static const char * const thp_enabled_strings[] = {19 "never",20 "always",21 "inherit",22 "madvise",23 NULL24};25 26static const char * const thp_defrag_strings[] = {27 "always",28 "defer",29 "defer+madvise",30 "madvise",31 "never",32 NULL33};34 35static const char * const shmem_enabled_strings[] = {36 "never",37 "always",38 "within_size",39 "advise",40 "inherit",41 "deny",42 "force",43 NULL44};45 46int read_file(const char *path, char *buf, size_t buflen)47{48 int fd;49 ssize_t numread;50 51 fd = open(path, O_RDONLY);52 if (fd == -1)53 return 0;54 55 numread = read(fd, buf, buflen - 1);56 if (numread < 1) {57 close(fd);58 return 0;59 }60 61 buf[numread] = '\0';62 close(fd);63 64 return (unsigned int) numread;65}66 67int write_file(const char *path, const char *buf, size_t buflen)68{69 int fd;70 ssize_t numwritten;71 72 fd = open(path, O_WRONLY);73 if (fd == -1) {74 printf("open(%s)\n", path);75 exit(EXIT_FAILURE);76 return 0;77 }78 79 numwritten = write(fd, buf, buflen - 1);80 close(fd);81 if (numwritten < 1) {82 printf("write(%s)\n", buf);83 exit(EXIT_FAILURE);84 return 0;85 }86 87 return (unsigned int) numwritten;88}89 90const unsigned long read_num(const char *path)91{92 char buf[21];93 94 if (read_file(path, buf, sizeof(buf)) < 0) {95 perror("read_file()");96 exit(EXIT_FAILURE);97 }98 99 return strtoul(buf, NULL, 10);100}101 102void write_num(const char *path, unsigned long num)103{104 char buf[21];105 106 sprintf(buf, "%ld", num);107 if (!write_file(path, buf, strlen(buf) + 1)) {108 perror(path);109 exit(EXIT_FAILURE);110 }111}112 113int thp_read_string(const char *name, const char * const strings[])114{115 char path[PATH_MAX];116 char buf[256];117 char *c;118 int ret;119 120 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);121 if (ret >= PATH_MAX) {122 printf("%s: Pathname is too long\n", __func__);123 exit(EXIT_FAILURE);124 }125 126 if (!read_file(path, buf, sizeof(buf))) {127 perror(path);128 exit(EXIT_FAILURE);129 }130 131 c = strchr(buf, '[');132 if (!c) {133 printf("%s: Parse failure\n", __func__);134 exit(EXIT_FAILURE);135 }136 137 c++;138 memmove(buf, c, sizeof(buf) - (c - buf));139 140 c = strchr(buf, ']');141 if (!c) {142 printf("%s: Parse failure\n", __func__);143 exit(EXIT_FAILURE);144 }145 *c = '\0';146 147 ret = 0;148 while (strings[ret]) {149 if (!strcmp(strings[ret], buf))150 return ret;151 ret++;152 }153 154 printf("Failed to parse %s\n", name);155 exit(EXIT_FAILURE);156}157 158void thp_write_string(const char *name, const char *val)159{160 char path[PATH_MAX];161 int ret;162 163 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);164 if (ret >= PATH_MAX) {165 printf("%s: Pathname is too long\n", __func__);166 exit(EXIT_FAILURE);167 }168 169 if (!write_file(path, val, strlen(val) + 1)) {170 perror(path);171 exit(EXIT_FAILURE);172 }173}174 175const unsigned long thp_read_num(const char *name)176{177 char path[PATH_MAX];178 int ret;179 180 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);181 if (ret >= PATH_MAX) {182 printf("%s: Pathname is too long\n", __func__);183 exit(EXIT_FAILURE);184 }185 return read_num(path);186}187 188void thp_write_num(const char *name, unsigned long num)189{190 char path[PATH_MAX];191 int ret;192 193 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);194 if (ret >= PATH_MAX) {195 printf("%s: Pathname is too long\n", __func__);196 exit(EXIT_FAILURE);197 }198 write_num(path, num);199}200 201void thp_read_settings(struct thp_settings *settings)202{203 unsigned long orders = thp_supported_orders();204 unsigned long shmem_orders = thp_shmem_supported_orders();205 char path[PATH_MAX];206 int i;207 208 *settings = (struct thp_settings) {209 .thp_enabled = thp_read_string("enabled", thp_enabled_strings),210 .thp_defrag = thp_read_string("defrag", thp_defrag_strings),211 .shmem_enabled =212 thp_read_string("shmem_enabled", shmem_enabled_strings),213 .use_zero_page = thp_read_num("use_zero_page"),214 };215 settings->khugepaged = (struct khugepaged_settings) {216 .defrag = thp_read_num("khugepaged/defrag"),217 .alloc_sleep_millisecs =218 thp_read_num("khugepaged/alloc_sleep_millisecs"),219 .scan_sleep_millisecs =220 thp_read_num("khugepaged/scan_sleep_millisecs"),221 .max_ptes_none = thp_read_num("khugepaged/max_ptes_none"),222 .max_ptes_swap = thp_read_num("khugepaged/max_ptes_swap"),223 .max_ptes_shared = thp_read_num("khugepaged/max_ptes_shared"),224 .pages_to_scan = thp_read_num("khugepaged/pages_to_scan"),225 };226 if (dev_queue_read_ahead_path[0])227 settings->read_ahead_kb = read_num(dev_queue_read_ahead_path);228 229 for (i = 0; i < NR_ORDERS; i++) {230 if (!((1 << i) & orders)) {231 settings->hugepages[i].enabled = THP_NEVER;232 continue;233 }234 snprintf(path, PATH_MAX, "hugepages-%ukB/enabled",235 (getpagesize() >> 10) << i);236 settings->hugepages[i].enabled =237 thp_read_string(path, thp_enabled_strings);238 }239 240 for (i = 0; i < NR_ORDERS; i++) {241 if (!((1 << i) & shmem_orders)) {242 settings->shmem_hugepages[i].enabled = SHMEM_NEVER;243 continue;244 }245 snprintf(path, PATH_MAX, "hugepages-%ukB/shmem_enabled",246 (getpagesize() >> 10) << i);247 settings->shmem_hugepages[i].enabled =248 thp_read_string(path, shmem_enabled_strings);249 }250}251 252void thp_write_settings(struct thp_settings *settings)253{254 struct khugepaged_settings *khugepaged = &settings->khugepaged;255 unsigned long orders = thp_supported_orders();256 unsigned long shmem_orders = thp_shmem_supported_orders();257 char path[PATH_MAX];258 int enabled;259 int i;260 261 thp_write_string("enabled", thp_enabled_strings[settings->thp_enabled]);262 thp_write_string("defrag", thp_defrag_strings[settings->thp_defrag]);263 thp_write_string("shmem_enabled",264 shmem_enabled_strings[settings->shmem_enabled]);265 thp_write_num("use_zero_page", settings->use_zero_page);266 267 thp_write_num("khugepaged/defrag", khugepaged->defrag);268 thp_write_num("khugepaged/alloc_sleep_millisecs",269 khugepaged->alloc_sleep_millisecs);270 thp_write_num("khugepaged/scan_sleep_millisecs",271 khugepaged->scan_sleep_millisecs);272 thp_write_num("khugepaged/max_ptes_none", khugepaged->max_ptes_none);273 thp_write_num("khugepaged/max_ptes_swap", khugepaged->max_ptes_swap);274 thp_write_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared);275 thp_write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan);276 277 if (dev_queue_read_ahead_path[0])278 write_num(dev_queue_read_ahead_path, settings->read_ahead_kb);279 280 for (i = 0; i < NR_ORDERS; i++) {281 if (!((1 << i) & orders))282 continue;283 snprintf(path, PATH_MAX, "hugepages-%ukB/enabled",284 (getpagesize() >> 10) << i);285 enabled = settings->hugepages[i].enabled;286 thp_write_string(path, thp_enabled_strings[enabled]);287 }288 289 for (i = 0; i < NR_ORDERS; i++) {290 if (!((1 << i) & shmem_orders))291 continue;292 snprintf(path, PATH_MAX, "hugepages-%ukB/shmem_enabled",293 (getpagesize() >> 10) << i);294 enabled = settings->shmem_hugepages[i].enabled;295 thp_write_string(path, shmem_enabled_strings[enabled]);296 }297}298 299struct thp_settings *thp_current_settings(void)300{301 if (!settings_index) {302 printf("Fail: No settings set");303 exit(EXIT_FAILURE);304 }305 return settings_stack + settings_index - 1;306}307 308void thp_push_settings(struct thp_settings *settings)309{310 if (settings_index >= MAX_SETTINGS_DEPTH) {311 printf("Fail: Settings stack exceeded");312 exit(EXIT_FAILURE);313 }314 settings_stack[settings_index++] = *settings;315 thp_write_settings(thp_current_settings());316}317 318void thp_pop_settings(void)319{320 if (settings_index <= 0) {321 printf("Fail: Settings stack empty");322 exit(EXIT_FAILURE);323 }324 --settings_index;325 thp_write_settings(thp_current_settings());326}327 328void thp_restore_settings(void)329{330 thp_write_settings(&saved_settings);331}332 333void thp_save_settings(void)334{335 thp_read_settings(&saved_settings);336}337 338void thp_set_read_ahead_path(char *path)339{340 if (!path) {341 dev_queue_read_ahead_path[0] = '\0';342 return;343 }344 345 strncpy(dev_queue_read_ahead_path, path,346 sizeof(dev_queue_read_ahead_path));347 dev_queue_read_ahead_path[sizeof(dev_queue_read_ahead_path) - 1] = '\0';348}349 350static unsigned long __thp_supported_orders(bool is_shmem)351{352 unsigned long orders = 0;353 char path[PATH_MAX];354 char buf[256];355 int ret, i;356 char anon_dir[] = "enabled";357 char shmem_dir[] = "shmem_enabled";358 359 for (i = 0; i < NR_ORDERS; i++) {360 ret = snprintf(path, PATH_MAX, THP_SYSFS "hugepages-%ukB/%s",361 (getpagesize() >> 10) << i, is_shmem ? shmem_dir : anon_dir);362 if (ret >= PATH_MAX) {363 printf("%s: Pathname is too long\n", __func__);364 exit(EXIT_FAILURE);365 }366 367 ret = read_file(path, buf, sizeof(buf));368 if (ret)369 orders |= 1UL << i;370 }371 372 return orders;373}374 375unsigned long thp_supported_orders(void)376{377 return __thp_supported_orders(false);378}379 380unsigned long thp_shmem_supported_orders(void)381{382 return __thp_supported_orders(true);383}384