273 lines · c
1#define __SANE_USERSPACE_TYPES__ // Use ll642#include <fcntl.h>3#include <errno.h>4#include <stdio.h>5#include <stdlib.h>6#include <unistd.h>7#include <dirent.h>8#include <sys/ioctl.h>9#include <sys/mman.h>10#include <sys/stat.h>11#include <sys/types.h>12#include <pthread.h>13#include <assert.h>14#include <mm/gup_test.h>15#include "../kselftest.h"16#include "vm_util.h"17 18#define MB (1UL << 20)19 20/* Just the flags we need, copied from mm.h: */21#define FOLL_WRITE 0x01 /* check pte is writable */22#define FOLL_TOUCH 0x02 /* mark page accessed */23 24#define GUP_TEST_FILE "/sys/kernel/debug/gup_test"25 26static unsigned long cmd = GUP_FAST_BENCHMARK;27static int gup_fd, repeats = 1;28static unsigned long size = 128 * MB;29/* Serialize prints */30static pthread_mutex_t print_mutex = PTHREAD_MUTEX_INITIALIZER;31 32static char *cmd_to_str(unsigned long cmd)33{34 switch (cmd) {35 case GUP_FAST_BENCHMARK:36 return "GUP_FAST_BENCHMARK";37 case PIN_FAST_BENCHMARK:38 return "PIN_FAST_BENCHMARK";39 case PIN_LONGTERM_BENCHMARK:40 return "PIN_LONGTERM_BENCHMARK";41 case GUP_BASIC_TEST:42 return "GUP_BASIC_TEST";43 case PIN_BASIC_TEST:44 return "PIN_BASIC_TEST";45 case DUMP_USER_PAGES_TEST:46 return "DUMP_USER_PAGES_TEST";47 }48 return "Unknown command";49}50 51void *gup_thread(void *data)52{53 struct gup_test gup = *(struct gup_test *)data;54 int i, status;55 56 /* Only report timing information on the *_BENCHMARK commands: */57 if ((cmd == PIN_FAST_BENCHMARK) || (cmd == GUP_FAST_BENCHMARK) ||58 (cmd == PIN_LONGTERM_BENCHMARK)) {59 for (i = 0; i < repeats; i++) {60 gup.size = size;61 status = ioctl(gup_fd, cmd, &gup);62 if (status)63 break;64 65 pthread_mutex_lock(&print_mutex);66 ksft_print_msg("%s: Time: get:%lld put:%lld us",67 cmd_to_str(cmd), gup.get_delta_usec,68 gup.put_delta_usec);69 if (gup.size != size)70 ksft_print_msg(", truncated (size: %lld)", gup.size);71 ksft_print_msg("\n");72 pthread_mutex_unlock(&print_mutex);73 }74 } else {75 gup.size = size;76 status = ioctl(gup_fd, cmd, &gup);77 if (status)78 goto return_;79 80 pthread_mutex_lock(&print_mutex);81 ksft_print_msg("%s: done\n", cmd_to_str(cmd));82 if (gup.size != size)83 ksft_print_msg("Truncated (size: %lld)\n", gup.size);84 pthread_mutex_unlock(&print_mutex);85 }86 87return_:88 ksft_test_result(!status, "ioctl status %d\n", status);89 return NULL;90}91 92int main(int argc, char **argv)93{94 struct gup_test gup = { 0 };95 int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret;96 int flags = MAP_PRIVATE, touch = 0;97 char *file = "/dev/zero";98 pthread_t *tid;99 char *p;100 101 while ((opt = getopt(argc, argv, "m:r:n:F:f:abcj:tTLUuwWSHpz")) != -1) {102 switch (opt) {103 case 'a':104 cmd = PIN_FAST_BENCHMARK;105 break;106 case 'b':107 cmd = PIN_BASIC_TEST;108 break;109 case 'L':110 cmd = PIN_LONGTERM_BENCHMARK;111 break;112 case 'c':113 cmd = DUMP_USER_PAGES_TEST;114 /*115 * Dump page 0 (index 1). May be overridden later, by116 * user's non-option arguments.117 *118 * .which_pages is zero-based, so that zero can mean "do119 * nothing".120 */121 gup.which_pages[0] = 1;122 break;123 case 'p':124 /* works only with DUMP_USER_PAGES_TEST */125 gup.test_flags |= GUP_TEST_FLAG_DUMP_PAGES_USE_PIN;126 break;127 case 'F':128 /* strtol, so you can pass flags in hex form */129 gup.gup_flags = strtol(optarg, 0, 0);130 break;131 case 'j':132 nthreads = atoi(optarg);133 break;134 case 'm':135 size = atoi(optarg) * MB;136 break;137 case 'r':138 repeats = atoi(optarg);139 break;140 case 'n':141 nr_pages = atoi(optarg);142 break;143 case 't':144 thp = 1;145 break;146 case 'T':147 thp = 0;148 break;149 case 'U':150 cmd = GUP_BASIC_TEST;151 break;152 case 'u':153 cmd = GUP_FAST_BENCHMARK;154 break;155 case 'w':156 write = 1;157 break;158 case 'W':159 write = 0;160 break;161 case 'f':162 file = optarg;163 break;164 case 'S':165 flags &= ~MAP_PRIVATE;166 flags |= MAP_SHARED;167 break;168 case 'H':169 flags |= (MAP_HUGETLB | MAP_ANONYMOUS);170 break;171 case 'z':172 /* fault pages in gup, do not fault in userland */173 touch = 1;174 break;175 default:176 ksft_exit_fail_msg("Wrong argument\n");177 }178 }179 180 if (optind < argc) {181 int extra_arg_count = 0;182 /*183 * For example:184 *185 * ./gup_test -c 0 1 0x1001186 *187 * ...to dump pages 0, 1, and 4097188 */189 190 while ((optind < argc) &&191 (extra_arg_count < GUP_TEST_MAX_PAGES_TO_DUMP)) {192 /*193 * Do the 1-based indexing here, so that the user can194 * use normal 0-based indexing on the command line.195 */196 long page_index = strtol(argv[optind], 0, 0) + 1;197 198 gup.which_pages[extra_arg_count] = page_index;199 extra_arg_count++;200 optind++;201 }202 }203 204 ksft_print_header();205 ksft_set_plan(nthreads);206 207 filed = open(file, O_RDWR|O_CREAT, 0664);208 if (filed < 0)209 ksft_exit_fail_msg("Unable to open %s: %s\n", file, strerror(errno));210 211 gup.nr_pages_per_call = nr_pages;212 if (write)213 gup.gup_flags |= FOLL_WRITE;214 215 gup_fd = open(GUP_TEST_FILE, O_RDWR);216 if (gup_fd == -1) {217 switch (errno) {218 case EACCES:219 if (getuid())220 ksft_print_msg("Please run this test as root\n");221 break;222 case ENOENT:223 if (opendir("/sys/kernel/debug") == NULL)224 ksft_print_msg("mount debugfs at /sys/kernel/debug\n");225 ksft_print_msg("check if CONFIG_GUP_TEST is enabled in kernel config\n");226 break;227 default:228 ksft_print_msg("failed to open %s: %s\n", GUP_TEST_FILE, strerror(errno));229 break;230 }231 ksft_test_result_skip("Please run this test as root\n");232 ksft_exit_pass();233 }234 235 p = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, filed, 0);236 if (p == MAP_FAILED)237 ksft_exit_fail_msg("mmap: %s\n", strerror(errno));238 gup.addr = (unsigned long)p;239 240 if (thp == 1)241 madvise(p, size, MADV_HUGEPAGE);242 else if (thp == 0)243 madvise(p, size, MADV_NOHUGEPAGE);244 245 /*246 * FOLL_TOUCH, in gup_test, is used as an either/or case: either247 * fault pages in from the kernel via FOLL_TOUCH, or fault them248 * in here, from user space. This allows comparison of performance249 * between those two cases.250 */251 if (touch) {252 gup.gup_flags |= FOLL_TOUCH;253 } else {254 for (; (unsigned long)p < gup.addr + size; p += psize())255 p[0] = 0;256 }257 258 tid = malloc(sizeof(pthread_t) * nthreads);259 assert(tid);260 for (i = 0; i < nthreads; i++) {261 ret = pthread_create(&tid[i], NULL, gup_thread, &gup);262 assert(ret == 0);263 }264 for (i = 0; i < nthreads; i++) {265 ret = pthread_join(tid[i], NULL);266 assert(ret == 0);267 }268 269 free(tid);270 271 ksft_exit_pass();272}273