469 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * GUP long-term page pinning tests.4 *5 * Copyright 2023, Red Hat, Inc.6 *7 * Author(s): David Hildenbrand <david@redhat.com>8 */9#define _GNU_SOURCE10#include <stdlib.h>11#include <string.h>12#include <stdbool.h>13#include <stdint.h>14#include <unistd.h>15#include <errno.h>16#include <fcntl.h>17#include <assert.h>18#include <sys/mman.h>19#include <sys/ioctl.h>20#include <sys/vfs.h>21#include <linux/magic.h>22#include <linux/memfd.h>23 24#include "local_config.h"25#ifdef LOCAL_CONFIG_HAVE_LIBURING26#include <liburing.h>27#endif /* LOCAL_CONFIG_HAVE_LIBURING */28 29#include "../../../../mm/gup_test.h"30#include "../kselftest.h"31#include "vm_util.h"32 33static size_t pagesize;34static int nr_hugetlbsizes;35static size_t hugetlbsizes[10];36static int gup_fd;37 38static __fsword_t get_fs_type(int fd)39{40 struct statfs fs;41 int ret;42 43 do {44 ret = fstatfs(fd, &fs);45 } while (ret && errno == EINTR);46 47 return ret ? 0 : fs.f_type;48}49 50static bool fs_is_unknown(__fsword_t fs_type)51{52 /*53 * We only support some filesystems in our tests when dealing with54 * R/W long-term pinning. For these filesystems, we can be fairly sure55 * whether they support it or not.56 */57 switch (fs_type) {58 case TMPFS_MAGIC:59 case HUGETLBFS_MAGIC:60 case BTRFS_SUPER_MAGIC:61 case EXT4_SUPER_MAGIC:62 case XFS_SUPER_MAGIC:63 return false;64 default:65 return true;66 }67}68 69static bool fs_supports_writable_longterm_pinning(__fsword_t fs_type)70{71 assert(!fs_is_unknown(fs_type));72 switch (fs_type) {73 case TMPFS_MAGIC:74 case HUGETLBFS_MAGIC:75 return true;76 default:77 return false;78 }79}80 81enum test_type {82 TEST_TYPE_RO,83 TEST_TYPE_RO_FAST,84 TEST_TYPE_RW,85 TEST_TYPE_RW_FAST,86#ifdef LOCAL_CONFIG_HAVE_LIBURING87 TEST_TYPE_IOURING,88#endif /* LOCAL_CONFIG_HAVE_LIBURING */89};90 91static void do_test(int fd, size_t size, enum test_type type, bool shared)92{93 __fsword_t fs_type = get_fs_type(fd);94 bool should_work;95 char *mem;96 int ret;97 98 if (ftruncate(fd, size)) {99 ksft_test_result_fail("ftruncate() failed\n");100 return;101 }102 103 if (fallocate(fd, 0, 0, size)) {104 if (size == pagesize)105 ksft_test_result_fail("fallocate() failed\n");106 else107 ksft_test_result_skip("need more free huge pages\n");108 return;109 }110 111 mem = mmap(NULL, size, PROT_READ | PROT_WRITE,112 shared ? MAP_SHARED : MAP_PRIVATE, fd, 0);113 if (mem == MAP_FAILED) {114 if (size == pagesize || shared)115 ksft_test_result_fail("mmap() failed\n");116 else117 ksft_test_result_skip("need more free huge pages\n");118 return;119 }120 121 /* Fault in the page such that GUP-fast can pin it directly. */122 memset(mem, 0, size);123 124 switch (type) {125 case TEST_TYPE_RO:126 case TEST_TYPE_RO_FAST:127 /*128 * Cover more cases regarding unsharing decisions when129 * long-term R/O pinning by mapping the page R/O.130 */131 ret = mprotect(mem, size, PROT_READ);132 if (ret) {133 ksft_test_result_fail("mprotect() failed\n");134 goto munmap;135 }136 /* FALLTHROUGH */137 case TEST_TYPE_RW:138 case TEST_TYPE_RW_FAST: {139 struct pin_longterm_test args;140 const bool fast = type == TEST_TYPE_RO_FAST ||141 type == TEST_TYPE_RW_FAST;142 const bool rw = type == TEST_TYPE_RW ||143 type == TEST_TYPE_RW_FAST;144 145 if (gup_fd < 0) {146 ksft_test_result_skip("gup_test not available\n");147 break;148 }149 150 if (rw && shared && fs_is_unknown(fs_type)) {151 ksft_test_result_skip("Unknown filesystem\n");152 return;153 }154 /*155 * R/O pinning or pinning in a private mapping is always156 * expected to work. Otherwise, we expect long-term R/W pinning157 * to only succeed for special fielesystems.158 */159 should_work = !shared || !rw ||160 fs_supports_writable_longterm_pinning(fs_type);161 162 args.addr = (__u64)(uintptr_t)mem;163 args.size = size;164 args.flags = fast ? PIN_LONGTERM_TEST_FLAG_USE_FAST : 0;165 args.flags |= rw ? PIN_LONGTERM_TEST_FLAG_USE_WRITE : 0;166 ret = ioctl(gup_fd, PIN_LONGTERM_TEST_START, &args);167 if (ret && errno == EINVAL) {168 ksft_test_result_skip("PIN_LONGTERM_TEST_START failed\n");169 break;170 } else if (ret && errno == EFAULT) {171 ksft_test_result(!should_work, "Should have failed\n");172 break;173 } else if (ret) {174 ksft_test_result_fail("PIN_LONGTERM_TEST_START failed\n");175 break;176 }177 178 if (ioctl(gup_fd, PIN_LONGTERM_TEST_STOP))179 ksft_print_msg("[INFO] PIN_LONGTERM_TEST_STOP failed\n");180 181 /*182 * TODO: if the kernel ever supports long-term R/W pinning on183 * some previously unsupported filesystems, we might want to184 * perform some additional tests for possible data corruptions.185 */186 ksft_test_result(should_work, "Should have worked\n");187 break;188 }189#ifdef LOCAL_CONFIG_HAVE_LIBURING190 case TEST_TYPE_IOURING: {191 struct io_uring ring;192 struct iovec iov;193 194 /* io_uring always pins pages writable. */195 if (shared && fs_is_unknown(fs_type)) {196 ksft_test_result_skip("Unknown filesystem\n");197 return;198 }199 should_work = !shared ||200 fs_supports_writable_longterm_pinning(fs_type);201 202 /* Skip on errors, as we might just lack kernel support. */203 ret = io_uring_queue_init(1, &ring, 0);204 if (ret < 0) {205 ksft_test_result_skip("io_uring_queue_init() failed\n");206 break;207 }208 /*209 * Register the range as a fixed buffer. This will FOLL_WRITE |210 * FOLL_PIN | FOLL_LONGTERM the range.211 */212 iov.iov_base = mem;213 iov.iov_len = size;214 ret = io_uring_register_buffers(&ring, &iov, 1);215 /* Only new kernels return EFAULT. */216 if (ret && (errno == ENOSPC || errno == EOPNOTSUPP ||217 errno == EFAULT)) {218 ksft_test_result(!should_work, "Should have failed\n");219 } else if (ret) {220 /*221 * We might just lack support or have insufficient222 * MEMLOCK limits.223 */224 ksft_test_result_skip("io_uring_register_buffers() failed\n");225 } else {226 ksft_test_result(should_work, "Should have worked\n");227 io_uring_unregister_buffers(&ring);228 }229 230 io_uring_queue_exit(&ring);231 break;232 }233#endif /* LOCAL_CONFIG_HAVE_LIBURING */234 default:235 assert(false);236 }237 238munmap:239 munmap(mem, size);240}241 242typedef void (*test_fn)(int fd, size_t size);243 244static void run_with_memfd(test_fn fn, const char *desc)245{246 int fd;247 248 ksft_print_msg("[RUN] %s ... with memfd\n", desc);249 250 fd = memfd_create("test", 0);251 if (fd < 0) {252 ksft_test_result_fail("memfd_create() failed\n");253 return;254 }255 256 fn(fd, pagesize);257 close(fd);258}259 260static void run_with_tmpfile(test_fn fn, const char *desc)261{262 FILE *file;263 int fd;264 265 ksft_print_msg("[RUN] %s ... with tmpfile\n", desc);266 267 file = tmpfile();268 if (!file) {269 ksft_test_result_fail("tmpfile() failed\n");270 return;271 }272 273 fd = fileno(file);274 if (fd < 0) {275 ksft_test_result_fail("fileno() failed\n");276 goto close;277 }278 279 fn(fd, pagesize);280close:281 fclose(file);282}283 284static void run_with_local_tmpfile(test_fn fn, const char *desc)285{286 char filename[] = __FILE__"_tmpfile_XXXXXX";287 int fd;288 289 ksft_print_msg("[RUN] %s ... with local tmpfile\n", desc);290 291 fd = mkstemp(filename);292 if (fd < 0) {293 ksft_test_result_fail("mkstemp() failed\n");294 return;295 }296 297 if (unlink(filename)) {298 ksft_test_result_fail("unlink() failed\n");299 goto close;300 }301 302 fn(fd, pagesize);303close:304 close(fd);305}306 307static void run_with_memfd_hugetlb(test_fn fn, const char *desc,308 size_t hugetlbsize)309{310 int flags = MFD_HUGETLB;311 int fd;312 313 ksft_print_msg("[RUN] %s ... with memfd hugetlb (%zu kB)\n", desc,314 hugetlbsize / 1024);315 316 flags |= __builtin_ctzll(hugetlbsize) << MFD_HUGE_SHIFT;317 318 fd = memfd_create("test", flags);319 if (fd < 0) {320 ksft_test_result_skip("memfd_create() failed\n");321 return;322 }323 324 fn(fd, hugetlbsize);325 close(fd);326}327 328struct test_case {329 const char *desc;330 test_fn fn;331};332 333static void test_shared_rw_pin(int fd, size_t size)334{335 do_test(fd, size, TEST_TYPE_RW, true);336}337 338static void test_shared_rw_fast_pin(int fd, size_t size)339{340 do_test(fd, size, TEST_TYPE_RW_FAST, true);341}342 343static void test_shared_ro_pin(int fd, size_t size)344{345 do_test(fd, size, TEST_TYPE_RO, true);346}347 348static void test_shared_ro_fast_pin(int fd, size_t size)349{350 do_test(fd, size, TEST_TYPE_RO_FAST, true);351}352 353static void test_private_rw_pin(int fd, size_t size)354{355 do_test(fd, size, TEST_TYPE_RW, false);356}357 358static void test_private_rw_fast_pin(int fd, size_t size)359{360 do_test(fd, size, TEST_TYPE_RW_FAST, false);361}362 363static void test_private_ro_pin(int fd, size_t size)364{365 do_test(fd, size, TEST_TYPE_RO, false);366}367 368static void test_private_ro_fast_pin(int fd, size_t size)369{370 do_test(fd, size, TEST_TYPE_RO_FAST, false);371}372 373#ifdef LOCAL_CONFIG_HAVE_LIBURING374static void test_shared_iouring(int fd, size_t size)375{376 do_test(fd, size, TEST_TYPE_IOURING, true);377}378 379static void test_private_iouring(int fd, size_t size)380{381 do_test(fd, size, TEST_TYPE_IOURING, false);382}383#endif /* LOCAL_CONFIG_HAVE_LIBURING */384 385static const struct test_case test_cases[] = {386 {387 "R/W longterm GUP pin in MAP_SHARED file mapping",388 test_shared_rw_pin,389 },390 {391 "R/W longterm GUP-fast pin in MAP_SHARED file mapping",392 test_shared_rw_fast_pin,393 },394 {395 "R/O longterm GUP pin in MAP_SHARED file mapping",396 test_shared_ro_pin,397 },398 {399 "R/O longterm GUP-fast pin in MAP_SHARED file mapping",400 test_shared_ro_fast_pin,401 },402 {403 "R/W longterm GUP pin in MAP_PRIVATE file mapping",404 test_private_rw_pin,405 },406 {407 "R/W longterm GUP-fast pin in MAP_PRIVATE file mapping",408 test_private_rw_fast_pin,409 },410 {411 "R/O longterm GUP pin in MAP_PRIVATE file mapping",412 test_private_ro_pin,413 },414 {415 "R/O longterm GUP-fast pin in MAP_PRIVATE file mapping",416 test_private_ro_fast_pin,417 },418#ifdef LOCAL_CONFIG_HAVE_LIBURING419 {420 "io_uring fixed buffer with MAP_SHARED file mapping",421 test_shared_iouring,422 },423 {424 "io_uring fixed buffer with MAP_PRIVATE file mapping",425 test_private_iouring,426 },427#endif /* LOCAL_CONFIG_HAVE_LIBURING */428};429 430static void run_test_case(struct test_case const *test_case)431{432 int i;433 434 run_with_memfd(test_case->fn, test_case->desc);435 run_with_tmpfile(test_case->fn, test_case->desc);436 run_with_local_tmpfile(test_case->fn, test_case->desc);437 for (i = 0; i < nr_hugetlbsizes; i++)438 run_with_memfd_hugetlb(test_case->fn, test_case->desc,439 hugetlbsizes[i]);440}441 442static int tests_per_test_case(void)443{444 return 3 + nr_hugetlbsizes;445}446 447int main(int argc, char **argv)448{449 int i, err;450 451 pagesize = getpagesize();452 nr_hugetlbsizes = detect_hugetlb_page_sizes(hugetlbsizes,453 ARRAY_SIZE(hugetlbsizes));454 455 ksft_print_header();456 ksft_set_plan(ARRAY_SIZE(test_cases) * tests_per_test_case());457 458 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);459 460 for (i = 0; i < ARRAY_SIZE(test_cases); i++)461 run_test_case(&test_cases[i]);462 463 err = ksft_get_fail_cnt();464 if (err)465 ksft_exit_fail_msg("%d out of %d tests failed\n",466 err, ksft_test_num());467 ksft_exit_pass();468}469