2203 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * fs/userfaultfd.c4 *5 * Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org>6 * Copyright (C) 2008-2009 Red Hat, Inc.7 * Copyright (C) 2015 Red Hat, Inc.8 *9 * Some part derived from fs/eventfd.c (anon inode setup) and10 * mm/ksm.c (mm hashing).11 */12 13#include <linux/list.h>14#include <linux/hashtable.h>15#include <linux/sched/signal.h>16#include <linux/sched/mm.h>17#include <linux/mm.h>18#include <linux/mm_inline.h>19#include <linux/mmu_notifier.h>20#include <linux/poll.h>21#include <linux/slab.h>22#include <linux/seq_file.h>23#include <linux/file.h>24#include <linux/bug.h>25#include <linux/anon_inodes.h>26#include <linux/syscalls.h>27#include <linux/userfaultfd_k.h>28#include <linux/mempolicy.h>29#include <linux/ioctl.h>30#include <linux/security.h>31#include <linux/hugetlb.h>32#include <linux/swapops.h>33#include <linux/miscdevice.h>34#include <linux/uio.h>35 36static int sysctl_unprivileged_userfaultfd __read_mostly;37 38#ifdef CONFIG_SYSCTL39static struct ctl_table vm_userfaultfd_table[] = {40 {41 .procname = "unprivileged_userfaultfd",42 .data = &sysctl_unprivileged_userfaultfd,43 .maxlen = sizeof(sysctl_unprivileged_userfaultfd),44 .mode = 0644,45 .proc_handler = proc_dointvec_minmax,46 .extra1 = SYSCTL_ZERO,47 .extra2 = SYSCTL_ONE,48 },49};50#endif51 52static struct kmem_cache *userfaultfd_ctx_cachep __ro_after_init;53 54struct userfaultfd_fork_ctx {55 struct userfaultfd_ctx *orig;56 struct userfaultfd_ctx *new;57 struct list_head list;58};59 60struct userfaultfd_unmap_ctx {61 struct userfaultfd_ctx *ctx;62 unsigned long start;63 unsigned long end;64 struct list_head list;65};66 67struct userfaultfd_wait_queue {68 struct uffd_msg msg;69 wait_queue_entry_t wq;70 struct userfaultfd_ctx *ctx;71 bool waken;72};73 74struct userfaultfd_wake_range {75 unsigned long start;76 unsigned long len;77};78 79/* internal indication that UFFD_API ioctl was successfully executed */80#define UFFD_FEATURE_INITIALIZED (1u << 31)81 82static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx)83{84 return ctx->features & UFFD_FEATURE_INITIALIZED;85}86 87static bool userfaultfd_wp_async_ctx(struct userfaultfd_ctx *ctx)88{89 return ctx && (ctx->features & UFFD_FEATURE_WP_ASYNC);90}91 92/*93 * Whether WP_UNPOPULATED is enabled on the uffd context. It is only94 * meaningful when userfaultfd_wp()==true on the vma and when it's95 * anonymous.96 */97bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma)98{99 struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;100 101 if (!ctx)102 return false;103 104 return ctx->features & UFFD_FEATURE_WP_UNPOPULATED;105}106 107static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode,108 int wake_flags, void *key)109{110 struct userfaultfd_wake_range *range = key;111 int ret;112 struct userfaultfd_wait_queue *uwq;113 unsigned long start, len;114 115 uwq = container_of(wq, struct userfaultfd_wait_queue, wq);116 ret = 0;117 /* len == 0 means wake all */118 start = range->start;119 len = range->len;120 if (len && (start > uwq->msg.arg.pagefault.address ||121 start + len <= uwq->msg.arg.pagefault.address))122 goto out;123 WRITE_ONCE(uwq->waken, true);124 /*125 * The Program-Order guarantees provided by the scheduler126 * ensure uwq->waken is visible before the task is woken.127 */128 ret = wake_up_state(wq->private, mode);129 if (ret) {130 /*131 * Wake only once, autoremove behavior.132 *133 * After the effect of list_del_init is visible to the other134 * CPUs, the waitqueue may disappear from under us, see the135 * !list_empty_careful() in handle_userfault().136 *137 * try_to_wake_up() has an implicit smp_mb(), and the138 * wq->private is read before calling the extern function139 * "wake_up_state" (which in turns calls try_to_wake_up).140 */141 list_del_init(&wq->entry);142 }143out:144 return ret;145}146 147/**148 * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd149 * context.150 * @ctx: [in] Pointer to the userfaultfd context.151 */152static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx)153{154 refcount_inc(&ctx->refcount);155}156 157/**158 * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd159 * context.160 * @ctx: [in] Pointer to userfaultfd context.161 *162 * The userfaultfd context reference must have been previously acquired either163 * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget().164 */165static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx)166{167 if (refcount_dec_and_test(&ctx->refcount)) {168 VM_BUG_ON(spin_is_locked(&ctx->fault_pending_wqh.lock));169 VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh));170 VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock));171 VM_BUG_ON(waitqueue_active(&ctx->fault_wqh));172 VM_BUG_ON(spin_is_locked(&ctx->event_wqh.lock));173 VM_BUG_ON(waitqueue_active(&ctx->event_wqh));174 VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock));175 VM_BUG_ON(waitqueue_active(&ctx->fd_wqh));176 mmdrop(ctx->mm);177 kmem_cache_free(userfaultfd_ctx_cachep, ctx);178 }179}180 181static inline void msg_init(struct uffd_msg *msg)182{183 BUILD_BUG_ON(sizeof(struct uffd_msg) != 32);184 /*185 * Must use memset to zero out the paddings or kernel data is186 * leaked to userland.187 */188 memset(msg, 0, sizeof(struct uffd_msg));189}190 191static inline struct uffd_msg userfault_msg(unsigned long address,192 unsigned long real_address,193 unsigned int flags,194 unsigned long reason,195 unsigned int features)196{197 struct uffd_msg msg;198 199 msg_init(&msg);200 msg.event = UFFD_EVENT_PAGEFAULT;201 202 msg.arg.pagefault.address = (features & UFFD_FEATURE_EXACT_ADDRESS) ?203 real_address : address;204 205 /*206 * These flags indicate why the userfault occurred:207 * - UFFD_PAGEFAULT_FLAG_WP indicates a write protect fault.208 * - UFFD_PAGEFAULT_FLAG_MINOR indicates a minor fault.209 * - Neither of these flags being set indicates a MISSING fault.210 *211 * Separately, UFFD_PAGEFAULT_FLAG_WRITE indicates it was a write212 * fault. Otherwise, it was a read fault.213 */214 if (flags & FAULT_FLAG_WRITE)215 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;216 if (reason & VM_UFFD_WP)217 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;218 if (reason & VM_UFFD_MINOR)219 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR;220 if (features & UFFD_FEATURE_THREAD_ID)221 msg.arg.pagefault.feat.ptid = task_pid_vnr(current);222 return msg;223}224 225#ifdef CONFIG_HUGETLB_PAGE226/*227 * Same functionality as userfaultfd_must_wait below with modifications for228 * hugepmd ranges.229 */230static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,231 struct vm_fault *vmf,232 unsigned long reason)233{234 struct vm_area_struct *vma = vmf->vma;235 pte_t *ptep, pte;236 bool ret = true;237 238 assert_fault_locked(vmf);239 240 ptep = hugetlb_walk(vma, vmf->address, vma_mmu_pagesize(vma));241 if (!ptep)242 goto out;243 244 ret = false;245 pte = huge_ptep_get(vma->vm_mm, vmf->address, ptep);246 247 /*248 * Lockless access: we're in a wait_event so it's ok if it249 * changes under us. PTE markers should be handled the same as none250 * ptes here.251 */252 if (huge_pte_none_mostly(pte))253 ret = true;254 if (!huge_pte_write(pte) && (reason & VM_UFFD_WP))255 ret = true;256out:257 return ret;258}259#else260static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,261 struct vm_fault *vmf,262 unsigned long reason)263{264 return false; /* should never get here */265}266#endif /* CONFIG_HUGETLB_PAGE */267 268/*269 * Verify the pagetables are still not ok after having reigstered into270 * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any271 * userfault that has already been resolved, if userfaultfd_read_iter and272 * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different273 * threads.274 */275static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,276 struct vm_fault *vmf,277 unsigned long reason)278{279 struct mm_struct *mm = ctx->mm;280 unsigned long address = vmf->address;281 pgd_t *pgd;282 p4d_t *p4d;283 pud_t *pud;284 pmd_t *pmd, _pmd;285 pte_t *pte;286 pte_t ptent;287 bool ret = true;288 289 assert_fault_locked(vmf);290 291 pgd = pgd_offset(mm, address);292 if (!pgd_present(*pgd))293 goto out;294 p4d = p4d_offset(pgd, address);295 if (!p4d_present(*p4d))296 goto out;297 pud = pud_offset(p4d, address);298 if (!pud_present(*pud))299 goto out;300 pmd = pmd_offset(pud, address);301again:302 _pmd = pmdp_get_lockless(pmd);303 if (pmd_none(_pmd))304 goto out;305 306 ret = false;307 if (!pmd_present(_pmd) || pmd_devmap(_pmd))308 goto out;309 310 if (pmd_trans_huge(_pmd)) {311 if (!pmd_write(_pmd) && (reason & VM_UFFD_WP))312 ret = true;313 goto out;314 }315 316 pte = pte_offset_map(pmd, address);317 if (!pte) {318 ret = true;319 goto again;320 }321 /*322 * Lockless access: we're in a wait_event so it's ok if it323 * changes under us. PTE markers should be handled the same as none324 * ptes here.325 */326 ptent = ptep_get(pte);327 if (pte_none_mostly(ptent))328 ret = true;329 if (!pte_write(ptent) && (reason & VM_UFFD_WP))330 ret = true;331 pte_unmap(pte);332 333out:334 return ret;335}336 337static inline unsigned int userfaultfd_get_blocking_state(unsigned int flags)338{339 if (flags & FAULT_FLAG_INTERRUPTIBLE)340 return TASK_INTERRUPTIBLE;341 342 if (flags & FAULT_FLAG_KILLABLE)343 return TASK_KILLABLE;344 345 return TASK_UNINTERRUPTIBLE;346}347 348/*349 * The locking rules involved in returning VM_FAULT_RETRY depending on350 * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and351 * FAULT_FLAG_KILLABLE are not straightforward. The "Caution"352 * recommendation in __lock_page_or_retry is not an understatement.353 *354 * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_lock must be released355 * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is356 * not set.357 *358 * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not359 * set, VM_FAULT_RETRY can still be returned if and only if there are360 * fatal_signal_pending()s, and the mmap_lock must be released before361 * returning it.362 */363vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)364{365 struct vm_area_struct *vma = vmf->vma;366 struct mm_struct *mm = vma->vm_mm;367 struct userfaultfd_ctx *ctx;368 struct userfaultfd_wait_queue uwq;369 vm_fault_t ret = VM_FAULT_SIGBUS;370 bool must_wait;371 unsigned int blocking_state;372 373 /*374 * We don't do userfault handling for the final child pid update375 * and when coredumping (faults triggered by get_dump_page()).376 */377 if (current->flags & (PF_EXITING|PF_DUMPCORE))378 goto out;379 380 assert_fault_locked(vmf);381 382 ctx = vma->vm_userfaultfd_ctx.ctx;383 if (!ctx)384 goto out;385 386 BUG_ON(ctx->mm != mm);387 388 /* Any unrecognized flag is a bug. */389 VM_BUG_ON(reason & ~__VM_UFFD_FLAGS);390 /* 0 or > 1 flags set is a bug; we expect exactly 1. */391 VM_BUG_ON(!reason || (reason & (reason - 1)));392 393 if (ctx->features & UFFD_FEATURE_SIGBUS)394 goto out;395 if (!(vmf->flags & FAULT_FLAG_USER) && (ctx->flags & UFFD_USER_MODE_ONLY))396 goto out;397 398 /*399 * If it's already released don't get it. This avoids to loop400 * in __get_user_pages if userfaultfd_release waits on the401 * caller of handle_userfault to release the mmap_lock.402 */403 if (unlikely(READ_ONCE(ctx->released))) {404 /*405 * Don't return VM_FAULT_SIGBUS in this case, so a non406 * cooperative manager can close the uffd after the407 * last UFFDIO_COPY, without risking to trigger an408 * involuntary SIGBUS if the process was starting the409 * userfaultfd while the userfaultfd was still armed410 * (but after the last UFFDIO_COPY). If the uffd411 * wasn't already closed when the userfault reached412 * this point, that would normally be solved by413 * userfaultfd_must_wait returning 'false'.414 *415 * If we were to return VM_FAULT_SIGBUS here, the non416 * cooperative manager would be instead forced to417 * always call UFFDIO_UNREGISTER before it can safely418 * close the uffd.419 */420 ret = VM_FAULT_NOPAGE;421 goto out;422 }423 424 /*425 * Check that we can return VM_FAULT_RETRY.426 *427 * NOTE: it should become possible to return VM_FAULT_RETRY428 * even if FAULT_FLAG_TRIED is set without leading to gup()429 * -EBUSY failures, if the userfaultfd is to be extended for430 * VM_UFFD_WP tracking and we intend to arm the userfault431 * without first stopping userland access to the memory. For432 * VM_UFFD_MISSING userfaults this is enough for now.433 */434 if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) {435 /*436 * Validate the invariant that nowait must allow retry437 * to be sure not to return SIGBUS erroneously on438 * nowait invocations.439 */440 BUG_ON(vmf->flags & FAULT_FLAG_RETRY_NOWAIT);441#ifdef CONFIG_DEBUG_VM442 if (printk_ratelimit()) {443 printk(KERN_WARNING444 "FAULT_FLAG_ALLOW_RETRY missing %x\n",445 vmf->flags);446 dump_stack();447 }448#endif449 goto out;450 }451 452 /*453 * Handle nowait, not much to do other than tell it to retry454 * and wait.455 */456 ret = VM_FAULT_RETRY;457 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)458 goto out;459 460 /* take the reference before dropping the mmap_lock */461 userfaultfd_ctx_get(ctx);462 463 init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function);464 uwq.wq.private = current;465 uwq.msg = userfault_msg(vmf->address, vmf->real_address, vmf->flags,466 reason, ctx->features);467 uwq.ctx = ctx;468 uwq.waken = false;469 470 blocking_state = userfaultfd_get_blocking_state(vmf->flags);471 472 /*473 * Take the vma lock now, in order to safely call474 * userfaultfd_huge_must_wait() later. Since acquiring the475 * (sleepable) vma lock can modify the current task state, that476 * must be before explicitly calling set_current_state().477 */478 if (is_vm_hugetlb_page(vma))479 hugetlb_vma_lock_read(vma);480 481 spin_lock_irq(&ctx->fault_pending_wqh.lock);482 /*483 * After the __add_wait_queue the uwq is visible to userland484 * through poll/read().485 */486 __add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq);487 /*488 * The smp_mb() after __set_current_state prevents the reads489 * following the spin_unlock to happen before the list_add in490 * __add_wait_queue.491 */492 set_current_state(blocking_state);493 spin_unlock_irq(&ctx->fault_pending_wqh.lock);494 495 if (!is_vm_hugetlb_page(vma))496 must_wait = userfaultfd_must_wait(ctx, vmf, reason);497 else498 must_wait = userfaultfd_huge_must_wait(ctx, vmf, reason);499 if (is_vm_hugetlb_page(vma))500 hugetlb_vma_unlock_read(vma);501 release_fault_lock(vmf);502 503 if (likely(must_wait && !READ_ONCE(ctx->released))) {504 wake_up_poll(&ctx->fd_wqh, EPOLLIN);505 schedule();506 }507 508 __set_current_state(TASK_RUNNING);509 510 /*511 * Here we race with the list_del; list_add in512 * userfaultfd_ctx_read(), however because we don't ever run513 * list_del_init() to refile across the two lists, the prev514 * and next pointers will never point to self. list_add also515 * would never let any of the two pointers to point to516 * self. So list_empty_careful won't risk to see both pointers517 * pointing to self at any time during the list refile. The518 * only case where list_del_init() is called is the full519 * removal in the wake function and there we don't re-list_add520 * and it's fine not to block on the spinlock. The uwq on this521 * kernel stack can be released after the list_del_init.522 */523 if (!list_empty_careful(&uwq.wq.entry)) {524 spin_lock_irq(&ctx->fault_pending_wqh.lock);525 /*526 * No need of list_del_init(), the uwq on the stack527 * will be freed shortly anyway.528 */529 list_del(&uwq.wq.entry);530 spin_unlock_irq(&ctx->fault_pending_wqh.lock);531 }532 533 /*534 * ctx may go away after this if the userfault pseudo fd is535 * already released.536 */537 userfaultfd_ctx_put(ctx);538 539out:540 return ret;541}542 543static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,544 struct userfaultfd_wait_queue *ewq)545{546 struct userfaultfd_ctx *release_new_ctx;547 548 if (WARN_ON_ONCE(current->flags & PF_EXITING))549 goto out;550 551 ewq->ctx = ctx;552 init_waitqueue_entry(&ewq->wq, current);553 release_new_ctx = NULL;554 555 spin_lock_irq(&ctx->event_wqh.lock);556 /*557 * After the __add_wait_queue the uwq is visible to userland558 * through poll/read().559 */560 __add_wait_queue(&ctx->event_wqh, &ewq->wq);561 for (;;) {562 set_current_state(TASK_KILLABLE);563 if (ewq->msg.event == 0)564 break;565 if (READ_ONCE(ctx->released) ||566 fatal_signal_pending(current)) {567 /*568 * &ewq->wq may be queued in fork_event, but569 * __remove_wait_queue ignores the head570 * parameter. It would be a problem if it571 * didn't.572 */573 __remove_wait_queue(&ctx->event_wqh, &ewq->wq);574 if (ewq->msg.event == UFFD_EVENT_FORK) {575 struct userfaultfd_ctx *new;576 577 new = (struct userfaultfd_ctx *)578 (unsigned long)579 ewq->msg.arg.reserved.reserved1;580 release_new_ctx = new;581 }582 break;583 }584 585 spin_unlock_irq(&ctx->event_wqh.lock);586 587 wake_up_poll(&ctx->fd_wqh, EPOLLIN);588 schedule();589 590 spin_lock_irq(&ctx->event_wqh.lock);591 }592 __set_current_state(TASK_RUNNING);593 spin_unlock_irq(&ctx->event_wqh.lock);594 595 if (release_new_ctx) {596 userfaultfd_release_new(release_new_ctx);597 userfaultfd_ctx_put(release_new_ctx);598 }599 600 /*601 * ctx may go away after this if the userfault pseudo fd is602 * already released.603 */604out:605 atomic_dec(&ctx->mmap_changing);606 VM_BUG_ON(atomic_read(&ctx->mmap_changing) < 0);607 userfaultfd_ctx_put(ctx);608}609 610static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx,611 struct userfaultfd_wait_queue *ewq)612{613 ewq->msg.event = 0;614 wake_up_locked(&ctx->event_wqh);615 __remove_wait_queue(&ctx->event_wqh, &ewq->wq);616}617 618int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs)619{620 struct userfaultfd_ctx *ctx = NULL, *octx;621 struct userfaultfd_fork_ctx *fctx;622 623 octx = vma->vm_userfaultfd_ctx.ctx;624 if (!octx)625 return 0;626 627 if (!(octx->features & UFFD_FEATURE_EVENT_FORK)) {628 userfaultfd_reset_ctx(vma);629 return 0;630 }631 632 list_for_each_entry(fctx, fcs, list)633 if (fctx->orig == octx) {634 ctx = fctx->new;635 break;636 }637 638 if (!ctx) {639 fctx = kmalloc(sizeof(*fctx), GFP_KERNEL);640 if (!fctx)641 return -ENOMEM;642 643 ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);644 if (!ctx) {645 kfree(fctx);646 return -ENOMEM;647 }648 649 refcount_set(&ctx->refcount, 1);650 ctx->flags = octx->flags;651 ctx->features = octx->features;652 ctx->released = false;653 init_rwsem(&ctx->map_changing_lock);654 atomic_set(&ctx->mmap_changing, 0);655 ctx->mm = vma->vm_mm;656 mmgrab(ctx->mm);657 658 userfaultfd_ctx_get(octx);659 down_write(&octx->map_changing_lock);660 atomic_inc(&octx->mmap_changing);661 up_write(&octx->map_changing_lock);662 fctx->orig = octx;663 fctx->new = ctx;664 list_add_tail(&fctx->list, fcs);665 }666 667 vma->vm_userfaultfd_ctx.ctx = ctx;668 return 0;669}670 671static void dup_fctx(struct userfaultfd_fork_ctx *fctx)672{673 struct userfaultfd_ctx *ctx = fctx->orig;674 struct userfaultfd_wait_queue ewq;675 676 msg_init(&ewq.msg);677 678 ewq.msg.event = UFFD_EVENT_FORK;679 ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new;680 681 userfaultfd_event_wait_completion(ctx, &ewq);682}683 684void dup_userfaultfd_complete(struct list_head *fcs)685{686 struct userfaultfd_fork_ctx *fctx, *n;687 688 list_for_each_entry_safe(fctx, n, fcs, list) {689 dup_fctx(fctx);690 list_del(&fctx->list);691 kfree(fctx);692 }693}694 695void dup_userfaultfd_fail(struct list_head *fcs)696{697 struct userfaultfd_fork_ctx *fctx, *n;698 699 /*700 * An error has occurred on fork, we will tear memory down, but have701 * allocated memory for fctx's and raised reference counts for both the702 * original and child contexts (and on the mm for each as a result).703 *704 * These would ordinarily be taken care of by a user handling the event,705 * but we are no longer doing so, so manually clean up here.706 *707 * mm tear down will take care of cleaning up VMA contexts.708 */709 list_for_each_entry_safe(fctx, n, fcs, list) {710 struct userfaultfd_ctx *octx = fctx->orig;711 struct userfaultfd_ctx *ctx = fctx->new;712 713 atomic_dec(&octx->mmap_changing);714 VM_BUG_ON(atomic_read(&octx->mmap_changing) < 0);715 userfaultfd_ctx_put(octx);716 userfaultfd_ctx_put(ctx);717 718 list_del(&fctx->list);719 kfree(fctx);720 }721}722 723void mremap_userfaultfd_prep(struct vm_area_struct *vma,724 struct vm_userfaultfd_ctx *vm_ctx)725{726 struct userfaultfd_ctx *ctx;727 728 ctx = vma->vm_userfaultfd_ctx.ctx;729 730 if (!ctx)731 return;732 733 if (ctx->features & UFFD_FEATURE_EVENT_REMAP) {734 vm_ctx->ctx = ctx;735 userfaultfd_ctx_get(ctx);736 down_write(&ctx->map_changing_lock);737 atomic_inc(&ctx->mmap_changing);738 up_write(&ctx->map_changing_lock);739 } else {740 /* Drop uffd context if remap feature not enabled */741 userfaultfd_reset_ctx(vma);742 }743}744 745void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx,746 unsigned long from, unsigned long to,747 unsigned long len)748{749 struct userfaultfd_ctx *ctx = vm_ctx->ctx;750 struct userfaultfd_wait_queue ewq;751 752 if (!ctx)753 return;754 755 if (to & ~PAGE_MASK) {756 userfaultfd_ctx_put(ctx);757 return;758 }759 760 msg_init(&ewq.msg);761 762 ewq.msg.event = UFFD_EVENT_REMAP;763 ewq.msg.arg.remap.from = from;764 ewq.msg.arg.remap.to = to;765 ewq.msg.arg.remap.len = len;766 767 userfaultfd_event_wait_completion(ctx, &ewq);768}769 770bool userfaultfd_remove(struct vm_area_struct *vma,771 unsigned long start, unsigned long end)772{773 struct mm_struct *mm = vma->vm_mm;774 struct userfaultfd_ctx *ctx;775 struct userfaultfd_wait_queue ewq;776 777 ctx = vma->vm_userfaultfd_ctx.ctx;778 if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE))779 return true;780 781 userfaultfd_ctx_get(ctx);782 down_write(&ctx->map_changing_lock);783 atomic_inc(&ctx->mmap_changing);784 up_write(&ctx->map_changing_lock);785 mmap_read_unlock(mm);786 787 msg_init(&ewq.msg);788 789 ewq.msg.event = UFFD_EVENT_REMOVE;790 ewq.msg.arg.remove.start = start;791 ewq.msg.arg.remove.end = end;792 793 userfaultfd_event_wait_completion(ctx, &ewq);794 795 return false;796}797 798static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,799 unsigned long start, unsigned long end)800{801 struct userfaultfd_unmap_ctx *unmap_ctx;802 803 list_for_each_entry(unmap_ctx, unmaps, list)804 if (unmap_ctx->ctx == ctx && unmap_ctx->start == start &&805 unmap_ctx->end == end)806 return true;807 808 return false;809}810 811int userfaultfd_unmap_prep(struct vm_area_struct *vma, unsigned long start,812 unsigned long end, struct list_head *unmaps)813{814 struct userfaultfd_unmap_ctx *unmap_ctx;815 struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;816 817 if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_UNMAP) ||818 has_unmap_ctx(ctx, unmaps, start, end))819 return 0;820 821 unmap_ctx = kzalloc(sizeof(*unmap_ctx), GFP_KERNEL);822 if (!unmap_ctx)823 return -ENOMEM;824 825 userfaultfd_ctx_get(ctx);826 down_write(&ctx->map_changing_lock);827 atomic_inc(&ctx->mmap_changing);828 up_write(&ctx->map_changing_lock);829 unmap_ctx->ctx = ctx;830 unmap_ctx->start = start;831 unmap_ctx->end = end;832 list_add_tail(&unmap_ctx->list, unmaps);833 834 return 0;835}836 837void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf)838{839 struct userfaultfd_unmap_ctx *ctx, *n;840 struct userfaultfd_wait_queue ewq;841 842 list_for_each_entry_safe(ctx, n, uf, list) {843 msg_init(&ewq.msg);844 845 ewq.msg.event = UFFD_EVENT_UNMAP;846 ewq.msg.arg.remove.start = ctx->start;847 ewq.msg.arg.remove.end = ctx->end;848 849 userfaultfd_event_wait_completion(ctx->ctx, &ewq);850 851 list_del(&ctx->list);852 kfree(ctx);853 }854}855 856static int userfaultfd_release(struct inode *inode, struct file *file)857{858 struct userfaultfd_ctx *ctx = file->private_data;859 struct mm_struct *mm = ctx->mm;860 /* len == 0 means wake all */861 struct userfaultfd_wake_range range = { .len = 0, };862 863 WRITE_ONCE(ctx->released, true);864 865 userfaultfd_release_all(mm, ctx);866 867 /*868 * After no new page faults can wait on this fault_*wqh, flush869 * the last page faults that may have been already waiting on870 * the fault_*wqh.871 */872 spin_lock_irq(&ctx->fault_pending_wqh.lock);873 __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range);874 __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range);875 spin_unlock_irq(&ctx->fault_pending_wqh.lock);876 877 /* Flush pending events that may still wait on event_wqh */878 wake_up_all(&ctx->event_wqh);879 880 wake_up_poll(&ctx->fd_wqh, EPOLLHUP);881 userfaultfd_ctx_put(ctx);882 return 0;883}884 885/* fault_pending_wqh.lock must be hold by the caller */886static inline struct userfaultfd_wait_queue *find_userfault_in(887 wait_queue_head_t *wqh)888{889 wait_queue_entry_t *wq;890 struct userfaultfd_wait_queue *uwq;891 892 lockdep_assert_held(&wqh->lock);893 894 uwq = NULL;895 if (!waitqueue_active(wqh))896 goto out;897 /* walk in reverse to provide FIFO behavior to read userfaults */898 wq = list_last_entry(&wqh->head, typeof(*wq), entry);899 uwq = container_of(wq, struct userfaultfd_wait_queue, wq);900out:901 return uwq;902}903 904static inline struct userfaultfd_wait_queue *find_userfault(905 struct userfaultfd_ctx *ctx)906{907 return find_userfault_in(&ctx->fault_pending_wqh);908}909 910static inline struct userfaultfd_wait_queue *find_userfault_evt(911 struct userfaultfd_ctx *ctx)912{913 return find_userfault_in(&ctx->event_wqh);914}915 916static __poll_t userfaultfd_poll(struct file *file, poll_table *wait)917{918 struct userfaultfd_ctx *ctx = file->private_data;919 __poll_t ret;920 921 poll_wait(file, &ctx->fd_wqh, wait);922 923 if (!userfaultfd_is_initialized(ctx))924 return EPOLLERR;925 926 /*927 * poll() never guarantees that read won't block.928 * userfaults can be waken before they're read().929 */930 if (unlikely(!(file->f_flags & O_NONBLOCK)))931 return EPOLLERR;932 /*933 * lockless access to see if there are pending faults934 * __pollwait last action is the add_wait_queue but935 * the spin_unlock would allow the waitqueue_active to936 * pass above the actual list_add inside937 * add_wait_queue critical section. So use a full938 * memory barrier to serialize the list_add write of939 * add_wait_queue() with the waitqueue_active read940 * below.941 */942 ret = 0;943 smp_mb();944 if (waitqueue_active(&ctx->fault_pending_wqh))945 ret = EPOLLIN;946 else if (waitqueue_active(&ctx->event_wqh))947 ret = EPOLLIN;948 949 return ret;950}951 952static const struct file_operations userfaultfd_fops;953 954static int resolve_userfault_fork(struct userfaultfd_ctx *new,955 struct inode *inode,956 struct uffd_msg *msg)957{958 int fd;959 960 fd = anon_inode_create_getfd("[userfaultfd]", &userfaultfd_fops, new,961 O_RDONLY | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode);962 if (fd < 0)963 return fd;964 965 msg->arg.reserved.reserved1 = 0;966 msg->arg.fork.ufd = fd;967 return 0;968}969 970static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait,971 struct uffd_msg *msg, struct inode *inode)972{973 ssize_t ret;974 DECLARE_WAITQUEUE(wait, current);975 struct userfaultfd_wait_queue *uwq;976 /*977 * Handling fork event requires sleeping operations, so978 * we drop the event_wqh lock, then do these ops, then979 * lock it back and wake up the waiter. While the lock is980 * dropped the ewq may go away so we keep track of it981 * carefully.982 */983 LIST_HEAD(fork_event);984 struct userfaultfd_ctx *fork_nctx = NULL;985 986 /* always take the fd_wqh lock before the fault_pending_wqh lock */987 spin_lock_irq(&ctx->fd_wqh.lock);988 __add_wait_queue(&ctx->fd_wqh, &wait);989 for (;;) {990 set_current_state(TASK_INTERRUPTIBLE);991 spin_lock(&ctx->fault_pending_wqh.lock);992 uwq = find_userfault(ctx);993 if (uwq) {994 /*995 * Use a seqcount to repeat the lockless check996 * in wake_userfault() to avoid missing997 * wakeups because during the refile both998 * waitqueue could become empty if this is the999 * only userfault.1000 */1001 write_seqcount_begin(&ctx->refile_seq);1002 1003 /*1004 * The fault_pending_wqh.lock prevents the uwq1005 * to disappear from under us.1006 *1007 * Refile this userfault from1008 * fault_pending_wqh to fault_wqh, it's not1009 * pending anymore after we read it.1010 *1011 * Use list_del() by hand (as1012 * userfaultfd_wake_function also uses1013 * list_del_init() by hand) to be sure nobody1014 * changes __remove_wait_queue() to use1015 * list_del_init() in turn breaking the1016 * !list_empty_careful() check in1017 * handle_userfault(). The uwq->wq.head list1018 * must never be empty at any time during the1019 * refile, or the waitqueue could disappear1020 * from under us. The "wait_queue_head_t"1021 * parameter of __remove_wait_queue() is unused1022 * anyway.1023 */1024 list_del(&uwq->wq.entry);1025 add_wait_queue(&ctx->fault_wqh, &uwq->wq);1026 1027 write_seqcount_end(&ctx->refile_seq);1028 1029 /* careful to always initialize msg if ret == 0 */1030 *msg = uwq->msg;1031 spin_unlock(&ctx->fault_pending_wqh.lock);1032 ret = 0;1033 break;1034 }1035 spin_unlock(&ctx->fault_pending_wqh.lock);1036 1037 spin_lock(&ctx->event_wqh.lock);1038 uwq = find_userfault_evt(ctx);1039 if (uwq) {1040 *msg = uwq->msg;1041 1042 if (uwq->msg.event == UFFD_EVENT_FORK) {1043 fork_nctx = (struct userfaultfd_ctx *)1044 (unsigned long)1045 uwq->msg.arg.reserved.reserved1;1046 list_move(&uwq->wq.entry, &fork_event);1047 /*1048 * fork_nctx can be freed as soon as1049 * we drop the lock, unless we take a1050 * reference on it.1051 */1052 userfaultfd_ctx_get(fork_nctx);1053 spin_unlock(&ctx->event_wqh.lock);1054 ret = 0;1055 break;1056 }1057 1058 userfaultfd_event_complete(ctx, uwq);1059 spin_unlock(&ctx->event_wqh.lock);1060 ret = 0;1061 break;1062 }1063 spin_unlock(&ctx->event_wqh.lock);1064 1065 if (signal_pending(current)) {1066 ret = -ERESTARTSYS;1067 break;1068 }1069 if (no_wait) {1070 ret = -EAGAIN;1071 break;1072 }1073 spin_unlock_irq(&ctx->fd_wqh.lock);1074 schedule();1075 spin_lock_irq(&ctx->fd_wqh.lock);1076 }1077 __remove_wait_queue(&ctx->fd_wqh, &wait);1078 __set_current_state(TASK_RUNNING);1079 spin_unlock_irq(&ctx->fd_wqh.lock);1080 1081 if (!ret && msg->event == UFFD_EVENT_FORK) {1082 ret = resolve_userfault_fork(fork_nctx, inode, msg);1083 spin_lock_irq(&ctx->event_wqh.lock);1084 if (!list_empty(&fork_event)) {1085 /*1086 * The fork thread didn't abort, so we can1087 * drop the temporary refcount.1088 */1089 userfaultfd_ctx_put(fork_nctx);1090 1091 uwq = list_first_entry(&fork_event,1092 typeof(*uwq),1093 wq.entry);1094 /*1095 * If fork_event list wasn't empty and in turn1096 * the event wasn't already released by fork1097 * (the event is allocated on fork kernel1098 * stack), put the event back to its place in1099 * the event_wq. fork_event head will be freed1100 * as soon as we return so the event cannot1101 * stay queued there no matter the current1102 * "ret" value.1103 */1104 list_del(&uwq->wq.entry);1105 __add_wait_queue(&ctx->event_wqh, &uwq->wq);1106 1107 /*1108 * Leave the event in the waitqueue and report1109 * error to userland if we failed to resolve1110 * the userfault fork.1111 */1112 if (likely(!ret))1113 userfaultfd_event_complete(ctx, uwq);1114 } else {1115 /*1116 * Here the fork thread aborted and the1117 * refcount from the fork thread on fork_nctx1118 * has already been released. We still hold1119 * the reference we took before releasing the1120 * lock above. If resolve_userfault_fork1121 * failed we've to drop it because the1122 * fork_nctx has to be freed in such case. If1123 * it succeeded we'll hold it because the new1124 * uffd references it.1125 */1126 if (ret)1127 userfaultfd_ctx_put(fork_nctx);1128 }1129 spin_unlock_irq(&ctx->event_wqh.lock);1130 }1131 1132 return ret;1133}1134 1135static ssize_t userfaultfd_read_iter(struct kiocb *iocb, struct iov_iter *to)1136{1137 struct file *file = iocb->ki_filp;1138 struct userfaultfd_ctx *ctx = file->private_data;1139 ssize_t _ret, ret = 0;1140 struct uffd_msg msg;1141 struct inode *inode = file_inode(file);1142 bool no_wait;1143 1144 if (!userfaultfd_is_initialized(ctx))1145 return -EINVAL;1146 1147 no_wait = file->f_flags & O_NONBLOCK || iocb->ki_flags & IOCB_NOWAIT;1148 for (;;) {1149 if (iov_iter_count(to) < sizeof(msg))1150 return ret ? ret : -EINVAL;1151 _ret = userfaultfd_ctx_read(ctx, no_wait, &msg, inode);1152 if (_ret < 0)1153 return ret ? ret : _ret;1154 _ret = !copy_to_iter_full(&msg, sizeof(msg), to);1155 if (_ret)1156 return ret ? ret : -EFAULT;1157 ret += sizeof(msg);1158 /*1159 * Allow to read more than one fault at time but only1160 * block if waiting for the very first one.1161 */1162 no_wait = true;1163 }1164}1165 1166static void __wake_userfault(struct userfaultfd_ctx *ctx,1167 struct userfaultfd_wake_range *range)1168{1169 spin_lock_irq(&ctx->fault_pending_wqh.lock);1170 /* wake all in the range and autoremove */1171 if (waitqueue_active(&ctx->fault_pending_wqh))1172 __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL,1173 range);1174 if (waitqueue_active(&ctx->fault_wqh))1175 __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, range);1176 spin_unlock_irq(&ctx->fault_pending_wqh.lock);1177}1178 1179static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx,1180 struct userfaultfd_wake_range *range)1181{1182 unsigned seq;1183 bool need_wakeup;1184 1185 /*1186 * To be sure waitqueue_active() is not reordered by the CPU1187 * before the pagetable update, use an explicit SMP memory1188 * barrier here. PT lock release or mmap_read_unlock(mm) still1189 * have release semantics that can allow the1190 * waitqueue_active() to be reordered before the pte update.1191 */1192 smp_mb();1193 1194 /*1195 * Use waitqueue_active because it's very frequent to1196 * change the address space atomically even if there are no1197 * userfaults yet. So we take the spinlock only when we're1198 * sure we've userfaults to wake.1199 */1200 do {1201 seq = read_seqcount_begin(&ctx->refile_seq);1202 need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) ||1203 waitqueue_active(&ctx->fault_wqh);1204 cond_resched();1205 } while (read_seqcount_retry(&ctx->refile_seq, seq));1206 if (need_wakeup)1207 __wake_userfault(ctx, range);1208}1209 1210static __always_inline int validate_unaligned_range(1211 struct mm_struct *mm, __u64 start, __u64 len)1212{1213 __u64 task_size = mm->task_size;1214 1215 if (len & ~PAGE_MASK)1216 return -EINVAL;1217 if (!len)1218 return -EINVAL;1219 if (start < mmap_min_addr)1220 return -EINVAL;1221 if (start >= task_size)1222 return -EINVAL;1223 if (len > task_size - start)1224 return -EINVAL;1225 if (start + len <= start)1226 return -EINVAL;1227 return 0;1228}1229 1230static __always_inline int validate_range(struct mm_struct *mm,1231 __u64 start, __u64 len)1232{1233 if (start & ~PAGE_MASK)1234 return -EINVAL;1235 1236 return validate_unaligned_range(mm, start, len);1237}1238 1239static int userfaultfd_register(struct userfaultfd_ctx *ctx,1240 unsigned long arg)1241{1242 struct mm_struct *mm = ctx->mm;1243 struct vm_area_struct *vma, *cur;1244 int ret;1245 struct uffdio_register uffdio_register;1246 struct uffdio_register __user *user_uffdio_register;1247 unsigned long vm_flags;1248 bool found;1249 bool basic_ioctls;1250 unsigned long start, end;1251 struct vma_iterator vmi;1252 bool wp_async = userfaultfd_wp_async_ctx(ctx);1253 1254 user_uffdio_register = (struct uffdio_register __user *) arg;1255 1256 ret = -EFAULT;1257 if (copy_from_user(&uffdio_register, user_uffdio_register,1258 sizeof(uffdio_register)-sizeof(__u64)))1259 goto out;1260 1261 ret = -EINVAL;1262 if (!uffdio_register.mode)1263 goto out;1264 if (uffdio_register.mode & ~UFFD_API_REGISTER_MODES)1265 goto out;1266 vm_flags = 0;1267 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING)1268 vm_flags |= VM_UFFD_MISSING;1269 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) {1270#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP1271 goto out;1272#endif1273 vm_flags |= VM_UFFD_WP;1274 }1275 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) {1276#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR1277 goto out;1278#endif1279 vm_flags |= VM_UFFD_MINOR;1280 }1281 1282 ret = validate_range(mm, uffdio_register.range.start,1283 uffdio_register.range.len);1284 if (ret)1285 goto out;1286 1287 start = uffdio_register.range.start;1288 end = start + uffdio_register.range.len;1289 1290 ret = -ENOMEM;1291 if (!mmget_not_zero(mm))1292 goto out;1293 1294 ret = -EINVAL;1295 mmap_write_lock(mm);1296 vma_iter_init(&vmi, mm, start);1297 vma = vma_find(&vmi, end);1298 if (!vma)1299 goto out_unlock;1300 1301 /*1302 * If the first vma contains huge pages, make sure start address1303 * is aligned to huge page size.1304 */1305 if (is_vm_hugetlb_page(vma)) {1306 unsigned long vma_hpagesize = vma_kernel_pagesize(vma);1307 1308 if (start & (vma_hpagesize - 1))1309 goto out_unlock;1310 }1311 1312 /*1313 * Search for not compatible vmas.1314 */1315 found = false;1316 basic_ioctls = false;1317 cur = vma;1318 do {1319 cond_resched();1320 1321 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^1322 !!(cur->vm_flags & __VM_UFFD_FLAGS));1323 1324 /* check not compatible vmas */1325 ret = -EINVAL;1326 if (!vma_can_userfault(cur, vm_flags, wp_async))1327 goto out_unlock;1328 1329 /*1330 * UFFDIO_COPY will fill file holes even without1331 * PROT_WRITE. This check enforces that if this is a1332 * MAP_SHARED, the process has write permission to the backing1333 * file. If VM_MAYWRITE is set it also enforces that on a1334 * MAP_SHARED vma: there is no F_WRITE_SEAL and no further1335 * F_WRITE_SEAL can be taken until the vma is destroyed.1336 */1337 ret = -EPERM;1338 if (unlikely(!(cur->vm_flags & VM_MAYWRITE)))1339 goto out_unlock;1340 1341 /*1342 * If this vma contains ending address, and huge pages1343 * check alignment.1344 */1345 if (is_vm_hugetlb_page(cur) && end <= cur->vm_end &&1346 end > cur->vm_start) {1347 unsigned long vma_hpagesize = vma_kernel_pagesize(cur);1348 1349 ret = -EINVAL;1350 1351 if (end & (vma_hpagesize - 1))1352 goto out_unlock;1353 }1354 if ((vm_flags & VM_UFFD_WP) && !(cur->vm_flags & VM_MAYWRITE))1355 goto out_unlock;1356 1357 /*1358 * Check that this vma isn't already owned by a1359 * different userfaultfd. We can't allow more than one1360 * userfaultfd to own a single vma simultaneously or we1361 * wouldn't know which one to deliver the userfaults to.1362 */1363 ret = -EBUSY;1364 if (cur->vm_userfaultfd_ctx.ctx &&1365 cur->vm_userfaultfd_ctx.ctx != ctx)1366 goto out_unlock;1367 1368 /*1369 * Note vmas containing huge pages1370 */1371 if (is_vm_hugetlb_page(cur))1372 basic_ioctls = true;1373 1374 found = true;1375 } for_each_vma_range(vmi, cur, end);1376 BUG_ON(!found);1377 1378 ret = userfaultfd_register_range(ctx, vma, vm_flags, start, end,1379 wp_async);1380 1381out_unlock:1382 mmap_write_unlock(mm);1383 mmput(mm);1384 if (!ret) {1385 __u64 ioctls_out;1386 1387 ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC :1388 UFFD_API_RANGE_IOCTLS;1389 1390 /*1391 * Declare the WP ioctl only if the WP mode is1392 * specified and all checks passed with the range1393 */1394 if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_WP))1395 ioctls_out &= ~((__u64)1 << _UFFDIO_WRITEPROTECT);1396 1397 /* CONTINUE ioctl is only supported for MINOR ranges. */1398 if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR))1399 ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE);1400 1401 /*1402 * Now that we scanned all vmas we can already tell1403 * userland which ioctls methods are guaranteed to1404 * succeed on this range.1405 */1406 if (put_user(ioctls_out, &user_uffdio_register->ioctls))1407 ret = -EFAULT;1408 }1409out:1410 return ret;1411}1412 1413static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,1414 unsigned long arg)1415{1416 struct mm_struct *mm = ctx->mm;1417 struct vm_area_struct *vma, *prev, *cur;1418 int ret;1419 struct uffdio_range uffdio_unregister;1420 bool found;1421 unsigned long start, end, vma_end;1422 const void __user *buf = (void __user *)arg;1423 struct vma_iterator vmi;1424 bool wp_async = userfaultfd_wp_async_ctx(ctx);1425 1426 ret = -EFAULT;1427 if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))1428 goto out;1429 1430 ret = validate_range(mm, uffdio_unregister.start,1431 uffdio_unregister.len);1432 if (ret)1433 goto out;1434 1435 start = uffdio_unregister.start;1436 end = start + uffdio_unregister.len;1437 1438 ret = -ENOMEM;1439 if (!mmget_not_zero(mm))1440 goto out;1441 1442 mmap_write_lock(mm);1443 ret = -EINVAL;1444 vma_iter_init(&vmi, mm, start);1445 vma = vma_find(&vmi, end);1446 if (!vma)1447 goto out_unlock;1448 1449 /*1450 * If the first vma contains huge pages, make sure start address1451 * is aligned to huge page size.1452 */1453 if (is_vm_hugetlb_page(vma)) {1454 unsigned long vma_hpagesize = vma_kernel_pagesize(vma);1455 1456 if (start & (vma_hpagesize - 1))1457 goto out_unlock;1458 }1459 1460 /*1461 * Search for not compatible vmas.1462 */1463 found = false;1464 cur = vma;1465 do {1466 cond_resched();1467 1468 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^1469 !!(cur->vm_flags & __VM_UFFD_FLAGS));1470 1471 /*1472 * Check not compatible vmas, not strictly required1473 * here as not compatible vmas cannot have an1474 * userfaultfd_ctx registered on them, but this1475 * provides for more strict behavior to notice1476 * unregistration errors.1477 */1478 if (!vma_can_userfault(cur, cur->vm_flags, wp_async))1479 goto out_unlock;1480 1481 found = true;1482 } for_each_vma_range(vmi, cur, end);1483 BUG_ON(!found);1484 1485 vma_iter_set(&vmi, start);1486 prev = vma_prev(&vmi);1487 if (vma->vm_start < start)1488 prev = vma;1489 1490 ret = 0;1491 for_each_vma_range(vmi, vma, end) {1492 cond_resched();1493 1494 BUG_ON(!vma_can_userfault(vma, vma->vm_flags, wp_async));1495 1496 /*1497 * Nothing to do: this vma is already registered into this1498 * userfaultfd and with the right tracking mode too.1499 */1500 if (!vma->vm_userfaultfd_ctx.ctx)1501 goto skip;1502 1503 WARN_ON(!(vma->vm_flags & VM_MAYWRITE));1504 1505 if (vma->vm_start > start)1506 start = vma->vm_start;1507 vma_end = min(end, vma->vm_end);1508 1509 if (userfaultfd_missing(vma)) {1510 /*1511 * Wake any concurrent pending userfault while1512 * we unregister, so they will not hang1513 * permanently and it avoids userland to call1514 * UFFDIO_WAKE explicitly.1515 */1516 struct userfaultfd_wake_range range;1517 range.start = start;1518 range.len = vma_end - start;1519 wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range);1520 }1521 1522 vma = userfaultfd_clear_vma(&vmi, prev, vma,1523 start, vma_end);1524 if (IS_ERR(vma)) {1525 ret = PTR_ERR(vma);1526 break;1527 }1528 1529 skip:1530 prev = vma;1531 start = vma->vm_end;1532 }1533 1534out_unlock:1535 mmap_write_unlock(mm);1536 mmput(mm);1537out:1538 return ret;1539}1540 1541/*1542 * userfaultfd_wake may be used in combination with the1543 * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches.1544 */1545static int userfaultfd_wake(struct userfaultfd_ctx *ctx,1546 unsigned long arg)1547{1548 int ret;1549 struct uffdio_range uffdio_wake;1550 struct userfaultfd_wake_range range;1551 const void __user *buf = (void __user *)arg;1552 1553 ret = -EFAULT;1554 if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake)))1555 goto out;1556 1557 ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len);1558 if (ret)1559 goto out;1560 1561 range.start = uffdio_wake.start;1562 range.len = uffdio_wake.len;1563 1564 /*1565 * len == 0 means wake all and we don't want to wake all here,1566 * so check it again to be sure.1567 */1568 VM_BUG_ON(!range.len);1569 1570 wake_userfault(ctx, &range);1571 ret = 0;1572 1573out:1574 return ret;1575}1576 1577static int userfaultfd_copy(struct userfaultfd_ctx *ctx,1578 unsigned long arg)1579{1580 __s64 ret;1581 struct uffdio_copy uffdio_copy;1582 struct uffdio_copy __user *user_uffdio_copy;1583 struct userfaultfd_wake_range range;1584 uffd_flags_t flags = 0;1585 1586 user_uffdio_copy = (struct uffdio_copy __user *) arg;1587 1588 ret = -EAGAIN;1589 if (atomic_read(&ctx->mmap_changing))1590 goto out;1591 1592 ret = -EFAULT;1593 if (copy_from_user(&uffdio_copy, user_uffdio_copy,1594 /* don't copy "copy" last field */1595 sizeof(uffdio_copy)-sizeof(__s64)))1596 goto out;1597 1598 ret = validate_unaligned_range(ctx->mm, uffdio_copy.src,1599 uffdio_copy.len);1600 if (ret)1601 goto out;1602 ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len);1603 if (ret)1604 goto out;1605 1606 ret = -EINVAL;1607 if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP))1608 goto out;1609 if (uffdio_copy.mode & UFFDIO_COPY_MODE_WP)1610 flags |= MFILL_ATOMIC_WP;1611 if (mmget_not_zero(ctx->mm)) {1612 ret = mfill_atomic_copy(ctx, uffdio_copy.dst, uffdio_copy.src,1613 uffdio_copy.len, flags);1614 mmput(ctx->mm);1615 } else {1616 return -ESRCH;1617 }1618 if (unlikely(put_user(ret, &user_uffdio_copy->copy)))1619 return -EFAULT;1620 if (ret < 0)1621 goto out;1622 BUG_ON(!ret);1623 /* len == 0 would wake all */1624 range.len = ret;1625 if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) {1626 range.start = uffdio_copy.dst;1627 wake_userfault(ctx, &range);1628 }1629 ret = range.len == uffdio_copy.len ? 0 : -EAGAIN;1630out:1631 return ret;1632}1633 1634static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx,1635 unsigned long arg)1636{1637 __s64 ret;1638 struct uffdio_zeropage uffdio_zeropage;1639 struct uffdio_zeropage __user *user_uffdio_zeropage;1640 struct userfaultfd_wake_range range;1641 1642 user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg;1643 1644 ret = -EAGAIN;1645 if (atomic_read(&ctx->mmap_changing))1646 goto out;1647 1648 ret = -EFAULT;1649 if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage,1650 /* don't copy "zeropage" last field */1651 sizeof(uffdio_zeropage)-sizeof(__s64)))1652 goto out;1653 1654 ret = validate_range(ctx->mm, uffdio_zeropage.range.start,1655 uffdio_zeropage.range.len);1656 if (ret)1657 goto out;1658 ret = -EINVAL;1659 if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE)1660 goto out;1661 1662 if (mmget_not_zero(ctx->mm)) {1663 ret = mfill_atomic_zeropage(ctx, uffdio_zeropage.range.start,1664 uffdio_zeropage.range.len);1665 mmput(ctx->mm);1666 } else {1667 return -ESRCH;1668 }1669 if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))1670 return -EFAULT;1671 if (ret < 0)1672 goto out;1673 /* len == 0 would wake all */1674 BUG_ON(!ret);1675 range.len = ret;1676 if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) {1677 range.start = uffdio_zeropage.range.start;1678 wake_userfault(ctx, &range);1679 }1680 ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN;1681out:1682 return ret;1683}1684 1685static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx,1686 unsigned long arg)1687{1688 int ret;1689 struct uffdio_writeprotect uffdio_wp;1690 struct uffdio_writeprotect __user *user_uffdio_wp;1691 struct userfaultfd_wake_range range;1692 bool mode_wp, mode_dontwake;1693 1694 if (atomic_read(&ctx->mmap_changing))1695 return -EAGAIN;1696 1697 user_uffdio_wp = (struct uffdio_writeprotect __user *) arg;1698 1699 if (copy_from_user(&uffdio_wp, user_uffdio_wp,1700 sizeof(struct uffdio_writeprotect)))1701 return -EFAULT;1702 1703 ret = validate_range(ctx->mm, uffdio_wp.range.start,1704 uffdio_wp.range.len);1705 if (ret)1706 return ret;1707 1708 if (uffdio_wp.mode & ~(UFFDIO_WRITEPROTECT_MODE_DONTWAKE |1709 UFFDIO_WRITEPROTECT_MODE_WP))1710 return -EINVAL;1711 1712 mode_wp = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_WP;1713 mode_dontwake = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE;1714 1715 if (mode_wp && mode_dontwake)1716 return -EINVAL;1717 1718 if (mmget_not_zero(ctx->mm)) {1719 ret = mwriteprotect_range(ctx, uffdio_wp.range.start,1720 uffdio_wp.range.len, mode_wp);1721 mmput(ctx->mm);1722 } else {1723 return -ESRCH;1724 }1725 1726 if (ret)1727 return ret;1728 1729 if (!mode_wp && !mode_dontwake) {1730 range.start = uffdio_wp.range.start;1731 range.len = uffdio_wp.range.len;1732 wake_userfault(ctx, &range);1733 }1734 return ret;1735}1736 1737static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg)1738{1739 __s64 ret;1740 struct uffdio_continue uffdio_continue;1741 struct uffdio_continue __user *user_uffdio_continue;1742 struct userfaultfd_wake_range range;1743 uffd_flags_t flags = 0;1744 1745 user_uffdio_continue = (struct uffdio_continue __user *)arg;1746 1747 ret = -EAGAIN;1748 if (atomic_read(&ctx->mmap_changing))1749 goto out;1750 1751 ret = -EFAULT;1752 if (copy_from_user(&uffdio_continue, user_uffdio_continue,1753 /* don't copy the output fields */1754 sizeof(uffdio_continue) - (sizeof(__s64))))1755 goto out;1756 1757 ret = validate_range(ctx->mm, uffdio_continue.range.start,1758 uffdio_continue.range.len);1759 if (ret)1760 goto out;1761 1762 ret = -EINVAL;1763 if (uffdio_continue.mode & ~(UFFDIO_CONTINUE_MODE_DONTWAKE |1764 UFFDIO_CONTINUE_MODE_WP))1765 goto out;1766 if (uffdio_continue.mode & UFFDIO_CONTINUE_MODE_WP)1767 flags |= MFILL_ATOMIC_WP;1768 1769 if (mmget_not_zero(ctx->mm)) {1770 ret = mfill_atomic_continue(ctx, uffdio_continue.range.start,1771 uffdio_continue.range.len, flags);1772 mmput(ctx->mm);1773 } else {1774 return -ESRCH;1775 }1776 1777 if (unlikely(put_user(ret, &user_uffdio_continue->mapped)))1778 return -EFAULT;1779 if (ret < 0)1780 goto out;1781 1782 /* len == 0 would wake all */1783 BUG_ON(!ret);1784 range.len = ret;1785 if (!(uffdio_continue.mode & UFFDIO_CONTINUE_MODE_DONTWAKE)) {1786 range.start = uffdio_continue.range.start;1787 wake_userfault(ctx, &range);1788 }1789 ret = range.len == uffdio_continue.range.len ? 0 : -EAGAIN;1790 1791out:1792 return ret;1793}1794 1795static inline int userfaultfd_poison(struct userfaultfd_ctx *ctx, unsigned long arg)1796{1797 __s64 ret;1798 struct uffdio_poison uffdio_poison;1799 struct uffdio_poison __user *user_uffdio_poison;1800 struct userfaultfd_wake_range range;1801 1802 user_uffdio_poison = (struct uffdio_poison __user *)arg;1803 1804 ret = -EAGAIN;1805 if (atomic_read(&ctx->mmap_changing))1806 goto out;1807 1808 ret = -EFAULT;1809 if (copy_from_user(&uffdio_poison, user_uffdio_poison,1810 /* don't copy the output fields */1811 sizeof(uffdio_poison) - (sizeof(__s64))))1812 goto out;1813 1814 ret = validate_range(ctx->mm, uffdio_poison.range.start,1815 uffdio_poison.range.len);1816 if (ret)1817 goto out;1818 1819 ret = -EINVAL;1820 if (uffdio_poison.mode & ~UFFDIO_POISON_MODE_DONTWAKE)1821 goto out;1822 1823 if (mmget_not_zero(ctx->mm)) {1824 ret = mfill_atomic_poison(ctx, uffdio_poison.range.start,1825 uffdio_poison.range.len, 0);1826 mmput(ctx->mm);1827 } else {1828 return -ESRCH;1829 }1830 1831 if (unlikely(put_user(ret, &user_uffdio_poison->updated)))1832 return -EFAULT;1833 if (ret < 0)1834 goto out;1835 1836 /* len == 0 would wake all */1837 BUG_ON(!ret);1838 range.len = ret;1839 if (!(uffdio_poison.mode & UFFDIO_POISON_MODE_DONTWAKE)) {1840 range.start = uffdio_poison.range.start;1841 wake_userfault(ctx, &range);1842 }1843 ret = range.len == uffdio_poison.range.len ? 0 : -EAGAIN;1844 1845out:1846 return ret;1847}1848 1849bool userfaultfd_wp_async(struct vm_area_struct *vma)1850{1851 return userfaultfd_wp_async_ctx(vma->vm_userfaultfd_ctx.ctx);1852}1853 1854static inline unsigned int uffd_ctx_features(__u64 user_features)1855{1856 /*1857 * For the current set of features the bits just coincide. Set1858 * UFFD_FEATURE_INITIALIZED to mark the features as enabled.1859 */1860 return (unsigned int)user_features | UFFD_FEATURE_INITIALIZED;1861}1862 1863static int userfaultfd_move(struct userfaultfd_ctx *ctx,1864 unsigned long arg)1865{1866 __s64 ret;1867 struct uffdio_move uffdio_move;1868 struct uffdio_move __user *user_uffdio_move;1869 struct userfaultfd_wake_range range;1870 struct mm_struct *mm = ctx->mm;1871 1872 user_uffdio_move = (struct uffdio_move __user *) arg;1873 1874 if (atomic_read(&ctx->mmap_changing))1875 return -EAGAIN;1876 1877 if (copy_from_user(&uffdio_move, user_uffdio_move,1878 /* don't copy "move" last field */1879 sizeof(uffdio_move)-sizeof(__s64)))1880 return -EFAULT;1881 1882 /* Do not allow cross-mm moves. */1883 if (mm != current->mm)1884 return -EINVAL;1885 1886 ret = validate_range(mm, uffdio_move.dst, uffdio_move.len);1887 if (ret)1888 return ret;1889 1890 ret = validate_range(mm, uffdio_move.src, uffdio_move.len);1891 if (ret)1892 return ret;1893 1894 if (uffdio_move.mode & ~(UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES|1895 UFFDIO_MOVE_MODE_DONTWAKE))1896 return -EINVAL;1897 1898 if (mmget_not_zero(mm)) {1899 ret = move_pages(ctx, uffdio_move.dst, uffdio_move.src,1900 uffdio_move.len, uffdio_move.mode);1901 mmput(mm);1902 } else {1903 return -ESRCH;1904 }1905 1906 if (unlikely(put_user(ret, &user_uffdio_move->move)))1907 return -EFAULT;1908 if (ret < 0)1909 goto out;1910 1911 /* len == 0 would wake all */1912 VM_WARN_ON(!ret);1913 range.len = ret;1914 if (!(uffdio_move.mode & UFFDIO_MOVE_MODE_DONTWAKE)) {1915 range.start = uffdio_move.dst;1916 wake_userfault(ctx, &range);1917 }1918 ret = range.len == uffdio_move.len ? 0 : -EAGAIN;1919 1920out:1921 return ret;1922}1923 1924/*1925 * userland asks for a certain API version and we return which bits1926 * and ioctl commands are implemented in this kernel for such API1927 * version or -EINVAL if unknown.1928 */1929static int userfaultfd_api(struct userfaultfd_ctx *ctx,1930 unsigned long arg)1931{1932 struct uffdio_api uffdio_api;1933 void __user *buf = (void __user *)arg;1934 unsigned int ctx_features;1935 int ret;1936 __u64 features;1937 1938 ret = -EFAULT;1939 if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))1940 goto out;1941 features = uffdio_api.features;1942 ret = -EINVAL;1943 if (uffdio_api.api != UFFD_API)1944 goto err_out;1945 ret = -EPERM;1946 if ((features & UFFD_FEATURE_EVENT_FORK) && !capable(CAP_SYS_PTRACE))1947 goto err_out;1948 1949 /* WP_ASYNC relies on WP_UNPOPULATED, choose it unconditionally */1950 if (features & UFFD_FEATURE_WP_ASYNC)1951 features |= UFFD_FEATURE_WP_UNPOPULATED;1952 1953 /* report all available features and ioctls to userland */1954 uffdio_api.features = UFFD_API_FEATURES;1955#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR1956 uffdio_api.features &=1957 ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM);1958#endif1959#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP1960 uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP;1961#endif1962#ifndef CONFIG_PTE_MARKER_UFFD_WP1963 uffdio_api.features &= ~UFFD_FEATURE_WP_HUGETLBFS_SHMEM;1964 uffdio_api.features &= ~UFFD_FEATURE_WP_UNPOPULATED;1965 uffdio_api.features &= ~UFFD_FEATURE_WP_ASYNC;1966#endif1967 1968 ret = -EINVAL;1969 if (features & ~uffdio_api.features)1970 goto err_out;1971 1972 uffdio_api.ioctls = UFFD_API_IOCTLS;1973 ret = -EFAULT;1974 if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))1975 goto out;1976 1977 /* only enable the requested features for this uffd context */1978 ctx_features = uffd_ctx_features(features);1979 ret = -EINVAL;1980 if (cmpxchg(&ctx->features, 0, ctx_features) != 0)1981 goto err_out;1982 1983 ret = 0;1984out:1985 return ret;1986err_out:1987 memset(&uffdio_api, 0, sizeof(uffdio_api));1988 if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))1989 ret = -EFAULT;1990 goto out;1991}1992 1993static long userfaultfd_ioctl(struct file *file, unsigned cmd,1994 unsigned long arg)1995{1996 int ret = -EINVAL;1997 struct userfaultfd_ctx *ctx = file->private_data;1998 1999 if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx))2000 return -EINVAL;2001 2002 switch(cmd) {2003 case UFFDIO_API:2004 ret = userfaultfd_api(ctx, arg);2005 break;2006 case UFFDIO_REGISTER:2007 ret = userfaultfd_register(ctx, arg);2008 break;2009 case UFFDIO_UNREGISTER:2010 ret = userfaultfd_unregister(ctx, arg);2011 break;2012 case UFFDIO_WAKE:2013 ret = userfaultfd_wake(ctx, arg);2014 break;2015 case UFFDIO_COPY:2016 ret = userfaultfd_copy(ctx, arg);2017 break;2018 case UFFDIO_ZEROPAGE:2019 ret = userfaultfd_zeropage(ctx, arg);2020 break;2021 case UFFDIO_MOVE:2022 ret = userfaultfd_move(ctx, arg);2023 break;2024 case UFFDIO_WRITEPROTECT:2025 ret = userfaultfd_writeprotect(ctx, arg);2026 break;2027 case UFFDIO_CONTINUE:2028 ret = userfaultfd_continue(ctx, arg);2029 break;2030 case UFFDIO_POISON:2031 ret = userfaultfd_poison(ctx, arg);2032 break;2033 }2034 return ret;2035}2036 2037#ifdef CONFIG_PROC_FS2038static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f)2039{2040 struct userfaultfd_ctx *ctx = f->private_data;2041 wait_queue_entry_t *wq;2042 unsigned long pending = 0, total = 0;2043 2044 spin_lock_irq(&ctx->fault_pending_wqh.lock);2045 list_for_each_entry(wq, &ctx->fault_pending_wqh.head, entry) {2046 pending++;2047 total++;2048 }2049 list_for_each_entry(wq, &ctx->fault_wqh.head, entry) {2050 total++;2051 }2052 spin_unlock_irq(&ctx->fault_pending_wqh.lock);2053 2054 /*2055 * If more protocols will be added, there will be all shown2056 * separated by a space. Like this:2057 * protocols: aa:... bb:...2058 */2059 seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n",2060 pending, total, UFFD_API, ctx->features,2061 UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS);2062}2063#endif2064 2065static const struct file_operations userfaultfd_fops = {2066#ifdef CONFIG_PROC_FS2067 .show_fdinfo = userfaultfd_show_fdinfo,2068#endif2069 .release = userfaultfd_release,2070 .poll = userfaultfd_poll,2071 .read_iter = userfaultfd_read_iter,2072 .unlocked_ioctl = userfaultfd_ioctl,2073 .compat_ioctl = compat_ptr_ioctl,2074 .llseek = noop_llseek,2075};2076 2077static void init_once_userfaultfd_ctx(void *mem)2078{2079 struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem;2080 2081 init_waitqueue_head(&ctx->fault_pending_wqh);2082 init_waitqueue_head(&ctx->fault_wqh);2083 init_waitqueue_head(&ctx->event_wqh);2084 init_waitqueue_head(&ctx->fd_wqh);2085 seqcount_spinlock_init(&ctx->refile_seq, &ctx->fault_pending_wqh.lock);2086}2087 2088static int new_userfaultfd(int flags)2089{2090 struct userfaultfd_ctx *ctx;2091 struct file *file;2092 int fd;2093 2094 BUG_ON(!current->mm);2095 2096 /* Check the UFFD_* constants for consistency. */2097 BUILD_BUG_ON(UFFD_USER_MODE_ONLY & UFFD_SHARED_FCNTL_FLAGS);2098 BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC);2099 BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK);2100 2101 if (flags & ~(UFFD_SHARED_FCNTL_FLAGS | UFFD_USER_MODE_ONLY))2102 return -EINVAL;2103 2104 ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);2105 if (!ctx)2106 return -ENOMEM;2107 2108 refcount_set(&ctx->refcount, 1);2109 ctx->flags = flags;2110 ctx->features = 0;2111 ctx->released = false;2112 init_rwsem(&ctx->map_changing_lock);2113 atomic_set(&ctx->mmap_changing, 0);2114 ctx->mm = current->mm;2115 2116 fd = get_unused_fd_flags(flags & UFFD_SHARED_FCNTL_FLAGS);2117 if (fd < 0)2118 goto err_out;2119 2120 /* Create a new inode so that the LSM can block the creation. */2121 file = anon_inode_create_getfile("[userfaultfd]", &userfaultfd_fops, ctx,2122 O_RDONLY | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL);2123 if (IS_ERR(file)) {2124 put_unused_fd(fd);2125 fd = PTR_ERR(file);2126 goto err_out;2127 }2128 /* prevent the mm struct to be freed */2129 mmgrab(ctx->mm);2130 file->f_mode |= FMODE_NOWAIT;2131 fd_install(fd, file);2132 return fd;2133err_out:2134 kmem_cache_free(userfaultfd_ctx_cachep, ctx);2135 return fd;2136}2137 2138static inline bool userfaultfd_syscall_allowed(int flags)2139{2140 /* Userspace-only page faults are always allowed */2141 if (flags & UFFD_USER_MODE_ONLY)2142 return true;2143 2144 /*2145 * The user is requesting a userfaultfd which can handle kernel faults.2146 * Privileged users are always allowed to do this.2147 */2148 if (capable(CAP_SYS_PTRACE))2149 return true;2150 2151 /* Otherwise, access to kernel fault handling is sysctl controlled. */2152 return sysctl_unprivileged_userfaultfd;2153}2154 2155SYSCALL_DEFINE1(userfaultfd, int, flags)2156{2157 if (!userfaultfd_syscall_allowed(flags))2158 return -EPERM;2159 2160 return new_userfaultfd(flags);2161}2162 2163static long userfaultfd_dev_ioctl(struct file *file, unsigned int cmd, unsigned long flags)2164{2165 if (cmd != USERFAULTFD_IOC_NEW)2166 return -EINVAL;2167 2168 return new_userfaultfd(flags);2169}2170 2171static const struct file_operations userfaultfd_dev_fops = {2172 .unlocked_ioctl = userfaultfd_dev_ioctl,2173 .compat_ioctl = userfaultfd_dev_ioctl,2174 .owner = THIS_MODULE,2175 .llseek = noop_llseek,2176};2177 2178static struct miscdevice userfaultfd_misc = {2179 .minor = MISC_DYNAMIC_MINOR,2180 .name = "userfaultfd",2181 .fops = &userfaultfd_dev_fops2182};2183 2184static int __init userfaultfd_init(void)2185{2186 int ret;2187 2188 ret = misc_register(&userfaultfd_misc);2189 if (ret)2190 return ret;2191 2192 userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache",2193 sizeof(struct userfaultfd_ctx),2194 0,2195 SLAB_HWCACHE_ALIGN|SLAB_PANIC,2196 init_once_userfaultfd_ctx);2197#ifdef CONFIG_SYSCTL2198 register_sysctl_init("vm", vm_userfaultfd_table);2199#endif2200 return 0;2201}2202__initcall(userfaultfd_init);2203