2621 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * fs/eventpoll.c (Efficient event retrieval implementation)4 * Copyright (C) 2001,...,2009 Davide Libenzi5 *6 * Davide Libenzi <davidel@xmailserver.org>7 */8 9#include <linux/init.h>10#include <linux/kernel.h>11#include <linux/sched/signal.h>12#include <linux/fs.h>13#include <linux/file.h>14#include <linux/signal.h>15#include <linux/errno.h>16#include <linux/mm.h>17#include <linux/slab.h>18#include <linux/poll.h>19#include <linux/string.h>20#include <linux/list.h>21#include <linux/hash.h>22#include <linux/spinlock.h>23#include <linux/syscalls.h>24#include <linux/rbtree.h>25#include <linux/wait.h>26#include <linux/eventpoll.h>27#include <linux/mount.h>28#include <linux/bitops.h>29#include <linux/mutex.h>30#include <linux/anon_inodes.h>31#include <linux/device.h>32#include <linux/uaccess.h>33#include <asm/io.h>34#include <asm/mman.h>35#include <linux/atomic.h>36#include <linux/proc_fs.h>37#include <linux/seq_file.h>38#include <linux/compat.h>39#include <linux/rculist.h>40#include <linux/capability.h>41#include <net/busy_poll.h>42 43/*44 * LOCKING:45 * There are three level of locking required by epoll :46 *47 * 1) epnested_mutex (mutex)48 * 2) ep->mtx (mutex)49 * 3) ep->lock (rwlock)50 *51 * The acquire order is the one listed above, from 1 to 3.52 * We need a rwlock (ep->lock) because we manipulate objects53 * from inside the poll callback, that might be triggered from54 * a wake_up() that in turn might be called from IRQ context.55 * So we can't sleep inside the poll callback and hence we need56 * a spinlock. During the event transfer loop (from kernel to57 * user space) we could end up sleeping due a copy_to_user(), so58 * we need a lock that will allow us to sleep. This lock is a59 * mutex (ep->mtx). It is acquired during the event transfer loop,60 * during epoll_ctl(EPOLL_CTL_DEL) and during eventpoll_release_file().61 * The epnested_mutex is acquired when inserting an epoll fd onto another62 * epoll fd. We do this so that we walk the epoll tree and ensure that this63 * insertion does not create a cycle of epoll file descriptors, which64 * could lead to deadlock. We need a global mutex to prevent two65 * simultaneous inserts (A into B and B into A) from racing and66 * constructing a cycle without either insert observing that it is67 * going to.68 * It is necessary to acquire multiple "ep->mtx"es at once in the69 * case when one epoll fd is added to another. In this case, we70 * always acquire the locks in the order of nesting (i.e. after71 * epoll_ctl(e1, EPOLL_CTL_ADD, e2), e1->mtx will always be acquired72 * before e2->mtx). Since we disallow cycles of epoll file73 * descriptors, this ensures that the mutexes are well-ordered. In74 * order to communicate this nesting to lockdep, when walking a tree75 * of epoll file descriptors, we use the current recursion depth as76 * the lockdep subkey.77 * It is possible to drop the "ep->mtx" and to use the global78 * mutex "epnested_mutex" (together with "ep->lock") to have it working,79 * but having "ep->mtx" will make the interface more scalable.80 * Events that require holding "epnested_mutex" are very rare, while for81 * normal operations the epoll private "ep->mtx" will guarantee82 * a better scalability.83 */84 85/* Epoll private bits inside the event mask */86#define EP_PRIVATE_BITS (EPOLLWAKEUP | EPOLLONESHOT | EPOLLET | EPOLLEXCLUSIVE)87 88#define EPOLLINOUT_BITS (EPOLLIN | EPOLLOUT)89 90#define EPOLLEXCLUSIVE_OK_BITS (EPOLLINOUT_BITS | EPOLLERR | EPOLLHUP | \91 EPOLLWAKEUP | EPOLLET | EPOLLEXCLUSIVE)92 93/* Maximum number of nesting allowed inside epoll sets */94#define EP_MAX_NESTS 495 96#define EP_MAX_EVENTS (INT_MAX / sizeof(struct epoll_event))97 98#define EP_UNACTIVE_PTR ((void *) -1L)99 100#define EP_ITEM_COST (sizeof(struct epitem) + sizeof(struct eppoll_entry))101 102struct epoll_filefd {103 struct file *file;104 int fd;105} __packed;106 107/* Wait structure used by the poll hooks */108struct eppoll_entry {109 /* List header used to link this structure to the "struct epitem" */110 struct eppoll_entry *next;111 112 /* The "base" pointer is set to the container "struct epitem" */113 struct epitem *base;114 115 /*116 * Wait queue item that will be linked to the target file wait117 * queue head.118 */119 wait_queue_entry_t wait;120 121 /* The wait queue head that linked the "wait" wait queue item */122 wait_queue_head_t *whead;123};124 125/*126 * Each file descriptor added to the eventpoll interface will127 * have an entry of this type linked to the "rbr" RB tree.128 * Avoid increasing the size of this struct, there can be many thousands129 * of these on a server and we do not want this to take another cache line.130 */131struct epitem {132 union {133 /* RB tree node links this structure to the eventpoll RB tree */134 struct rb_node rbn;135 /* Used to free the struct epitem */136 struct rcu_head rcu;137 };138 139 /* List header used to link this structure to the eventpoll ready list */140 struct list_head rdllink;141 142 /*143 * Works together "struct eventpoll"->ovflist in keeping the144 * single linked chain of items.145 */146 struct epitem *next;147 148 /* The file descriptor information this item refers to */149 struct epoll_filefd ffd;150 151 /*152 * Protected by file->f_lock, true for to-be-released epitem already153 * removed from the "struct file" items list; together with154 * eventpoll->refcount orchestrates "struct eventpoll" disposal155 */156 bool dying;157 158 /* List containing poll wait queues */159 struct eppoll_entry *pwqlist;160 161 /* The "container" of this item */162 struct eventpoll *ep;163 164 /* List header used to link this item to the "struct file" items list */165 struct hlist_node fllink;166 167 /* wakeup_source used when EPOLLWAKEUP is set */168 struct wakeup_source __rcu *ws;169 170 /* The structure that describe the interested events and the source fd */171 struct epoll_event event;172};173 174/*175 * This structure is stored inside the "private_data" member of the file176 * structure and represents the main data structure for the eventpoll177 * interface.178 */179struct eventpoll {180 /*181 * This mutex is used to ensure that files are not removed182 * while epoll is using them. This is held during the event183 * collection loop, the file cleanup path, the epoll file exit184 * code and the ctl operations.185 */186 struct mutex mtx;187 188 /* Wait queue used by sys_epoll_wait() */189 wait_queue_head_t wq;190 191 /* Wait queue used by file->poll() */192 wait_queue_head_t poll_wait;193 194 /* List of ready file descriptors */195 struct list_head rdllist;196 197 /* Lock which protects rdllist and ovflist */198 rwlock_t lock;199 200 /* RB tree root used to store monitored fd structs */201 struct rb_root_cached rbr;202 203 /*204 * This is a single linked list that chains all the "struct epitem" that205 * happened while transferring ready events to userspace w/out206 * holding ->lock.207 */208 struct epitem *ovflist;209 210 /* wakeup_source used when ep_send_events or __ep_eventpoll_poll is running */211 struct wakeup_source *ws;212 213 /* The user that created the eventpoll descriptor */214 struct user_struct *user;215 216 struct file *file;217 218 /* used to optimize loop detection check */219 u64 gen;220 struct hlist_head refs;221 222 /*223 * usage count, used together with epitem->dying to224 * orchestrate the disposal of this struct225 */226 refcount_t refcount;227 228#ifdef CONFIG_NET_RX_BUSY_POLL229 /* used to track busy poll napi_id */230 unsigned int napi_id;231 /* busy poll timeout */232 u32 busy_poll_usecs;233 /* busy poll packet budget */234 u16 busy_poll_budget;235 bool prefer_busy_poll;236#endif237 238#ifdef CONFIG_DEBUG_LOCK_ALLOC239 /* tracks wakeup nests for lockdep validation */240 u8 nests;241#endif242};243 244/* Wrapper struct used by poll queueing */245struct ep_pqueue {246 poll_table pt;247 struct epitem *epi;248};249 250/*251 * Configuration options available inside /proc/sys/fs/epoll/252 */253/* Maximum number of epoll watched descriptors, per user */254static long max_user_watches __read_mostly;255 256/* Used for cycles detection */257static DEFINE_MUTEX(epnested_mutex);258 259static u64 loop_check_gen = 0;260 261/* Used to check for epoll file descriptor inclusion loops */262static struct eventpoll *inserting_into;263 264/* Slab cache used to allocate "struct epitem" */265static struct kmem_cache *epi_cache __ro_after_init;266 267/* Slab cache used to allocate "struct eppoll_entry" */268static struct kmem_cache *pwq_cache __ro_after_init;269 270/*271 * List of files with newly added links, where we may need to limit the number272 * of emanating paths. Protected by the epnested_mutex.273 */274struct epitems_head {275 struct hlist_head epitems;276 struct epitems_head *next;277};278static struct epitems_head *tfile_check_list = EP_UNACTIVE_PTR;279 280static struct kmem_cache *ephead_cache __ro_after_init;281 282static inline void free_ephead(struct epitems_head *head)283{284 if (head)285 kmem_cache_free(ephead_cache, head);286}287 288static void list_file(struct file *file)289{290 struct epitems_head *head;291 292 head = container_of(file->f_ep, struct epitems_head, epitems);293 if (!head->next) {294 head->next = tfile_check_list;295 tfile_check_list = head;296 }297}298 299static void unlist_file(struct epitems_head *head)300{301 struct epitems_head *to_free = head;302 struct hlist_node *p = rcu_dereference(hlist_first_rcu(&head->epitems));303 if (p) {304 struct epitem *epi= container_of(p, struct epitem, fllink);305 spin_lock(&epi->ffd.file->f_lock);306 if (!hlist_empty(&head->epitems))307 to_free = NULL;308 head->next = NULL;309 spin_unlock(&epi->ffd.file->f_lock);310 }311 free_ephead(to_free);312}313 314#ifdef CONFIG_SYSCTL315 316#include <linux/sysctl.h>317 318static long long_zero;319static long long_max = LONG_MAX;320 321static struct ctl_table epoll_table[] = {322 {323 .procname = "max_user_watches",324 .data = &max_user_watches,325 .maxlen = sizeof(max_user_watches),326 .mode = 0644,327 .proc_handler = proc_doulongvec_minmax,328 .extra1 = &long_zero,329 .extra2 = &long_max,330 },331};332 333static void __init epoll_sysctls_init(void)334{335 register_sysctl("fs/epoll", epoll_table);336}337#else338#define epoll_sysctls_init() do { } while (0)339#endif /* CONFIG_SYSCTL */340 341static const struct file_operations eventpoll_fops;342 343static inline int is_file_epoll(struct file *f)344{345 return f->f_op == &eventpoll_fops;346}347 348/* Setup the structure that is used as key for the RB tree */349static inline void ep_set_ffd(struct epoll_filefd *ffd,350 struct file *file, int fd)351{352 ffd->file = file;353 ffd->fd = fd;354}355 356/* Compare RB tree keys */357static inline int ep_cmp_ffd(struct epoll_filefd *p1,358 struct epoll_filefd *p2)359{360 return (p1->file > p2->file ? +1:361 (p1->file < p2->file ? -1 : p1->fd - p2->fd));362}363 364/* Tells us if the item is currently linked */365static inline int ep_is_linked(struct epitem *epi)366{367 return !list_empty(&epi->rdllink);368}369 370static inline struct eppoll_entry *ep_pwq_from_wait(wait_queue_entry_t *p)371{372 return container_of(p, struct eppoll_entry, wait);373}374 375/* Get the "struct epitem" from a wait queue pointer */376static inline struct epitem *ep_item_from_wait(wait_queue_entry_t *p)377{378 return container_of(p, struct eppoll_entry, wait)->base;379}380 381/**382 * ep_events_available - Checks if ready events might be available.383 *384 * @ep: Pointer to the eventpoll context.385 *386 * Return: a value different than %zero if ready events are available,387 * or %zero otherwise.388 */389static inline int ep_events_available(struct eventpoll *ep)390{391 return !list_empty_careful(&ep->rdllist) ||392 READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR;393}394 395#ifdef CONFIG_NET_RX_BUSY_POLL396/**397 * busy_loop_ep_timeout - check if busy poll has timed out. The timeout value398 * from the epoll instance ep is preferred, but if it is not set fallback to399 * the system-wide global via busy_loop_timeout.400 *401 * @start_time: The start time used to compute the remaining time until timeout.402 * @ep: Pointer to the eventpoll context.403 *404 * Return: true if the timeout has expired, false otherwise.405 */406static bool busy_loop_ep_timeout(unsigned long start_time,407 struct eventpoll *ep)408{409 unsigned long bp_usec = READ_ONCE(ep->busy_poll_usecs);410 411 if (bp_usec) {412 unsigned long end_time = start_time + bp_usec;413 unsigned long now = busy_loop_current_time();414 415 return time_after(now, end_time);416 } else {417 return busy_loop_timeout(start_time);418 }419}420 421static bool ep_busy_loop_on(struct eventpoll *ep)422{423 return !!READ_ONCE(ep->busy_poll_usecs) || net_busy_loop_on();424}425 426static bool ep_busy_loop_end(void *p, unsigned long start_time)427{428 struct eventpoll *ep = p;429 430 return ep_events_available(ep) || busy_loop_ep_timeout(start_time, ep);431}432 433/*434 * Busy poll if globally on and supporting sockets found && no events,435 * busy loop will return if need_resched or ep_events_available.436 *437 * we must do our busy polling with irqs enabled438 */439static bool ep_busy_loop(struct eventpoll *ep, int nonblock)440{441 unsigned int napi_id = READ_ONCE(ep->napi_id);442 u16 budget = READ_ONCE(ep->busy_poll_budget);443 bool prefer_busy_poll = READ_ONCE(ep->prefer_busy_poll);444 445 if (!budget)446 budget = BUSY_POLL_BUDGET;447 448 if (napi_id >= MIN_NAPI_ID && ep_busy_loop_on(ep)) {449 napi_busy_loop(napi_id, nonblock ? NULL : ep_busy_loop_end,450 ep, prefer_busy_poll, budget);451 if (ep_events_available(ep))452 return true;453 /*454 * Busy poll timed out. Drop NAPI ID for now, we can add455 * it back in when we have moved a socket with a valid NAPI456 * ID onto the ready list.457 */458 ep->napi_id = 0;459 return false;460 }461 return false;462}463 464/*465 * Set epoll busy poll NAPI ID from sk.466 */467static inline void ep_set_busy_poll_napi_id(struct epitem *epi)468{469 struct eventpoll *ep = epi->ep;470 unsigned int napi_id;471 struct socket *sock;472 struct sock *sk;473 474 if (!ep_busy_loop_on(ep))475 return;476 477 sock = sock_from_file(epi->ffd.file);478 if (!sock)479 return;480 481 sk = sock->sk;482 if (!sk)483 return;484 485 napi_id = READ_ONCE(sk->sk_napi_id);486 487 /* Non-NAPI IDs can be rejected488 * or489 * Nothing to do if we already have this ID490 */491 if (napi_id < MIN_NAPI_ID || napi_id == ep->napi_id)492 return;493 494 /* record NAPI ID for use in next busy poll */495 ep->napi_id = napi_id;496}497 498static long ep_eventpoll_bp_ioctl(struct file *file, unsigned int cmd,499 unsigned long arg)500{501 struct eventpoll *ep = file->private_data;502 void __user *uarg = (void __user *)arg;503 struct epoll_params epoll_params;504 505 switch (cmd) {506 case EPIOCSPARAMS:507 if (copy_from_user(&epoll_params, uarg, sizeof(epoll_params)))508 return -EFAULT;509 510 /* pad byte must be zero */511 if (epoll_params.__pad)512 return -EINVAL;513 514 if (epoll_params.busy_poll_usecs > S32_MAX)515 return -EINVAL;516 517 if (epoll_params.prefer_busy_poll > 1)518 return -EINVAL;519 520 if (epoll_params.busy_poll_budget > NAPI_POLL_WEIGHT &&521 !capable(CAP_NET_ADMIN))522 return -EPERM;523 524 WRITE_ONCE(ep->busy_poll_usecs, epoll_params.busy_poll_usecs);525 WRITE_ONCE(ep->busy_poll_budget, epoll_params.busy_poll_budget);526 WRITE_ONCE(ep->prefer_busy_poll, epoll_params.prefer_busy_poll);527 return 0;528 case EPIOCGPARAMS:529 memset(&epoll_params, 0, sizeof(epoll_params));530 epoll_params.busy_poll_usecs = READ_ONCE(ep->busy_poll_usecs);531 epoll_params.busy_poll_budget = READ_ONCE(ep->busy_poll_budget);532 epoll_params.prefer_busy_poll = READ_ONCE(ep->prefer_busy_poll);533 if (copy_to_user(uarg, &epoll_params, sizeof(epoll_params)))534 return -EFAULT;535 return 0;536 default:537 return -ENOIOCTLCMD;538 }539}540 541#else542 543static inline bool ep_busy_loop(struct eventpoll *ep, int nonblock)544{545 return false;546}547 548static inline void ep_set_busy_poll_napi_id(struct epitem *epi)549{550}551 552static long ep_eventpoll_bp_ioctl(struct file *file, unsigned int cmd,553 unsigned long arg)554{555 return -EOPNOTSUPP;556}557 558#endif /* CONFIG_NET_RX_BUSY_POLL */559 560/*561 * As described in commit 0ccf831cb lockdep: annotate epoll562 * the use of wait queues used by epoll is done in a very controlled563 * manner. Wake ups can nest inside each other, but are never done564 * with the same locking. For example:565 *566 * dfd = socket(...);567 * efd1 = epoll_create();568 * efd2 = epoll_create();569 * epoll_ctl(efd1, EPOLL_CTL_ADD, dfd, ...);570 * epoll_ctl(efd2, EPOLL_CTL_ADD, efd1, ...);571 *572 * When a packet arrives to the device underneath "dfd", the net code will573 * issue a wake_up() on its poll wake list. Epoll (efd1) has installed a574 * callback wakeup entry on that queue, and the wake_up() performed by the575 * "dfd" net code will end up in ep_poll_callback(). At this point epoll576 * (efd1) notices that it may have some event ready, so it needs to wake up577 * the waiters on its poll wait list (efd2). So it calls ep_poll_safewake()578 * that ends up in another wake_up(), after having checked about the579 * recursion constraints. That are, no more than EP_MAX_NESTS, to avoid580 * stack blasting.581 *582 * When CONFIG_DEBUG_LOCK_ALLOC is enabled, make sure lockdep can handle583 * this special case of epoll.584 */585#ifdef CONFIG_DEBUG_LOCK_ALLOC586 587static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi,588 unsigned pollflags)589{590 struct eventpoll *ep_src;591 unsigned long flags;592 u8 nests = 0;593 594 /*595 * To set the subclass or nesting level for spin_lock_irqsave_nested()596 * it might be natural to create a per-cpu nest count. However, since597 * we can recurse on ep->poll_wait.lock, and a non-raw spinlock can598 * schedule() in the -rt kernel, the per-cpu variable are no longer599 * protected. Thus, we are introducing a per eventpoll nest field.600 * If we are not being call from ep_poll_callback(), epi is NULL and601 * we are at the first level of nesting, 0. Otherwise, we are being602 * called from ep_poll_callback() and if a previous wakeup source is603 * not an epoll file itself, we are at depth 1 since the wakeup source604 * is depth 0. If the wakeup source is a previous epoll file in the605 * wakeup chain then we use its nests value and record ours as606 * nests + 1. The previous epoll file nests value is stable since its607 * already holding its own poll_wait.lock.608 */609 if (epi) {610 if ((is_file_epoll(epi->ffd.file))) {611 ep_src = epi->ffd.file->private_data;612 nests = ep_src->nests;613 } else {614 nests = 1;615 }616 }617 spin_lock_irqsave_nested(&ep->poll_wait.lock, flags, nests);618 ep->nests = nests + 1;619 wake_up_locked_poll(&ep->poll_wait, EPOLLIN | pollflags);620 ep->nests = 0;621 spin_unlock_irqrestore(&ep->poll_wait.lock, flags);622}623 624#else625 626static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi,627 __poll_t pollflags)628{629 wake_up_poll(&ep->poll_wait, EPOLLIN | pollflags);630}631 632#endif633 634static void ep_remove_wait_queue(struct eppoll_entry *pwq)635{636 wait_queue_head_t *whead;637 638 rcu_read_lock();639 /*640 * If it is cleared by POLLFREE, it should be rcu-safe.641 * If we read NULL we need a barrier paired with642 * smp_store_release() in ep_poll_callback(), otherwise643 * we rely on whead->lock.644 */645 whead = smp_load_acquire(&pwq->whead);646 if (whead)647 remove_wait_queue(whead, &pwq->wait);648 rcu_read_unlock();649}650 651/*652 * This function unregisters poll callbacks from the associated file653 * descriptor. Must be called with "mtx" held.654 */655static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi)656{657 struct eppoll_entry **p = &epi->pwqlist;658 struct eppoll_entry *pwq;659 660 while ((pwq = *p) != NULL) {661 *p = pwq->next;662 ep_remove_wait_queue(pwq);663 kmem_cache_free(pwq_cache, pwq);664 }665}666 667/* call only when ep->mtx is held */668static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi)669{670 return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx));671}672 673/* call only when ep->mtx is held */674static inline void ep_pm_stay_awake(struct epitem *epi)675{676 struct wakeup_source *ws = ep_wakeup_source(epi);677 678 if (ws)679 __pm_stay_awake(ws);680}681 682static inline bool ep_has_wakeup_source(struct epitem *epi)683{684 return rcu_access_pointer(epi->ws) ? true : false;685}686 687/* call when ep->mtx cannot be held (ep_poll_callback) */688static inline void ep_pm_stay_awake_rcu(struct epitem *epi)689{690 struct wakeup_source *ws;691 692 rcu_read_lock();693 ws = rcu_dereference(epi->ws);694 if (ws)695 __pm_stay_awake(ws);696 rcu_read_unlock();697}698 699 700/*701 * ep->mutex needs to be held because we could be hit by702 * eventpoll_release_file() and epoll_ctl().703 */704static void ep_start_scan(struct eventpoll *ep, struct list_head *txlist)705{706 /*707 * Steal the ready list, and re-init the original one to the708 * empty list. Also, set ep->ovflist to NULL so that events709 * happening while looping w/out locks, are not lost. We cannot710 * have the poll callback to queue directly on ep->rdllist,711 * because we want the "sproc" callback to be able to do it712 * in a lockless way.713 */714 lockdep_assert_irqs_enabled();715 write_lock_irq(&ep->lock);716 list_splice_init(&ep->rdllist, txlist);717 WRITE_ONCE(ep->ovflist, NULL);718 write_unlock_irq(&ep->lock);719}720 721static HWJS_SUSPENDS void ep_done_scan(struct eventpoll *ep,722 struct list_head *txlist)723{724 struct epitem *epi, *nepi;725 726 write_lock_irq(&ep->lock);727 /*728 * During the time we spent inside the "sproc" callback, some729 * other events might have been queued by the poll callback.730 * We re-insert them inside the main ready-list here.731 */732 for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL;733 nepi = epi->next, epi->next = EP_UNACTIVE_PTR) {734 /*735 * We need to check if the item is already in the list.736 * During the "sproc" callback execution time, items are737 * queued into ->ovflist but the "txlist" might already738 * contain them, and the list_splice() below takes care of them.739 */740 if (!ep_is_linked(epi)) {741 /*742 * ->ovflist is LIFO, so we have to reverse it in order743 * to keep in FIFO.744 */745 list_add(&epi->rdllink, &ep->rdllist);746 ep_pm_stay_awake(epi);747 }748 }749 /*750 * We need to set back ep->ovflist to EP_UNACTIVE_PTR, so that after751 * releasing the lock, events will be queued in the normal way inside752 * ep->rdllist.753 */754 WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR);755 756 /*757 * Quickly re-inject items left on "txlist".758 */759 list_splice(txlist, &ep->rdllist);760 __pm_relax(ep->ws);761 762 if (!list_empty(&ep->rdllist)) {763 if (waitqueue_active(&ep->wq))764 wake_up(&ep->wq);765 }766 767 write_unlock_irq(&ep->lock);768}769 770static void ep_get(struct eventpoll *ep)771{772 refcount_inc(&ep->refcount);773}774 775/*776 * Returns true if the event poll can be disposed777 */778static bool ep_refcount_dec_and_test(struct eventpoll *ep)779{780 if (!refcount_dec_and_test(&ep->refcount))781 return false;782 783 WARN_ON_ONCE(!RB_EMPTY_ROOT(&ep->rbr.rb_root));784 return true;785}786 787static void ep_free(struct eventpoll *ep)788{789 mutex_destroy(&ep->mtx);790 free_uid(ep->user);791 wakeup_source_unregister(ep->ws);792 kfree(ep);793}794 795/*796 * Removes a "struct epitem" from the eventpoll RB tree and deallocates797 * all the associated resources. Must be called with "mtx" held.798 * If the dying flag is set, do the removal only if force is true.799 * This prevents ep_clear_and_put() from dropping all the ep references800 * while running concurrently with eventpoll_release_file().801 * Returns true if the eventpoll can be disposed.802 */803static HWJS_SUSPENDS bool __ep_remove(struct eventpoll *ep, struct epitem *epi, bool force)804{805 struct file *file = epi->ffd.file;806 struct epitems_head *to_free;807 struct hlist_head *head;808 809 lockdep_assert_irqs_enabled();810 811 /*812 * Removes poll wait queue hooks.813 */814 ep_unregister_pollwait(ep, epi);815 816 /* Remove the current item from the list of epoll hooks */817 spin_lock(&file->f_lock);818 if (epi->dying && !force) {819 spin_unlock(&file->f_lock);820 return false;821 }822 823 to_free = NULL;824 head = file->f_ep;825 if (head->first == &epi->fllink && !epi->fllink.next) {826 file->f_ep = NULL;827 if (!is_file_epoll(file)) {828 struct epitems_head *v;829 v = container_of(head, struct epitems_head, epitems);830 if (!smp_load_acquire(&v->next))831 to_free = v;832 }833 }834 hlist_del_rcu(&epi->fllink);835 spin_unlock(&file->f_lock);836 free_ephead(to_free);837 838 rb_erase_cached(&epi->rbn, &ep->rbr);839 840 write_lock_irq(&ep->lock);841 if (ep_is_linked(epi))842 list_del_init(&epi->rdllink);843 write_unlock_irq(&ep->lock);844 845 wakeup_source_unregister(ep_wakeup_source(epi));846 /*847 * At this point it is safe to free the eventpoll item. Use the union848 * field epi->rcu, since we are trying to minimize the size of849 * 'struct epitem'. The 'rbn' field is no longer in use. Protected by850 * ep->mtx. The rcu read side, reverse_path_check_proc(), does not make851 * use of the rbn field.852 */853 kfree_rcu(epi, rcu);854 855 percpu_counter_dec(&ep->user->epoll_watches);856 return ep_refcount_dec_and_test(ep);857}858 859/*860 * ep_remove variant for callers owing an additional reference to the ep861 */862static HWJS_SUSPENDS void ep_remove_safe(struct eventpoll *ep, struct epitem *epi)863{864 WARN_ON_ONCE(__ep_remove(ep, epi, false));865}866 867static HWJS_SUSPENDS void ep_clear_and_put(struct eventpoll *ep)868{869 struct rb_node *rbp, *next;870 struct epitem *epi;871 bool dispose;872 873 /* We need to release all tasks waiting for these file */874 if (waitqueue_active(&ep->poll_wait))875 ep_poll_safewake(ep, NULL, 0);876 877 mutex_lock(&ep->mtx);878 879 /*880 * Walks through the whole tree by unregistering poll callbacks.881 */882 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {883 epi = rb_entry(rbp, struct epitem, rbn);884 885 ep_unregister_pollwait(ep, epi);886 cond_resched();887 }888 889 /*890 * Walks through the whole tree and try to free each "struct epitem".891 * Note that ep_remove_safe() will not remove the epitem in case of a892 * racing eventpoll_release_file(); the latter will do the removal.893 * At this point we are sure no poll callbacks will be lingering around.894 * Since we still own a reference to the eventpoll struct, the loop can't895 * dispose it.896 */897 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = next) {898 next = rb_next(rbp);899 epi = rb_entry(rbp, struct epitem, rbn);900 ep_remove_safe(ep, epi);901 cond_resched();902 }903 904 dispose = ep_refcount_dec_and_test(ep);905 mutex_unlock(&ep->mtx);906 907 if (dispose)908 ep_free(ep);909}910 911static long ep_eventpoll_ioctl(struct file *file, unsigned int cmd,912 unsigned long arg)913{914 int ret;915 916 if (!is_file_epoll(file))917 return -EINVAL;918 919 switch (cmd) {920 case EPIOCSPARAMS:921 case EPIOCGPARAMS:922 ret = ep_eventpoll_bp_ioctl(file, cmd, arg);923 break;924 default:925 ret = -EINVAL;926 break;927 }928 929 return ret;930}931 932static HWJS_SUSPENDS int ep_eventpoll_release(struct inode *inode, struct file *file)933{934 struct eventpoll *ep = file->private_data;935 936 if (ep)937 ep_clear_and_put(ep);938 939 return 0;940}941 942static HWJS_SUSPENDS __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt, int depth);943 944static HWJS_SUSPENDS __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int depth)945{946 struct eventpoll *ep = file->private_data;947 LIST_HEAD(txlist);948 struct epitem *epi, *tmp;949 poll_table pt;950 __poll_t res = 0;951 952 init_poll_funcptr(&pt, NULL);953 954 /* Insert inside our poll wait queue */955 poll_wait(file, &ep->poll_wait, wait);956 957 /*958 * Proceed to find out if wanted events are really available inside959 * the ready list.960 */961 mutex_lock_nested(&ep->mtx, depth);962 ep_start_scan(ep, &txlist);963 list_for_each_entry_safe(epi, tmp, &txlist, rdllink) {964 if (ep_item_poll(epi, &pt, depth + 1)) {965 res = EPOLLIN | EPOLLRDNORM;966 break;967 } else {968 /*969 * Item has been dropped into the ready list by the poll970 * callback, but it's not actually ready, as far as971 * caller requested events goes. We can remove it here.972 */973 __pm_relax(ep_wakeup_source(epi));974 list_del_init(&epi->rdllink);975 }976 }977 ep_done_scan(ep, &txlist);978 mutex_unlock(&ep->mtx);979 return res;980}981 982/*983 * The ffd.file pointer may be in the process of being torn down due to984 * being closed, but we may not have finished eventpoll_release() yet.985 *986 * Normally, even with the atomic_long_inc_not_zero, the file may have987 * been free'd and then gotten re-allocated to something else (since988 * files are not RCU-delayed, they are SLAB_TYPESAFE_BY_RCU).989 *990 * But for epoll, users hold the ep->mtx mutex, and as such any file in991 * the process of being free'd will block in eventpoll_release_file()992 * and thus the underlying file allocation will not be free'd, and the993 * file re-use cannot happen.994 *995 * For the same reason we can avoid a rcu_read_lock() around the996 * operation - 'ffd.file' cannot go away even if the refcount has997 * reached zero (but we must still not call out to ->poll() functions998 * etc).999 */1000static struct file *epi_fget(const struct epitem *epi)1001{1002 struct file *file;1003 1004 file = epi->ffd.file;1005 if (!atomic_long_inc_not_zero(&file->f_count))1006 file = NULL;1007 return file;1008}1009 1010/*1011 * Differs from ep_eventpoll_poll() in that internal callers already have1012 * the ep->mtx so we need to start from depth=1, such that mutex_lock_nested()1013 * is correctly annotated.1014 */1015static HWJS_SUSPENDS __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt,1016 int depth)1017{1018 struct file *file = epi_fget(epi);1019 __poll_t res;1020 1021 /*1022 * We could return EPOLLERR | EPOLLHUP or something, but let's1023 * treat this more as "file doesn't exist, poll didn't happen".1024 */1025 if (!file)1026 return 0;1027 1028 pt->_key = epi->event.events;1029 if (!is_file_epoll(file))1030 res = vfs_poll(file, pt);1031 else1032 res = __ep_eventpoll_poll(file, pt, depth);1033 fput(file);1034 return res & epi->event.events;1035}1036 1037static HWJS_SUSPENDS __poll_t ep_eventpoll_poll(struct file *file, poll_table *wait)1038{1039 return __ep_eventpoll_poll(file, wait, 0);1040}1041 1042#ifdef CONFIG_PROC_FS1043static void ep_show_fdinfo(struct seq_file *m, struct file *f)1044{1045 struct eventpoll *ep = f->private_data;1046 struct rb_node *rbp;1047 1048 mutex_lock(&ep->mtx);1049 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {1050 struct epitem *epi = rb_entry(rbp, struct epitem, rbn);1051 struct inode *inode = file_inode(epi->ffd.file);1052 1053 seq_printf(m, "tfd: %8d events: %8x data: %16llx "1054 " pos:%lli ino:%lx sdev:%x\n",1055 epi->ffd.fd, epi->event.events,1056 (long long)epi->event.data,1057 (long long)epi->ffd.file->f_pos,1058 inode->i_ino, inode->i_sb->s_dev);1059 if (seq_has_overflowed(m))1060 break;1061 }1062 mutex_unlock(&ep->mtx);1063}1064#endif1065 1066/* File callbacks that implement the eventpoll file behaviour */1067static const struct file_operations eventpoll_fops = {1068#ifdef CONFIG_PROC_FS1069 .show_fdinfo = ep_show_fdinfo,1070#endif1071 .release = ep_eventpoll_release,1072 .poll = ep_eventpoll_poll,1073 .llseek = noop_llseek,1074 .unlocked_ioctl = ep_eventpoll_ioctl,1075 .compat_ioctl = compat_ptr_ioctl,1076};1077 1078/*1079 * This is called from eventpoll_release() to unlink files from the eventpoll1080 * interface. We need to have this facility to cleanup correctly files that are1081 * closed without being removed from the eventpoll interface.1082 */1083void eventpoll_release_file(struct file *file)1084{1085 struct eventpoll *ep;1086 struct epitem *epi;1087 bool dispose;1088 1089 /*1090 * Use the 'dying' flag to prevent a concurrent ep_clear_and_put() from1091 * touching the epitems list before eventpoll_release_file() can access1092 * the ep->mtx.1093 */1094again:1095 spin_lock(&file->f_lock);1096 if (file->f_ep && file->f_ep->first) {1097 epi = hlist_entry(file->f_ep->first, struct epitem, fllink);1098 epi->dying = true;1099 spin_unlock(&file->f_lock);1100 1101 /*1102 * ep access is safe as we still own a reference to the ep1103 * struct1104 */1105 ep = epi->ep;1106 mutex_lock(&ep->mtx);1107 dispose = __ep_remove(ep, epi, true);1108 mutex_unlock(&ep->mtx);1109 1110 if (dispose)1111 ep_free(ep);1112 goto again;1113 }1114 spin_unlock(&file->f_lock);1115}1116 1117static int ep_alloc(struct eventpoll **pep)1118{1119 struct eventpoll *ep;1120 1121 ep = kzalloc(sizeof(*ep), GFP_KERNEL);1122 if (unlikely(!ep))1123 return -ENOMEM;1124 1125 mutex_init(&ep->mtx);1126 rwlock_init(&ep->lock);1127 init_waitqueue_head(&ep->wq);1128 init_waitqueue_head(&ep->poll_wait);1129 INIT_LIST_HEAD(&ep->rdllist);1130 ep->rbr = RB_ROOT_CACHED;1131 ep->ovflist = EP_UNACTIVE_PTR;1132 ep->user = get_current_user();1133 refcount_set(&ep->refcount, 1);1134 1135 *pep = ep;1136 1137 return 0;1138}1139 1140/*1141 * Search the file inside the eventpoll tree. The RB tree operations1142 * are protected by the "mtx" mutex, and ep_find() must be called with1143 * "mtx" held.1144 */1145static struct epitem *ep_find(struct eventpoll *ep, struct file *file, int fd)1146{1147 int kcmp;1148 struct rb_node *rbp;1149 struct epitem *epi, *epir = NULL;1150 struct epoll_filefd ffd;1151 1152 ep_set_ffd(&ffd, file, fd);1153 for (rbp = ep->rbr.rb_root.rb_node; rbp; ) {1154 epi = rb_entry(rbp, struct epitem, rbn);1155 kcmp = ep_cmp_ffd(&ffd, &epi->ffd);1156 if (kcmp > 0)1157 rbp = rbp->rb_right;1158 else if (kcmp < 0)1159 rbp = rbp->rb_left;1160 else {1161 epir = epi;1162 break;1163 }1164 }1165 1166 return epir;1167}1168 1169#ifdef CONFIG_KCMP1170static struct epitem *ep_find_tfd(struct eventpoll *ep, int tfd, unsigned long toff)1171{1172 struct rb_node *rbp;1173 struct epitem *epi;1174 1175 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {1176 epi = rb_entry(rbp, struct epitem, rbn);1177 if (epi->ffd.fd == tfd) {1178 if (toff == 0)1179 return epi;1180 else1181 toff--;1182 }1183 cond_resched();1184 }1185 1186 return NULL;1187}1188 1189struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd,1190 unsigned long toff)1191{1192 struct file *file_raw;1193 struct eventpoll *ep;1194 struct epitem *epi;1195 1196 if (!is_file_epoll(file))1197 return ERR_PTR(-EINVAL);1198 1199 ep = file->private_data;1200 1201 mutex_lock(&ep->mtx);1202 epi = ep_find_tfd(ep, tfd, toff);1203 if (epi)1204 file_raw = epi->ffd.file;1205 else1206 file_raw = ERR_PTR(-ENOENT);1207 mutex_unlock(&ep->mtx);1208 1209 return file_raw;1210}1211#endif /* CONFIG_KCMP */1212 1213/*1214 * Adds a new entry to the tail of the list in a lockless way, i.e.1215 * multiple CPUs are allowed to call this function concurrently.1216 *1217 * Beware: it is necessary to prevent any other modifications of the1218 * existing list until all changes are completed, in other words1219 * concurrent list_add_tail_lockless() calls should be protected1220 * with a read lock, where write lock acts as a barrier which1221 * makes sure all list_add_tail_lockless() calls are fully1222 * completed.1223 *1224 * Also an element can be locklessly added to the list only in one1225 * direction i.e. either to the tail or to the head, otherwise1226 * concurrent access will corrupt the list.1227 *1228 * Return: %false if element has been already added to the list, %true1229 * otherwise.1230 */1231static inline bool list_add_tail_lockless(struct list_head *new,1232 struct list_head *head)1233{1234 struct list_head *prev;1235 1236 /*1237 * This is simple 'new->next = head' operation, but cmpxchg()1238 * is used in order to detect that same element has been just1239 * added to the list from another CPU: the winner observes1240 * new->next == new.1241 */1242 if (!try_cmpxchg(&new->next, &new, head))1243 return false;1244 1245 /*1246 * Initially ->next of a new element must be updated with the head1247 * (we are inserting to the tail) and only then pointers are atomically1248 * exchanged. XCHG guarantees memory ordering, thus ->next should be1249 * updated before pointers are actually swapped and pointers are1250 * swapped before prev->next is updated.1251 */1252 1253 prev = xchg(&head->prev, new);1254 1255 /*1256 * It is safe to modify prev->next and new->prev, because a new element1257 * is added only to the tail and new->next is updated before XCHG.1258 */1259 1260 prev->next = new;1261 new->prev = prev;1262 1263 return true;1264}1265 1266/*1267 * Chains a new epi entry to the tail of the ep->ovflist in a lockless way,1268 * i.e. multiple CPUs are allowed to call this function concurrently.1269 *1270 * Return: %false if epi element has been already chained, %true otherwise.1271 */1272static inline bool chain_epi_lockless(struct epitem *epi)1273{1274 struct eventpoll *ep = epi->ep;1275 1276 /* Fast preliminary check */1277 if (epi->next != EP_UNACTIVE_PTR)1278 return false;1279 1280 /* Check that the same epi has not been just chained from another CPU */1281 if (cmpxchg(&epi->next, EP_UNACTIVE_PTR, NULL) != EP_UNACTIVE_PTR)1282 return false;1283 1284 /* Atomically exchange tail */1285 epi->next = xchg(&ep->ovflist, epi);1286 1287 return true;1288}1289 1290/*1291 * This is the callback that is passed to the wait queue wakeup1292 * mechanism. It is called by the stored file descriptors when they1293 * have events to report.1294 *1295 * This callback takes a read lock in order not to contend with concurrent1296 * events from another file descriptor, thus all modifications to ->rdllist1297 * or ->ovflist are lockless. Read lock is paired with the write lock from1298 * ep_start/done_scan(), which stops all list modifications and guarantees1299 * that lists state is seen correctly.1300 *1301 * Another thing worth to mention is that ep_poll_callback() can be called1302 * concurrently for the same @epi from different CPUs if poll table was inited1303 * with several wait queues entries. Plural wakeup from different CPUs of a1304 * single wait queue is serialized by wq.lock, but the case when multiple wait1305 * queues are used should be detected accordingly. This is detected using1306 * cmpxchg() operation.1307 */1308static HWJS_SUSPENDS int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)1309{1310 int pwake = 0;1311 struct epitem *epi = ep_item_from_wait(wait);1312 struct eventpoll *ep = epi->ep;1313 __poll_t pollflags = key_to_poll(key);1314 unsigned long flags;1315 int ewake = 0;1316 1317 read_lock_irqsave(&ep->lock, flags);1318 1319 ep_set_busy_poll_napi_id(epi);1320 1321 /*1322 * If the event mask does not contain any poll(2) event, we consider the1323 * descriptor to be disabled. This condition is likely the effect of the1324 * EPOLLONESHOT bit that disables the descriptor when an event is received,1325 * until the next EPOLL_CTL_MOD will be issued.1326 */1327 if (!(epi->event.events & ~EP_PRIVATE_BITS))1328 goto out_unlock;1329 1330 /*1331 * Check the events coming with the callback. At this stage, not1332 * every device reports the events in the "key" parameter of the1333 * callback. We need to be able to handle both cases here, hence the1334 * test for "key" != NULL before the event match test.1335 */1336 if (pollflags && !(pollflags & epi->event.events))1337 goto out_unlock;1338 1339 /*1340 * If we are transferring events to userspace, we can hold no locks1341 * (because we're accessing user memory, and because of linux f_op->poll()1342 * semantics). All the events that happen during that period of time are1343 * chained in ep->ovflist and requeued later on.1344 */1345 if (READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR) {1346 if (chain_epi_lockless(epi))1347 ep_pm_stay_awake_rcu(epi);1348 } else if (!ep_is_linked(epi)) {1349 /* In the usual case, add event to ready list. */1350 if (list_add_tail_lockless(&epi->rdllink, &ep->rdllist))1351 ep_pm_stay_awake_rcu(epi);1352 }1353 1354 /*1355 * Wake up ( if active ) both the eventpoll wait list and the ->poll()1356 * wait list.1357 */1358 if (waitqueue_active(&ep->wq)) {1359 if ((epi->event.events & EPOLLEXCLUSIVE) &&1360 !(pollflags & POLLFREE)) {1361 switch (pollflags & EPOLLINOUT_BITS) {1362 case EPOLLIN:1363 if (epi->event.events & EPOLLIN)1364 ewake = 1;1365 break;1366 case EPOLLOUT:1367 if (epi->event.events & EPOLLOUT)1368 ewake = 1;1369 break;1370 case 0:1371 ewake = 1;1372 break;1373 }1374 }1375 wake_up(&ep->wq);1376 }1377 if (waitqueue_active(&ep->poll_wait))1378 pwake++;1379 1380out_unlock:1381 read_unlock_irqrestore(&ep->lock, flags);1382 1383 /* We have to call this outside the lock */1384 if (pwake)1385 ep_poll_safewake(ep, epi, pollflags & EPOLL_URING_WAKE);1386 1387 if (!(epi->event.events & EPOLLEXCLUSIVE))1388 ewake = 1;1389 1390 if (pollflags & POLLFREE) {1391 /*1392 * If we race with ep_remove_wait_queue() it can miss1393 * ->whead = NULL and do another remove_wait_queue() after1394 * us, so we can't use __remove_wait_queue().1395 */1396 list_del_init(&wait->entry);1397 /*1398 * ->whead != NULL protects us from the race with1399 * ep_clear_and_put() or ep_remove(), ep_remove_wait_queue()1400 * takes whead->lock held by the caller. Once we nullify it,1401 * nothing protects ep/epi or even wait.1402 */1403 smp_store_release(&ep_pwq_from_wait(wait)->whead, NULL);1404 }1405 1406 return ewake;1407}1408 1409/*1410 * This is the callback that is used to add our wait queue to the1411 * target file wakeup lists.1412 */1413static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead,1414 poll_table *pt)1415{1416 struct ep_pqueue *epq = container_of(pt, struct ep_pqueue, pt);1417 struct epitem *epi = epq->epi;1418 struct eppoll_entry *pwq;1419 1420 if (unlikely(!epi)) // an earlier allocation has failed1421 return;1422 1423 pwq = kmem_cache_alloc(pwq_cache, GFP_KERNEL);1424 if (unlikely(!pwq)) {1425 epq->epi = NULL;1426 return;1427 }1428 1429 init_waitqueue_func_entry(&pwq->wait, ep_poll_callback);1430 pwq->whead = whead;1431 pwq->base = epi;1432 if (epi->event.events & EPOLLEXCLUSIVE)1433 add_wait_queue_exclusive(whead, &pwq->wait);1434 else1435 add_wait_queue(whead, &pwq->wait);1436 pwq->next = epi->pwqlist;1437 epi->pwqlist = pwq;1438}1439 1440static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi)1441{1442 int kcmp;1443 struct rb_node **p = &ep->rbr.rb_root.rb_node, *parent = NULL;1444 struct epitem *epic;1445 bool leftmost = true;1446 1447 while (*p) {1448 parent = *p;1449 epic = rb_entry(parent, struct epitem, rbn);1450 kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd);1451 if (kcmp > 0) {1452 p = &parent->rb_right;1453 leftmost = false;1454 } else1455 p = &parent->rb_left;1456 }1457 rb_link_node(&epi->rbn, parent, p);1458 rb_insert_color_cached(&epi->rbn, &ep->rbr, leftmost);1459}1460 1461 1462 1463#define PATH_ARR_SIZE 51464/*1465 * These are the number paths of length 1 to 5, that we are allowing to emanate1466 * from a single file of interest. For example, we allow 1000 paths of length1467 * 1, to emanate from each file of interest. This essentially represents the1468 * potential wakeup paths, which need to be limited in order to avoid massive1469 * uncontrolled wakeup storms. The common use case should be a single ep which1470 * is connected to n file sources. In this case each file source has 1 path1471 * of length 1. Thus, the numbers below should be more than sufficient. These1472 * path limits are enforced during an EPOLL_CTL_ADD operation, since a modify1473 * and delete can't add additional paths. Protected by the epnested_mutex.1474 */1475static const int path_limits[PATH_ARR_SIZE] = { 1000, 500, 100, 50, 10 };1476static int path_count[PATH_ARR_SIZE];1477 1478static int path_count_inc(int nests)1479{1480 /* Allow an arbitrary number of depth 1 paths */1481 if (nests == 0)1482 return 0;1483 1484 if (++path_count[nests] > path_limits[nests])1485 return -1;1486 return 0;1487}1488 1489static void path_count_init(void)1490{1491 int i;1492 1493 for (i = 0; i < PATH_ARR_SIZE; i++)1494 path_count[i] = 0;1495}1496 1497static int reverse_path_check_proc(struct hlist_head *refs, int depth)1498{1499 int error = 0;1500 struct epitem *epi;1501 1502 if (depth > EP_MAX_NESTS) /* too deep nesting */1503 return -1;1504 1505 /* CTL_DEL can remove links here, but that can't increase our count */1506 hlist_for_each_entry_rcu(epi, refs, fllink) {1507 struct hlist_head *refs = &epi->ep->refs;1508 if (hlist_empty(refs))1509 error = path_count_inc(depth);1510 else1511 error = reverse_path_check_proc(refs, depth + 1);1512 if (error != 0)1513 break;1514 }1515 return error;1516}1517 1518/**1519 * reverse_path_check - The tfile_check_list is list of epitem_head, which have1520 * links that are proposed to be newly added. We need to1521 * make sure that those added links don't add too many1522 * paths such that we will spend all our time waking up1523 * eventpoll objects.1524 *1525 * Return: %zero if the proposed links don't create too many paths,1526 * %-1 otherwise.1527 */1528static int reverse_path_check(void)1529{1530 struct epitems_head *p;1531 1532 for (p = tfile_check_list; p != EP_UNACTIVE_PTR; p = p->next) {1533 int error;1534 path_count_init();1535 rcu_read_lock();1536 error = reverse_path_check_proc(&p->epitems, 0);1537 rcu_read_unlock();1538 if (error)1539 return error;1540 }1541 return 0;1542}1543 1544static int ep_create_wakeup_source(struct epitem *epi)1545{1546 struct name_snapshot n;1547 struct wakeup_source *ws;1548 1549 if (!epi->ep->ws) {1550 epi->ep->ws = wakeup_source_register(NULL, "eventpoll");1551 if (!epi->ep->ws)1552 return -ENOMEM;1553 }1554 1555 take_dentry_name_snapshot(&n, epi->ffd.file->f_path.dentry);1556 ws = wakeup_source_register(NULL, n.name.name);1557 release_dentry_name_snapshot(&n);1558 1559 if (!ws)1560 return -ENOMEM;1561 rcu_assign_pointer(epi->ws, ws);1562 1563 return 0;1564}1565 1566/* rare code path, only used when EPOLL_CTL_MOD removes a wakeup source */1567static HWJS_SUSPENDS noinline void ep_destroy_wakeup_source(struct epitem *epi)1568{1569 struct wakeup_source *ws = ep_wakeup_source(epi);1570 1571 RCU_INIT_POINTER(epi->ws, NULL);1572 1573 /*1574 * wait for ep_pm_stay_awake_rcu to finish, synchronize_rcu is1575 * used internally by wakeup_source_remove, too (called by1576 * wakeup_source_unregister), so we cannot use call_rcu1577 */1578 synchronize_rcu();1579 wakeup_source_unregister(ws);1580}1581 1582static int attach_epitem(struct file *file, struct epitem *epi)1583{1584 struct epitems_head *to_free = NULL;1585 struct hlist_head *head = NULL;1586 struct eventpoll *ep = NULL;1587 1588 if (is_file_epoll(file))1589 ep = file->private_data;1590 1591 if (ep) {1592 head = &ep->refs;1593 } else if (!READ_ONCE(file->f_ep)) {1594allocate:1595 to_free = kmem_cache_zalloc(ephead_cache, GFP_KERNEL);1596 if (!to_free)1597 return -ENOMEM;1598 head = &to_free->epitems;1599 }1600 spin_lock(&file->f_lock);1601 if (!file->f_ep) {1602 if (unlikely(!head)) {1603 spin_unlock(&file->f_lock);1604 goto allocate;1605 }1606 file->f_ep = head;1607 to_free = NULL;1608 }1609 hlist_add_head_rcu(&epi->fllink, file->f_ep);1610 spin_unlock(&file->f_lock);1611 free_ephead(to_free);1612 return 0;1613}1614 1615/*1616 * Must be called with "mtx" held.1617 */1618static HWJS_SUSPENDS int ep_insert(struct eventpoll *ep, const struct epoll_event *event,1619 struct file *tfile, int fd, int full_check)1620{1621 int error, pwake = 0;1622 __poll_t revents;1623 struct epitem *epi;1624 struct ep_pqueue epq;1625 struct eventpoll *tep = NULL;1626 1627 if (is_file_epoll(tfile))1628 tep = tfile->private_data;1629 1630 lockdep_assert_irqs_enabled();1631 1632 if (unlikely(percpu_counter_compare(&ep->user->epoll_watches,1633 max_user_watches) >= 0))1634 return -ENOSPC;1635 percpu_counter_inc(&ep->user->epoll_watches);1636 1637 if (!(epi = kmem_cache_zalloc(epi_cache, GFP_KERNEL))) {1638 percpu_counter_dec(&ep->user->epoll_watches);1639 return -ENOMEM;1640 }1641 1642 /* Item initialization follow here ... */1643 INIT_LIST_HEAD(&epi->rdllink);1644 epi->ep = ep;1645 ep_set_ffd(&epi->ffd, tfile, fd);1646 epi->event = *event;1647 epi->next = EP_UNACTIVE_PTR;1648 1649 if (tep)1650 mutex_lock_nested(&tep->mtx, 1);1651 /* Add the current item to the list of active epoll hook for this file */1652 if (unlikely(attach_epitem(tfile, epi) < 0)) {1653 if (tep)1654 mutex_unlock(&tep->mtx);1655 kmem_cache_free(epi_cache, epi);1656 percpu_counter_dec(&ep->user->epoll_watches);1657 return -ENOMEM;1658 }1659 1660 if (full_check && !tep)1661 list_file(tfile);1662 1663 /*1664 * Add the current item to the RB tree. All RB tree operations are1665 * protected by "mtx", and ep_insert() is called with "mtx" held.1666 */1667 ep_rbtree_insert(ep, epi);1668 if (tep)1669 mutex_unlock(&tep->mtx);1670 1671 /*1672 * ep_remove_safe() calls in the later error paths can't lead to1673 * ep_free() as the ep file itself still holds an ep reference.1674 */1675 ep_get(ep);1676 1677 /* now check if we've created too many backpaths */1678 if (unlikely(full_check && reverse_path_check())) {1679 ep_remove_safe(ep, epi);1680 return -EINVAL;1681 }1682 1683 if (epi->event.events & EPOLLWAKEUP) {1684 error = ep_create_wakeup_source(epi);1685 if (error) {1686 ep_remove_safe(ep, epi);1687 return error;1688 }1689 }1690 1691 /* Initialize the poll table using the queue callback */1692 epq.epi = epi;1693 init_poll_funcptr(&epq.pt, ep_ptable_queue_proc);1694 1695 /*1696 * Attach the item to the poll hooks and get current event bits.1697 * We can safely use the file* here because its usage count has1698 * been increased by the caller of this function. Note that after1699 * this operation completes, the poll callback can start hitting1700 * the new item.1701 */1702 revents = ep_item_poll(epi, &epq.pt, 1);1703 1704 /*1705 * We have to check if something went wrong during the poll wait queue1706 * install process. Namely an allocation for a wait queue failed due1707 * high memory pressure.1708 */1709 if (unlikely(!epq.epi)) {1710 ep_remove_safe(ep, epi);1711 return -ENOMEM;1712 }1713 1714 /* We have to drop the new item inside our item list to keep track of it */1715 write_lock_irq(&ep->lock);1716 1717 /* record NAPI ID of new item if present */1718 ep_set_busy_poll_napi_id(epi);1719 1720 /* If the file is already "ready" we drop it inside the ready list */1721 if (revents && !ep_is_linked(epi)) {1722 list_add_tail(&epi->rdllink, &ep->rdllist);1723 ep_pm_stay_awake(epi);1724 1725 /* Notify waiting tasks that events are available */1726 if (waitqueue_active(&ep->wq))1727 wake_up(&ep->wq);1728 if (waitqueue_active(&ep->poll_wait))1729 pwake++;1730 }1731 1732 write_unlock_irq(&ep->lock);1733 1734 /* We have to call this outside the lock */1735 if (pwake)1736 ep_poll_safewake(ep, NULL, 0);1737 1738 return 0;1739}1740 1741/*1742 * Modify the interest event mask by dropping an event if the new mask1743 * has a match in the current file status. Must be called with "mtx" held.1744 */1745static HWJS_SUSPENDS int ep_modify(struct eventpoll *ep, struct epitem *epi,1746 const struct epoll_event *event)1747{1748 int pwake = 0;1749 poll_table pt;1750 1751 lockdep_assert_irqs_enabled();1752 1753 init_poll_funcptr(&pt, NULL);1754 1755 /*1756 * Set the new event interest mask before calling f_op->poll();1757 * otherwise we might miss an event that happens between the1758 * f_op->poll() call and the new event set registering.1759 */1760 epi->event.events = event->events; /* need barrier below */1761 epi->event.data = event->data; /* protected by mtx */1762 if (epi->event.events & EPOLLWAKEUP) {1763 if (!ep_has_wakeup_source(epi))1764 ep_create_wakeup_source(epi);1765 } else if (ep_has_wakeup_source(epi)) {1766 ep_destroy_wakeup_source(epi);1767 }1768 1769 /*1770 * The following barrier has two effects:1771 *1772 * 1) Flush epi changes above to other CPUs. This ensures1773 * we do not miss events from ep_poll_callback if an1774 * event occurs immediately after we call f_op->poll().1775 * We need this because we did not take ep->lock while1776 * changing epi above (but ep_poll_callback does take1777 * ep->lock).1778 *1779 * 2) We also need to ensure we do not miss _past_ events1780 * when calling f_op->poll(). This barrier also1781 * pairs with the barrier in wq_has_sleeper (see1782 * comments for wq_has_sleeper).1783 *1784 * This barrier will now guarantee ep_poll_callback or f_op->poll1785 * (or both) will notice the readiness of an item.1786 */1787 smp_mb();1788 1789 /*1790 * Get current event bits. We can safely use the file* here because1791 * its usage count has been increased by the caller of this function.1792 * If the item is "hot" and it is not registered inside the ready1793 * list, push it inside.1794 */1795 if (ep_item_poll(epi, &pt, 1)) {1796 write_lock_irq(&ep->lock);1797 if (!ep_is_linked(epi)) {1798 list_add_tail(&epi->rdllink, &ep->rdllist);1799 ep_pm_stay_awake(epi);1800 1801 /* Notify waiting tasks that events are available */1802 if (waitqueue_active(&ep->wq))1803 wake_up(&ep->wq);1804 if (waitqueue_active(&ep->poll_wait))1805 pwake++;1806 }1807 write_unlock_irq(&ep->lock);1808 }1809 1810 /* We have to call this outside the lock */1811 if (pwake)1812 ep_poll_safewake(ep, NULL, 0);1813 1814 return 0;1815}1816 1817static HWJS_SUSPENDS int ep_send_events(struct eventpoll *ep,1818 struct epoll_event __user *events, int maxevents)1819{1820 struct epitem *epi, *tmp;1821 LIST_HEAD(txlist);1822 poll_table pt;1823 int res = 0;1824 1825 /*1826 * Always short-circuit for fatal signals to allow threads to make a1827 * timely exit without the chance of finding more events available and1828 * fetching repeatedly.1829 */1830 if (fatal_signal_pending(current))1831 return -EINTR;1832 1833 init_poll_funcptr(&pt, NULL);1834 1835 mutex_lock(&ep->mtx);1836 ep_start_scan(ep, &txlist);1837 1838 /*1839 * We can loop without lock because we are passed a task private list.1840 * Items cannot vanish during the loop we are holding ep->mtx.1841 */1842 list_for_each_entry_safe(epi, tmp, &txlist, rdllink) {1843 struct wakeup_source *ws;1844 __poll_t revents;1845 1846 if (res >= maxevents)1847 break;1848 1849 /*1850 * Activate ep->ws before deactivating epi->ws to prevent1851 * triggering auto-suspend here (in case we reactive epi->ws1852 * below).1853 *1854 * This could be rearranged to delay the deactivation of epi->ws1855 * instead, but then epi->ws would temporarily be out of sync1856 * with ep_is_linked().1857 */1858 ws = ep_wakeup_source(epi);1859 if (ws) {1860 if (ws->active)1861 __pm_stay_awake(ep->ws);1862 __pm_relax(ws);1863 }1864 1865 list_del_init(&epi->rdllink);1866 1867 /*1868 * If the event mask intersect the caller-requested one,1869 * deliver the event to userspace. Again, we are holding ep->mtx,1870 * so no operations coming from userspace can change the item.1871 */1872 revents = ep_item_poll(epi, &pt, 1);1873 if (!revents)1874 continue;1875 1876 events = epoll_put_uevent(revents, epi->event.data, events);1877 if (!events) {1878 list_add(&epi->rdllink, &txlist);1879 ep_pm_stay_awake(epi);1880 if (!res)1881 res = -EFAULT;1882 break;1883 }1884 res++;1885 if (epi->event.events & EPOLLONESHOT)1886 epi->event.events &= EP_PRIVATE_BITS;1887 else if (!(epi->event.events & EPOLLET)) {1888 /*1889 * If this file has been added with Level1890 * Trigger mode, we need to insert back inside1891 * the ready list, so that the next call to1892 * epoll_wait() will check again the events1893 * availability. At this point, no one can insert1894 * into ep->rdllist besides us. The epoll_ctl()1895 * callers are locked out by1896 * ep_send_events() holding "mtx" and the1897 * poll callback will queue them in ep->ovflist.1898 */1899 list_add_tail(&epi->rdllink, &ep->rdllist);1900 ep_pm_stay_awake(epi);1901 }1902 }1903 ep_done_scan(ep, &txlist);1904 mutex_unlock(&ep->mtx);1905 1906 return res;1907}1908 1909static struct timespec64 *ep_timeout_to_timespec(struct timespec64 *to, long ms)1910{1911 struct timespec64 now;1912 1913 if (ms < 0)1914 return NULL;1915 1916 if (!ms) {1917 to->tv_sec = 0;1918 to->tv_nsec = 0;1919 return to;1920 }1921 1922 to->tv_sec = ms / MSEC_PER_SEC;1923 to->tv_nsec = NSEC_PER_MSEC * (ms % MSEC_PER_SEC);1924 1925 ktime_get_ts64(&now);1926 *to = timespec64_add_safe(now, *to);1927 return to;1928}1929 1930/*1931 * autoremove_wake_function, but remove even on failure to wake up, because we1932 * know that default_wake_function/ttwu will only fail if the thread is already1933 * woken, and in that case the ep_poll loop will remove the entry anyways, not1934 * try to reuse it.1935 */1936static HWJS_SUSPENDS int ep_autoremove_wake_function(struct wait_queue_entry *wq_entry,1937 unsigned int mode, int sync, void *key)1938{1939 int ret = default_wake_function(wq_entry, mode, sync, key);1940 1941 /*1942 * Pairs with list_empty_careful in ep_poll, and ensures future loop1943 * iterations see the cause of this wakeup.1944 */1945 list_del_init_careful(&wq_entry->entry);1946 return ret;1947}1948 1949/**1950 * ep_poll - Retrieves ready events, and delivers them to the caller-supplied1951 * event buffer.1952 *1953 * @ep: Pointer to the eventpoll context.1954 * @events: Pointer to the userspace buffer where the ready events should be1955 * stored.1956 * @maxevents: Size (in terms of number of events) of the caller event buffer.1957 * @timeout: Maximum timeout for the ready events fetch operation, in1958 * timespec. If the timeout is zero, the function will not block,1959 * while if the @timeout ptr is NULL, the function will block1960 * until at least one event has been retrieved (or an error1961 * occurred).1962 *1963 * Return: the number of ready events which have been fetched, or an1964 * error code, in case of error.1965 */1966static HWJS_SUSPENDS int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,1967 int maxevents, struct timespec64 *timeout)1968{1969 int res, eavail, timed_out = 0;1970 u64 slack = 0;1971 wait_queue_entry_t wait;1972 ktime_t expires, *to = NULL;1973 1974 lockdep_assert_irqs_enabled();1975 1976 if (timeout && (timeout->tv_sec | timeout->tv_nsec)) {1977 slack = select_estimate_accuracy(timeout);1978 to = &expires;1979 *to = timespec64_to_ktime(*timeout);1980 } else if (timeout) {1981 /*1982 * Avoid the unnecessary trip to the wait queue loop, if the1983 * caller specified a non blocking operation.1984 */1985 timed_out = 1;1986 }1987 1988 /*1989 * This call is racy: We may or may not see events that are being added1990 * to the ready list under the lock (e.g., in IRQ callbacks). For cases1991 * with a non-zero timeout, this thread will check the ready list under1992 * lock and will add to the wait queue. For cases with a zero1993 * timeout, the user by definition should not care and will have to1994 * recheck again.1995 */1996 eavail = ep_events_available(ep);1997 1998 while (1) {1999 if (eavail) {2000 /*2001 * Try to transfer events to user space. In case we get2002 * 0 events and there's still timeout left over, we go2003 * trying again in search of more luck.2004 */2005 res = ep_send_events(ep, events, maxevents);2006 if (res)2007 return res;2008 }2009 2010 if (timed_out)2011 return 0;2012 2013 eavail = ep_busy_loop(ep, timed_out);2014 if (eavail)2015 continue;2016 2017 if (signal_pending(current))2018 return -EINTR;2019 2020 /*2021 * Internally init_wait() uses autoremove_wake_function(),2022 * thus wait entry is removed from the wait queue on each2023 * wakeup. Why it is important? In case of several waiters2024 * each new wakeup will hit the next waiter, giving it the2025 * chance to harvest new event. Otherwise wakeup can be2026 * lost. This is also good performance-wise, because on2027 * normal wakeup path no need to call __remove_wait_queue()2028 * explicitly, thus ep->lock is not taken, which halts the2029 * event delivery.2030 *2031 * In fact, we now use an even more aggressive function that2032 * unconditionally removes, because we don't reuse the wait2033 * entry between loop iterations. This lets us also avoid the2034 * performance issue if a process is killed, causing all of its2035 * threads to wake up without being removed normally.2036 */2037 init_wait(&wait);2038 wait.func = ep_autoremove_wake_function;2039 2040 write_lock_irq(&ep->lock);2041 /*2042 * Barrierless variant, waitqueue_active() is called under2043 * the same lock on wakeup ep_poll_callback() side, so it2044 * is safe to avoid an explicit barrier.2045 */2046 __set_current_state(TASK_INTERRUPTIBLE);2047 2048 /*2049 * Do the final check under the lock. ep_start/done_scan()2050 * plays with two lists (->rdllist and ->ovflist) and there2051 * is always a race when both lists are empty for short2052 * period of time although events are pending, so lock is2053 * important.2054 */2055 eavail = ep_events_available(ep);2056 if (!eavail)2057 __add_wait_queue_exclusive(&ep->wq, &wait);2058 2059 write_unlock_irq(&ep->lock);2060 2061 if (!eavail)2062 timed_out = !schedule_hrtimeout_range(to, slack,2063 HRTIMER_MODE_ABS);2064 __set_current_state(TASK_RUNNING);2065 2066 /*2067 * We were woken up, thus go and try to harvest some events.2068 * If timed out and still on the wait queue, recheck eavail2069 * carefully under lock, below.2070 */2071 eavail = 1;2072 2073 if (!list_empty_careful(&wait.entry)) {2074 write_lock_irq(&ep->lock);2075 /*2076 * If the thread timed out and is not on the wait queue,2077 * it means that the thread was woken up after its2078 * timeout expired before it could reacquire the lock.2079 * Thus, when wait.entry is empty, it needs to harvest2080 * events.2081 */2082 if (timed_out)2083 eavail = list_empty(&wait.entry);2084 __remove_wait_queue(&ep->wq, &wait);2085 write_unlock_irq(&ep->lock);2086 }2087 }2088}2089 2090/**2091 * ep_loop_check_proc - verify that adding an epoll file inside another2092 * epoll structure does not violate the constraints, in2093 * terms of closed loops, or too deep chains (which can2094 * result in excessive stack usage).2095 *2096 * @ep: the &struct eventpoll to be currently checked.2097 * @depth: Current depth of the path being checked.2098 *2099 * Return: %zero if adding the epoll @file inside current epoll2100 * structure @ep does not violate the constraints, or %-1 otherwise.2101 */2102static HWJS_SUSPENDS int ep_loop_check_proc(struct eventpoll *ep, int depth)2103{2104 int error = 0;2105 struct rb_node *rbp;2106 struct epitem *epi;2107 2108 mutex_lock_nested(&ep->mtx, depth + 1);2109 ep->gen = loop_check_gen;2110 for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {2111 epi = rb_entry(rbp, struct epitem, rbn);2112 if (unlikely(is_file_epoll(epi->ffd.file))) {2113 struct eventpoll *ep_tovisit;2114 ep_tovisit = epi->ffd.file->private_data;2115 if (ep_tovisit->gen == loop_check_gen)2116 continue;2117 if (ep_tovisit == inserting_into || depth > EP_MAX_NESTS)2118 error = -1;2119 else2120 error = ep_loop_check_proc(ep_tovisit, depth + 1);2121 if (error != 0)2122 break;2123 } else {2124 /*2125 * If we've reached a file that is not associated with2126 * an ep, then we need to check if the newly added2127 * links are going to add too many wakeup paths. We do2128 * this by adding it to the tfile_check_list, if it's2129 * not already there, and calling reverse_path_check()2130 * during ep_insert().2131 */2132 list_file(epi->ffd.file);2133 }2134 }2135 mutex_unlock(&ep->mtx);2136 2137 return error;2138}2139 2140/**2141 * ep_loop_check - Performs a check to verify that adding an epoll file (@to)2142 * into another epoll file (represented by @ep) does not create2143 * closed loops or too deep chains.2144 *2145 * @ep: Pointer to the epoll we are inserting into.2146 * @to: Pointer to the epoll to be inserted.2147 *2148 * Return: %zero if adding the epoll @to inside the epoll @from2149 * does not violate the constraints, or %-1 otherwise.2150 */2151static HWJS_SUSPENDS int ep_loop_check(struct eventpoll *ep, struct eventpoll *to)2152{2153 inserting_into = ep;2154 return ep_loop_check_proc(to, 0);2155}2156 2157static void clear_tfile_check_list(void)2158{2159 rcu_read_lock();2160 while (tfile_check_list != EP_UNACTIVE_PTR) {2161 struct epitems_head *head = tfile_check_list;2162 tfile_check_list = head->next;2163 unlist_file(head);2164 }2165 rcu_read_unlock();2166}2167 2168/*2169 * Open an eventpoll file descriptor.2170 */2171static HWJS_SUSPENDS int do_epoll_create(int flags)2172{2173 int error, fd;2174 struct eventpoll *ep = NULL;2175 struct file *file;2176 2177 /* Check the EPOLL_* constant for consistency. */2178 BUILD_BUG_ON(EPOLL_CLOEXEC != O_CLOEXEC);2179 2180 if (flags & ~EPOLL_CLOEXEC)2181 return -EINVAL;2182 /*2183 * Create the internal data structure ("struct eventpoll").2184 */2185 error = ep_alloc(&ep);2186 if (error < 0)2187 return error;2188 /*2189 * Creates all the items needed to setup an eventpoll file. That is,2190 * a file structure and a free file descriptor.2191 */2192 fd = get_unused_fd_flags(O_RDWR | (flags & O_CLOEXEC));2193 if (fd < 0) {2194 error = fd;2195 goto out_free_ep;2196 }2197 file = anon_inode_getfile("[eventpoll]", &eventpoll_fops, ep,2198 O_RDWR | (flags & O_CLOEXEC));2199 if (IS_ERR(file)) {2200 error = PTR_ERR(file);2201 goto out_free_fd;2202 }2203 ep->file = file;2204 fd_install(fd, file);2205 return fd;2206 2207out_free_fd:2208 put_unused_fd(fd);2209out_free_ep:2210 ep_clear_and_put(ep);2211 return error;2212}2213 2214SYSCALL_DEFINE1(epoll_create1, int, flags)2215{2216 return do_epoll_create(flags);2217}2218 2219SYSCALL_DEFINE1(epoll_create, int, size)2220{2221 if (size <= 0)2222 return -EINVAL;2223 2224 return do_epoll_create(0);2225}2226 2227#ifdef CONFIG_PM_SLEEP2228static inline void ep_take_care_of_epollwakeup(struct epoll_event *epev)2229{2230 if ((epev->events & EPOLLWAKEUP) && !capable(CAP_BLOCK_SUSPEND))2231 epev->events &= ~EPOLLWAKEUP;2232}2233#else2234static inline void ep_take_care_of_epollwakeup(struct epoll_event *epev)2235{2236 epev->events &= ~EPOLLWAKEUP;2237}2238#endif2239 2240static inline int epoll_mutex_lock(struct mutex *mutex, int depth,2241 bool nonblock)2242{2243 if (!nonblock) {2244 mutex_lock_nested(mutex, depth);2245 return 0;2246 }2247 if (mutex_trylock(mutex))2248 return 0;2249 return -EAGAIN;2250}2251 2252int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds,2253 bool nonblock)2254{2255 int error;2256 int full_check = 0;2257 struct fd f, tf;2258 struct eventpoll *ep;2259 struct epitem *epi;2260 struct eventpoll *tep = NULL;2261 2262 error = -EBADF;2263 f = fdget(epfd);2264 if (!fd_file(f))2265 goto error_return;2266 2267 /* Get the "struct file *" for the target file */2268 tf = fdget(fd);2269 if (!fd_file(tf))2270 goto error_fput;2271 2272 /* The target file descriptor must support poll */2273 error = -EPERM;2274 if (!file_can_poll(fd_file(tf)))2275 goto error_tgt_fput;2276 2277 /* Check if EPOLLWAKEUP is allowed */2278 if (ep_op_has_event(op))2279 ep_take_care_of_epollwakeup(epds);2280 2281 /*2282 * We have to check that the file structure underneath the file descriptor2283 * the user passed to us _is_ an eventpoll file. And also we do not permit2284 * adding an epoll file descriptor inside itself.2285 */2286 error = -EINVAL;2287 if (fd_file(f) == fd_file(tf) || !is_file_epoll(fd_file(f)))2288 goto error_tgt_fput;2289 2290 /*2291 * epoll adds to the wakeup queue at EPOLL_CTL_ADD time only,2292 * so EPOLLEXCLUSIVE is not allowed for a EPOLL_CTL_MOD operation.2293 * Also, we do not currently supported nested exclusive wakeups.2294 */2295 if (ep_op_has_event(op) && (epds->events & EPOLLEXCLUSIVE)) {2296 if (op == EPOLL_CTL_MOD)2297 goto error_tgt_fput;2298 if (op == EPOLL_CTL_ADD && (is_file_epoll(fd_file(tf)) ||2299 (epds->events & ~EPOLLEXCLUSIVE_OK_BITS)))2300 goto error_tgt_fput;2301 }2302 2303 /*2304 * At this point it is safe to assume that the "private_data" contains2305 * our own data structure.2306 */2307 ep = fd_file(f)->private_data;2308 2309 /*2310 * When we insert an epoll file descriptor inside another epoll file2311 * descriptor, there is the chance of creating closed loops, which are2312 * better be handled here, than in more critical paths. While we are2313 * checking for loops we also determine the list of files reachable2314 * and hang them on the tfile_check_list, so we can check that we2315 * haven't created too many possible wakeup paths.2316 *2317 * We do not need to take the global 'epumutex' on EPOLL_CTL_ADD when2318 * the epoll file descriptor is attaching directly to a wakeup source,2319 * unless the epoll file descriptor is nested. The purpose of taking the2320 * 'epnested_mutex' on add is to prevent complex toplogies such as loops and2321 * deep wakeup paths from forming in parallel through multiple2322 * EPOLL_CTL_ADD operations.2323 */2324 error = epoll_mutex_lock(&ep->mtx, 0, nonblock);2325 if (error)2326 goto error_tgt_fput;2327 if (op == EPOLL_CTL_ADD) {2328 if (READ_ONCE(fd_file(f)->f_ep) || ep->gen == loop_check_gen ||2329 is_file_epoll(fd_file(tf))) {2330 mutex_unlock(&ep->mtx);2331 error = epoll_mutex_lock(&epnested_mutex, 0, nonblock);2332 if (error)2333 goto error_tgt_fput;2334 loop_check_gen++;2335 full_check = 1;2336 if (is_file_epoll(fd_file(tf))) {2337 tep = fd_file(tf)->private_data;2338 error = -ELOOP;2339 if (ep_loop_check(ep, tep) != 0)2340 goto error_tgt_fput;2341 }2342 error = epoll_mutex_lock(&ep->mtx, 0, nonblock);2343 if (error)2344 goto error_tgt_fput;2345 }2346 }2347 2348 /*2349 * Try to lookup the file inside our RB tree. Since we grabbed "mtx"2350 * above, we can be sure to be able to use the item looked up by2351 * ep_find() till we release the mutex.2352 */2353 epi = ep_find(ep, fd_file(tf), fd);2354 2355 error = -EINVAL;2356 switch (op) {2357 case EPOLL_CTL_ADD:2358 if (!epi) {2359 epds->events |= EPOLLERR | EPOLLHUP;2360 error = ep_insert(ep, epds, fd_file(tf), fd, full_check);2361 } else2362 error = -EEXIST;2363 break;2364 case EPOLL_CTL_DEL:2365 if (epi) {2366 /*2367 * The eventpoll itself is still alive: the refcount2368 * can't go to zero here.2369 */2370 ep_remove_safe(ep, epi);2371 error = 0;2372 } else {2373 error = -ENOENT;2374 }2375 break;2376 case EPOLL_CTL_MOD:2377 if (epi) {2378 if (!(epi->event.events & EPOLLEXCLUSIVE)) {2379 epds->events |= EPOLLERR | EPOLLHUP;2380 error = ep_modify(ep, epi, epds);2381 }2382 } else2383 error = -ENOENT;2384 break;2385 }2386 mutex_unlock(&ep->mtx);2387 2388error_tgt_fput:2389 if (full_check) {2390 clear_tfile_check_list();2391 loop_check_gen++;2392 mutex_unlock(&epnested_mutex);2393 }2394 2395 fdput(tf);2396error_fput:2397 fdput(f);2398error_return:2399 2400 return error;2401}2402 2403/*2404 * The following function implements the controller interface for2405 * the eventpoll file that enables the insertion/removal/change of2406 * file descriptors inside the interest set.2407 */2408SYSCALL_DEFINE4(epoll_ctl, int, epfd, int, op, int, fd,2409 struct epoll_event __user *, event)2410{2411 struct epoll_event epds;2412 2413 if (ep_op_has_event(op) &&2414 copy_from_user(&epds, event, sizeof(struct epoll_event)))2415 return -EFAULT;2416 2417 return do_epoll_ctl(epfd, op, fd, &epds, false);2418}2419 2420/*2421 * Implement the event wait interface for the eventpoll file. It is the kernel2422 * part of the user space epoll_wait(2).2423 */2424static HWJS_SUSPENDS int do_epoll_wait(int epfd, struct epoll_event __user *events,2425 int maxevents, struct timespec64 *to)2426{2427 int error;2428 struct fd f;2429 struct eventpoll *ep;2430 2431 /* The maximum number of event must be greater than zero */2432 if (maxevents <= 0 || maxevents > EP_MAX_EVENTS)2433 return -EINVAL;2434 2435 /* Verify that the area passed by the user is writeable */2436 if (!access_ok(events, maxevents * sizeof(struct epoll_event)))2437 return -EFAULT;2438 2439 /* Get the "struct file *" for the eventpoll file */2440 f = fdget(epfd);2441 if (!fd_file(f))2442 return -EBADF;2443 2444 /*2445 * We have to check that the file structure underneath the fd2446 * the user passed to us _is_ an eventpoll file.2447 */2448 error = -EINVAL;2449 if (!is_file_epoll(fd_file(f)))2450 goto error_fput;2451 2452 /*2453 * At this point it is safe to assume that the "private_data" contains2454 * our own data structure.2455 */2456 ep = fd_file(f)->private_data;2457 2458 /* Time to fish for events ... */2459 error = ep_poll(ep, events, maxevents, to);2460 2461error_fput:2462 fdput(f);2463 return error;2464}2465 2466SYSCALL_DEFINE4(epoll_wait, int, epfd, struct epoll_event __user *, events,2467 int, maxevents, int, timeout)2468{2469 struct timespec64 to;2470 2471 return do_epoll_wait(epfd, events, maxevents,2472 ep_timeout_to_timespec(&to, timeout));2473}2474 2475/*2476 * Implement the event wait interface for the eventpoll file. It is the kernel2477 * part of the user space epoll_pwait(2).2478 */2479static HWJS_SUSPENDS int do_epoll_pwait(int epfd, struct epoll_event __user *events,2480 int maxevents, struct timespec64 *to,2481 const sigset_t __user *sigmask, size_t sigsetsize)2482{2483 int error;2484 2485 /*2486 * If the caller wants a certain signal mask to be set during the wait,2487 * we apply it here.2488 */2489 error = set_user_sigmask(sigmask, sigsetsize);2490 if (error)2491 return error;2492 2493 error = do_epoll_wait(epfd, events, maxevents, to);2494 2495 restore_saved_sigmask_unless(error == -EINTR);2496 2497 return error;2498}2499 2500SYSCALL_DEFINE6(epoll_pwait, int, epfd, struct epoll_event __user *, events,2501 int, maxevents, int, timeout, const sigset_t __user *, sigmask,2502 size_t, sigsetsize)2503{2504 struct timespec64 to;2505 2506 return do_epoll_pwait(epfd, events, maxevents,2507 ep_timeout_to_timespec(&to, timeout),2508 sigmask, sigsetsize);2509}2510 2511SYSCALL_DEFINE6(epoll_pwait2, int, epfd, struct epoll_event __user *, events,2512 int, maxevents, const struct __kernel_timespec __user *, timeout,2513 const sigset_t __user *, sigmask, size_t, sigsetsize)2514{2515 struct timespec64 ts, *to = NULL;2516 2517 if (timeout) {2518 if (get_timespec64(&ts, timeout))2519 return -EFAULT;2520 to = &ts;2521 if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec))2522 return -EINVAL;2523 }2524 2525 return do_epoll_pwait(epfd, events, maxevents, to,2526 sigmask, sigsetsize);2527}2528 2529#ifdef CONFIG_COMPAT2530static int do_compat_epoll_pwait(int epfd, struct epoll_event __user *events,2531 int maxevents, struct timespec64 *timeout,2532 const compat_sigset_t __user *sigmask,2533 compat_size_t sigsetsize)2534{2535 long err;2536 2537 /*2538 * If the caller wants a certain signal mask to be set during the wait,2539 * we apply it here.2540 */2541 err = set_compat_user_sigmask(sigmask, sigsetsize);2542 if (err)2543 return err;2544 2545 err = do_epoll_wait(epfd, events, maxevents, timeout);2546 2547 restore_saved_sigmask_unless(err == -EINTR);2548 2549 return err;2550}2551 2552COMPAT_SYSCALL_DEFINE6(epoll_pwait, int, epfd,2553 struct epoll_event __user *, events,2554 int, maxevents, int, timeout,2555 const compat_sigset_t __user *, sigmask,2556 compat_size_t, sigsetsize)2557{2558 struct timespec64 to;2559 2560 return do_compat_epoll_pwait(epfd, events, maxevents,2561 ep_timeout_to_timespec(&to, timeout),2562 sigmask, sigsetsize);2563}2564 2565COMPAT_SYSCALL_DEFINE6(epoll_pwait2, int, epfd,2566 struct epoll_event __user *, events,2567 int, maxevents,2568 const struct __kernel_timespec __user *, timeout,2569 const compat_sigset_t __user *, sigmask,2570 compat_size_t, sigsetsize)2571{2572 struct timespec64 ts, *to = NULL;2573 2574 if (timeout) {2575 if (get_timespec64(&ts, timeout))2576 return -EFAULT;2577 to = &ts;2578 if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec))2579 return -EINVAL;2580 }2581 2582 return do_compat_epoll_pwait(epfd, events, maxevents, to,2583 sigmask, sigsetsize);2584}2585 2586#endif2587 2588static int __init eventpoll_init(void)2589{2590 struct sysinfo si;2591 2592 si_meminfo(&si);2593 /*2594 * Allows top 4% of lomem to be allocated for epoll watches (per user).2595 */2596 max_user_watches = (((si.totalram - si.totalhigh) / 25) << PAGE_SHIFT) /2597 EP_ITEM_COST;2598 BUG_ON(max_user_watches < 0);2599 2600 /*2601 * We can have many thousands of epitems, so prevent this from2602 * using an extra cache line on 64-bit (and smaller) CPUs2603 */2604 BUILD_BUG_ON(sizeof(void *) <= 8 && sizeof(struct epitem) > 128);2605 2606 /* Allocates slab cache used to allocate "struct epitem" items */2607 epi_cache = kmem_cache_create("eventpoll_epi", sizeof(struct epitem),2608 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL);2609 2610 /* Allocates slab cache used to allocate "struct eppoll_entry" */2611 pwq_cache = kmem_cache_create("eventpoll_pwq",2612 sizeof(struct eppoll_entry), 0, SLAB_PANIC|SLAB_ACCOUNT, NULL);2613 epoll_sysctls_init();2614 2615 ephead_cache = kmem_cache_create("ep_head",2616 sizeof(struct epitems_head), 0, SLAB_PANIC|SLAB_ACCOUNT, NULL);2617 2618 return 0;2619}2620fs_initcall(eventpoll_init);2621