1591 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * linux/fs/pipe.c4 *5 * Copyright (C) 1991, 1992, 1999 Linus Torvalds6 */7 8#include <linux/mm.h>9#include <linux/file.h>10#include <linux/poll.h>11#include <linux/slab.h>12#include <linux/module.h>13#include <linux/init.h>14#include <linux/fs.h>15#include <linux/log2.h>16#include <linux/mount.h>17#include <linux/pseudo_fs.h>18#include <linux/magic.h>19#include <linux/pipe_fs_i.h>20#include <linux/uio.h>21#include <linux/highmem.h>22#include <linux/pagemap.h>23#include <linux/audit.h>24#include <linux/syscalls.h>25#include <linux/fcntl.h>26#include <linux/memcontrol.h>27#include <linux/watch_queue.h>28#include <linux/sysctl.h>29 30#include <linux/uaccess.h>31#include <asm/ioctls.h>32 33#include "internal.h"34 35#ifdef CONFIG_WASM3236/*37 * K13-B (cross-Worker pipe-DATA): a blocked pipe endpoint in a process38 * Worker cannot use wait_event_interruptible_exclusive()->schedule() (no39 * preemptive scheduler in a process Worker) and its waker (a peer write /40 * close / signal in another Worker) cannot use try_to_wake_up() (no remote41 * rq advancement). Swap, under CONFIG_WASM32, the parked entry's wake42 * function and park primitive for the wait4/tty-shaped per-task wake word43 * (arch/wasm32/kernel/pipe_wait.c): the endpoint parks on44 * current->thread.pipe_wake via __builtin_wasm_memory_atomic_wait32, and45 * the existing wake_up_interruptible*(&pipe->rd_wait / wr_wait) sites route46 * through the swapped .func to a per-task atomic notify of the SPECIFIC47 * sleeper. No wake-site changes are needed (__wake_up_common already48 * dispatches via .func). See pipe_read()/pipe_write() below.49 */50extern int wasm32_pipe_wake_function(struct wait_queue_entry *wq_entry,51 unsigned int mode, int sync, void *key);52extern void wasm32_pipe_wait(struct wait_queue_entry *wq_entry);53#endif54 55/*56 * New pipe buffers will be restricted to this size while the user is exceeding57 * their pipe buffer quota. The general pipe use case needs at least two58 * buffers: one for data yet to be read, and one for new data. If this is less59 * than two, then a write to a non-empty pipe may block even if the pipe is not60 * full. This can occur with GNU make jobserver or similar uses of pipes as61 * semaphores: multiple processes may be waiting to write tokens back to the62 * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/.63 *64 * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their65 * own risk, namely: pipe writes to non-full pipes may block until the pipe is66 * emptied.67 */68#define PIPE_MIN_DEF_BUFFERS 269 70/*71 * The max size that a non-root user is allowed to grow the pipe. Can72 * be set by root in /proc/sys/fs/pipe-max-size73 */74static unsigned int pipe_max_size = 1048576;75 76/* Maximum allocatable pages per user. Hard limit is unset by default, soft77 * matches default values.78 */79static unsigned long pipe_user_pages_hard;80static unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR;81 82/*83 * We use head and tail indices that aren't masked off, except at the point of84 * dereference, but rather they're allowed to wrap naturally. This means there85 * isn't a dead spot in the buffer, but the ring has to be a power of two and86 * <= 2^31.87 * -- David Howells 2019-09-23.88 *89 * Reads with count = 0 should always return 0.90 * -- Julian Bradfield 1999-06-07.91 *92 * FIFOs and Pipes now generate SIGIO for both readers and writers.93 * -- Jeremy Elson <jelson@circlemud.org> 2001-08-1694 *95 * pipe_read & write cleanup96 * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-0997 */98 99#define cmp_int(l, r) ((l > r) - (l < r))100 101#ifdef CONFIG_PROVE_LOCKING102static int pipe_lock_cmp_fn(const struct lockdep_map *a,103 const struct lockdep_map *b)104{105 return cmp_int((unsigned long) a, (unsigned long) b);106}107#endif108 109void pipe_lock(struct pipe_inode_info *pipe)110{111 if (pipe->files)112 mutex_lock(&pipe->mutex);113}114EXPORT_SYMBOL(pipe_lock);115 116void pipe_unlock(struct pipe_inode_info *pipe)117{118 if (pipe->files)119 mutex_unlock(&pipe->mutex);120}121EXPORT_SYMBOL(pipe_unlock);122 123void pipe_double_lock(struct pipe_inode_info *pipe1,124 struct pipe_inode_info *pipe2)125{126 BUG_ON(pipe1 == pipe2);127 128 if (pipe1 > pipe2)129 swap(pipe1, pipe2);130 131 pipe_lock(pipe1);132 pipe_lock(pipe2);133}134 135static void anon_pipe_buf_release(struct pipe_inode_info *pipe,136 struct pipe_buffer *buf)137{138 struct page *page = buf->page;139 140 /*141 * If nobody else uses this page, and we don't already have a142 * temporary page, let's keep track of it as a one-deep143 * allocation cache. (Otherwise just release our reference to it)144 */145 if (page_count(page) == 1 && !pipe->tmp_page)146 pipe->tmp_page = page;147 else148 put_page(page);149}150 151static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,152 struct pipe_buffer *buf)153{154 struct page *page = buf->page;155 156 if (page_count(page) != 1)157 return false;158 memcg_kmem_uncharge_page(page, 0);159 __SetPageLocked(page);160 return true;161}162 163/**164 * generic_pipe_buf_try_steal - attempt to take ownership of a &pipe_buffer165 * @pipe: the pipe that the buffer belongs to166 * @buf: the buffer to attempt to steal167 *168 * Description:169 * This function attempts to steal the &struct page attached to170 * @buf. If successful, this function returns 0 and returns with171 * the page locked. The caller may then reuse the page for whatever172 * he wishes; the typical use is insertion into a different file173 * page cache.174 */175bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,176 struct pipe_buffer *buf)177{178 struct page *page = buf->page;179 180 /*181 * A reference of one is golden, that means that the owner of this182 * page is the only one holding a reference to it. lock the page183 * and return OK.184 */185 if (page_count(page) == 1) {186 lock_page(page);187 return true;188 }189 return false;190}191EXPORT_SYMBOL(generic_pipe_buf_try_steal);192 193/**194 * generic_pipe_buf_get - get a reference to a &struct pipe_buffer195 * @pipe: the pipe that the buffer belongs to196 * @buf: the buffer to get a reference to197 *198 * Description:199 * This function grabs an extra reference to @buf. It's used in200 * the tee() system call, when we duplicate the buffers in one201 * pipe into another.202 */203bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)204{205 return try_get_page(buf->page);206}207EXPORT_SYMBOL(generic_pipe_buf_get);208 209/**210 * generic_pipe_buf_release - put a reference to a &struct pipe_buffer211 * @pipe: the pipe that the buffer belongs to212 * @buf: the buffer to put a reference to213 *214 * Description:215 * This function releases a reference to @buf.216 */217void generic_pipe_buf_release(struct pipe_inode_info *pipe,218 struct pipe_buffer *buf)219{220 put_page(buf->page);221}222EXPORT_SYMBOL(generic_pipe_buf_release);223 224static const struct pipe_buf_operations anon_pipe_buf_ops = {225 .release = anon_pipe_buf_release,226 .try_steal = anon_pipe_buf_try_steal,227 .get = generic_pipe_buf_get,228};229 230/* Done while waiting without holding the pipe lock - thus the READ_ONCE() */231static inline bool pipe_readable(const struct pipe_inode_info *pipe)232{233 unsigned int head = READ_ONCE(pipe->head);234 unsigned int tail = READ_ONCE(pipe->tail);235 unsigned int writers = READ_ONCE(pipe->writers);236 237 return !pipe_empty(head, tail) || !writers;238}239 240static inline unsigned int pipe_update_tail(struct pipe_inode_info *pipe,241 struct pipe_buffer *buf,242 unsigned int tail)243{244 pipe_buf_release(pipe, buf);245 246 /*247 * If the pipe has a watch_queue, we need additional protection248 * by the spinlock because notifications get posted with only249 * this spinlock, no mutex250 */251 if (pipe_has_watch_queue(pipe)) {252 spin_lock_irq(&pipe->rd_wait.lock);253#ifdef CONFIG_WATCH_QUEUE254 if (buf->flags & PIPE_BUF_FLAG_LOSS)255 pipe->note_loss = true;256#endif257 pipe->tail = ++tail;258 spin_unlock_irq(&pipe->rd_wait.lock);259 return tail;260 }261 262 /*263 * Without a watch_queue, we can simply increment the tail264 * without the spinlock - the mutex is enough.265 */266 pipe->tail = ++tail;267 return tail;268}269 270static HWJS_SUSPENDS ssize_t271pipe_read(struct kiocb *iocb, struct iov_iter *to)272{273 size_t total_len = iov_iter_count(to);274 struct file *filp = iocb->ki_filp;275 struct pipe_inode_info *pipe = filp->private_data;276 bool was_full, wake_next_reader = false;277 ssize_t ret;278 279 /* Null read succeeds. */280 if (unlikely(total_len == 0))281 return 0;282 283 ret = 0;284 mutex_lock(&pipe->mutex);285 286 /*287 * We only wake up writers if the pipe was full when we started288 * reading in order to avoid unnecessary wakeups.289 *290 * But when we do wake up writers, we do so using a sync wakeup291 * (WF_SYNC), because we want them to get going and generate more292 * data for us.293 */294 was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);295 for (;;) {296 /* Read ->head with a barrier vs post_one_notification() */297 unsigned int head = smp_load_acquire(&pipe->head);298 unsigned int tail = pipe->tail;299 unsigned int mask = pipe->ring_size - 1;300 301#ifdef CONFIG_WATCH_QUEUE302 if (pipe->note_loss) {303 struct watch_notification n;304 305 if (total_len < 8) {306 if (ret == 0)307 ret = -ENOBUFS;308 break;309 }310 311 n.type = WATCH_TYPE_META;312 n.subtype = WATCH_META_LOSS_NOTIFICATION;313 n.info = watch_sizeof(n);314 if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) {315 if (ret == 0)316 ret = -EFAULT;317 break;318 }319 ret += sizeof(n);320 total_len -= sizeof(n);321 pipe->note_loss = false;322 }323#endif324 325 if (!pipe_empty(head, tail)) {326 struct pipe_buffer *buf = &pipe->bufs[tail & mask];327 size_t chars = buf->len;328 size_t written;329 int error;330 331 if (chars > total_len) {332 if (buf->flags & PIPE_BUF_FLAG_WHOLE) {333 if (ret == 0)334 ret = -ENOBUFS;335 break;336 }337 chars = total_len;338 }339 340 error = pipe_buf_confirm(pipe, buf);341 if (error) {342 if (!ret)343 ret = error;344 break;345 }346 347 written = copy_page_to_iter(buf->page, buf->offset, chars, to);348 if (unlikely(written < chars)) {349 if (!ret)350 ret = -EFAULT;351 break;352 }353 ret += chars;354 buf->offset += chars;355 buf->len -= chars;356 357 /* Was it a packet buffer? Clean up and exit */358 if (buf->flags & PIPE_BUF_FLAG_PACKET) {359 total_len = chars;360 buf->len = 0;361 }362 363 if (!buf->len)364 tail = pipe_update_tail(pipe, buf, tail);365 total_len -= chars;366 if (!total_len)367 break; /* common path: read succeeded */368 if (!pipe_empty(head, tail)) /* More to do? */369 continue;370 }371 372 if (!pipe->writers)373 break;374 if (ret)375 break;376 if ((filp->f_flags & O_NONBLOCK) ||377 (iocb->ki_flags & IOCB_NOWAIT)) {378 ret = -EAGAIN;379 break;380 }381 mutex_unlock(&pipe->mutex);382 383 /*384 * We only get here if we didn't actually read anything.385 *386 * However, we could have seen (and removed) a zero-sized387 * pipe buffer, and might have made space in the buffers388 * that way.389 *390 * You can't make zero-sized pipe buffers by doing an empty391 * write (not even in packet mode), but they can happen if392 * the writer gets an EFAULT when trying to fill a buffer393 * that already got allocated and inserted in the buffer394 * array.395 *396 * So we still need to wake up any pending writers in the397 * _very_ unlikely case that the pipe was full, but we got398 * no data.399 */400 if (unlikely(was_full))401 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);402 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);403 404 /*405 * But because we didn't read anything, at this point we can406 * just return directly with -ERESTARTSYS if we're interrupted,407 * since we've done any required wakeups and there's no need408 * to mark anything accessed. And we've dropped the lock.409 */410#ifdef CONFIG_WASM32411 {412 /* K13-B: park on the per-task wake word instead of413 * schedule(); the peer writer/closer in another Worker414 * routes its wake_up_interruptible(&pipe->rd_wait) through415 * this entry's .func == wasm32_pipe_wake_function. */416 DEFINE_WAIT_FUNC(__wwait, wasm32_pipe_wake_function);417 int __wintr = 0;418 419 for (;;) {420 prepare_to_wait_exclusive(&pipe->rd_wait, &__wwait,421 TASK_INTERRUPTIBLE);422 if (pipe_readable(pipe))423 break;424 if (signal_pending(current)) {425 __wintr = 1;426 break;427 }428 wasm32_pipe_wait(&__wwait);429 }430 finish_wait(&pipe->rd_wait, &__wwait);431 if (__wintr)432 return -ERESTARTSYS;433 }434#else435 if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)436 return -ERESTARTSYS;437#endif438 439 mutex_lock(&pipe->mutex);440 was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);441 wake_next_reader = true;442 }443 if (pipe_empty(pipe->head, pipe->tail))444 wake_next_reader = false;445 mutex_unlock(&pipe->mutex);446 447 if (was_full)448 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);449 if (wake_next_reader)450 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);451 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);452 if (ret > 0)453 file_accessed(filp);454 return ret;455}456 457static inline int is_packetized(struct file *file)458{459 return (file->f_flags & O_DIRECT) != 0;460}461 462/* Done while waiting without holding the pipe lock - thus the READ_ONCE() */463static inline bool pipe_writable(const struct pipe_inode_info *pipe)464{465 unsigned int head = READ_ONCE(pipe->head);466 unsigned int tail = READ_ONCE(pipe->tail);467 unsigned int max_usage = READ_ONCE(pipe->max_usage);468 469 return !pipe_full(head, tail, max_usage) ||470 !READ_ONCE(pipe->readers);471}472 473static HWJS_SUSPENDS ssize_t474pipe_write(struct kiocb *iocb, struct iov_iter *from)475{476 struct file *filp = iocb->ki_filp;477 struct pipe_inode_info *pipe = filp->private_data;478 unsigned int head;479 ssize_t ret = 0;480 size_t total_len = iov_iter_count(from);481 ssize_t chars;482 bool was_empty = false;483 bool wake_next_writer = false;484 485 /*486 * Reject writing to watch queue pipes before the point where we lock487 * the pipe.488 * Otherwise, lockdep would be unhappy if the caller already has another489 * pipe locked.490 * If we had to support locking a normal pipe and a notification pipe at491 * the same time, we could set up lockdep annotations for that, but492 * since we don't actually need that, it's simpler to just bail here.493 */494 if (pipe_has_watch_queue(pipe))495 return -EXDEV;496 497 /* Null write succeeds. */498 if (unlikely(total_len == 0))499 return 0;500 501 mutex_lock(&pipe->mutex);502 503 if (!pipe->readers) {504 send_sig(SIGPIPE, current, 0);505 ret = -EPIPE;506 goto out;507 }508 509 /*510 * If it wasn't empty we try to merge new data into511 * the last buffer.512 *513 * That naturally merges small writes, but it also514 * page-aligns the rest of the writes for large writes515 * spanning multiple pages.516 */517 head = pipe->head;518 was_empty = pipe_empty(head, pipe->tail);519 chars = total_len & (PAGE_SIZE-1);520 if (chars && !was_empty) {521 unsigned int mask = pipe->ring_size - 1;522 struct pipe_buffer *buf = &pipe->bufs[(head - 1) & mask];523 int offset = buf->offset + buf->len;524 525 if ((buf->flags & PIPE_BUF_FLAG_CAN_MERGE) &&526 offset + chars <= PAGE_SIZE) {527 ret = pipe_buf_confirm(pipe, buf);528 if (ret)529 goto out;530 531 ret = copy_page_from_iter(buf->page, offset, chars, from);532 if (unlikely(ret < chars)) {533 ret = -EFAULT;534 goto out;535 }536 537 buf->len += ret;538 if (!iov_iter_count(from))539 goto out;540 }541 }542 543 for (;;) {544 if (!pipe->readers) {545 send_sig(SIGPIPE, current, 0);546 if (!ret)547 ret = -EPIPE;548 break;549 }550 551 head = pipe->head;552 if (!pipe_full(head, pipe->tail, pipe->max_usage)) {553 unsigned int mask = pipe->ring_size - 1;554 struct pipe_buffer *buf;555 struct page *page = pipe->tmp_page;556 int copied;557 558 if (!page) {559 page = alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);560 if (unlikely(!page)) {561 ret = ret ? : -ENOMEM;562 break;563 }564 pipe->tmp_page = page;565 }566 567 /* Allocate a slot in the ring in advance and attach an568 * empty buffer. If we fault or otherwise fail to use569 * it, either the reader will consume it or it'll still570 * be there for the next write.571 */572 pipe->head = head + 1;573 574 /* Insert it into the buffer array */575 buf = &pipe->bufs[head & mask];576 buf->page = page;577 buf->ops = &anon_pipe_buf_ops;578 buf->offset = 0;579 buf->len = 0;580 if (is_packetized(filp))581 buf->flags = PIPE_BUF_FLAG_PACKET;582 else583 buf->flags = PIPE_BUF_FLAG_CAN_MERGE;584 pipe->tmp_page = NULL;585 586 copied = copy_page_from_iter(page, 0, PAGE_SIZE, from);587 if (unlikely(copied < PAGE_SIZE && iov_iter_count(from))) {588 if (!ret)589 ret = -EFAULT;590 break;591 }592 ret += copied;593 buf->len = copied;594 595 if (!iov_iter_count(from))596 break;597 }598 599 if (!pipe_full(head, pipe->tail, pipe->max_usage))600 continue;601 602 /* Wait for buffer space to become available. */603 if ((filp->f_flags & O_NONBLOCK) ||604 (iocb->ki_flags & IOCB_NOWAIT)) {605 if (!ret)606 ret = -EAGAIN;607 break;608 }609 if (signal_pending(current)) {610 if (!ret)611 ret = -ERESTARTSYS;612 break;613 }614 615 /*616 * We're going to release the pipe lock and wait for more617 * space. We wake up any readers if necessary, and then618 * after waiting we need to re-check whether the pipe619 * become empty while we dropped the lock.620 */621 mutex_unlock(&pipe->mutex);622 if (was_empty)623 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);624 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);625#ifdef CONFIG_WASM32626 {627 /* K13-B: park on the per-task wake word instead of628 * schedule(); the peer reader/closer in another Worker629 * routes its wake_up_interruptible(&pipe->wr_wait) through630 * this entry's .func. A signal breaks the park; the loop631 * top re-checks signal_pending() -> -ERESTARTSYS. */632 DEFINE_WAIT_FUNC(__wwait, wasm32_pipe_wake_function);633 634 for (;;) {635 prepare_to_wait_exclusive(&pipe->wr_wait, &__wwait,636 TASK_INTERRUPTIBLE);637 if (pipe_writable(pipe))638 break;639 if (signal_pending(current))640 break;641 wasm32_pipe_wait(&__wwait);642 }643 finish_wait(&pipe->wr_wait, &__wwait);644 }645#else646 wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe));647#endif648 mutex_lock(&pipe->mutex);649 was_empty = pipe_empty(pipe->head, pipe->tail);650 wake_next_writer = true;651 }652out:653 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))654 wake_next_writer = false;655 mutex_unlock(&pipe->mutex);656 657 /*658 * If we do do a wakeup event, we do a 'sync' wakeup, because we659 * want the reader to start processing things asap, rather than660 * leave the data pending.661 *662 * This is particularly important for small writes, because of663 * how (for example) the GNU make jobserver uses small writes to664 * wake up pending jobs665 *666 * Epoll nonsensically wants a wakeup whether the pipe667 * was already empty or not.668 */669 if (was_empty || pipe->poll_usage)670 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);671 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);672 if (wake_next_writer)673 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);674 if (ret > 0 && sb_start_write_trylock(file_inode(filp)->i_sb)) {675 int err = file_update_time(filp);676 if (err)677 ret = err;678 sb_end_write(file_inode(filp)->i_sb);679 }680 return ret;681}682 683static HWJS_SUSPENDS long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)684{685 struct pipe_inode_info *pipe = filp->private_data;686 unsigned int count, head, tail, mask;687 688 switch (cmd) {689 case FIONREAD:690 mutex_lock(&pipe->mutex);691 count = 0;692 head = pipe->head;693 tail = pipe->tail;694 mask = pipe->ring_size - 1;695 696 while (tail != head) {697 count += pipe->bufs[tail & mask].len;698 tail++;699 }700 mutex_unlock(&pipe->mutex);701 702 return put_user(count, (int __user *)arg);703 704#ifdef CONFIG_WATCH_QUEUE705 case IOC_WATCH_QUEUE_SET_SIZE: {706 int ret;707 mutex_lock(&pipe->mutex);708 ret = watch_queue_set_size(pipe, arg);709 mutex_unlock(&pipe->mutex);710 return ret;711 }712 713 case IOC_WATCH_QUEUE_SET_FILTER:714 return watch_queue_set_filter(715 pipe, (struct watch_notification_filter __user *)arg);716#endif717 718 default:719 return -ENOIOCTLCMD;720 }721}722 723/* No kernel lock held - fine */724static __poll_t725pipe_poll(struct file *filp, poll_table *wait)726{727 __poll_t mask;728 struct pipe_inode_info *pipe = filp->private_data;729 unsigned int head, tail;730 731 /* Epoll has some historical nasty semantics, this enables them */732 WRITE_ONCE(pipe->poll_usage, true);733 734 /*735 * Reading pipe state only -- no need for acquiring the semaphore.736 *737 * But because this is racy, the code has to add the738 * entry to the poll table _first_ ..739 */740 if (filp->f_mode & FMODE_READ)741 poll_wait(filp, &pipe->rd_wait, wait);742 if (filp->f_mode & FMODE_WRITE)743 poll_wait(filp, &pipe->wr_wait, wait);744 745 /*746 * .. and only then can you do the racy tests. That way,747 * if something changes and you got it wrong, the poll748 * table entry will wake you up and fix it.749 */750 head = READ_ONCE(pipe->head);751 tail = READ_ONCE(pipe->tail);752 753 mask = 0;754 if (filp->f_mode & FMODE_READ) {755 if (!pipe_empty(head, tail))756 mask |= EPOLLIN | EPOLLRDNORM;757 if (!pipe->writers && filp->f_pipe != pipe->w_counter)758 mask |= EPOLLHUP;759 }760 761 if (filp->f_mode & FMODE_WRITE) {762 if (!pipe_full(head, tail, pipe->max_usage))763 mask |= EPOLLOUT | EPOLLWRNORM;764 /*765 * Most Unices do not set EPOLLERR for FIFOs but on Linux they766 * behave exactly like pipes for poll().767 */768 if (!pipe->readers)769 mask |= EPOLLERR;770 }771 772 return mask;773}774 775static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe)776{777 int kill = 0;778 779 spin_lock(&inode->i_lock);780 if (!--pipe->files) {781 inode->i_pipe = NULL;782 kill = 1;783 }784 spin_unlock(&inode->i_lock);785 786 if (kill)787 free_pipe_info(pipe);788}789 790static HWJS_SUSPENDS int791pipe_release(struct inode *inode, struct file *file)792{793 struct pipe_inode_info *pipe = file->private_data;794 795 mutex_lock(&pipe->mutex);796 if (file->f_mode & FMODE_READ)797 pipe->readers--;798 if (file->f_mode & FMODE_WRITE)799 pipe->writers--;800 801 /* Was that the last reader or writer, but not the other side? */802 if (!pipe->readers != !pipe->writers) {803 wake_up_interruptible_all(&pipe->rd_wait);804 wake_up_interruptible_all(&pipe->wr_wait);805 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);806 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);807 }808 mutex_unlock(&pipe->mutex);809 810 put_pipe_info(inode, pipe);811 return 0;812}813 814static HWJS_SUSPENDS int815pipe_fasync(int fd, struct file *filp, int on)816{817 struct pipe_inode_info *pipe = filp->private_data;818 int retval = 0;819 820 mutex_lock(&pipe->mutex);821 if (filp->f_mode & FMODE_READ)822 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);823 if ((filp->f_mode & FMODE_WRITE) && retval >= 0) {824 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);825 if (retval < 0 && (filp->f_mode & FMODE_READ))826 /* this can happen only if on == T */827 fasync_helper(-1, filp, 0, &pipe->fasync_readers);828 }829 mutex_unlock(&pipe->mutex);830 return retval;831}832 833unsigned long account_pipe_buffers(struct user_struct *user,834 unsigned long old, unsigned long new)835{836 return atomic_long_add_return(new - old, &user->pipe_bufs);837}838 839bool too_many_pipe_buffers_soft(unsigned long user_bufs)840{841 unsigned long soft_limit = READ_ONCE(pipe_user_pages_soft);842 843 return soft_limit && user_bufs > soft_limit;844}845 846bool too_many_pipe_buffers_hard(unsigned long user_bufs)847{848 unsigned long hard_limit = READ_ONCE(pipe_user_pages_hard);849 850 return hard_limit && user_bufs > hard_limit;851}852 853bool pipe_is_unprivileged_user(void)854{855 return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN);856}857 858struct pipe_inode_info *alloc_pipe_info(void)859{860 struct pipe_inode_info *pipe;861 unsigned long pipe_bufs = PIPE_DEF_BUFFERS;862 struct user_struct *user = get_current_user();863 unsigned long user_bufs;864 unsigned int max_size = READ_ONCE(pipe_max_size);865 866 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT);867 if (pipe == NULL)868 goto out_free_uid;869 870 if (pipe_bufs * PAGE_SIZE > max_size && !capable(CAP_SYS_RESOURCE))871 pipe_bufs = max_size >> PAGE_SHIFT;872 873 user_bufs = account_pipe_buffers(user, 0, pipe_bufs);874 875 if (too_many_pipe_buffers_soft(user_bufs) && pipe_is_unprivileged_user()) {876 user_bufs = account_pipe_buffers(user, pipe_bufs, PIPE_MIN_DEF_BUFFERS);877 pipe_bufs = PIPE_MIN_DEF_BUFFERS;878 }879 880 if (too_many_pipe_buffers_hard(user_bufs) && pipe_is_unprivileged_user())881 goto out_revert_acct;882 883 pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer),884 GFP_KERNEL_ACCOUNT);885 886 if (pipe->bufs) {887 init_waitqueue_head(&pipe->rd_wait);888 init_waitqueue_head(&pipe->wr_wait);889 pipe->r_counter = pipe->w_counter = 1;890 pipe->max_usage = pipe_bufs;891 pipe->ring_size = pipe_bufs;892 pipe->nr_accounted = pipe_bufs;893 pipe->user = user;894 mutex_init(&pipe->mutex);895 lock_set_cmp_fn(&pipe->mutex, pipe_lock_cmp_fn, NULL);896 return pipe;897 }898 899out_revert_acct:900 (void) account_pipe_buffers(user, pipe_bufs, 0);901 kfree(pipe);902out_free_uid:903 free_uid(user);904 return NULL;905}906 907void free_pipe_info(struct pipe_inode_info *pipe)908{909 unsigned int i;910 911#ifdef CONFIG_WATCH_QUEUE912 if (pipe->watch_queue)913 watch_queue_clear(pipe->watch_queue);914#endif915 916 (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0);917 free_uid(pipe->user);918 for (i = 0; i < pipe->ring_size; i++) {919 struct pipe_buffer *buf = pipe->bufs + i;920 if (buf->ops)921 pipe_buf_release(pipe, buf);922 }923#ifdef CONFIG_WATCH_QUEUE924 if (pipe->watch_queue)925 put_watch_queue(pipe->watch_queue);926#endif927 if (pipe->tmp_page)928 __free_page(pipe->tmp_page);929 kfree(pipe->bufs);930 kfree(pipe);931}932 933static struct vfsmount *pipe_mnt __ro_after_init;934 935/*936 * pipefs_dname() is called from d_path().937 */938static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen)939{940 return dynamic_dname(buffer, buflen, "pipe:[%lu]",941 d_inode(dentry)->i_ino);942}943 944static const struct dentry_operations pipefs_dentry_operations = {945 .d_dname = pipefs_dname,946};947 948static HWJS_SUSPENDS struct inode * get_pipe_inode(void)949{950 struct inode *inode = new_inode_pseudo(pipe_mnt->mnt_sb);951 struct pipe_inode_info *pipe;952 953 if (!inode)954 goto fail_inode;955 956 inode->i_ino = get_next_ino();957 958 pipe = alloc_pipe_info();959 if (!pipe)960 goto fail_iput;961 962 inode->i_pipe = pipe;963 pipe->files = 2;964 pipe->readers = pipe->writers = 1;965 inode->i_fop = &pipefifo_fops;966 967 /*968 * Mark the inode dirty from the very beginning,969 * that way it will never be moved to the dirty970 * list because "mark_inode_dirty()" will think971 * that it already _is_ on the dirty list.972 */973 inode->i_state = I_DIRTY;974 inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR;975 inode->i_uid = current_fsuid();976 inode->i_gid = current_fsgid();977 simple_inode_init_ts(inode);978 979 return inode;980 981fail_iput:982 iput(inode);983 984fail_inode:985 return NULL;986}987 988int create_pipe_files(struct file **res, int flags)989{990 struct inode *inode = get_pipe_inode();991 struct file *f;992 int error;993 994 if (!inode)995 return -ENFILE;996 997 if (flags & O_NOTIFICATION_PIPE) {998 error = watch_queue_init(inode->i_pipe);999 if (error) {1000 free_pipe_info(inode->i_pipe);1001 iput(inode);1002 return error;1003 }1004 }1005 1006 f = alloc_file_pseudo(inode, pipe_mnt, "",1007 O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT)),1008 &pipefifo_fops);1009 if (IS_ERR(f)) {1010 free_pipe_info(inode->i_pipe);1011 iput(inode);1012 return PTR_ERR(f);1013 }1014 1015 f->private_data = inode->i_pipe;1016 f->f_pipe = 0;1017 1018 res[0] = alloc_file_clone(f, O_RDONLY | (flags & O_NONBLOCK),1019 &pipefifo_fops);1020 if (IS_ERR(res[0])) {1021 put_pipe_info(inode, inode->i_pipe);1022 fput(f);1023 return PTR_ERR(res[0]);1024 }1025 res[0]->private_data = inode->i_pipe;1026 res[0]->f_pipe = 0;1027 res[1] = f;1028 stream_open(inode, res[0]);1029 stream_open(inode, res[1]);1030 return 0;1031}1032 1033static HWJS_SUSPENDS int __do_pipe_flags(int *fd, struct file **files, int flags)1034{1035 int error;1036 int fdw, fdr;1037 1038 if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE))1039 return -EINVAL;1040 1041 error = create_pipe_files(files, flags);1042 if (error)1043 return error;1044 1045 error = get_unused_fd_flags(flags);1046 if (error < 0)1047 goto err_read_pipe;1048 fdr = error;1049 1050 error = get_unused_fd_flags(flags);1051 if (error < 0)1052 goto err_fdr;1053 fdw = error;1054 1055 audit_fd_pair(fdr, fdw);1056 fd[0] = fdr;1057 fd[1] = fdw;1058 /* pipe groks IOCB_NOWAIT */1059 files[0]->f_mode |= FMODE_NOWAIT;1060 files[1]->f_mode |= FMODE_NOWAIT;1061 return 0;1062 1063 err_fdr:1064 put_unused_fd(fdr);1065 err_read_pipe:1066 fput(files[0]);1067 fput(files[1]);1068 return error;1069}1070 1071int do_pipe_flags(int *fd, int flags)1072{1073 struct file *files[2];1074 int error = __do_pipe_flags(fd, files, flags);1075 if (!error) {1076 fd_install(fd[0], files[0]);1077 fd_install(fd[1], files[1]);1078 }1079 return error;1080}1081 1082/*1083 * sys_pipe() is the normal C calling standard for creating1084 * a pipe. It's not the way Unix traditionally does this, though.1085 */1086static HWJS_SUSPENDS int do_pipe2(int __user *fildes, int flags)1087{1088 struct file *files[2];1089 int fd[2];1090 int error;1091 1092 error = __do_pipe_flags(fd, files, flags);1093 if (!error) {1094 if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) {1095 fput(files[0]);1096 fput(files[1]);1097 put_unused_fd(fd[0]);1098 put_unused_fd(fd[1]);1099 error = -EFAULT;1100 } else {1101 fd_install(fd[0], files[0]);1102 fd_install(fd[1], files[1]);1103 }1104 }1105 return error;1106}1107 1108SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags)1109{1110 return do_pipe2(fildes, flags);1111}1112 1113SYSCALL_DEFINE1(pipe, int __user *, fildes)1114{1115 return do_pipe2(fildes, 0);1116}1117 1118/*1119 * This is the stupid "wait for pipe to be readable or writable"1120 * model.1121 *1122 * See pipe_read/write() for the proper kind of exclusive wait,1123 * but that requires that we wake up any other readers/writers1124 * if we then do not end up reading everything (ie the whole1125 * "wake_next_reader/writer" logic in pipe_read/write()).1126 */1127void pipe_wait_readable(struct pipe_inode_info *pipe)1128{1129 pipe_unlock(pipe);1130 wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe));1131 pipe_lock(pipe);1132}1133 1134void pipe_wait_writable(struct pipe_inode_info *pipe)1135{1136 pipe_unlock(pipe);1137 wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe));1138 pipe_lock(pipe);1139}1140 1141/*1142 * This depends on both the wait (here) and the wakeup (wake_up_partner)1143 * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot1144 * race with the count check and waitqueue prep.1145 *1146 * Normally in order to avoid races, you'd do the prepare_to_wait() first,1147 * then check the condition you're waiting for, and only then sleep. But1148 * because of the pipe lock, we can check the condition before being on1149 * the wait queue.1150 *1151 * We use the 'rd_wait' waitqueue for pipe partner waiting.1152 */1153static HWJS_SUSPENDS int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt)1154{1155 DEFINE_WAIT(rdwait);1156 int cur = *cnt;1157 1158 while (cur == *cnt) {1159 prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE);1160 pipe_unlock(pipe);1161 schedule();1162 finish_wait(&pipe->rd_wait, &rdwait);1163 pipe_lock(pipe);1164 if (signal_pending(current))1165 break;1166 }1167 return cur == *cnt ? -ERESTARTSYS : 0;1168}1169 1170static HWJS_SUSPENDS void wake_up_partner(struct pipe_inode_info *pipe)1171{1172 wake_up_interruptible_all(&pipe->rd_wait);1173}1174 1175static HWJS_SUSPENDS int fifo_open(struct inode *inode, struct file *filp)1176{1177 struct pipe_inode_info *pipe;1178 bool is_pipe = inode->i_sb->s_magic == PIPEFS_MAGIC;1179 int ret;1180 1181 filp->f_pipe = 0;1182 1183 spin_lock(&inode->i_lock);1184 if (inode->i_pipe) {1185 pipe = inode->i_pipe;1186 pipe->files++;1187 spin_unlock(&inode->i_lock);1188 } else {1189 spin_unlock(&inode->i_lock);1190 pipe = alloc_pipe_info();1191 if (!pipe)1192 return -ENOMEM;1193 pipe->files = 1;1194 spin_lock(&inode->i_lock);1195 if (unlikely(inode->i_pipe)) {1196 inode->i_pipe->files++;1197 spin_unlock(&inode->i_lock);1198 free_pipe_info(pipe);1199 pipe = inode->i_pipe;1200 } else {1201 inode->i_pipe = pipe;1202 spin_unlock(&inode->i_lock);1203 }1204 }1205 filp->private_data = pipe;1206 /* OK, we have a pipe and it's pinned down */1207 1208 mutex_lock(&pipe->mutex);1209 1210 /* We can only do regular read/write on fifos */1211 stream_open(inode, filp);1212 1213 switch (filp->f_mode & (FMODE_READ | FMODE_WRITE)) {1214 case FMODE_READ:1215 /*1216 * O_RDONLY1217 * POSIX.1 says that O_NONBLOCK means return with the FIFO1218 * opened, even when there is no process writing the FIFO.1219 */1220 pipe->r_counter++;1221 if (pipe->readers++ == 0)1222 wake_up_partner(pipe);1223 1224 if (!is_pipe && !pipe->writers) {1225 if ((filp->f_flags & O_NONBLOCK)) {1226 /* suppress EPOLLHUP until we have1227 * seen a writer */1228 filp->f_pipe = pipe->w_counter;1229 } else {1230 if (wait_for_partner(pipe, &pipe->w_counter))1231 goto err_rd;1232 }1233 }1234 break;1235 1236 case FMODE_WRITE:1237 /*1238 * O_WRONLY1239 * POSIX.1 says that O_NONBLOCK means return -1 with1240 * errno=ENXIO when there is no process reading the FIFO.1241 */1242 ret = -ENXIO;1243 if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers)1244 goto err;1245 1246 pipe->w_counter++;1247 if (!pipe->writers++)1248 wake_up_partner(pipe);1249 1250 if (!is_pipe && !pipe->readers) {1251 if (wait_for_partner(pipe, &pipe->r_counter))1252 goto err_wr;1253 }1254 break;1255 1256 case FMODE_READ | FMODE_WRITE:1257 /*1258 * O_RDWR1259 * POSIX.1 leaves this case "undefined" when O_NONBLOCK is set.1260 * This implementation will NEVER block on a O_RDWR open, since1261 * the process can at least talk to itself.1262 */1263 1264 pipe->readers++;1265 pipe->writers++;1266 pipe->r_counter++;1267 pipe->w_counter++;1268 if (pipe->readers == 1 || pipe->writers == 1)1269 wake_up_partner(pipe);1270 break;1271 1272 default:1273 ret = -EINVAL;1274 goto err;1275 }1276 1277 /* Ok! */1278 mutex_unlock(&pipe->mutex);1279 return 0;1280 1281err_rd:1282 if (!--pipe->readers)1283 wake_up_interruptible(&pipe->wr_wait);1284 ret = -ERESTARTSYS;1285 goto err;1286 1287err_wr:1288 if (!--pipe->writers)1289 wake_up_interruptible_all(&pipe->rd_wait);1290 ret = -ERESTARTSYS;1291 goto err;1292 1293err:1294 mutex_unlock(&pipe->mutex);1295 1296 put_pipe_info(inode, pipe);1297 return ret;1298}1299 1300const struct file_operations pipefifo_fops = {1301 .open = fifo_open,1302 .read_iter = pipe_read,1303 .write_iter = pipe_write,1304 .poll = pipe_poll,1305 .unlocked_ioctl = pipe_ioctl,1306 .release = pipe_release,1307 .fasync = pipe_fasync,1308 .splice_write = iter_file_splice_write,1309};1310 1311/*1312 * Currently we rely on the pipe array holding a power-of-2 number1313 * of pages. Returns 0 on error.1314 */1315unsigned int round_pipe_size(unsigned int size)1316{1317 if (size > (1U << 31))1318 return 0;1319 1320 /* Minimum pipe size, as required by POSIX */1321 if (size < PAGE_SIZE)1322 return PAGE_SIZE;1323 1324 return roundup_pow_of_two(size);1325}1326 1327/*1328 * Resize the pipe ring to a number of slots.1329 *1330 * Note the pipe can be reduced in capacity, but only if the current1331 * occupancy doesn't exceed nr_slots; if it does, EBUSY will be1332 * returned instead.1333 */1334int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots)1335{1336 struct pipe_buffer *bufs;1337 unsigned int head, tail, mask, n;1338 1339 bufs = kcalloc(nr_slots, sizeof(*bufs),1340 GFP_KERNEL_ACCOUNT | __GFP_NOWARN);1341 if (unlikely(!bufs))1342 return -ENOMEM;1343 1344 spin_lock_irq(&pipe->rd_wait.lock);1345 mask = pipe->ring_size - 1;1346 head = pipe->head;1347 tail = pipe->tail;1348 1349 n = pipe_occupancy(head, tail);1350 if (nr_slots < n) {1351 spin_unlock_irq(&pipe->rd_wait.lock);1352 kfree(bufs);1353 return -EBUSY;1354 }1355 1356 /*1357 * The pipe array wraps around, so just start the new one at zero1358 * and adjust the indices.1359 */1360 if (n > 0) {1361 unsigned int h = head & mask;1362 unsigned int t = tail & mask;1363 if (h > t) {1364 memcpy(bufs, pipe->bufs + t,1365 n * sizeof(struct pipe_buffer));1366 } else {1367 unsigned int tsize = pipe->ring_size - t;1368 if (h > 0)1369 memcpy(bufs + tsize, pipe->bufs,1370 h * sizeof(struct pipe_buffer));1371 memcpy(bufs, pipe->bufs + t,1372 tsize * sizeof(struct pipe_buffer));1373 }1374 }1375 1376 head = n;1377 tail = 0;1378 1379 kfree(pipe->bufs);1380 pipe->bufs = bufs;1381 pipe->ring_size = nr_slots;1382 if (pipe->max_usage > nr_slots)1383 pipe->max_usage = nr_slots;1384 pipe->tail = tail;1385 pipe->head = head;1386 1387 if (!pipe_has_watch_queue(pipe)) {1388 pipe->max_usage = nr_slots;1389 pipe->nr_accounted = nr_slots;1390 }1391 1392 spin_unlock_irq(&pipe->rd_wait.lock);1393 1394 /* This might have made more room for writers */1395 wake_up_interruptible(&pipe->wr_wait);1396 return 0;1397}1398 1399/*1400 * Allocate a new array of pipe buffers and copy the info over. Returns the1401 * pipe size if successful, or return -ERROR on error.1402 */1403static HWJS_SUSPENDS long pipe_set_size(struct pipe_inode_info *pipe, unsigned int arg)1404{1405 unsigned long user_bufs;1406 unsigned int nr_slots, size;1407 long ret = 0;1408 1409 if (pipe_has_watch_queue(pipe))1410 return -EBUSY;1411 1412 size = round_pipe_size(arg);1413 nr_slots = size >> PAGE_SHIFT;1414 1415 if (!nr_slots)1416 return -EINVAL;1417 1418 /*1419 * If trying to increase the pipe capacity, check that an1420 * unprivileged user is not trying to exceed various limits1421 * (soft limit check here, hard limit check just below).1422 * Decreasing the pipe capacity is always permitted, even1423 * if the user is currently over a limit.1424 */1425 if (nr_slots > pipe->max_usage &&1426 size > pipe_max_size && !capable(CAP_SYS_RESOURCE))1427 return -EPERM;1428 1429 user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots);1430 1431 if (nr_slots > pipe->max_usage &&1432 (too_many_pipe_buffers_hard(user_bufs) ||1433 too_many_pipe_buffers_soft(user_bufs)) &&1434 pipe_is_unprivileged_user()) {1435 ret = -EPERM;1436 goto out_revert_acct;1437 }1438 1439 ret = pipe_resize_ring(pipe, nr_slots);1440 if (ret < 0)1441 goto out_revert_acct;1442 1443 return pipe->max_usage * PAGE_SIZE;1444 1445out_revert_acct:1446 (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted);1447 return ret;1448}1449 1450/*1451 * Note that i_pipe and i_cdev share the same location, so checking ->i_pipe is1452 * not enough to verify that this is a pipe.1453 */1454struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice)1455{1456 struct pipe_inode_info *pipe = file->private_data;1457 1458 if (file->f_op != &pipefifo_fops || !pipe)1459 return NULL;1460 if (for_splice && pipe_has_watch_queue(pipe))1461 return NULL;1462 return pipe;1463}1464 1465long pipe_fcntl(struct file *file, unsigned int cmd, unsigned int arg)1466{1467 struct pipe_inode_info *pipe;1468 long ret;1469 1470 pipe = get_pipe_info(file, false);1471 if (!pipe)1472 return -EBADF;1473 1474 mutex_lock(&pipe->mutex);1475 1476 switch (cmd) {1477 case F_SETPIPE_SZ:1478 ret = pipe_set_size(pipe, arg);1479 break;1480 case F_GETPIPE_SZ:1481 ret = pipe->max_usage * PAGE_SIZE;1482 break;1483 default:1484 ret = -EINVAL;1485 break;1486 }1487 1488 mutex_unlock(&pipe->mutex);1489 return ret;1490}1491 1492static const struct super_operations pipefs_ops = {1493 .destroy_inode = free_inode_nonrcu,1494 .statfs = simple_statfs,1495};1496 1497/*1498 * pipefs should _never_ be mounted by userland - too much of security hassle,1499 * no real gain from having the whole file system mounted. So we don't need1500 * any operations on the root directory. However, we need a non-trivial1501 * d_name - pipe: will go nicely and kill the special-casing in procfs.1502 */1503 1504static int pipefs_init_fs_context(struct fs_context *fc)1505{1506 struct pseudo_fs_context *ctx = init_pseudo(fc, PIPEFS_MAGIC);1507 if (!ctx)1508 return -ENOMEM;1509 ctx->ops = &pipefs_ops;1510 ctx->dops = &pipefs_dentry_operations;1511 return 0;1512}1513 1514static struct file_system_type pipe_fs_type = {1515 .name = "pipefs",1516 .init_fs_context = pipefs_init_fs_context,1517 .kill_sb = kill_anon_super,1518};1519 1520#ifdef CONFIG_SYSCTL1521static int do_proc_dopipe_max_size_conv(unsigned long *lvalp,1522 unsigned int *valp,1523 int write, void *data)1524{1525 if (write) {1526 unsigned int val;1527 1528 val = round_pipe_size(*lvalp);1529 if (val == 0)1530 return -EINVAL;1531 1532 *valp = val;1533 } else {1534 unsigned int val = *valp;1535 *lvalp = (unsigned long) val;1536 }1537 1538 return 0;1539}1540 1541static int proc_dopipe_max_size(const struct ctl_table *table, int write,1542 void *buffer, size_t *lenp, loff_t *ppos)1543{1544 return do_proc_douintvec(table, write, buffer, lenp, ppos,1545 do_proc_dopipe_max_size_conv, NULL);1546}1547 1548static struct ctl_table fs_pipe_sysctls[] = {1549 {1550 .procname = "pipe-max-size",1551 .data = &pipe_max_size,1552 .maxlen = sizeof(pipe_max_size),1553 .mode = 0644,1554 .proc_handler = proc_dopipe_max_size,1555 },1556 {1557 .procname = "pipe-user-pages-hard",1558 .data = &pipe_user_pages_hard,1559 .maxlen = sizeof(pipe_user_pages_hard),1560 .mode = 0644,1561 .proc_handler = proc_doulongvec_minmax,1562 },1563 {1564 .procname = "pipe-user-pages-soft",1565 .data = &pipe_user_pages_soft,1566 .maxlen = sizeof(pipe_user_pages_soft),1567 .mode = 0644,1568 .proc_handler = proc_doulongvec_minmax,1569 },1570};1571#endif1572 1573static int __init init_pipe_fs(void)1574{1575 int err = register_filesystem(&pipe_fs_type);1576 1577 if (!err) {1578 pipe_mnt = kern_mount(&pipe_fs_type);1579 if (IS_ERR(pipe_mnt)) {1580 err = PTR_ERR(pipe_mnt);1581 unregister_filesystem(&pipe_fs_type);1582 }1583 }1584#ifdef CONFIG_SYSCTL1585 register_sysctl_init("fs", fs_pipe_sysctls);1586#endif1587 return err;1588}1589 1590fs_initcall(init_pipe_fs);1591