2030 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * "splice": joining two ropes together by interweaving their strands.4 *5 * This is the "extended pipe" functionality, where a pipe is used as6 * an arbitrary in-memory buffer. Think of a pipe as a small kernel7 * buffer that you can use to transfer data from one end to the other.8 *9 * The traditional unix read/write is extended with a "splice()" operation10 * that transfers data buffers to or from a pipe buffer.11 *12 * Named by Larry McVoy, original implementation from Linus, extended by13 * Jens to support splicing to files, network, direct splicing, etc and14 * fixing lots of bugs.15 *16 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>17 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>18 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>19 *20 */21#include <linux/bvec.h>22#include <linux/fs.h>23#include <linux/file.h>24#include <linux/pagemap.h>25#include <linux/splice.h>26#include <linux/memcontrol.h>27#include <linux/mm_inline.h>28#include <linux/swap.h>29#include <linux/writeback.h>30#include <linux/export.h>31#include <linux/syscalls.h>32#include <linux/uio.h>33#include <linux/fsnotify.h>34#include <linux/security.h>35#include <linux/gfp.h>36#include <linux/net.h>37#include <linux/socket.h>38#include <linux/sched/signal.h>39 40#include "internal.h"41 42/*43 * Splice doesn't support FMODE_NOWAIT. Since pipes may set this flag to44 * indicate they support non-blocking reads or writes, we must clear it45 * here if set to avoid blocking other users of this pipe if splice is46 * being done on it.47 */48static noinline void noinline pipe_clear_nowait(struct file *file)49{50 fmode_t fmode = READ_ONCE(file->f_mode);51 52 do {53 if (!(fmode & FMODE_NOWAIT))54 break;55 } while (!try_cmpxchg(&file->f_mode, &fmode, fmode & ~FMODE_NOWAIT));56}57 58/*59 * Attempt to steal a page from a pipe buffer. This should perhaps go into60 * a vm helper function, it's already simplified quite a bit by the61 * addition of remove_mapping(). If success is returned, the caller may62 * attempt to reuse this page for another destination.63 */64static HWJS_SUSPENDS bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,65 struct pipe_buffer *buf)66{67 struct folio *folio = page_folio(buf->page);68 struct address_space *mapping;69 70 folio_lock(folio);71 72 mapping = folio_mapping(folio);73 if (mapping) {74 WARN_ON(!folio_test_uptodate(folio));75 76 /*77 * At least for ext2 with nobh option, we need to wait on78 * writeback completing on this folio, since we'll remove it79 * from the pagecache. Otherwise truncate wont wait on the80 * folio, allowing the disk blocks to be reused by someone else81 * before we actually wrote our data to them. fs corruption82 * ensues.83 */84 folio_wait_writeback(folio);85 86 if (!filemap_release_folio(folio, GFP_KERNEL))87 goto out_unlock;88 89 /*90 * If we succeeded in removing the mapping, set LRU flag91 * and return good.92 */93 if (remove_mapping(mapping, folio)) {94 buf->flags |= PIPE_BUF_FLAG_LRU;95 return true;96 }97 }98 99 /*100 * Raced with truncate or failed to remove folio from current101 * address space, unlock and return failure.102 */103out_unlock:104 folio_unlock(folio);105 return false;106}107 108static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,109 struct pipe_buffer *buf)110{111 put_page(buf->page);112 buf->flags &= ~PIPE_BUF_FLAG_LRU;113}114 115/*116 * Check whether the contents of buf is OK to access. Since the content117 * is a page cache page, IO may be in flight.118 */119static HWJS_SUSPENDS int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,120 struct pipe_buffer *buf)121{122 struct folio *folio = page_folio(buf->page);123 int err;124 125 if (!folio_test_uptodate(folio)) {126 folio_lock(folio);127 128 /*129 * Folio got truncated/unhashed. This will cause a 0-byte130 * splice, if this is the first page.131 */132 if (!folio->mapping) {133 err = -ENODATA;134 goto error;135 }136 137 /*138 * Uh oh, read-error from disk.139 */140 if (!folio_test_uptodate(folio)) {141 err = -EIO;142 goto error;143 }144 145 /* Folio is ok after all, we are done */146 folio_unlock(folio);147 }148 149 return 0;150error:151 folio_unlock(folio);152 return err;153}154 155const struct pipe_buf_operations page_cache_pipe_buf_ops = {156 .confirm = page_cache_pipe_buf_confirm,157 .release = page_cache_pipe_buf_release,158 .try_steal = page_cache_pipe_buf_try_steal,159 .get = generic_pipe_buf_get,160};161 162static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe,163 struct pipe_buffer *buf)164{165 if (!(buf->flags & PIPE_BUF_FLAG_GIFT))166 return false;167 168 buf->flags |= PIPE_BUF_FLAG_LRU;169 return generic_pipe_buf_try_steal(pipe, buf);170}171 172static const struct pipe_buf_operations user_page_pipe_buf_ops = {173 .release = page_cache_pipe_buf_release,174 .try_steal = user_page_pipe_buf_try_steal,175 .get = generic_pipe_buf_get,176};177 178static HWJS_SUSPENDS void wakeup_pipe_readers(struct pipe_inode_info *pipe)179{180 smp_mb();181 if (waitqueue_active(&pipe->rd_wait))182 wake_up_interruptible(&pipe->rd_wait);183 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);184}185 186/**187 * splice_to_pipe - fill passed data into a pipe188 * @pipe: pipe to fill189 * @spd: data to fill190 *191 * Description:192 * @spd contains a map of pages and len/offset tuples, along with193 * the struct pipe_buf_operations associated with these pages. This194 * function will link that data to the pipe.195 *196 */197ssize_t splice_to_pipe(struct pipe_inode_info *pipe,198 struct splice_pipe_desc *spd)199{200 unsigned int spd_pages = spd->nr_pages;201 unsigned int tail = pipe->tail;202 unsigned int head = pipe->head;203 unsigned int mask = pipe->ring_size - 1;204 ssize_t ret = 0;205 int page_nr = 0;206 207 if (!spd_pages)208 return 0;209 210 if (unlikely(!pipe->readers)) {211 send_sig(SIGPIPE, current, 0);212 ret = -EPIPE;213 goto out;214 }215 216 while (!pipe_full(head, tail, pipe->max_usage)) {217 struct pipe_buffer *buf = &pipe->bufs[head & mask];218 219 buf->page = spd->pages[page_nr];220 buf->offset = spd->partial[page_nr].offset;221 buf->len = spd->partial[page_nr].len;222 buf->private = spd->partial[page_nr].private;223 buf->ops = spd->ops;224 buf->flags = 0;225 226 head++;227 pipe->head = head;228 page_nr++;229 ret += buf->len;230 231 if (!--spd->nr_pages)232 break;233 }234 235 if (!ret)236 ret = -EAGAIN;237 238out:239 while (page_nr < spd_pages)240 spd->spd_release(spd, page_nr++);241 242 return ret;243}244EXPORT_SYMBOL_GPL(splice_to_pipe);245 246ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)247{248 unsigned int head = pipe->head;249 unsigned int tail = pipe->tail;250 unsigned int mask = pipe->ring_size - 1;251 int ret;252 253 if (unlikely(!pipe->readers)) {254 send_sig(SIGPIPE, current, 0);255 ret = -EPIPE;256 } else if (pipe_full(head, tail, pipe->max_usage)) {257 ret = -EAGAIN;258 } else {259 pipe->bufs[head & mask] = *buf;260 pipe->head = head + 1;261 return buf->len;262 }263 pipe_buf_release(pipe, buf);264 return ret;265}266EXPORT_SYMBOL(add_to_pipe);267 268/*269 * Check if we need to grow the arrays holding pages and partial page270 * descriptions.271 */272int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)273{274 unsigned int max_usage = READ_ONCE(pipe->max_usage);275 276 spd->nr_pages_max = max_usage;277 if (max_usage <= PIPE_DEF_BUFFERS)278 return 0;279 280 spd->pages = kmalloc_array(max_usage, sizeof(struct page *), GFP_KERNEL);281 spd->partial = kmalloc_array(max_usage, sizeof(struct partial_page),282 GFP_KERNEL);283 284 if (spd->pages && spd->partial)285 return 0;286 287 kfree(spd->pages);288 kfree(spd->partial);289 return -ENOMEM;290}291 292void splice_shrink_spd(struct splice_pipe_desc *spd)293{294 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)295 return;296 297 kfree(spd->pages);298 kfree(spd->partial);299}300 301/**302 * copy_splice_read - Copy data from a file and splice the copy into a pipe303 * @in: The file to read from304 * @ppos: Pointer to the file position to read from305 * @pipe: The pipe to splice into306 * @len: The amount to splice307 * @flags: The SPLICE_F_* flags308 *309 * This function allocates a bunch of pages sufficient to hold the requested310 * amount of data (but limited by the remaining pipe capacity), passes it to311 * the file's ->read_iter() to read into and then splices the used pages into312 * the pipe.313 *314 * Return: On success, the number of bytes read will be returned and *@ppos315 * will be updated if appropriate; 0 will be returned if there is no more data316 * to be read; -EAGAIN will be returned if the pipe had no space, and some317 * other negative error code will be returned on error. A short read may occur318 * if the pipe has insufficient space, we reach the end of the data or we hit a319 * hole.320 */321ssize_t copy_splice_read(struct file *in, loff_t *ppos,322 struct pipe_inode_info *pipe,323 size_t len, unsigned int flags)324{325 struct iov_iter to;326 struct bio_vec *bv;327 struct kiocb kiocb;328 struct page **pages;329 ssize_t ret;330 size_t used, npages, chunk, remain, keep = 0;331 int i;332 333 /* Work out how much data we can actually add into the pipe */334 used = pipe_occupancy(pipe->head, pipe->tail);335 npages = max_t(ssize_t, pipe->max_usage - used, 0);336 len = min_t(size_t, len, npages * PAGE_SIZE);337 npages = DIV_ROUND_UP(len, PAGE_SIZE);338 339 bv = kzalloc(array_size(npages, sizeof(bv[0])) +340 array_size(npages, sizeof(struct page *)), GFP_KERNEL);341 if (!bv)342 return -ENOMEM;343 344 pages = (struct page **)(bv + npages);345 npages = alloc_pages_bulk_array(GFP_USER, npages, pages);346 if (!npages) {347 kfree(bv);348 return -ENOMEM;349 }350 351 remain = len = min_t(size_t, len, npages * PAGE_SIZE);352 353 for (i = 0; i < npages; i++) {354 chunk = min_t(size_t, PAGE_SIZE, remain);355 bv[i].bv_page = pages[i];356 bv[i].bv_offset = 0;357 bv[i].bv_len = chunk;358 remain -= chunk;359 }360 361 /* Do the I/O */362 iov_iter_bvec(&to, ITER_DEST, bv, npages, len);363 init_sync_kiocb(&kiocb, in);364 kiocb.ki_pos = *ppos;365 ret = in->f_op->read_iter(&kiocb, &to);366 367 if (ret > 0) {368 keep = DIV_ROUND_UP(ret, PAGE_SIZE);369 *ppos = kiocb.ki_pos;370 }371 372 /*373 * Callers of ->splice_read() expect -EAGAIN on "can't put anything in374 * there", rather than -EFAULT.375 */376 if (ret == -EFAULT)377 ret = -EAGAIN;378 379 /* Free any pages that didn't get touched at all. */380 if (keep < npages)381 release_pages(pages + keep, npages - keep);382 383 /* Push the remaining pages into the pipe. */384 remain = ret;385 for (i = 0; i < keep; i++) {386 struct pipe_buffer *buf = pipe_head_buf(pipe);387 388 chunk = min_t(size_t, remain, PAGE_SIZE);389 *buf = (struct pipe_buffer) {390 .ops = &default_pipe_buf_ops,391 .page = bv[i].bv_page,392 .offset = 0,393 .len = chunk,394 };395 pipe->head++;396 remain -= chunk;397 }398 399 kfree(bv);400 return ret;401}402EXPORT_SYMBOL(copy_splice_read);403 404const struct pipe_buf_operations default_pipe_buf_ops = {405 .release = generic_pipe_buf_release,406 .try_steal = generic_pipe_buf_try_steal,407 .get = generic_pipe_buf_get,408};409 410/* Pipe buffer operations for a socket and similar. */411const struct pipe_buf_operations nosteal_pipe_buf_ops = {412 .release = generic_pipe_buf_release,413 .get = generic_pipe_buf_get,414};415EXPORT_SYMBOL(nosteal_pipe_buf_ops);416 417static HWJS_SUSPENDS void wakeup_pipe_writers(struct pipe_inode_info *pipe)418{419 smp_mb();420 if (waitqueue_active(&pipe->wr_wait))421 wake_up_interruptible(&pipe->wr_wait);422 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);423}424 425/**426 * splice_from_pipe_feed - feed available data from a pipe to a file427 * @pipe: pipe to splice from428 * @sd: information to @actor429 * @actor: handler that splices the data430 *431 * Description:432 * This function loops over the pipe and calls @actor to do the433 * actual moving of a single struct pipe_buffer to the desired434 * destination. It returns when there's no more buffers left in435 * the pipe or if the requested number of bytes (@sd->total_len)436 * have been copied. It returns a positive number (one) if the437 * pipe needs to be filled with more data, zero if the required438 * number of bytes have been copied and -errno on error.439 *440 * This, together with splice_from_pipe_{begin,end,next}, may be441 * used to implement the functionality of __splice_from_pipe() when442 * locking is required around copying the pipe buffers to the443 * destination.444 */445static HWJS_SUSPENDS int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,446 splice_actor *actor)447{448 unsigned int head = pipe->head;449 unsigned int tail = pipe->tail;450 unsigned int mask = pipe->ring_size - 1;451 int ret;452 453 while (!pipe_empty(head, tail)) {454 struct pipe_buffer *buf = &pipe->bufs[tail & mask];455 456 sd->len = buf->len;457 if (sd->len > sd->total_len)458 sd->len = sd->total_len;459 460 ret = pipe_buf_confirm(pipe, buf);461 if (unlikely(ret)) {462 if (ret == -ENODATA)463 ret = 0;464 return ret;465 }466 467 ret = actor(pipe, buf, sd);468 if (ret <= 0)469 return ret;470 471 buf->offset += ret;472 buf->len -= ret;473 474 sd->num_spliced += ret;475 sd->len -= ret;476 sd->pos += ret;477 sd->total_len -= ret;478 479 if (!buf->len) {480 pipe_buf_release(pipe, buf);481 tail++;482 pipe->tail = tail;483 if (pipe->files)484 sd->need_wakeup = true;485 }486 487 if (!sd->total_len)488 return 0;489 }490 491 return 1;492}493 494/* We know we have a pipe buffer, but maybe it's empty? */495static inline bool eat_empty_buffer(struct pipe_inode_info *pipe)496{497 unsigned int tail = pipe->tail;498 unsigned int mask = pipe->ring_size - 1;499 struct pipe_buffer *buf = &pipe->bufs[tail & mask];500 501 if (unlikely(!buf->len)) {502 pipe_buf_release(pipe, buf);503 pipe->tail = tail+1;504 return true;505 }506 507 return false;508}509 510/**511 * splice_from_pipe_next - wait for some data to splice from512 * @pipe: pipe to splice from513 * @sd: information about the splice operation514 *515 * Description:516 * This function will wait for some data and return a positive517 * value (one) if pipe buffers are available. It will return zero518 * or -errno if no more data needs to be spliced.519 */520static HWJS_SUSPENDS int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)521{522 /*523 * Check for signal early to make process killable when there are524 * always buffers available525 */526 if (signal_pending(current))527 return -ERESTARTSYS;528 529repeat:530 while (pipe_empty(pipe->head, pipe->tail)) {531 if (!pipe->writers)532 return 0;533 534 if (sd->num_spliced)535 return 0;536 537 if (sd->flags & SPLICE_F_NONBLOCK)538 return -EAGAIN;539 540 if (signal_pending(current))541 return -ERESTARTSYS;542 543 if (sd->need_wakeup) {544 wakeup_pipe_writers(pipe);545 sd->need_wakeup = false;546 }547 548 pipe_wait_readable(pipe);549 }550 551 if (eat_empty_buffer(pipe))552 goto repeat;553 554 return 1;555}556 557/**558 * splice_from_pipe_begin - start splicing from pipe559 * @sd: information about the splice operation560 *561 * Description:562 * This function should be called before a loop containing563 * splice_from_pipe_next() and splice_from_pipe_feed() to564 * initialize the necessary fields of @sd.565 */566static void splice_from_pipe_begin(struct splice_desc *sd)567{568 sd->num_spliced = 0;569 sd->need_wakeup = false;570}571 572/**573 * splice_from_pipe_end - finish splicing from pipe574 * @pipe: pipe to splice from575 * @sd: information about the splice operation576 *577 * Description:578 * This function will wake up pipe writers if necessary. It should579 * be called after a loop containing splice_from_pipe_next() and580 * splice_from_pipe_feed().581 */582static HWJS_SUSPENDS void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)583{584 if (sd->need_wakeup)585 wakeup_pipe_writers(pipe);586}587 588/**589 * __splice_from_pipe - splice data from a pipe to given actor590 * @pipe: pipe to splice from591 * @sd: information to @actor592 * @actor: handler that splices the data593 *594 * Description:595 * This function does little more than loop over the pipe and call596 * @actor to do the actual moving of a single struct pipe_buffer to597 * the desired destination. See pipe_to_file, pipe_to_sendmsg, or598 * pipe_to_user.599 *600 */601ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,602 splice_actor *actor)603{604 int ret;605 606 splice_from_pipe_begin(sd);607 do {608 cond_resched();609 ret = splice_from_pipe_next(pipe, sd);610 if (ret > 0)611 ret = splice_from_pipe_feed(pipe, sd, actor);612 } while (ret > 0);613 splice_from_pipe_end(pipe, sd);614 615 return sd->num_spliced ? sd->num_spliced : ret;616}617EXPORT_SYMBOL(__splice_from_pipe);618 619/**620 * splice_from_pipe - splice data from a pipe to a file621 * @pipe: pipe to splice from622 * @out: file to splice to623 * @ppos: position in @out624 * @len: how many bytes to splice625 * @flags: splice modifier flags626 * @actor: handler that splices the data627 *628 * Description:629 * See __splice_from_pipe. This function locks the pipe inode,630 * otherwise it's identical to __splice_from_pipe().631 *632 */633ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,634 loff_t *ppos, size_t len, unsigned int flags,635 splice_actor *actor)636{637 ssize_t ret;638 struct splice_desc sd = {639 .total_len = len,640 .flags = flags,641 .pos = *ppos,642 .u.file = out,643 };644 645 pipe_lock(pipe);646 ret = __splice_from_pipe(pipe, &sd, actor);647 pipe_unlock(pipe);648 649 return ret;650}651 652/**653 * iter_file_splice_write - splice data from a pipe to a file654 * @pipe: pipe info655 * @out: file to write to656 * @ppos: position in @out657 * @len: number of bytes to splice658 * @flags: splice modifier flags659 *660 * Description:661 * Will either move or copy pages (determined by @flags options) from662 * the given pipe inode to the given file.663 * This one is ->write_iter-based.664 *665 */666ssize_t667iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,668 loff_t *ppos, size_t len, unsigned int flags)669{670 struct splice_desc sd = {671 .total_len = len,672 .flags = flags,673 .pos = *ppos,674 .u.file = out,675 };676 int nbufs = pipe->max_usage;677 struct bio_vec *array;678 ssize_t ret;679 680 if (!out->f_op->write_iter)681 return -EINVAL;682 683 array = kcalloc(nbufs, sizeof(struct bio_vec), GFP_KERNEL);684 if (unlikely(!array))685 return -ENOMEM;686 687 pipe_lock(pipe);688 689 splice_from_pipe_begin(&sd);690 while (sd.total_len) {691 struct kiocb kiocb;692 struct iov_iter from;693 unsigned int head, tail, mask;694 size_t left;695 int n;696 697 ret = splice_from_pipe_next(pipe, &sd);698 if (ret <= 0)699 break;700 701 if (unlikely(nbufs < pipe->max_usage)) {702 kfree(array);703 nbufs = pipe->max_usage;704 array = kcalloc(nbufs, sizeof(struct bio_vec),705 GFP_KERNEL);706 if (!array) {707 ret = -ENOMEM;708 break;709 }710 }711 712 head = pipe->head;713 tail = pipe->tail;714 mask = pipe->ring_size - 1;715 716 /* build the vector */717 left = sd.total_len;718 for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++) {719 struct pipe_buffer *buf = &pipe->bufs[tail & mask];720 size_t this_len = buf->len;721 722 /* zero-length bvecs are not supported, skip them */723 if (!this_len)724 continue;725 this_len = min(this_len, left);726 727 ret = pipe_buf_confirm(pipe, buf);728 if (unlikely(ret)) {729 if (ret == -ENODATA)730 ret = 0;731 goto done;732 }733 734 bvec_set_page(&array[n], buf->page, this_len,735 buf->offset);736 left -= this_len;737 n++;738 }739 740 iov_iter_bvec(&from, ITER_SOURCE, array, n, sd.total_len - left);741 init_sync_kiocb(&kiocb, out);742 kiocb.ki_pos = sd.pos;743 ret = out->f_op->write_iter(&kiocb, &from);744 sd.pos = kiocb.ki_pos;745 if (ret <= 0)746 break;747 748 sd.num_spliced += ret;749 sd.total_len -= ret;750 *ppos = sd.pos;751 752 /* dismiss the fully eaten buffers, adjust the partial one */753 tail = pipe->tail;754 while (ret) {755 struct pipe_buffer *buf = &pipe->bufs[tail & mask];756 if (ret >= buf->len) {757 ret -= buf->len;758 buf->len = 0;759 pipe_buf_release(pipe, buf);760 tail++;761 pipe->tail = tail;762 if (pipe->files)763 sd.need_wakeup = true;764 } else {765 buf->offset += ret;766 buf->len -= ret;767 ret = 0;768 }769 }770 }771done:772 kfree(array);773 splice_from_pipe_end(pipe, &sd);774 775 pipe_unlock(pipe);776 777 if (sd.num_spliced)778 ret = sd.num_spliced;779 780 return ret;781}782 783EXPORT_SYMBOL(iter_file_splice_write);784 785#ifdef CONFIG_NET786/**787 * splice_to_socket - splice data from a pipe to a socket788 * @pipe: pipe to splice from789 * @out: socket to write to790 * @ppos: position in @out791 * @len: number of bytes to splice792 * @flags: splice modifier flags793 *794 * Description:795 * Will send @len bytes from the pipe to a network socket. No data copying796 * is involved.797 *798 */799ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out,800 loff_t *ppos, size_t len, unsigned int flags)801{802 struct socket *sock = sock_from_file(out);803 struct bio_vec bvec[16];804 struct msghdr msg = {};805 ssize_t ret = 0;806 size_t spliced = 0;807 bool need_wakeup = false;808 809 pipe_lock(pipe);810 811 while (len > 0) {812 unsigned int head, tail, mask, bc = 0;813 size_t remain = len;814 815 /*816 * Check for signal early to make process killable when there817 * are always buffers available818 */819 ret = -ERESTARTSYS;820 if (signal_pending(current))821 break;822 823 while (pipe_empty(pipe->head, pipe->tail)) {824 ret = 0;825 if (!pipe->writers)826 goto out;827 828 if (spliced)829 goto out;830 831 ret = -EAGAIN;832 if (flags & SPLICE_F_NONBLOCK)833 goto out;834 835 ret = -ERESTARTSYS;836 if (signal_pending(current))837 goto out;838 839 if (need_wakeup) {840 wakeup_pipe_writers(pipe);841 need_wakeup = false;842 }843 844 pipe_wait_readable(pipe);845 }846 847 head = pipe->head;848 tail = pipe->tail;849 mask = pipe->ring_size - 1;850 851 while (!pipe_empty(head, tail)) {852 struct pipe_buffer *buf = &pipe->bufs[tail & mask];853 size_t seg;854 855 if (!buf->len) {856 tail++;857 continue;858 }859 860 seg = min_t(size_t, remain, buf->len);861 862 ret = pipe_buf_confirm(pipe, buf);863 if (unlikely(ret)) {864 if (ret == -ENODATA)865 ret = 0;866 break;867 }868 869 bvec_set_page(&bvec[bc++], buf->page, seg, buf->offset);870 remain -= seg;871 if (remain == 0 || bc >= ARRAY_SIZE(bvec))872 break;873 tail++;874 }875 876 if (!bc)877 break;878 879 msg.msg_flags = MSG_SPLICE_PAGES;880 if (flags & SPLICE_F_MORE)881 msg.msg_flags |= MSG_MORE;882 if (remain && pipe_occupancy(pipe->head, tail) > 0)883 msg.msg_flags |= MSG_MORE;884 if (out->f_flags & O_NONBLOCK)885 msg.msg_flags |= MSG_DONTWAIT;886 887 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, bvec, bc,888 len - remain);889 ret = sock_sendmsg(sock, &msg);890 if (ret <= 0)891 break;892 893 spliced += ret;894 len -= ret;895 tail = pipe->tail;896 while (ret > 0) {897 struct pipe_buffer *buf = &pipe->bufs[tail & mask];898 size_t seg = min_t(size_t, ret, buf->len);899 900 buf->offset += seg;901 buf->len -= seg;902 ret -= seg;903 904 if (!buf->len) {905 pipe_buf_release(pipe, buf);906 tail++;907 }908 }909 910 if (tail != pipe->tail) {911 pipe->tail = tail;912 if (pipe->files)913 need_wakeup = true;914 }915 }916 917out:918 pipe_unlock(pipe);919 if (need_wakeup)920 wakeup_pipe_writers(pipe);921 return spliced ?: ret;922}923#endif924 925static int warn_unsupported(struct file *file, const char *op)926{927 pr_debug_ratelimited(928 "splice %s not supported for file %pD4 (pid: %d comm: %.20s)\n",929 op, file, current->pid, current->comm);930 return -EINVAL;931}932 933/*934 * Attempt to initiate a splice from pipe to file.935 */936static ssize_t do_splice_from(struct pipe_inode_info *pipe, struct file *out,937 loff_t *ppos, size_t len, unsigned int flags)938{939 if (unlikely(!out->f_op->splice_write))940 return warn_unsupported(out, "write");941 return out->f_op->splice_write(pipe, out, ppos, len, flags);942}943 944/*945 * Indicate to the caller that there was a premature EOF when reading from the946 * source and the caller didn't indicate they would be sending more data after947 * this.948 */949static void do_splice_eof(struct splice_desc *sd)950{951 if (sd->splice_eof)952 sd->splice_eof(sd);953}954 955/*956 * Callers already called rw_verify_area() on the entire range.957 * No need to call it for sub ranges.958 */959static HWJS_SUSPENDS ssize_t do_splice_read(struct file *in, loff_t *ppos,960 struct pipe_inode_info *pipe, size_t len,961 unsigned int flags)962{963 unsigned int p_space;964 965 if (unlikely(!(in->f_mode & FMODE_READ)))966 return -EBADF;967 if (!len)968 return 0;969 970 /* Don't try to read more the pipe has space for. */971 p_space = pipe->max_usage - pipe_occupancy(pipe->head, pipe->tail);972 len = min_t(size_t, len, p_space << PAGE_SHIFT);973 974 if (unlikely(len > MAX_RW_COUNT))975 len = MAX_RW_COUNT;976 977 if (unlikely(!in->f_op->splice_read))978 return warn_unsupported(in, "read");979 /*980 * O_DIRECT and DAX don't deal with the pagecache, so we allocate a981 * buffer, copy into it and splice that into the pipe.982 */983 if ((in->f_flags & O_DIRECT) || IS_DAX(in->f_mapping->host))984 return copy_splice_read(in, ppos, pipe, len, flags);985 return in->f_op->splice_read(in, ppos, pipe, len, flags);986}987 988/**989 * vfs_splice_read - Read data from a file and splice it into a pipe990 * @in: File to splice from991 * @ppos: Input file offset992 * @pipe: Pipe to splice to993 * @len: Number of bytes to splice994 * @flags: Splice modifier flags (SPLICE_F_*)995 *996 * Splice the requested amount of data from the input file to the pipe. This997 * is synchronous as the caller must hold the pipe lock across the entire998 * operation.999 *1000 * If successful, it returns the amount of data spliced, 0 if it hit the EOF or1001 * a hole and a negative error code otherwise.1002 */1003ssize_t vfs_splice_read(struct file *in, loff_t *ppos,1004 struct pipe_inode_info *pipe, size_t len,1005 unsigned int flags)1006{1007 ssize_t ret;1008 1009 ret = rw_verify_area(READ, in, ppos, len);1010 if (unlikely(ret < 0))1011 return ret;1012 1013 return do_splice_read(in, ppos, pipe, len, flags);1014}1015EXPORT_SYMBOL_GPL(vfs_splice_read);1016 1017/**1018 * splice_direct_to_actor - splices data directly between two non-pipes1019 * @in: file to splice from1020 * @sd: actor information on where to splice to1021 * @actor: handles the data splicing1022 *1023 * Description:1024 * This is a special case helper to splice directly between two1025 * points, without requiring an explicit pipe. Internally an allocated1026 * pipe is cached in the process, and reused during the lifetime of1027 * that process.1028 *1029 */1030ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,1031 splice_direct_actor *actor)1032{1033 struct pipe_inode_info *pipe;1034 ssize_t ret, bytes;1035 size_t len;1036 int i, flags, more;1037 1038 /*1039 * We require the input to be seekable, as we don't want to randomly1040 * drop data for eg socket -> socket splicing. Use the piped splicing1041 * for that!1042 */1043 if (unlikely(!(in->f_mode & FMODE_LSEEK)))1044 return -EINVAL;1045 1046 /*1047 * neither in nor out is a pipe, setup an internal pipe attached to1048 * 'out' and transfer the wanted data from 'in' to 'out' through that1049 */1050 pipe = current->splice_pipe;1051 if (unlikely(!pipe)) {1052 pipe = alloc_pipe_info();1053 if (!pipe)1054 return -ENOMEM;1055 1056 /*1057 * We don't have an immediate reader, but we'll read the stuff1058 * out of the pipe right after the splice_to_pipe(). So set1059 * PIPE_READERS appropriately.1060 */1061 pipe->readers = 1;1062 1063 current->splice_pipe = pipe;1064 }1065 1066 /*1067 * Do the splice.1068 */1069 bytes = 0;1070 len = sd->total_len;1071 1072 /* Don't block on output, we have to drain the direct pipe. */1073 flags = sd->flags;1074 sd->flags &= ~SPLICE_F_NONBLOCK;1075 1076 /*1077 * We signal MORE until we've read sufficient data to fulfill the1078 * request and we keep signalling it if the caller set it.1079 */1080 more = sd->flags & SPLICE_F_MORE;1081 sd->flags |= SPLICE_F_MORE;1082 1083 WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail));1084 1085 while (len) {1086 size_t read_len;1087 loff_t pos = sd->pos, prev_pos = pos;1088 1089 ret = do_splice_read(in, &pos, pipe, len, flags);1090 if (unlikely(ret <= 0))1091 goto read_failure;1092 1093 read_len = ret;1094 sd->total_len = read_len;1095 1096 /*1097 * If we now have sufficient data to fulfill the request then1098 * we clear SPLICE_F_MORE if it was not set initially.1099 */1100 if (read_len >= len && !more)1101 sd->flags &= ~SPLICE_F_MORE;1102 1103 /*1104 * NOTE: nonblocking mode only applies to the input. We1105 * must not do the output in nonblocking mode as then we1106 * could get stuck data in the internal pipe:1107 */1108 ret = actor(pipe, sd);1109 if (unlikely(ret <= 0)) {1110 sd->pos = prev_pos;1111 goto out_release;1112 }1113 1114 bytes += ret;1115 len -= ret;1116 sd->pos = pos;1117 1118 if (ret < read_len) {1119 sd->pos = prev_pos + ret;1120 goto out_release;1121 }1122 }1123 1124done:1125 pipe->tail = pipe->head = 0;1126 file_accessed(in);1127 return bytes;1128 1129read_failure:1130 /*1131 * If the user did *not* set SPLICE_F_MORE *and* we didn't hit that1132 * "use all of len" case that cleared SPLICE_F_MORE, *and* we did a1133 * "->splice_in()" that returned EOF (ie zero) *and* we have sent at1134 * least 1 byte *then* we will also do the ->splice_eof() call.1135 */1136 if (ret == 0 && !more && len > 0 && bytes)1137 do_splice_eof(sd);1138out_release:1139 /*1140 * If we did an incomplete transfer we must release1141 * the pipe buffers in question:1142 */1143 for (i = 0; i < pipe->ring_size; i++) {1144 struct pipe_buffer *buf = &pipe->bufs[i];1145 1146 if (buf->ops)1147 pipe_buf_release(pipe, buf);1148 }1149 1150 if (!bytes)1151 bytes = ret;1152 1153 goto done;1154}1155EXPORT_SYMBOL(splice_direct_to_actor);1156 1157static HWJS_SUSPENDS int direct_splice_actor(struct pipe_inode_info *pipe,1158 struct splice_desc *sd)1159{1160 struct file *file = sd->u.file;1161 long ret;1162 1163 file_start_write(file);1164 ret = do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags);1165 file_end_write(file);1166 return ret;1167}1168 1169static int splice_file_range_actor(struct pipe_inode_info *pipe,1170 struct splice_desc *sd)1171{1172 struct file *file = sd->u.file;1173 1174 return do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags);1175}1176 1177static void direct_file_splice_eof(struct splice_desc *sd)1178{1179 struct file *file = sd->u.file;1180 1181 if (file->f_op->splice_eof)1182 file->f_op->splice_eof(file);1183}1184 1185static HWJS_SUSPENDS ssize_t do_splice_direct_actor(struct file *in, loff_t *ppos,1186 struct file *out, loff_t *opos,1187 size_t len, unsigned int flags,1188 splice_direct_actor *actor)1189{1190 struct splice_desc sd = {1191 .len = len,1192 .total_len = len,1193 .flags = flags,1194 .pos = *ppos,1195 .u.file = out,1196 .splice_eof = direct_file_splice_eof,1197 .opos = opos,1198 };1199 ssize_t ret;1200 1201 if (unlikely(!(out->f_mode & FMODE_WRITE)))1202 return -EBADF;1203 1204 if (unlikely(out->f_flags & O_APPEND))1205 return -EINVAL;1206 1207 ret = splice_direct_to_actor(in, &sd, actor);1208 if (ret > 0)1209 *ppos = sd.pos;1210 1211 return ret;1212}1213/**1214 * do_splice_direct - splices data directly between two files1215 * @in: file to splice from1216 * @ppos: input file offset1217 * @out: file to splice to1218 * @opos: output file offset1219 * @len: number of bytes to splice1220 * @flags: splice modifier flags1221 *1222 * Description:1223 * For use by do_sendfile(). splice can easily emulate sendfile, but1224 * doing it in the application would incur an extra system call1225 * (splice in + splice out, as compared to just sendfile()). So this helper1226 * can splice directly through a process-private pipe.1227 *1228 * Callers already called rw_verify_area() on the entire range.1229 */1230ssize_t do_splice_direct(struct file *in, loff_t *ppos, struct file *out,1231 loff_t *opos, size_t len, unsigned int flags)1232{1233 return do_splice_direct_actor(in, ppos, out, opos, len, flags,1234 direct_splice_actor);1235}1236EXPORT_SYMBOL(do_splice_direct);1237 1238/**1239 * splice_file_range - splices data between two files for copy_file_range()1240 * @in: file to splice from1241 * @ppos: input file offset1242 * @out: file to splice to1243 * @opos: output file offset1244 * @len: number of bytes to splice1245 *1246 * Description:1247 * For use by ->copy_file_range() methods.1248 * Like do_splice_direct(), but vfs_copy_file_range() already holds1249 * start_file_write() on @out file.1250 *1251 * Callers already called rw_verify_area() on the entire range.1252 */1253ssize_t splice_file_range(struct file *in, loff_t *ppos, struct file *out,1254 loff_t *opos, size_t len)1255{1256 lockdep_assert(file_write_started(out));1257 1258 return do_splice_direct_actor(in, ppos, out, opos,1259 min_t(size_t, len, MAX_RW_COUNT),1260 0, splice_file_range_actor);1261}1262EXPORT_SYMBOL(splice_file_range);1263 1264static HWJS_SUSPENDS int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)1265{1266 for (;;) {1267 if (unlikely(!pipe->readers)) {1268 send_sig(SIGPIPE, current, 0);1269 return -EPIPE;1270 }1271 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))1272 return 0;1273 if (flags & SPLICE_F_NONBLOCK)1274 return -EAGAIN;1275 if (signal_pending(current))1276 return -ERESTARTSYS;1277 pipe_wait_writable(pipe);1278 }1279}1280 1281static HWJS_SUSPENDS int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,1282 struct pipe_inode_info *opipe,1283 size_t len, unsigned int flags);1284 1285ssize_t splice_file_to_pipe(struct file *in,1286 struct pipe_inode_info *opipe,1287 loff_t *offset,1288 size_t len, unsigned int flags)1289{1290 ssize_t ret;1291 1292 pipe_lock(opipe);1293 ret = wait_for_space(opipe, flags);1294 if (!ret)1295 ret = do_splice_read(in, offset, opipe, len, flags);1296 pipe_unlock(opipe);1297 if (ret > 0)1298 wakeup_pipe_readers(opipe);1299 return ret;1300}1301 1302/*1303 * Determine where to splice to/from.1304 */1305ssize_t do_splice(struct file *in, loff_t *off_in, struct file *out,1306 loff_t *off_out, size_t len, unsigned int flags)1307{1308 struct pipe_inode_info *ipipe;1309 struct pipe_inode_info *opipe;1310 loff_t offset;1311 ssize_t ret;1312 1313 if (unlikely(!(in->f_mode & FMODE_READ) ||1314 !(out->f_mode & FMODE_WRITE)))1315 return -EBADF;1316 1317 ipipe = get_pipe_info(in, true);1318 opipe = get_pipe_info(out, true);1319 1320 if (ipipe && opipe) {1321 if (off_in || off_out)1322 return -ESPIPE;1323 1324 /* Splicing to self would be fun, but... */1325 if (ipipe == opipe)1326 return -EINVAL;1327 1328 if ((in->f_flags | out->f_flags) & O_NONBLOCK)1329 flags |= SPLICE_F_NONBLOCK;1330 1331 ret = splice_pipe_to_pipe(ipipe, opipe, len, flags);1332 } else if (ipipe) {1333 if (off_in)1334 return -ESPIPE;1335 if (off_out) {1336 if (!(out->f_mode & FMODE_PWRITE))1337 return -EINVAL;1338 offset = *off_out;1339 } else {1340 offset = out->f_pos;1341 }1342 1343 if (unlikely(out->f_flags & O_APPEND))1344 return -EINVAL;1345 1346 ret = rw_verify_area(WRITE, out, &offset, len);1347 if (unlikely(ret < 0))1348 return ret;1349 1350 if (in->f_flags & O_NONBLOCK)1351 flags |= SPLICE_F_NONBLOCK;1352 1353 file_start_write(out);1354 ret = do_splice_from(ipipe, out, &offset, len, flags);1355 file_end_write(out);1356 1357 if (!off_out)1358 out->f_pos = offset;1359 else1360 *off_out = offset;1361 } else if (opipe) {1362 if (off_out)1363 return -ESPIPE;1364 if (off_in) {1365 if (!(in->f_mode & FMODE_PREAD))1366 return -EINVAL;1367 offset = *off_in;1368 } else {1369 offset = in->f_pos;1370 }1371 1372 ret = rw_verify_area(READ, in, &offset, len);1373 if (unlikely(ret < 0))1374 return ret;1375 1376 if (out->f_flags & O_NONBLOCK)1377 flags |= SPLICE_F_NONBLOCK;1378 1379 ret = splice_file_to_pipe(in, opipe, &offset, len, flags);1380 1381 if (!off_in)1382 in->f_pos = offset;1383 else1384 *off_in = offset;1385 } else {1386 ret = -EINVAL;1387 }1388 1389 if (ret > 0) {1390 /*1391 * Generate modify out before access in:1392 * do_splice_from() may've already sent modify out,1393 * and this ensures the events get merged.1394 */1395 fsnotify_modify(out);1396 fsnotify_access(in);1397 }1398 1399 return ret;1400}1401 1402static HWJS_SUSPENDS ssize_t __do_splice(struct file *in, loff_t __user *off_in,1403 struct file *out, loff_t __user *off_out,1404 size_t len, unsigned int flags)1405{1406 struct pipe_inode_info *ipipe;1407 struct pipe_inode_info *opipe;1408 loff_t offset, *__off_in = NULL, *__off_out = NULL;1409 ssize_t ret;1410 1411 ipipe = get_pipe_info(in, true);1412 opipe = get_pipe_info(out, true);1413 1414 if (ipipe) {1415 if (off_in)1416 return -ESPIPE;1417 pipe_clear_nowait(in);1418 }1419 if (opipe) {1420 if (off_out)1421 return -ESPIPE;1422 pipe_clear_nowait(out);1423 }1424 1425 if (off_out) {1426 if (copy_from_user(&offset, off_out, sizeof(loff_t)))1427 return -EFAULT;1428 __off_out = &offset;1429 }1430 if (off_in) {1431 if (copy_from_user(&offset, off_in, sizeof(loff_t)))1432 return -EFAULT;1433 __off_in = &offset;1434 }1435 1436 ret = do_splice(in, __off_in, out, __off_out, len, flags);1437 if (ret < 0)1438 return ret;1439 1440 if (__off_out && copy_to_user(off_out, __off_out, sizeof(loff_t)))1441 return -EFAULT;1442 if (__off_in && copy_to_user(off_in, __off_in, sizeof(loff_t)))1443 return -EFAULT;1444 1445 return ret;1446}1447 1448static HWJS_SUSPENDS ssize_t iter_to_pipe(struct iov_iter *from,1449 struct pipe_inode_info *pipe,1450 unsigned int flags)1451{1452 struct pipe_buffer buf = {1453 .ops = &user_page_pipe_buf_ops,1454 .flags = flags1455 };1456 size_t total = 0;1457 ssize_t ret = 0;1458 1459 while (iov_iter_count(from)) {1460 struct page *pages[16];1461 ssize_t left;1462 size_t start;1463 int i, n;1464 1465 left = iov_iter_get_pages2(from, pages, ~0UL, 16, &start);1466 if (left <= 0) {1467 ret = left;1468 break;1469 }1470 1471 n = DIV_ROUND_UP(left + start, PAGE_SIZE);1472 for (i = 0; i < n; i++) {1473 int size = min_t(int, left, PAGE_SIZE - start);1474 1475 buf.page = pages[i];1476 buf.offset = start;1477 buf.len = size;1478 ret = add_to_pipe(pipe, &buf);1479 if (unlikely(ret < 0)) {1480 iov_iter_revert(from, left);1481 // this one got dropped by add_to_pipe()1482 while (++i < n)1483 put_page(pages[i]);1484 goto out;1485 }1486 total += ret;1487 left -= size;1488 start = 0;1489 }1490 }1491out:1492 return total ? total : ret;1493}1494 1495static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,1496 struct splice_desc *sd)1497{1498 int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);1499 return n == sd->len ? n : -EFAULT;1500}1501 1502/*1503 * For lack of a better implementation, implement vmsplice() to userspace1504 * as a simple copy of the pipes pages to the user iov.1505 */1506static HWJS_SUSPENDS ssize_t vmsplice_to_user(struct file *file, struct iov_iter *iter,1507 unsigned int flags)1508{1509 struct pipe_inode_info *pipe = get_pipe_info(file, true);1510 struct splice_desc sd = {1511 .total_len = iov_iter_count(iter),1512 .flags = flags,1513 .u.data = iter1514 };1515 ssize_t ret = 0;1516 1517 if (!pipe)1518 return -EBADF;1519 1520 pipe_clear_nowait(file);1521 1522 if (sd.total_len) {1523 pipe_lock(pipe);1524 ret = __splice_from_pipe(pipe, &sd, pipe_to_user);1525 pipe_unlock(pipe);1526 }1527 1528 if (ret > 0)1529 fsnotify_access(file);1530 1531 return ret;1532}1533 1534/*1535 * vmsplice splices a user address range into a pipe. It can be thought of1536 * as splice-from-memory, where the regular splice is splice-from-file (or1537 * to file). In both cases the output is a pipe, naturally.1538 */1539static HWJS_SUSPENDS ssize_t vmsplice_to_pipe(struct file *file, struct iov_iter *iter,1540 unsigned int flags)1541{1542 struct pipe_inode_info *pipe;1543 ssize_t ret = 0;1544 unsigned buf_flag = 0;1545 1546 if (flags & SPLICE_F_GIFT)1547 buf_flag = PIPE_BUF_FLAG_GIFT;1548 1549 pipe = get_pipe_info(file, true);1550 if (!pipe)1551 return -EBADF;1552 1553 pipe_clear_nowait(file);1554 1555 pipe_lock(pipe);1556 ret = wait_for_space(pipe, flags);1557 if (!ret)1558 ret = iter_to_pipe(iter, pipe, buf_flag);1559 pipe_unlock(pipe);1560 if (ret > 0) {1561 wakeup_pipe_readers(pipe);1562 fsnotify_modify(file);1563 }1564 return ret;1565}1566 1567static int vmsplice_type(struct fd f, int *type)1568{1569 if (!fd_file(f))1570 return -EBADF;1571 if (fd_file(f)->f_mode & FMODE_WRITE) {1572 *type = ITER_SOURCE;1573 } else if (fd_file(f)->f_mode & FMODE_READ) {1574 *type = ITER_DEST;1575 } else {1576 fdput(f);1577 return -EBADF;1578 }1579 return 0;1580}1581 1582/*1583 * Note that vmsplice only really supports true splicing _from_ user memory1584 * to a pipe, not the other way around. Splicing from user memory is a simple1585 * operation that can be supported without any funky alignment restrictions1586 * or nasty vm tricks. We simply map in the user memory and fill them into1587 * a pipe. The reverse isn't quite as easy, though. There are two possible1588 * solutions for that:1589 *1590 * - memcpy() the data internally, at which point we might as well just1591 * do a regular read() on the buffer anyway.1592 * - Lots of nasty vm tricks, that are neither fast nor flexible (it1593 * has restriction limitations on both ends of the pipe).1594 *1595 * Currently we punt and implement it as a normal copy, see pipe_to_user().1596 *1597 */1598SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov,1599 unsigned long, nr_segs, unsigned int, flags)1600{1601 struct iovec iovstack[UIO_FASTIOV];1602 struct iovec *iov = iovstack;1603 struct iov_iter iter;1604 ssize_t error;1605 struct fd f;1606 int type;1607 1608 if (unlikely(flags & ~SPLICE_F_ALL))1609 return -EINVAL;1610 1611 f = fdget(fd);1612 error = vmsplice_type(f, &type);1613 if (error)1614 return error;1615 1616 error = import_iovec(type, uiov, nr_segs,1617 ARRAY_SIZE(iovstack), &iov, &iter);1618 if (error < 0)1619 goto out_fdput;1620 1621 if (!iov_iter_count(&iter))1622 error = 0;1623 else if (type == ITER_SOURCE)1624 error = vmsplice_to_pipe(fd_file(f), &iter, flags);1625 else1626 error = vmsplice_to_user(fd_file(f), &iter, flags);1627 1628 kfree(iov);1629out_fdput:1630 fdput(f);1631 return error;1632}1633 1634SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,1635 int, fd_out, loff_t __user *, off_out,1636 size_t, len, unsigned int, flags)1637{1638 struct fd in, out;1639 ssize_t error;1640 1641 if (unlikely(!len))1642 return 0;1643 1644 if (unlikely(flags & ~SPLICE_F_ALL))1645 return -EINVAL;1646 1647 error = -EBADF;1648 in = fdget(fd_in);1649 if (fd_file(in)) {1650 out = fdget(fd_out);1651 if (fd_file(out)) {1652 error = __do_splice(fd_file(in), off_in, fd_file(out), off_out,1653 len, flags);1654 fdput(out);1655 }1656 fdput(in);1657 }1658 return error;1659}1660 1661/*1662 * Make sure there's data to read. Wait for input if we can, otherwise1663 * return an appropriate error.1664 */1665static HWJS_SUSPENDS int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)1666{1667 int ret;1668 1669 /*1670 * Check the pipe occupancy without the inode lock first. This function1671 * is speculative anyways, so missing one is ok.1672 */1673 if (!pipe_empty(pipe->head, pipe->tail))1674 return 0;1675 1676 ret = 0;1677 pipe_lock(pipe);1678 1679 while (pipe_empty(pipe->head, pipe->tail)) {1680 if (signal_pending(current)) {1681 ret = -ERESTARTSYS;1682 break;1683 }1684 if (!pipe->writers)1685 break;1686 if (flags & SPLICE_F_NONBLOCK) {1687 ret = -EAGAIN;1688 break;1689 }1690 pipe_wait_readable(pipe);1691 }1692 1693 pipe_unlock(pipe);1694 return ret;1695}1696 1697/*1698 * Make sure there's writeable room. Wait for room if we can, otherwise1699 * return an appropriate error.1700 */1701static HWJS_SUSPENDS int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)1702{1703 int ret;1704 1705 /*1706 * Check pipe occupancy without the inode lock first. This function1707 * is speculative anyways, so missing one is ok.1708 */1709 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))1710 return 0;1711 1712 ret = 0;1713 pipe_lock(pipe);1714 1715 while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) {1716 if (!pipe->readers) {1717 send_sig(SIGPIPE, current, 0);1718 ret = -EPIPE;1719 break;1720 }1721 if (flags & SPLICE_F_NONBLOCK) {1722 ret = -EAGAIN;1723 break;1724 }1725 if (signal_pending(current)) {1726 ret = -ERESTARTSYS;1727 break;1728 }1729 pipe_wait_writable(pipe);1730 }1731 1732 pipe_unlock(pipe);1733 return ret;1734}1735 1736/*1737 * Splice contents of ipipe to opipe.1738 */1739static HWJS_SUSPENDS int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,1740 struct pipe_inode_info *opipe,1741 size_t len, unsigned int flags)1742{1743 struct pipe_buffer *ibuf, *obuf;1744 unsigned int i_head, o_head;1745 unsigned int i_tail, o_tail;1746 unsigned int i_mask, o_mask;1747 int ret = 0;1748 bool input_wakeup = false;1749 1750 1751retry:1752 ret = ipipe_prep(ipipe, flags);1753 if (ret)1754 return ret;1755 1756 ret = opipe_prep(opipe, flags);1757 if (ret)1758 return ret;1759 1760 /*1761 * Potential ABBA deadlock, work around it by ordering lock1762 * grabbing by pipe info address. Otherwise two different processes1763 * could deadlock (one doing tee from A -> B, the other from B -> A).1764 */1765 pipe_double_lock(ipipe, opipe);1766 1767 i_tail = ipipe->tail;1768 i_mask = ipipe->ring_size - 1;1769 o_head = opipe->head;1770 o_mask = opipe->ring_size - 1;1771 1772 do {1773 size_t o_len;1774 1775 if (!opipe->readers) {1776 send_sig(SIGPIPE, current, 0);1777 if (!ret)1778 ret = -EPIPE;1779 break;1780 }1781 1782 i_head = ipipe->head;1783 o_tail = opipe->tail;1784 1785 if (pipe_empty(i_head, i_tail) && !ipipe->writers)1786 break;1787 1788 /*1789 * Cannot make any progress, because either the input1790 * pipe is empty or the output pipe is full.1791 */1792 if (pipe_empty(i_head, i_tail) ||1793 pipe_full(o_head, o_tail, opipe->max_usage)) {1794 /* Already processed some buffers, break */1795 if (ret)1796 break;1797 1798 if (flags & SPLICE_F_NONBLOCK) {1799 ret = -EAGAIN;1800 break;1801 }1802 1803 /*1804 * We raced with another reader/writer and haven't1805 * managed to process any buffers. A zero return1806 * value means EOF, so retry instead.1807 */1808 pipe_unlock(ipipe);1809 pipe_unlock(opipe);1810 goto retry;1811 }1812 1813 ibuf = &ipipe->bufs[i_tail & i_mask];1814 obuf = &opipe->bufs[o_head & o_mask];1815 1816 if (len >= ibuf->len) {1817 /*1818 * Simply move the whole buffer from ipipe to opipe1819 */1820 *obuf = *ibuf;1821 ibuf->ops = NULL;1822 i_tail++;1823 ipipe->tail = i_tail;1824 input_wakeup = true;1825 o_len = obuf->len;1826 o_head++;1827 opipe->head = o_head;1828 } else {1829 /*1830 * Get a reference to this pipe buffer,1831 * so we can copy the contents over.1832 */1833 if (!pipe_buf_get(ipipe, ibuf)) {1834 if (ret == 0)1835 ret = -EFAULT;1836 break;1837 }1838 *obuf = *ibuf;1839 1840 /*1841 * Don't inherit the gift and merge flags, we need to1842 * prevent multiple steals of this page.1843 */1844 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;1845 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;1846 1847 obuf->len = len;1848 ibuf->offset += len;1849 ibuf->len -= len;1850 o_len = len;1851 o_head++;1852 opipe->head = o_head;1853 }1854 ret += o_len;1855 len -= o_len;1856 } while (len);1857 1858 pipe_unlock(ipipe);1859 pipe_unlock(opipe);1860 1861 /*1862 * If we put data in the output pipe, wakeup any potential readers.1863 */1864 if (ret > 0)1865 wakeup_pipe_readers(opipe);1866 1867 if (input_wakeup)1868 wakeup_pipe_writers(ipipe);1869 1870 return ret;1871}1872 1873/*1874 * Link contents of ipipe to opipe.1875 */1876static HWJS_SUSPENDS ssize_t link_pipe(struct pipe_inode_info *ipipe,1877 struct pipe_inode_info *opipe,1878 size_t len, unsigned int flags)1879{1880 struct pipe_buffer *ibuf, *obuf;1881 unsigned int i_head, o_head;1882 unsigned int i_tail, o_tail;1883 unsigned int i_mask, o_mask;1884 ssize_t ret = 0;1885 1886 /*1887 * Potential ABBA deadlock, work around it by ordering lock1888 * grabbing by pipe info address. Otherwise two different processes1889 * could deadlock (one doing tee from A -> B, the other from B -> A).1890 */1891 pipe_double_lock(ipipe, opipe);1892 1893 i_tail = ipipe->tail;1894 i_mask = ipipe->ring_size - 1;1895 o_head = opipe->head;1896 o_mask = opipe->ring_size - 1;1897 1898 do {1899 if (!opipe->readers) {1900 send_sig(SIGPIPE, current, 0);1901 if (!ret)1902 ret = -EPIPE;1903 break;1904 }1905 1906 i_head = ipipe->head;1907 o_tail = opipe->tail;1908 1909 /*1910 * If we have iterated all input buffers or run out of1911 * output room, break.1912 */1913 if (pipe_empty(i_head, i_tail) ||1914 pipe_full(o_head, o_tail, opipe->max_usage))1915 break;1916 1917 ibuf = &ipipe->bufs[i_tail & i_mask];1918 obuf = &opipe->bufs[o_head & o_mask];1919 1920 /*1921 * Get a reference to this pipe buffer,1922 * so we can copy the contents over.1923 */1924 if (!pipe_buf_get(ipipe, ibuf)) {1925 if (ret == 0)1926 ret = -EFAULT;1927 break;1928 }1929 1930 *obuf = *ibuf;1931 1932 /*1933 * Don't inherit the gift and merge flag, we need to prevent1934 * multiple steals of this page.1935 */1936 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;1937 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;1938 1939 if (obuf->len > len)1940 obuf->len = len;1941 ret += obuf->len;1942 len -= obuf->len;1943 1944 o_head++;1945 opipe->head = o_head;1946 i_tail++;1947 } while (len);1948 1949 pipe_unlock(ipipe);1950 pipe_unlock(opipe);1951 1952 /*1953 * If we put data in the output pipe, wakeup any potential readers.1954 */1955 if (ret > 0)1956 wakeup_pipe_readers(opipe);1957 1958 return ret;1959}1960 1961/*1962 * This is a tee(1) implementation that works on pipes. It doesn't copy1963 * any data, it simply references the 'in' pages on the 'out' pipe.1964 * The 'flags' used are the SPLICE_F_* variants, currently the only1965 * applicable one is SPLICE_F_NONBLOCK.1966 */1967ssize_t do_tee(struct file *in, struct file *out, size_t len,1968 unsigned int flags)1969{1970 struct pipe_inode_info *ipipe = get_pipe_info(in, true);1971 struct pipe_inode_info *opipe = get_pipe_info(out, true);1972 ssize_t ret = -EINVAL;1973 1974 if (unlikely(!(in->f_mode & FMODE_READ) ||1975 !(out->f_mode & FMODE_WRITE)))1976 return -EBADF;1977 1978 /*1979 * Duplicate the contents of ipipe to opipe without actually1980 * copying the data.1981 */1982 if (ipipe && opipe && ipipe != opipe) {1983 if ((in->f_flags | out->f_flags) & O_NONBLOCK)1984 flags |= SPLICE_F_NONBLOCK;1985 1986 /*1987 * Keep going, unless we encounter an error. The ipipe/opipe1988 * ordering doesn't really matter.1989 */1990 ret = ipipe_prep(ipipe, flags);1991 if (!ret) {1992 ret = opipe_prep(opipe, flags);1993 if (!ret)1994 ret = link_pipe(ipipe, opipe, len, flags);1995 }1996 }1997 1998 if (ret > 0) {1999 fsnotify_access(in);2000 fsnotify_modify(out);2001 }2002 2003 return ret;2004}2005 2006SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)2007{2008 struct fd in, out;2009 ssize_t error;2010 2011 if (unlikely(flags & ~SPLICE_F_ALL))2012 return -EINVAL;2013 2014 if (unlikely(!len))2015 return 0;2016 2017 error = -EBADF;2018 in = fdget(fdin);2019 if (fd_file(in)) {2020 out = fdget(fdout);2021 if (fd_file(out)) {2022 error = do_tee(fd_file(in), fd_file(out), len, flags);2023 fdput(out);2024 }2025 fdput(in);2026 }2027 2028 return error;2029}2030