1082 lines · c
1// SPDX-License-Identifier: GPL-2.02#ifndef NO_BCACHEFS_FS3 4#include "bcachefs.h"5#include "alloc_foreground.h"6#include "bkey_buf.h"7#include "fs-io.h"8#include "fs-io-buffered.h"9#include "fs-io-direct.h"10#include "fs-io-pagecache.h"11#include "io_read.h"12#include "io_write.h"13 14#include <linux/backing-dev.h>15#include <linux/pagemap.h>16#include <linux/writeback.h>17 18static inline bool bio_full(struct bio *bio, unsigned len)19{20 if (bio->bi_vcnt >= bio->bi_max_vecs)21 return true;22 if (bio->bi_iter.bi_size > UINT_MAX - len)23 return true;24 return false;25}26 27/* readpage(s): */28 29static void bch2_readpages_end_io(struct bio *bio)30{31 struct folio_iter fi;32 33 bio_for_each_folio_all(fi, bio)34 folio_end_read(fi.folio, bio->bi_status == BLK_STS_OK);35 36 bio_put(bio);37}38 39struct readpages_iter {40 struct address_space *mapping;41 unsigned idx;42 folios folios;43};44 45static int readpages_iter_init(struct readpages_iter *iter,46 struct readahead_control *ractl)47{48 struct folio *folio;49 50 *iter = (struct readpages_iter) { ractl->mapping };51 52 while ((folio = __readahead_folio(ractl))) {53 if (!bch2_folio_create(folio, GFP_KERNEL) ||54 darray_push(&iter->folios, folio)) {55 bch2_folio_release(folio);56 ractl->_nr_pages += folio_nr_pages(folio);57 ractl->_index -= folio_nr_pages(folio);58 return iter->folios.nr ? 0 : -ENOMEM;59 }60 61 folio_put(folio);62 }63 64 return 0;65}66 67static inline struct folio *readpage_iter_peek(struct readpages_iter *iter)68{69 if (iter->idx >= iter->folios.nr)70 return NULL;71 return iter->folios.data[iter->idx];72}73 74static inline void readpage_iter_advance(struct readpages_iter *iter)75{76 iter->idx++;77}78 79static bool extent_partial_reads_expensive(struct bkey_s_c k)80{81 struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);82 struct bch_extent_crc_unpacked crc;83 const union bch_extent_entry *i;84 85 bkey_for_each_crc(k.k, ptrs, crc, i)86 if (crc.csum_type || crc.compression_type)87 return true;88 return false;89}90 91static int readpage_bio_extend(struct btree_trans *trans,92 struct readpages_iter *iter,93 struct bio *bio,94 unsigned sectors_this_extent,95 bool get_more)96{97 /* Don't hold btree locks while allocating memory: */98 bch2_trans_unlock(trans);99 100 while (bio_sectors(bio) < sectors_this_extent &&101 bio->bi_vcnt < bio->bi_max_vecs) {102 struct folio *folio = readpage_iter_peek(iter);103 int ret;104 105 if (folio) {106 readpage_iter_advance(iter);107 } else {108 pgoff_t folio_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT;109 110 if (!get_more)111 break;112 113 folio = xa_load(&iter->mapping->i_pages, folio_offset);114 if (folio && !xa_is_value(folio))115 break;116 117 folio = filemap_alloc_folio(readahead_gfp_mask(iter->mapping), 0);118 if (!folio)119 break;120 121 if (!__bch2_folio_create(folio, GFP_KERNEL)) {122 folio_put(folio);123 break;124 }125 126 ret = filemap_add_folio(iter->mapping, folio, folio_offset, GFP_KERNEL);127 if (ret) {128 __bch2_folio_release(folio);129 folio_put(folio);130 break;131 }132 133 folio_put(folio);134 }135 136 BUG_ON(folio_sector(folio) != bio_end_sector(bio));137 138 BUG_ON(!bio_add_folio(bio, folio, folio_size(folio), 0));139 }140 141 return bch2_trans_relock(trans);142}143 144static void bchfs_read(struct btree_trans *trans,145 struct bch_read_bio *rbio,146 subvol_inum inum,147 struct readpages_iter *readpages_iter)148{149 struct bch_fs *c = trans->c;150 struct btree_iter iter;151 struct bkey_buf sk;152 int flags = BCH_READ_RETRY_IF_STALE|153 BCH_READ_MAY_PROMOTE;154 int ret = 0;155 156 rbio->c = c;157 rbio->start_time = local_clock();158 rbio->subvol = inum.subvol;159 160 bch2_bkey_buf_init(&sk);161 bch2_trans_begin(trans);162 bch2_trans_iter_init(trans, &iter, BTREE_ID_extents,163 POS(inum.inum, rbio->bio.bi_iter.bi_sector),164 BTREE_ITER_slots);165 while (1) {166 struct bkey_s_c k;167 unsigned bytes, sectors, offset_into_extent;168 enum btree_id data_btree = BTREE_ID_extents;169 170 bch2_trans_begin(trans);171 172 u32 snapshot;173 ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot);174 if (ret)175 goto err;176 177 bch2_btree_iter_set_snapshot(&iter, snapshot);178 179 bch2_btree_iter_set_pos(&iter,180 POS(inum.inum, rbio->bio.bi_iter.bi_sector));181 182 k = bch2_btree_iter_peek_slot(&iter);183 ret = bkey_err(k);184 if (ret)185 goto err;186 187 offset_into_extent = iter.pos.offset -188 bkey_start_offset(k.k);189 sectors = k.k->size - offset_into_extent;190 191 bch2_bkey_buf_reassemble(&sk, c, k);192 193 ret = bch2_read_indirect_extent(trans, &data_btree,194 &offset_into_extent, &sk);195 if (ret)196 goto err;197 198 k = bkey_i_to_s_c(sk.k);199 200 sectors = min(sectors, k.k->size - offset_into_extent);201 202 if (readpages_iter) {203 ret = readpage_bio_extend(trans, readpages_iter, &rbio->bio, sectors,204 extent_partial_reads_expensive(k));205 if (ret)206 goto err;207 }208 209 bytes = min(sectors, bio_sectors(&rbio->bio)) << 9;210 swap(rbio->bio.bi_iter.bi_size, bytes);211 212 if (rbio->bio.bi_iter.bi_size == bytes)213 flags |= BCH_READ_LAST_FRAGMENT;214 215 bch2_bio_page_state_set(&rbio->bio, k);216 217 bch2_read_extent(trans, rbio, iter.pos,218 data_btree, k, offset_into_extent, flags);219 220 if (flags & BCH_READ_LAST_FRAGMENT)221 break;222 223 swap(rbio->bio.bi_iter.bi_size, bytes);224 bio_advance(&rbio->bio, bytes);225err:226 if (ret &&227 !bch2_err_matches(ret, BCH_ERR_transaction_restart))228 break;229 }230 bch2_trans_iter_exit(trans, &iter);231 232 if (ret) {233 bch_err_inum_offset_ratelimited(c,234 iter.pos.inode,235 iter.pos.offset << 9,236 "read error %i from btree lookup", ret);237 rbio->bio.bi_status = BLK_STS_IOERR;238 bio_endio(&rbio->bio);239 }240 241 bch2_bkey_buf_exit(&sk, c);242}243 244void bch2_readahead(struct readahead_control *ractl)245{246 struct bch_inode_info *inode = to_bch_ei(ractl->mapping->host);247 struct bch_fs *c = inode->v.i_sb->s_fs_info;248 struct bch_io_opts opts;249 struct folio *folio;250 struct readpages_iter readpages_iter;251 252 bch2_inode_opts_get(&opts, c, &inode->ei_inode);253 254 int ret = readpages_iter_init(&readpages_iter, ractl);255 if (ret)256 return;257 258 bch2_pagecache_add_get(inode);259 260 struct btree_trans *trans = bch2_trans_get(c);261 while ((folio = readpage_iter_peek(&readpages_iter))) {262 unsigned n = min_t(unsigned,263 readpages_iter.folios.nr -264 readpages_iter.idx,265 BIO_MAX_VECS);266 struct bch_read_bio *rbio =267 rbio_init(bio_alloc_bioset(NULL, n, REQ_OP_READ,268 GFP_KERNEL, &c->bio_read),269 opts);270 271 readpage_iter_advance(&readpages_iter);272 273 rbio->bio.bi_iter.bi_sector = folio_sector(folio);274 rbio->bio.bi_end_io = bch2_readpages_end_io;275 BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));276 277 bchfs_read(trans, rbio, inode_inum(inode),278 &readpages_iter);279 bch2_trans_unlock(trans);280 }281 bch2_trans_put(trans);282 283 bch2_pagecache_add_put(inode);284 285 darray_exit(&readpages_iter.folios);286}287 288static void bch2_read_single_folio_end_io(struct bio *bio)289{290 complete(bio->bi_private);291}292 293int bch2_read_single_folio(struct folio *folio, struct address_space *mapping)294{295 struct bch_inode_info *inode = to_bch_ei(mapping->host);296 struct bch_fs *c = inode->v.i_sb->s_fs_info;297 struct bch_read_bio *rbio;298 struct bch_io_opts opts;299 int ret;300 DECLARE_COMPLETION_ONSTACK(done);301 302 if (!bch2_folio_create(folio, GFP_KERNEL))303 return -ENOMEM;304 305 bch2_inode_opts_get(&opts, c, &inode->ei_inode);306 307 rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_KERNEL, &c->bio_read),308 opts);309 rbio->bio.bi_private = &done;310 rbio->bio.bi_end_io = bch2_read_single_folio_end_io;311 312 rbio->bio.bi_opf = REQ_OP_READ|REQ_SYNC;313 rbio->bio.bi_iter.bi_sector = folio_sector(folio);314 BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));315 316 bch2_trans_run(c, (bchfs_read(trans, rbio, inode_inum(inode), NULL), 0));317 wait_for_completion(&done);318 319 ret = blk_status_to_errno(rbio->bio.bi_status);320 bio_put(&rbio->bio);321 322 if (ret < 0)323 return ret;324 325 folio_mark_uptodate(folio);326 return 0;327}328 329int bch2_read_folio(struct file *file, struct folio *folio)330{331 int ret;332 333 ret = bch2_read_single_folio(folio, folio->mapping);334 folio_unlock(folio);335 return bch2_err_class(ret);336}337 338/* writepages: */339 340struct bch_writepage_io {341 struct bch_inode_info *inode;342 343 /* must be last: */344 struct bch_write_op op;345};346 347struct bch_writepage_state {348 struct bch_writepage_io *io;349 struct bch_io_opts opts;350 struct bch_folio_sector *tmp;351 unsigned tmp_sectors;352};353 354static inline struct bch_writepage_state bch_writepage_state_init(struct bch_fs *c,355 struct bch_inode_info *inode)356{357 struct bch_writepage_state ret = { 0 };358 359 bch2_inode_opts_get(&ret.opts, c, &inode->ei_inode);360 return ret;361}362 363/*364 * Determine when a writepage io is full. We have to limit writepage bios to a365 * single page per bvec (i.e. 1MB with 4k pages) because that is the limit to366 * what the bounce path in bch2_write_extent() can handle. In theory we could367 * loosen this restriction for non-bounce I/O, but we don't have that context368 * here. Ideally, we can up this limit and make it configurable in the future369 * when the bounce path can be enhanced to accommodate larger source bios.370 */371static inline bool bch_io_full(struct bch_writepage_io *io, unsigned len)372{373 struct bio *bio = &io->op.wbio.bio;374 return bio_full(bio, len) ||375 (bio->bi_iter.bi_size + len > BIO_MAX_VECS * PAGE_SIZE);376}377 378static void bch2_writepage_io_done(struct bch_write_op *op)379{380 struct bch_writepage_io *io =381 container_of(op, struct bch_writepage_io, op);382 struct bch_fs *c = io->op.c;383 struct bio *bio = &io->op.wbio.bio;384 struct folio_iter fi;385 unsigned i;386 387 if (io->op.error) {388 set_bit(EI_INODE_ERROR, &io->inode->ei_flags);389 390 bio_for_each_folio_all(fi, bio) {391 struct bch_folio *s;392 393 mapping_set_error(fi.folio->mapping, -EIO);394 395 s = __bch2_folio(fi.folio);396 spin_lock(&s->lock);397 for (i = 0; i < folio_sectors(fi.folio); i++)398 s->s[i].nr_replicas = 0;399 spin_unlock(&s->lock);400 }401 }402 403 if (io->op.flags & BCH_WRITE_WROTE_DATA_INLINE) {404 bio_for_each_folio_all(fi, bio) {405 struct bch_folio *s;406 407 s = __bch2_folio(fi.folio);408 spin_lock(&s->lock);409 for (i = 0; i < folio_sectors(fi.folio); i++)410 s->s[i].nr_replicas = 0;411 spin_unlock(&s->lock);412 }413 }414 415 /*416 * racing with fallocate can cause us to add fewer sectors than417 * expected - but we shouldn't add more sectors than expected:418 */419 WARN_ON_ONCE(io->op.i_sectors_delta > 0);420 421 /*422 * (error (due to going RO) halfway through a page can screw that up423 * slightly)424 * XXX wtf?425 BUG_ON(io->op.op.i_sectors_delta >= PAGE_SECTORS);426 */427 428 /*429 * The writeback flag is effectively our ref on the inode -430 * fixup i_blocks before calling folio_end_writeback:431 */432 bch2_i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta);433 434 bio_for_each_folio_all(fi, bio) {435 struct bch_folio *s = __bch2_folio(fi.folio);436 437 if (atomic_dec_and_test(&s->write_count))438 folio_end_writeback(fi.folio);439 }440 441 bio_put(&io->op.wbio.bio);442}443 444static void bch2_writepage_do_io(struct bch_writepage_state *w)445{446 struct bch_writepage_io *io = w->io;447 448 w->io = NULL;449 closure_call(&io->op.cl, bch2_write, NULL, NULL);450}451 452/*453 * Get a bch_writepage_io and add @page to it - appending to an existing one if454 * possible, else allocating a new one:455 */456static void bch2_writepage_io_alloc(struct bch_fs *c,457 struct writeback_control *wbc,458 struct bch_writepage_state *w,459 struct bch_inode_info *inode,460 u64 sector,461 unsigned nr_replicas)462{463 struct bch_write_op *op;464 465 w->io = container_of(bio_alloc_bioset(NULL, BIO_MAX_VECS,466 REQ_OP_WRITE,467 GFP_KERNEL,468 &c->writepage_bioset),469 struct bch_writepage_io, op.wbio.bio);470 471 w->io->inode = inode;472 op = &w->io->op;473 bch2_write_op_init(op, c, w->opts);474 op->target = w->opts.foreground_target;475 op->nr_replicas = nr_replicas;476 op->res.nr_replicas = nr_replicas;477 op->write_point = writepoint_hashed(inode->ei_last_dirtied);478 op->subvol = inode->ei_inum.subvol;479 op->pos = POS(inode->v.i_ino, sector);480 op->end_io = bch2_writepage_io_done;481 op->devs_need_flush = &inode->ei_devs_need_flush;482 op->wbio.bio.bi_iter.bi_sector = sector;483 op->wbio.bio.bi_opf = wbc_to_write_flags(wbc);484}485 486static int __bch2_writepage(struct folio *folio,487 struct writeback_control *wbc,488 void *data)489{490 struct bch_inode_info *inode = to_bch_ei(folio->mapping->host);491 struct bch_fs *c = inode->v.i_sb->s_fs_info;492 struct bch_writepage_state *w = data;493 struct bch_folio *s;494 unsigned i, offset, f_sectors, nr_replicas_this_write = U32_MAX;495 loff_t i_size = i_size_read(&inode->v);496 int ret;497 498 EBUG_ON(!folio_test_uptodate(folio));499 500 /* Is the folio fully inside i_size? */501 if (folio_end_pos(folio) <= i_size)502 goto do_io;503 504 /* Is the folio fully outside i_size? (truncate in progress) */505 if (folio_pos(folio) >= i_size) {506 folio_unlock(folio);507 return 0;508 }509 510 /*511 * The folio straddles i_size. It must be zeroed out on each and every512 * writepage invocation because it may be mmapped. "A file is mapped513 * in multiples of the folio size. For a file that is not a multiple of514 * the folio size, the remaining memory is zeroed when mapped, and515 * writes to that region are not written out to the file."516 */517 folio_zero_segment(folio,518 i_size - folio_pos(folio),519 folio_size(folio));520do_io:521 f_sectors = folio_sectors(folio);522 s = bch2_folio(folio);523 524 if (f_sectors > w->tmp_sectors) {525 kfree(w->tmp);526 w->tmp = kcalloc(f_sectors, sizeof(struct bch_folio_sector), GFP_NOFS|__GFP_NOFAIL);527 w->tmp_sectors = f_sectors;528 }529 530 /*531 * Things get really hairy with errors during writeback:532 */533 ret = bch2_get_folio_disk_reservation(c, inode, folio, false);534 BUG_ON(ret);535 536 /* Before unlocking the page, get copy of reservations: */537 spin_lock(&s->lock);538 memcpy(w->tmp, s->s, sizeof(struct bch_folio_sector) * f_sectors);539 540 for (i = 0; i < f_sectors; i++) {541 if (s->s[i].state < SECTOR_dirty)542 continue;543 544 nr_replicas_this_write =545 min_t(unsigned, nr_replicas_this_write,546 s->s[i].nr_replicas +547 s->s[i].replicas_reserved);548 }549 550 for (i = 0; i < f_sectors; i++) {551 if (s->s[i].state < SECTOR_dirty)552 continue;553 554 s->s[i].nr_replicas = w->opts.compression555 ? 0 : nr_replicas_this_write;556 557 s->s[i].replicas_reserved = 0;558 bch2_folio_sector_set(folio, s, i, SECTOR_allocated);559 }560 spin_unlock(&s->lock);561 562 BUG_ON(atomic_read(&s->write_count));563 atomic_set(&s->write_count, 1);564 565 BUG_ON(folio_test_writeback(folio));566 folio_start_writeback(folio);567 568 folio_unlock(folio);569 570 offset = 0;571 while (1) {572 unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0;573 u64 sector;574 575 while (offset < f_sectors &&576 w->tmp[offset].state < SECTOR_dirty)577 offset++;578 579 if (offset == f_sectors)580 break;581 582 while (offset + sectors < f_sectors &&583 w->tmp[offset + sectors].state >= SECTOR_dirty) {584 reserved_sectors += w->tmp[offset + sectors].replicas_reserved;585 dirty_sectors += w->tmp[offset + sectors].state == SECTOR_dirty;586 sectors++;587 }588 BUG_ON(!sectors);589 590 sector = folio_sector(folio) + offset;591 592 if (w->io &&593 (w->io->op.res.nr_replicas != nr_replicas_this_write ||594 bch_io_full(w->io, sectors << 9) ||595 bio_end_sector(&w->io->op.wbio.bio) != sector))596 bch2_writepage_do_io(w);597 598 if (!w->io)599 bch2_writepage_io_alloc(c, wbc, w, inode, sector,600 nr_replicas_this_write);601 602 atomic_inc(&s->write_count);603 604 BUG_ON(inode != w->io->inode);605 BUG_ON(!bio_add_folio(&w->io->op.wbio.bio, folio,606 sectors << 9, offset << 9));607 608 /* Check for writing past i_size: */609 WARN_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) >610 round_up(i_size, block_bytes(c)) &&611 !test_bit(BCH_FS_emergency_ro, &c->flags),612 "writing past i_size: %llu > %llu (unrounded %llu)\n",613 bio_end_sector(&w->io->op.wbio.bio) << 9,614 round_up(i_size, block_bytes(c)),615 i_size);616 617 w->io->op.res.sectors += reserved_sectors;618 w->io->op.i_sectors_delta -= dirty_sectors;619 w->io->op.new_i_size = i_size;620 621 offset += sectors;622 }623 624 if (atomic_dec_and_test(&s->write_count))625 folio_end_writeback(folio);626 627 return 0;628}629 630int bch2_writepages(struct address_space *mapping, struct writeback_control *wbc)631{632 struct bch_fs *c = mapping->host->i_sb->s_fs_info;633 struct bch_writepage_state w =634 bch_writepage_state_init(c, to_bch_ei(mapping->host));635 struct blk_plug plug;636 int ret;637 638 blk_start_plug(&plug);639 ret = write_cache_pages(mapping, wbc, __bch2_writepage, &w);640 if (w.io)641 bch2_writepage_do_io(&w);642 blk_finish_plug(&plug);643 kfree(w.tmp);644 return bch2_err_class(ret);645}646 647/* buffered writes: */648 649int bch2_write_begin(struct file *file, struct address_space *mapping,650 loff_t pos, unsigned len,651 struct folio **foliop, void **fsdata)652{653 struct bch_inode_info *inode = to_bch_ei(mapping->host);654 struct bch_fs *c = inode->v.i_sb->s_fs_info;655 struct bch2_folio_reservation *res;656 struct folio *folio;657 unsigned offset;658 int ret = -ENOMEM;659 660 res = kmalloc(sizeof(*res), GFP_KERNEL);661 if (!res)662 return -ENOMEM;663 664 bch2_folio_reservation_init(c, inode, res);665 *fsdata = res;666 667 bch2_pagecache_add_get(inode);668 669 folio = __filemap_get_folio(mapping, pos >> PAGE_SHIFT,670 FGP_WRITEBEGIN | fgf_set_order(len),671 mapping_gfp_mask(mapping));672 if (IS_ERR_OR_NULL(folio))673 goto err_unlock;674 675 offset = pos - folio_pos(folio);676 len = min_t(size_t, len, folio_end_pos(folio) - pos);677 678 if (folio_test_uptodate(folio))679 goto out;680 681 /* If we're writing entire folio, don't need to read it in first: */682 if (!offset && len == folio_size(folio))683 goto out;684 685 if (!offset && pos + len >= inode->v.i_size) {686 folio_zero_segment(folio, len, folio_size(folio));687 flush_dcache_folio(folio);688 goto out;689 }690 691 if (folio_pos(folio) >= inode->v.i_size) {692 folio_zero_segments(folio, 0, offset, offset + len, folio_size(folio));693 flush_dcache_folio(folio);694 goto out;695 }696readpage:697 ret = bch2_read_single_folio(folio, mapping);698 if (ret)699 goto err;700out:701 ret = bch2_folio_set(c, inode_inum(inode), &folio, 1);702 if (ret)703 goto err;704 705 ret = bch2_folio_reservation_get(c, inode, folio, res, offset, len);706 if (ret) {707 if (!folio_test_uptodate(folio)) {708 /*709 * If the folio hasn't been read in, we won't know if we710 * actually need a reservation - we don't actually need711 * to read here, we just need to check if the folio is712 * fully backed by uncompressed data:713 */714 goto readpage;715 }716 717 goto err;718 }719 720 *foliop = folio;721 return 0;722err:723 folio_unlock(folio);724 folio_put(folio);725err_unlock:726 bch2_pagecache_add_put(inode);727 kfree(res);728 *fsdata = NULL;729 return bch2_err_class(ret);730}731 732int bch2_write_end(struct file *file, struct address_space *mapping,733 loff_t pos, unsigned len, unsigned copied,734 struct folio *folio, void *fsdata)735{736 struct bch_inode_info *inode = to_bch_ei(mapping->host);737 struct bch_fs *c = inode->v.i_sb->s_fs_info;738 struct bch2_folio_reservation *res = fsdata;739 unsigned offset = pos - folio_pos(folio);740 741 lockdep_assert_held(&inode->v.i_rwsem);742 BUG_ON(offset + copied > folio_size(folio));743 744 if (unlikely(copied < len && !folio_test_uptodate(folio))) {745 /*746 * The folio needs to be read in, but that would destroy747 * our partial write - simplest thing is to just force748 * userspace to redo the write:749 */750 folio_zero_range(folio, 0, folio_size(folio));751 flush_dcache_folio(folio);752 copied = 0;753 }754 755 spin_lock(&inode->v.i_lock);756 if (pos + copied > inode->v.i_size)757 i_size_write(&inode->v, pos + copied);758 spin_unlock(&inode->v.i_lock);759 760 if (copied) {761 if (!folio_test_uptodate(folio))762 folio_mark_uptodate(folio);763 764 bch2_set_folio_dirty(c, inode, folio, res, offset, copied);765 766 inode->ei_last_dirtied = (unsigned long) current;767 }768 769 folio_unlock(folio);770 folio_put(folio);771 bch2_pagecache_add_put(inode);772 773 bch2_folio_reservation_put(c, inode, res);774 kfree(res);775 776 return copied;777}778 779static noinline void folios_trunc(folios *fs, struct folio **fi)780{781 while (fs->data + fs->nr > fi) {782 struct folio *f = darray_pop(fs);783 784 folio_unlock(f);785 folio_put(f);786 }787}788 789static int __bch2_buffered_write(struct bch_inode_info *inode,790 struct address_space *mapping,791 struct iov_iter *iter,792 loff_t pos, unsigned len)793{794 struct bch_fs *c = inode->v.i_sb->s_fs_info;795 struct bch2_folio_reservation res;796 folios fs;797 struct folio *f;798 unsigned copied = 0, f_offset, f_copied;799 u64 end = pos + len, f_pos, f_len;800 loff_t last_folio_pos = inode->v.i_size;801 int ret = 0;802 803 BUG_ON(!len);804 805 bch2_folio_reservation_init(c, inode, &res);806 darray_init(&fs);807 808 ret = bch2_filemap_get_contig_folios_d(mapping, pos, end,809 FGP_WRITEBEGIN | fgf_set_order(len),810 mapping_gfp_mask(mapping), &fs);811 if (ret)812 goto out;813 814 BUG_ON(!fs.nr);815 816 f = darray_first(fs);817 if (pos != folio_pos(f) && !folio_test_uptodate(f)) {818 ret = bch2_read_single_folio(f, mapping);819 if (ret)820 goto out;821 }822 823 f = darray_last(fs);824 end = min(end, folio_end_pos(f));825 last_folio_pos = folio_pos(f);826 if (end != folio_end_pos(f) && !folio_test_uptodate(f)) {827 if (end >= inode->v.i_size) {828 folio_zero_range(f, 0, folio_size(f));829 } else {830 ret = bch2_read_single_folio(f, mapping);831 if (ret)832 goto out;833 }834 }835 836 ret = bch2_folio_set(c, inode_inum(inode), fs.data, fs.nr);837 if (ret)838 goto out;839 840 f_pos = pos;841 f_offset = pos - folio_pos(darray_first(fs));842 darray_for_each(fs, fi) {843 ssize_t f_reserved;844 845 f = *fi;846 f_len = min(end, folio_end_pos(f)) - f_pos;847 f_reserved = bch2_folio_reservation_get_partial(c, inode, f, &res, f_offset, f_len);848 849 if (unlikely(f_reserved != f_len)) {850 if (f_reserved < 0) {851 if (f == darray_first(fs)) {852 ret = f_reserved;853 goto out;854 }855 856 folios_trunc(&fs, fi);857 end = min(end, folio_end_pos(darray_last(fs)));858 } else {859 if (!folio_test_uptodate(f)) {860 ret = bch2_read_single_folio(f, mapping);861 if (ret)862 goto out;863 }864 865 folios_trunc(&fs, fi + 1);866 end = f_pos + f_reserved;867 }868 869 break;870 }871 872 f_pos = folio_end_pos(f);873 f_offset = 0;874 }875 876 if (mapping_writably_mapped(mapping))877 darray_for_each(fs, fi)878 flush_dcache_folio(*fi);879 880 f_pos = pos;881 f_offset = pos - folio_pos(darray_first(fs));882 darray_for_each(fs, fi) {883 f = *fi;884 f_len = min(end, folio_end_pos(f)) - f_pos;885 f_copied = copy_folio_from_iter_atomic(f, f_offset, f_len, iter);886 if (!f_copied) {887 folios_trunc(&fs, fi);888 break;889 }890 891 if (!folio_test_uptodate(f) &&892 f_copied != folio_size(f) &&893 pos + copied + f_copied < inode->v.i_size) {894 iov_iter_revert(iter, f_copied);895 folio_zero_range(f, 0, folio_size(f));896 folios_trunc(&fs, fi);897 break;898 }899 900 flush_dcache_folio(f);901 copied += f_copied;902 903 if (f_copied != f_len) {904 folios_trunc(&fs, fi + 1);905 break;906 }907 908 f_pos = folio_end_pos(f);909 f_offset = 0;910 }911 912 if (!copied)913 goto out;914 915 end = pos + copied;916 917 spin_lock(&inode->v.i_lock);918 if (end > inode->v.i_size)919 i_size_write(&inode->v, end);920 spin_unlock(&inode->v.i_lock);921 922 f_pos = pos;923 f_offset = pos - folio_pos(darray_first(fs));924 darray_for_each(fs, fi) {925 f = *fi;926 f_len = min(end, folio_end_pos(f)) - f_pos;927 928 if (!folio_test_uptodate(f))929 folio_mark_uptodate(f);930 931 bch2_set_folio_dirty(c, inode, f, &res, f_offset, f_len);932 933 f_pos = folio_end_pos(f);934 f_offset = 0;935 }936 937 inode->ei_last_dirtied = (unsigned long) current;938out:939 darray_for_each(fs, fi) {940 folio_unlock(*fi);941 folio_put(*fi);942 }943 944 /*945 * If the last folio added to the mapping starts beyond current EOF, we946 * performed a short write but left around at least one post-EOF folio.947 * Clean up the mapping before we return.948 */949 if (last_folio_pos >= inode->v.i_size)950 truncate_pagecache(&inode->v, inode->v.i_size);951 952 darray_exit(&fs);953 bch2_folio_reservation_put(c, inode, &res);954 955 return copied ?: ret;956}957 958static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter)959{960 struct file *file = iocb->ki_filp;961 struct address_space *mapping = file->f_mapping;962 struct bch_inode_info *inode = file_bch_inode(file);963 loff_t pos = iocb->ki_pos;964 ssize_t written = 0;965 int ret = 0;966 967 bch2_pagecache_add_get(inode);968 969 do {970 unsigned offset = pos & (PAGE_SIZE - 1);971 unsigned bytes = iov_iter_count(iter);972again:973 /*974 * Bring in the user page that we will copy from _first_.975 * Otherwise there's a nasty deadlock on copying from the976 * same page as we're writing to, without it being marked977 * up-to-date.978 *979 * Not only is this an optimisation, but it is also required980 * to check that the address is actually valid, when atomic981 * usercopies are used, below.982 */983 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {984 bytes = min_t(unsigned long, iov_iter_count(iter),985 PAGE_SIZE - offset);986 987 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {988 ret = -EFAULT;989 break;990 }991 }992 993 if (unlikely(fatal_signal_pending(current))) {994 ret = -EINTR;995 break;996 }997 998 ret = __bch2_buffered_write(inode, mapping, iter, pos, bytes);999 if (unlikely(ret < 0))1000 break;1001 1002 cond_resched();1003 1004 if (unlikely(ret == 0)) {1005 /*1006 * If we were unable to copy any data at all, we must1007 * fall back to a single segment length write.1008 *1009 * If we didn't fallback here, we could livelock1010 * because not all segments in the iov can be copied at1011 * once without a pagefault.1012 */1013 bytes = min_t(unsigned long, PAGE_SIZE - offset,1014 iov_iter_single_seg_count(iter));1015 goto again;1016 }1017 pos += ret;1018 written += ret;1019 ret = 0;1020 1021 balance_dirty_pages_ratelimited(mapping);1022 } while (iov_iter_count(iter));1023 1024 bch2_pagecache_add_put(inode);1025 1026 return written ? written : ret;1027}1028 1029ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *from)1030{1031 struct file *file = iocb->ki_filp;1032 struct bch_inode_info *inode = file_bch_inode(file);1033 ssize_t ret;1034 1035 if (iocb->ki_flags & IOCB_DIRECT) {1036 ret = bch2_direct_write(iocb, from);1037 goto out;1038 }1039 1040 inode_lock(&inode->v);1041 1042 ret = generic_write_checks(iocb, from);1043 if (ret <= 0)1044 goto unlock;1045 1046 ret = file_remove_privs(file);1047 if (ret)1048 goto unlock;1049 1050 ret = file_update_time(file);1051 if (ret)1052 goto unlock;1053 1054 ret = bch2_buffered_write(iocb, from);1055 if (likely(ret > 0))1056 iocb->ki_pos += ret;1057unlock:1058 inode_unlock(&inode->v);1059 1060 if (ret > 0)1061 ret = generic_write_sync(iocb, ret);1062out:1063 return bch2_err_class(ret);1064}1065 1066void bch2_fs_fs_io_buffered_exit(struct bch_fs *c)1067{1068 bioset_exit(&c->writepage_bioset);1069}1070 1071int bch2_fs_fs_io_buffered_init(struct bch_fs *c)1072{1073 if (bioset_init(&c->writepage_bioset,1074 4, offsetof(struct bch_writepage_io, op.wbio.bio),1075 BIOSET_NEED_BVECS))1076 return -BCH_ERR_ENOMEM_writepage_bioset_init;1077 1078 return 0;1079}1080 1081#endif /* NO_BCACHEFS_FS */1082