1029 lines · c
1// SPDX-License-Identifier: GPL-2.02#ifndef NO_BCACHEFS_FS3 4#include "bcachefs.h"5#include "alloc_foreground.h"6#include "bkey_buf.h"7#include "btree_update.h"8#include "buckets.h"9#include "clock.h"10#include "error.h"11#include "extents.h"12#include "extent_update.h"13#include "fs.h"14#include "fs-io.h"15#include "fs-io-buffered.h"16#include "fs-io-pagecache.h"17#include "fsck.h"18#include "inode.h"19#include "journal.h"20#include "io_misc.h"21#include "keylist.h"22#include "quota.h"23#include "reflink.h"24#include "trace.h"25 26#include <linux/aio.h>27#include <linux/backing-dev.h>28#include <linux/falloc.h>29#include <linux/migrate.h>30#include <linux/mmu_context.h>31#include <linux/pagevec.h>32#include <linux/rmap.h>33#include <linux/sched/signal.h>34#include <linux/task_io_accounting_ops.h>35#include <linux/uio.h>36 37#include <trace/events/writeback.h>38 39struct nocow_flush {40 struct closure *cl;41 struct bch_dev *ca;42 struct bio bio;43};44 45static void nocow_flush_endio(struct bio *_bio)46{47 48 struct nocow_flush *bio = container_of(_bio, struct nocow_flush, bio);49 50 closure_put(bio->cl);51 percpu_ref_put(&bio->ca->io_ref);52 bio_put(&bio->bio);53}54 55void bch2_inode_flush_nocow_writes_async(struct bch_fs *c,56 struct bch_inode_info *inode,57 struct closure *cl)58{59 struct nocow_flush *bio;60 struct bch_dev *ca;61 struct bch_devs_mask devs;62 unsigned dev;63 64 dev = find_first_bit(inode->ei_devs_need_flush.d, BCH_SB_MEMBERS_MAX);65 if (dev == BCH_SB_MEMBERS_MAX)66 return;67 68 devs = inode->ei_devs_need_flush;69 memset(&inode->ei_devs_need_flush, 0, sizeof(inode->ei_devs_need_flush));70 71 for_each_set_bit(dev, devs.d, BCH_SB_MEMBERS_MAX) {72 rcu_read_lock();73 ca = rcu_dereference(c->devs[dev]);74 if (ca && !percpu_ref_tryget(&ca->io_ref))75 ca = NULL;76 rcu_read_unlock();77 78 if (!ca)79 continue;80 81 bio = container_of(bio_alloc_bioset(ca->disk_sb.bdev, 0,82 REQ_OP_WRITE|REQ_PREFLUSH,83 GFP_KERNEL,84 &c->nocow_flush_bioset),85 struct nocow_flush, bio);86 bio->cl = cl;87 bio->ca = ca;88 bio->bio.bi_end_io = nocow_flush_endio;89 closure_bio_submit(&bio->bio, cl);90 }91}92 93static int bch2_inode_flush_nocow_writes(struct bch_fs *c,94 struct bch_inode_info *inode)95{96 struct closure cl;97 98 closure_init_stack(&cl);99 bch2_inode_flush_nocow_writes_async(c, inode, &cl);100 closure_sync(&cl);101 102 return 0;103}104 105/* i_size updates: */106 107struct inode_new_size {108 loff_t new_size;109 u64 now;110 unsigned fields;111};112 113static int inode_set_size(struct btree_trans *trans,114 struct bch_inode_info *inode,115 struct bch_inode_unpacked *bi,116 void *p)117{118 struct inode_new_size *s = p;119 120 bi->bi_size = s->new_size;121 if (s->fields & ATTR_ATIME)122 bi->bi_atime = s->now;123 if (s->fields & ATTR_MTIME)124 bi->bi_mtime = s->now;125 if (s->fields & ATTR_CTIME)126 bi->bi_ctime = s->now;127 128 return 0;129}130 131int __must_check bch2_write_inode_size(struct bch_fs *c,132 struct bch_inode_info *inode,133 loff_t new_size, unsigned fields)134{135 struct inode_new_size s = {136 .new_size = new_size,137 .now = bch2_current_time(c),138 .fields = fields,139 };140 141 return bch2_write_inode(c, inode, inode_set_size, &s, fields);142}143 144void __bch2_i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode,145 struct quota_res *quota_res, s64 sectors)146{147 bch2_fs_inconsistent_on((s64) inode->v.i_blocks + sectors < 0, c,148 "inode %lu i_blocks underflow: %llu + %lli < 0 (ondisk %lli)",149 inode->v.i_ino, (u64) inode->v.i_blocks, sectors,150 inode->ei_inode.bi_sectors);151 inode->v.i_blocks += sectors;152 153#ifdef CONFIG_BCACHEFS_QUOTA154 if (quota_res &&155 !test_bit(EI_INODE_SNAPSHOT, &inode->ei_flags) &&156 sectors > 0) {157 BUG_ON(sectors > quota_res->sectors);158 BUG_ON(sectors > inode->ei_quota_reserved);159 160 quota_res->sectors -= sectors;161 inode->ei_quota_reserved -= sectors;162 } else {163 bch2_quota_acct(c, inode->ei_qid, Q_SPC, sectors, KEY_TYPE_QUOTA_WARN);164 }165#endif166}167 168/* fsync: */169 170/*171 * inode->ei_inode.bi_journal_seq won't be up to date since it's set in an172 * insert trigger: look up the btree inode instead173 */174static int bch2_flush_inode(struct bch_fs *c,175 struct bch_inode_info *inode)176{177 if (c->opts.journal_flush_disabled)178 return 0;179 180 if (!bch2_write_ref_tryget(c, BCH_WRITE_REF_fsync))181 return -EROFS;182 183 struct bch_inode_unpacked u;184 int ret = bch2_inode_find_by_inum(c, inode_inum(inode), &u) ?:185 bch2_journal_flush_seq(&c->journal, u.bi_journal_seq, TASK_INTERRUPTIBLE) ?:186 bch2_inode_flush_nocow_writes(c, inode);187 bch2_write_ref_put(c, BCH_WRITE_REF_fsync);188 return ret;189}190 191int bch2_fsync(struct file *file, loff_t start, loff_t end, int datasync)192{193 struct bch_inode_info *inode = file_bch_inode(file);194 struct bch_fs *c = inode->v.i_sb->s_fs_info;195 int ret, err;196 197 trace_bch2_fsync(file, datasync);198 199 ret = file_write_and_wait_range(file, start, end);200 if (ret)201 goto out;202 ret = sync_inode_metadata(&inode->v, 1);203 if (ret)204 goto out;205 ret = bch2_flush_inode(c, inode);206out:207 ret = bch2_err_class(ret);208 if (ret == -EROFS)209 ret = -EIO;210 211 err = file_check_and_advance_wb_err(file);212 if (!ret)213 ret = err;214 215 return ret;216}217 218/* truncate: */219 220static inline int range_has_data(struct bch_fs *c, u32 subvol,221 struct bpos start,222 struct bpos end)223{224 return bch2_trans_run(c,225 for_each_btree_key_in_subvolume_upto(trans, iter, BTREE_ID_extents, start, end,226 subvol, 0, k, ({227 bkey_extent_is_data(k.k) && !bkey_extent_is_unwritten(k);228 })));229}230 231static int __bch2_truncate_folio(struct bch_inode_info *inode,232 pgoff_t index, loff_t start, loff_t end)233{234 struct bch_fs *c = inode->v.i_sb->s_fs_info;235 struct address_space *mapping = inode->v.i_mapping;236 struct bch_folio *s;237 unsigned start_offset;238 unsigned end_offset;239 unsigned i;240 struct folio *folio;241 s64 i_sectors_delta = 0;242 int ret = 0;243 u64 end_pos;244 245 folio = filemap_lock_folio(mapping, index);246 if (IS_ERR_OR_NULL(folio)) {247 /*248 * XXX: we're doing two index lookups when we end up reading the249 * folio250 */251 ret = range_has_data(c, inode->ei_inum.subvol,252 POS(inode->v.i_ino, (index << PAGE_SECTORS_SHIFT)),253 POS(inode->v.i_ino, (index << PAGE_SECTORS_SHIFT) + PAGE_SECTORS));254 if (ret <= 0)255 return ret;256 257 folio = __filemap_get_folio(mapping, index,258 FGP_LOCK|FGP_CREAT, GFP_KERNEL);259 if (IS_ERR_OR_NULL(folio)) {260 ret = -ENOMEM;261 goto out;262 }263 }264 265 BUG_ON(start >= folio_end_pos(folio));266 BUG_ON(end <= folio_pos(folio));267 268 start_offset = max(start, folio_pos(folio)) - folio_pos(folio);269 end_offset = min_t(u64, end, folio_end_pos(folio)) - folio_pos(folio);270 271 /* Folio boundary? Nothing to do */272 if (start_offset == 0 &&273 end_offset == folio_size(folio)) {274 ret = 0;275 goto unlock;276 }277 278 s = bch2_folio_create(folio, 0);279 if (!s) {280 ret = -ENOMEM;281 goto unlock;282 }283 284 if (!folio_test_uptodate(folio)) {285 ret = bch2_read_single_folio(folio, mapping);286 if (ret)287 goto unlock;288 }289 290 ret = bch2_folio_set(c, inode_inum(inode), &folio, 1);291 if (ret)292 goto unlock;293 294 for (i = round_up(start_offset, block_bytes(c)) >> 9;295 i < round_down(end_offset, block_bytes(c)) >> 9;296 i++) {297 s->s[i].nr_replicas = 0;298 299 i_sectors_delta -= s->s[i].state == SECTOR_dirty;300 bch2_folio_sector_set(folio, s, i, SECTOR_unallocated);301 }302 303 bch2_i_sectors_acct(c, inode, NULL, i_sectors_delta);304 305 /*306 * Caller needs to know whether this folio will be written out by307 * writeback - doing an i_size update if necessary - or whether it will308 * be responsible for the i_size update.309 *310 * Note that we shouldn't ever see a folio beyond EOF, but check and311 * warn if so. This has been observed by failure to clean up folios312 * after a short write and there's still a chance reclaim will fix313 * things up.314 */315 WARN_ON_ONCE(folio_pos(folio) >= inode->v.i_size);316 end_pos = folio_end_pos(folio);317 if (inode->v.i_size > folio_pos(folio))318 end_pos = min_t(u64, inode->v.i_size, end_pos);319 ret = s->s[folio_pos_to_s(folio, end_pos - 1)].state >= SECTOR_dirty;320 321 folio_zero_segment(folio, start_offset, end_offset);322 323 /*324 * Bit of a hack - we don't want truncate to fail due to -ENOSPC.325 *326 * XXX: because we aren't currently tracking whether the folio has actual327 * data in it (vs. just 0s, or only partially written) this wrong. ick.328 */329 BUG_ON(bch2_get_folio_disk_reservation(c, inode, folio, false));330 331 /*332 * This removes any writeable userspace mappings; we need to force333 * .page_mkwrite to be called again before any mmapped writes, to334 * redirty the full page:335 */336 folio_mkclean(folio);337 filemap_dirty_folio(mapping, folio);338unlock:339 folio_unlock(folio);340 folio_put(folio);341out:342 return ret;343}344 345static int bch2_truncate_folio(struct bch_inode_info *inode, loff_t from)346{347 return __bch2_truncate_folio(inode, from >> PAGE_SHIFT,348 from, ANYSINT_MAX(loff_t));349}350 351static int bch2_truncate_folios(struct bch_inode_info *inode,352 loff_t start, loff_t end)353{354 int ret = __bch2_truncate_folio(inode, start >> PAGE_SHIFT,355 start, end);356 357 if (ret >= 0 &&358 start >> PAGE_SHIFT != end >> PAGE_SHIFT)359 ret = __bch2_truncate_folio(inode,360 (end - 1) >> PAGE_SHIFT,361 start, end);362 return ret;363}364 365static int bch2_extend(struct mnt_idmap *idmap,366 struct bch_inode_info *inode,367 struct bch_inode_unpacked *inode_u,368 struct iattr *iattr)369{370 struct address_space *mapping = inode->v.i_mapping;371 int ret;372 373 /*374 * sync appends:375 *376 * this has to be done _before_ extending i_size:377 */378 ret = filemap_write_and_wait_range(mapping, inode_u->bi_size, S64_MAX);379 if (ret)380 return ret;381 382 truncate_setsize(&inode->v, iattr->ia_size);383 384 return bch2_setattr_nonsize(idmap, inode, iattr);385}386 387int bchfs_truncate(struct mnt_idmap *idmap,388 struct bch_inode_info *inode, struct iattr *iattr)389{390 struct bch_fs *c = inode->v.i_sb->s_fs_info;391 struct address_space *mapping = inode->v.i_mapping;392 struct bch_inode_unpacked inode_u;393 s64 i_sectors_delta = 0;394 int ret = 0;395 396 /*397 * If the truncate call with change the size of the file, the398 * cmtimes should be updated. If the size will not change, we399 * do not need to update the cmtimes.400 */401 if (iattr->ia_size != inode->v.i_size) {402 if (!(iattr->ia_valid & ATTR_MTIME))403 ktime_get_coarse_real_ts64(&iattr->ia_mtime);404 if (!(iattr->ia_valid & ATTR_CTIME))405 ktime_get_coarse_real_ts64(&iattr->ia_ctime);406 iattr->ia_valid |= ATTR_MTIME|ATTR_CTIME;407 }408 409 inode_dio_wait(&inode->v);410 bch2_pagecache_block_get(inode);411 412 ret = bch2_inode_find_by_inum(c, inode_inum(inode), &inode_u);413 if (ret)414 goto err;415 416 /*417 * check this before next assertion; on filesystem error our normal418 * invariants are a bit broken (truncate has to truncate the page cache419 * before the inode).420 */421 ret = bch2_journal_error(&c->journal);422 if (ret)423 goto err;424 425 WARN_ONCE(!test_bit(EI_INODE_ERROR, &inode->ei_flags) &&426 inode->v.i_size < inode_u.bi_size,427 "truncate spotted in mem i_size < btree i_size: %llu < %llu\n",428 (u64) inode->v.i_size, inode_u.bi_size);429 430 if (iattr->ia_size > inode->v.i_size) {431 ret = bch2_extend(idmap, inode, &inode_u, iattr);432 goto err;433 }434 435 iattr->ia_valid &= ~ATTR_SIZE;436 437 ret = bch2_truncate_folio(inode, iattr->ia_size);438 if (unlikely(ret < 0))439 goto err;440 441 truncate_setsize(&inode->v, iattr->ia_size);442 443 /*444 * When extending, we're going to write the new i_size to disk445 * immediately so we need to flush anything above the current on disk446 * i_size first:447 *448 * Also, when extending we need to flush the page that i_size currently449 * straddles - if it's mapped to userspace, we need to ensure that450 * userspace has to redirty it and call .mkwrite -> set_page_dirty451 * again to allocate the part of the page that was extended.452 */453 if (iattr->ia_size > inode_u.bi_size)454 ret = filemap_write_and_wait_range(mapping,455 inode_u.bi_size,456 iattr->ia_size - 1);457 else if (iattr->ia_size & (PAGE_SIZE - 1))458 ret = filemap_write_and_wait_range(mapping,459 round_down(iattr->ia_size, PAGE_SIZE),460 iattr->ia_size - 1);461 if (ret)462 goto err;463 464 ret = bch2_truncate(c, inode_inum(inode), iattr->ia_size, &i_sectors_delta);465 bch2_i_sectors_acct(c, inode, NULL, i_sectors_delta);466 467 if (unlikely(ret)) {468 /*469 * If we error here, VFS caches are now inconsistent with btree470 */471 set_bit(EI_INODE_ERROR, &inode->ei_flags);472 goto err;473 }474 475 bch2_fs_inconsistent_on(!inode->v.i_size && inode->v.i_blocks &&476 !bch2_journal_error(&c->journal), c,477 "inode %lu truncated to 0 but i_blocks %llu (ondisk %lli)",478 inode->v.i_ino, (u64) inode->v.i_blocks,479 inode->ei_inode.bi_sectors);480 481 ret = bch2_setattr_nonsize(idmap, inode, iattr);482err:483 bch2_pagecache_block_put(inode);484 return bch2_err_class(ret);485}486 487/* fallocate: */488 489static int inode_update_times_fn(struct btree_trans *trans,490 struct bch_inode_info *inode,491 struct bch_inode_unpacked *bi, void *p)492{493 struct bch_fs *c = inode->v.i_sb->s_fs_info;494 495 bi->bi_mtime = bi->bi_ctime = bch2_current_time(c);496 return 0;497}498 499static noinline long bchfs_fpunch(struct bch_inode_info *inode, loff_t offset, loff_t len)500{501 struct bch_fs *c = inode->v.i_sb->s_fs_info;502 u64 end = offset + len;503 u64 block_start = round_up(offset, block_bytes(c));504 u64 block_end = round_down(end, block_bytes(c));505 bool truncated_last_page;506 int ret = 0;507 508 ret = bch2_truncate_folios(inode, offset, end);509 if (unlikely(ret < 0))510 goto err;511 512 truncated_last_page = ret;513 514 truncate_pagecache_range(&inode->v, offset, end - 1);515 516 if (block_start < block_end) {517 s64 i_sectors_delta = 0;518 519 ret = bch2_fpunch(c, inode_inum(inode),520 block_start >> 9, block_end >> 9,521 &i_sectors_delta);522 bch2_i_sectors_acct(c, inode, NULL, i_sectors_delta);523 }524 525 mutex_lock(&inode->ei_update_lock);526 if (end >= inode->v.i_size && !truncated_last_page) {527 ret = bch2_write_inode_size(c, inode, inode->v.i_size,528 ATTR_MTIME|ATTR_CTIME);529 } else {530 ret = bch2_write_inode(c, inode, inode_update_times_fn, NULL,531 ATTR_MTIME|ATTR_CTIME);532 }533 mutex_unlock(&inode->ei_update_lock);534err:535 return ret;536}537 538static noinline long bchfs_fcollapse_finsert(struct bch_inode_info *inode,539 loff_t offset, loff_t len,540 bool insert)541{542 struct bch_fs *c = inode->v.i_sb->s_fs_info;543 struct address_space *mapping = inode->v.i_mapping;544 s64 i_sectors_delta = 0;545 int ret = 0;546 547 if ((offset | len) & (block_bytes(c) - 1))548 return -EINVAL;549 550 if (insert) {551 if (offset >= inode->v.i_size)552 return -EINVAL;553 } else {554 if (offset + len >= inode->v.i_size)555 return -EINVAL;556 }557 558 ret = bch2_write_invalidate_inode_pages_range(mapping, offset, LLONG_MAX);559 if (ret)560 return ret;561 562 if (insert)563 i_size_write(&inode->v, inode->v.i_size + len);564 565 ret = bch2_fcollapse_finsert(c, inode_inum(inode), offset >> 9, len >> 9,566 insert, &i_sectors_delta);567 if (!ret && !insert)568 i_size_write(&inode->v, inode->v.i_size - len);569 bch2_i_sectors_acct(c, inode, NULL, i_sectors_delta);570 571 return ret;572}573 574static noinline int __bchfs_fallocate(struct bch_inode_info *inode, int mode,575 u64 start_sector, u64 end_sector)576{577 struct bch_fs *c = inode->v.i_sb->s_fs_info;578 struct btree_trans *trans = bch2_trans_get(c);579 struct btree_iter iter;580 struct bpos end_pos = POS(inode->v.i_ino, end_sector);581 struct bch_io_opts opts;582 int ret = 0;583 584 bch2_inode_opts_get(&opts, c, &inode->ei_inode);585 586 bch2_trans_iter_init(trans, &iter, BTREE_ID_extents,587 POS(inode->v.i_ino, start_sector),588 BTREE_ITER_slots|BTREE_ITER_intent);589 590 while (!ret) {591 s64 i_sectors_delta = 0;592 struct quota_res quota_res = { 0 };593 struct bkey_s_c k;594 unsigned sectors;595 bool is_allocation;596 u64 hole_start, hole_end;597 u32 snapshot;598 599 bch2_trans_begin(trans);600 601 if (bkey_ge(iter.pos, end_pos))602 break;603 604 ret = bch2_subvolume_get_snapshot(trans,605 inode->ei_inum.subvol, &snapshot);606 if (ret)607 goto bkey_err;608 609 bch2_btree_iter_set_snapshot(&iter, snapshot);610 611 k = bch2_btree_iter_peek_slot(&iter);612 if ((ret = bkey_err(k)))613 goto bkey_err;614 615 hole_start = iter.pos.offset;616 hole_end = bpos_min(k.k->p, end_pos).offset;617 is_allocation = bkey_extent_is_allocation(k.k);618 619 /* already reserved */620 if (bkey_extent_is_reservation(k) &&621 bch2_bkey_nr_ptrs_fully_allocated(k) >= opts.data_replicas) {622 bch2_btree_iter_advance(&iter);623 continue;624 }625 626 if (bkey_extent_is_data(k.k) &&627 !(mode & FALLOC_FL_ZERO_RANGE)) {628 bch2_btree_iter_advance(&iter);629 continue;630 }631 632 if (!(mode & FALLOC_FL_ZERO_RANGE)) {633 /*634 * Lock ordering - can't be holding btree locks while635 * blocking on a folio lock:636 */637 if (bch2_clamp_data_hole(&inode->v,638 &hole_start,639 &hole_end,640 opts.data_replicas, true)) {641 ret = drop_locks_do(trans,642 (bch2_clamp_data_hole(&inode->v,643 &hole_start,644 &hole_end,645 opts.data_replicas, false), 0));646 if (ret)647 goto bkey_err;648 }649 bch2_btree_iter_set_pos(&iter, POS(iter.pos.inode, hole_start));650 651 if (ret)652 goto bkey_err;653 654 if (hole_start == hole_end)655 continue;656 }657 658 sectors = hole_end - hole_start;659 660 if (!is_allocation) {661 ret = bch2_quota_reservation_add(c, inode,662 "a_res, sectors, true);663 if (unlikely(ret))664 goto bkey_err;665 }666 667 ret = bch2_extent_fallocate(trans, inode_inum(inode), &iter,668 sectors, opts, &i_sectors_delta,669 writepoint_hashed((unsigned long) current));670 if (ret)671 goto bkey_err;672 673 bch2_i_sectors_acct(c, inode, "a_res, i_sectors_delta);674 675 if (bch2_mark_pagecache_reserved(inode, &hole_start,676 iter.pos.offset, true)) {677 ret = drop_locks_do(trans,678 bch2_mark_pagecache_reserved(inode, &hole_start,679 iter.pos.offset, false));680 if (ret)681 goto bkey_err;682 }683bkey_err:684 bch2_quota_reservation_put(c, inode, "a_res);685 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))686 ret = 0;687 }688 689 if (bch2_err_matches(ret, ENOSPC) && (mode & FALLOC_FL_ZERO_RANGE)) {690 struct quota_res quota_res = { 0 };691 s64 i_sectors_delta = 0;692 693 bch2_fpunch_at(trans, &iter, inode_inum(inode),694 end_sector, &i_sectors_delta);695 bch2_i_sectors_acct(c, inode, "a_res, i_sectors_delta);696 bch2_quota_reservation_put(c, inode, "a_res);697 }698 699 bch2_trans_iter_exit(trans, &iter);700 bch2_trans_put(trans);701 return ret;702}703 704static noinline long bchfs_fallocate(struct bch_inode_info *inode, int mode,705 loff_t offset, loff_t len)706{707 struct bch_fs *c = inode->v.i_sb->s_fs_info;708 u64 end = offset + len;709 u64 block_start = round_down(offset, block_bytes(c));710 u64 block_end = round_up(end, block_bytes(c));711 bool truncated_last_page = false;712 int ret, ret2 = 0;713 714 if (!(mode & FALLOC_FL_KEEP_SIZE) && end > inode->v.i_size) {715 ret = inode_newsize_ok(&inode->v, end);716 if (ret)717 return ret;718 }719 720 if (mode & FALLOC_FL_ZERO_RANGE) {721 ret = bch2_truncate_folios(inode, offset, end);722 if (unlikely(ret < 0))723 return ret;724 725 truncated_last_page = ret;726 727 truncate_pagecache_range(&inode->v, offset, end - 1);728 729 block_start = round_up(offset, block_bytes(c));730 block_end = round_down(end, block_bytes(c));731 }732 733 ret = __bchfs_fallocate(inode, mode, block_start >> 9, block_end >> 9);734 735 /*736 * On -ENOSPC in ZERO_RANGE mode, we still want to do the inode update,737 * so that the VFS cache i_size is consistent with the btree i_size:738 */739 if (ret &&740 !(bch2_err_matches(ret, ENOSPC) && (mode & FALLOC_FL_ZERO_RANGE)))741 return ret;742 743 if (mode & FALLOC_FL_KEEP_SIZE && end > inode->v.i_size)744 end = inode->v.i_size;745 746 if (end >= inode->v.i_size &&747 (((mode & FALLOC_FL_ZERO_RANGE) && !truncated_last_page) ||748 !(mode & FALLOC_FL_KEEP_SIZE))) {749 spin_lock(&inode->v.i_lock);750 i_size_write(&inode->v, end);751 spin_unlock(&inode->v.i_lock);752 753 mutex_lock(&inode->ei_update_lock);754 ret2 = bch2_write_inode_size(c, inode, end, 0);755 mutex_unlock(&inode->ei_update_lock);756 }757 758 return ret ?: ret2;759}760 761long bch2_fallocate_dispatch(struct file *file, int mode,762 loff_t offset, loff_t len)763{764 struct bch_inode_info *inode = file_bch_inode(file);765 struct bch_fs *c = inode->v.i_sb->s_fs_info;766 long ret;767 768 if (!bch2_write_ref_tryget(c, BCH_WRITE_REF_fallocate))769 return -EROFS;770 771 inode_lock(&inode->v);772 inode_dio_wait(&inode->v);773 bch2_pagecache_block_get(inode);774 775 ret = file_modified(file);776 if (ret)777 goto err;778 779 if (!(mode & ~(FALLOC_FL_KEEP_SIZE|FALLOC_FL_ZERO_RANGE)))780 ret = bchfs_fallocate(inode, mode, offset, len);781 else if (mode == (FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE))782 ret = bchfs_fpunch(inode, offset, len);783 else if (mode == FALLOC_FL_INSERT_RANGE)784 ret = bchfs_fcollapse_finsert(inode, offset, len, true);785 else if (mode == FALLOC_FL_COLLAPSE_RANGE)786 ret = bchfs_fcollapse_finsert(inode, offset, len, false);787 else788 ret = -EOPNOTSUPP;789err:790 bch2_pagecache_block_put(inode);791 inode_unlock(&inode->v);792 bch2_write_ref_put(c, BCH_WRITE_REF_fallocate);793 794 return bch2_err_class(ret);795}796 797/*798 * Take a quota reservation for unallocated blocks in a given file range799 * Does not check pagecache800 */801static int quota_reserve_range(struct bch_inode_info *inode,802 struct quota_res *res,803 u64 start, u64 end)804{805 struct bch_fs *c = inode->v.i_sb->s_fs_info;806 u64 sectors = end - start;807 808 int ret = bch2_trans_run(c,809 for_each_btree_key_in_subvolume_upto(trans, iter,810 BTREE_ID_extents,811 POS(inode->v.i_ino, start),812 POS(inode->v.i_ino, end - 1),813 inode->ei_inum.subvol, 0, k, ({814 if (bkey_extent_is_allocation(k.k)) {815 u64 s = min(end, k.k->p.offset) -816 max(start, bkey_start_offset(k.k));817 BUG_ON(s > sectors);818 sectors -= s;819 }820 821 0;822 })));823 824 return ret ?: bch2_quota_reservation_add(c, inode, res, sectors, true);825}826 827loff_t bch2_remap_file_range(struct file *file_src, loff_t pos_src,828 struct file *file_dst, loff_t pos_dst,829 loff_t len, unsigned remap_flags)830{831 struct bch_inode_info *src = file_bch_inode(file_src);832 struct bch_inode_info *dst = file_bch_inode(file_dst);833 struct bch_fs *c = src->v.i_sb->s_fs_info;834 struct quota_res quota_res = { 0 };835 s64 i_sectors_delta = 0;836 u64 aligned_len;837 loff_t ret = 0;838 839 if (remap_flags & ~(REMAP_FILE_DEDUP|REMAP_FILE_ADVISORY))840 return -EINVAL;841 842 if ((pos_src & (block_bytes(c) - 1)) ||843 (pos_dst & (block_bytes(c) - 1)))844 return -EINVAL;845 846 if (src == dst &&847 abs(pos_src - pos_dst) < len)848 return -EINVAL;849 850 lock_two_nondirectories(&src->v, &dst->v);851 bch2_lock_inodes(INODE_PAGECACHE_BLOCK, src, dst);852 853 inode_dio_wait(&src->v);854 inode_dio_wait(&dst->v);855 856 ret = generic_remap_file_range_prep(file_src, pos_src,857 file_dst, pos_dst,858 &len, remap_flags);859 if (ret < 0 || len == 0)860 goto err;861 862 aligned_len = round_up((u64) len, block_bytes(c));863 864 ret = bch2_write_invalidate_inode_pages_range(dst->v.i_mapping,865 pos_dst, pos_dst + len - 1);866 if (ret)867 goto err;868 869 ret = quota_reserve_range(dst, "a_res, pos_dst >> 9,870 (pos_dst + aligned_len) >> 9);871 if (ret)872 goto err;873 874 if (!(remap_flags & REMAP_FILE_DEDUP))875 file_update_time(file_dst);876 877 bch2_mark_pagecache_unallocated(src, pos_src >> 9,878 (pos_src + aligned_len) >> 9);879 880 ret = bch2_remap_range(c,881 inode_inum(dst), pos_dst >> 9,882 inode_inum(src), pos_src >> 9,883 aligned_len >> 9,884 pos_dst + len, &i_sectors_delta);885 if (ret < 0)886 goto err;887 888 /*889 * due to alignment, we might have remapped slightly more than requsted890 */891 ret = min((u64) ret << 9, (u64) len);892 893 bch2_i_sectors_acct(c, dst, "a_res, i_sectors_delta);894 895 spin_lock(&dst->v.i_lock);896 if (pos_dst + ret > dst->v.i_size)897 i_size_write(&dst->v, pos_dst + ret);898 spin_unlock(&dst->v.i_lock);899 900 if ((file_dst->f_flags & (__O_SYNC | O_DSYNC)) ||901 IS_SYNC(file_inode(file_dst)))902 ret = bch2_flush_inode(c, dst);903err:904 bch2_quota_reservation_put(c, dst, "a_res);905 bch2_unlock_inodes(INODE_PAGECACHE_BLOCK, src, dst);906 unlock_two_nondirectories(&src->v, &dst->v);907 908 return bch2_err_class(ret);909}910 911/* fseek: */912 913static loff_t bch2_seek_data(struct file *file, u64 offset)914{915 struct bch_inode_info *inode = file_bch_inode(file);916 struct bch_fs *c = inode->v.i_sb->s_fs_info;917 subvol_inum inum = inode_inum(inode);918 u64 isize, next_data = MAX_LFS_FILESIZE;919 920 isize = i_size_read(&inode->v);921 if (offset >= isize)922 return -ENXIO;923 924 int ret = bch2_trans_run(c,925 for_each_btree_key_in_subvolume_upto(trans, iter, BTREE_ID_extents,926 POS(inode->v.i_ino, offset >> 9),927 POS(inode->v.i_ino, U64_MAX),928 inum.subvol, 0, k, ({929 if (bkey_extent_is_data(k.k)) {930 next_data = max(offset, bkey_start_offset(k.k) << 9);931 break;932 } else if (k.k->p.offset >> 9 > isize)933 break;934 0;935 })));936 if (ret)937 return ret;938 939 if (next_data > offset)940 next_data = bch2_seek_pagecache_data(&inode->v,941 offset, next_data, 0, false);942 943 if (next_data >= isize)944 return -ENXIO;945 946 return vfs_setpos(file, next_data, MAX_LFS_FILESIZE);947}948 949static loff_t bch2_seek_hole(struct file *file, u64 offset)950{951 struct bch_inode_info *inode = file_bch_inode(file);952 struct bch_fs *c = inode->v.i_sb->s_fs_info;953 subvol_inum inum = inode_inum(inode);954 u64 isize, next_hole = MAX_LFS_FILESIZE;955 956 isize = i_size_read(&inode->v);957 if (offset >= isize)958 return -ENXIO;959 960 int ret = bch2_trans_run(c,961 for_each_btree_key_in_subvolume_upto(trans, iter, BTREE_ID_extents,962 POS(inode->v.i_ino, offset >> 9),963 POS(inode->v.i_ino, U64_MAX),964 inum.subvol, BTREE_ITER_slots, k, ({965 if (k.k->p.inode != inode->v.i_ino) {966 next_hole = bch2_seek_pagecache_hole(&inode->v,967 offset, MAX_LFS_FILESIZE, 0, false);968 break;969 } else if (!bkey_extent_is_data(k.k)) {970 next_hole = bch2_seek_pagecache_hole(&inode->v,971 max(offset, bkey_start_offset(k.k) << 9),972 k.k->p.offset << 9, 0, false);973 974 if (next_hole < k.k->p.offset << 9)975 break;976 } else {977 offset = max(offset, bkey_start_offset(k.k) << 9);978 }979 0;980 })));981 if (ret)982 return ret;983 984 if (next_hole > isize)985 next_hole = isize;986 987 return vfs_setpos(file, next_hole, MAX_LFS_FILESIZE);988}989 990loff_t bch2_llseek(struct file *file, loff_t offset, int whence)991{992 loff_t ret;993 994 switch (whence) {995 case SEEK_SET:996 case SEEK_CUR:997 case SEEK_END:998 ret = generic_file_llseek(file, offset, whence);999 break;1000 case SEEK_DATA:1001 ret = bch2_seek_data(file, offset);1002 break;1003 case SEEK_HOLE:1004 ret = bch2_seek_hole(file, offset);1005 break;1006 default:1007 ret = -EINVAL;1008 break;1009 }1010 1011 return bch2_err_class(ret);1012}1013 1014void bch2_fs_fsio_exit(struct bch_fs *c)1015{1016 bioset_exit(&c->nocow_flush_bioset);1017}1018 1019int bch2_fs_fsio_init(struct bch_fs *c)1020{1021 if (bioset_init(&c->nocow_flush_bioset,1022 1, offsetof(struct nocow_flush, bio), 0))1023 return -BCH_ERR_ENOMEM_nocow_flush_bioset_init;1024 1025 return 0;1026}1027 1028#endif /* NO_BCACHEFS_FS */1029