10218 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (C) 2007 Oracle. All rights reserved.4 */5 6#include <crypto/hash.h>7#include <linux/kernel.h>8#include <linux/bio.h>9#include <linux/blk-cgroup.h>10#include <linux/file.h>11#include <linux/fs.h>12#include <linux/pagemap.h>13#include <linux/highmem.h>14#include <linux/time.h>15#include <linux/init.h>16#include <linux/string.h>17#include <linux/backing-dev.h>18#include <linux/writeback.h>19#include <linux/compat.h>20#include <linux/xattr.h>21#include <linux/posix_acl.h>22#include <linux/falloc.h>23#include <linux/slab.h>24#include <linux/ratelimit.h>25#include <linux/btrfs.h>26#include <linux/blkdev.h>27#include <linux/posix_acl_xattr.h>28#include <linux/uio.h>29#include <linux/magic.h>30#include <linux/iversion.h>31#include <linux/swap.h>32#include <linux/migrate.h>33#include <linux/sched/mm.h>34#include <linux/iomap.h>35#include <linux/unaligned.h>36#include <linux/fsverity.h>37#include "misc.h"38#include "ctree.h"39#include "disk-io.h"40#include "transaction.h"41#include "btrfs_inode.h"42#include "ordered-data.h"43#include "xattr.h"44#include "tree-log.h"45#include "bio.h"46#include "compression.h"47#include "locking.h"48#include "props.h"49#include "qgroup.h"50#include "delalloc-space.h"51#include "block-group.h"52#include "space-info.h"53#include "zoned.h"54#include "subpage.h"55#include "inode-item.h"56#include "fs.h"57#include "accessors.h"58#include "extent-tree.h"59#include "root-tree.h"60#include "defrag.h"61#include "dir-item.h"62#include "file-item.h"63#include "uuid-tree.h"64#include "ioctl.h"65#include "file.h"66#include "acl.h"67#include "relocation.h"68#include "verity.h"69#include "super.h"70#include "orphan.h"71#include "backref.h"72#include "raid-stripe-tree.h"73#include "fiemap.h"74 75struct btrfs_iget_args {76 u64 ino;77 struct btrfs_root *root;78};79 80struct btrfs_rename_ctx {81 /* Output field. Stores the index number of the old directory entry. */82 u64 index;83};84 85/*86 * Used by data_reloc_print_warning_inode() to pass needed info for filename87 * resolution and output of error message.88 */89struct data_reloc_warn {90 struct btrfs_path path;91 struct btrfs_fs_info *fs_info;92 u64 extent_item_size;93 u64 logical;94 int mirror_num;95};96 97/*98 * For the file_extent_tree, we want to hold the inode lock when we lookup and99 * update the disk_i_size, but lockdep will complain because our io_tree we hold100 * the tree lock and get the inode lock when setting delalloc. These two things101 * are unrelated, so make a class for the file_extent_tree so we don't get the102 * two locking patterns mixed up.103 */104static struct lock_class_key file_extent_tree_class;105 106static const struct inode_operations btrfs_dir_inode_operations;107static const struct inode_operations btrfs_symlink_inode_operations;108static const struct inode_operations btrfs_special_inode_operations;109static const struct inode_operations btrfs_file_inode_operations;110static const struct address_space_operations btrfs_aops;111static const struct file_operations btrfs_dir_file_operations;112 113static struct kmem_cache *btrfs_inode_cachep;114 115static int btrfs_setsize(struct inode *inode, struct iattr *attr);116static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback);117 118static noinline int run_delalloc_cow(struct btrfs_inode *inode,119 struct folio *locked_folio, u64 start,120 u64 end, struct writeback_control *wbc,121 bool pages_dirty);122 123static int data_reloc_print_warning_inode(u64 inum, u64 offset, u64 num_bytes,124 u64 root, void *warn_ctx)125{126 struct data_reloc_warn *warn = warn_ctx;127 struct btrfs_fs_info *fs_info = warn->fs_info;128 struct extent_buffer *eb;129 struct btrfs_inode_item *inode_item;130 struct inode_fs_paths *ipath = NULL;131 struct btrfs_root *local_root;132 struct btrfs_key key;133 unsigned int nofs_flag;134 u32 nlink;135 int ret;136 137 local_root = btrfs_get_fs_root(fs_info, root, true);138 if (IS_ERR(local_root)) {139 ret = PTR_ERR(local_root);140 goto err;141 }142 143 /* This makes the path point to (inum INODE_ITEM ioff). */144 key.objectid = inum;145 key.type = BTRFS_INODE_ITEM_KEY;146 key.offset = 0;147 148 ret = btrfs_search_slot(NULL, local_root, &key, &warn->path, 0, 0);149 if (ret) {150 btrfs_put_root(local_root);151 btrfs_release_path(&warn->path);152 goto err;153 }154 155 eb = warn->path.nodes[0];156 inode_item = btrfs_item_ptr(eb, warn->path.slots[0], struct btrfs_inode_item);157 nlink = btrfs_inode_nlink(eb, inode_item);158 btrfs_release_path(&warn->path);159 160 nofs_flag = memalloc_nofs_save();161 ipath = init_ipath(4096, local_root, &warn->path);162 memalloc_nofs_restore(nofs_flag);163 if (IS_ERR(ipath)) {164 btrfs_put_root(local_root);165 ret = PTR_ERR(ipath);166 ipath = NULL;167 /*168 * -ENOMEM, not a critical error, just output an generic error169 * without filename.170 */171 btrfs_warn(fs_info,172"checksum error at logical %llu mirror %u root %llu, inode %llu offset %llu",173 warn->logical, warn->mirror_num, root, inum, offset);174 return ret;175 }176 ret = paths_from_inode(inum, ipath);177 if (ret < 0)178 goto err;179 180 /*181 * We deliberately ignore the bit ipath might have been too small to182 * hold all of the paths here183 */184 for (int i = 0; i < ipath->fspath->elem_cnt; i++) {185 btrfs_warn(fs_info,186"checksum error at logical %llu mirror %u root %llu inode %llu offset %llu length %u links %u (path: %s)",187 warn->logical, warn->mirror_num, root, inum, offset,188 fs_info->sectorsize, nlink,189 (char *)(unsigned long)ipath->fspath->val[i]);190 }191 192 btrfs_put_root(local_root);193 free_ipath(ipath);194 return 0;195 196err:197 btrfs_warn(fs_info,198"checksum error at logical %llu mirror %u root %llu inode %llu offset %llu, path resolving failed with ret=%d",199 warn->logical, warn->mirror_num, root, inum, offset, ret);200 201 free_ipath(ipath);202 return ret;203}204 205/*206 * Do extra user-friendly error output (e.g. lookup all the affected files).207 *208 * Return true if we succeeded doing the backref lookup.209 * Return false if such lookup failed, and has to fallback to the old error message.210 */211static void print_data_reloc_error(const struct btrfs_inode *inode, u64 file_off,212 const u8 *csum, const u8 *csum_expected,213 int mirror_num)214{215 struct btrfs_fs_info *fs_info = inode->root->fs_info;216 struct btrfs_path path = { 0 };217 struct btrfs_key found_key = { 0 };218 struct extent_buffer *eb;219 struct btrfs_extent_item *ei;220 const u32 csum_size = fs_info->csum_size;221 u64 logical;222 u64 flags;223 u32 item_size;224 int ret;225 226 mutex_lock(&fs_info->reloc_mutex);227 logical = btrfs_get_reloc_bg_bytenr(fs_info);228 mutex_unlock(&fs_info->reloc_mutex);229 230 if (logical == U64_MAX) {231 btrfs_warn_rl(fs_info, "has data reloc tree but no running relocation");232 btrfs_warn_rl(fs_info,233"csum failed root %lld ino %llu off %llu csum " CSUM_FMT " expected csum " CSUM_FMT " mirror %d",234 btrfs_root_id(inode->root), btrfs_ino(inode), file_off,235 CSUM_FMT_VALUE(csum_size, csum),236 CSUM_FMT_VALUE(csum_size, csum_expected),237 mirror_num);238 return;239 }240 241 logical += file_off;242 btrfs_warn_rl(fs_info,243"csum failed root %lld ino %llu off %llu logical %llu csum " CSUM_FMT " expected csum " CSUM_FMT " mirror %d",244 btrfs_root_id(inode->root),245 btrfs_ino(inode), file_off, logical,246 CSUM_FMT_VALUE(csum_size, csum),247 CSUM_FMT_VALUE(csum_size, csum_expected),248 mirror_num);249 250 ret = extent_from_logical(fs_info, logical, &path, &found_key, &flags);251 if (ret < 0) {252 btrfs_err_rl(fs_info, "failed to lookup extent item for logical %llu: %d",253 logical, ret);254 return;255 }256 eb = path.nodes[0];257 ei = btrfs_item_ptr(eb, path.slots[0], struct btrfs_extent_item);258 item_size = btrfs_item_size(eb, path.slots[0]);259 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {260 unsigned long ptr = 0;261 u64 ref_root;262 u8 ref_level;263 264 while (true) {265 ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,266 item_size, &ref_root,267 &ref_level);268 if (ret < 0) {269 btrfs_warn_rl(fs_info,270 "failed to resolve tree backref for logical %llu: %d",271 logical, ret);272 break;273 }274 if (ret > 0)275 break;276 277 btrfs_warn_rl(fs_info,278"csum error at logical %llu mirror %u: metadata %s (level %d) in tree %llu",279 logical, mirror_num,280 (ref_level ? "node" : "leaf"),281 ref_level, ref_root);282 }283 btrfs_release_path(&path);284 } else {285 struct btrfs_backref_walk_ctx ctx = { 0 };286 struct data_reloc_warn reloc_warn = { 0 };287 288 btrfs_release_path(&path);289 290 ctx.bytenr = found_key.objectid;291 ctx.extent_item_pos = logical - found_key.objectid;292 ctx.fs_info = fs_info;293 294 reloc_warn.logical = logical;295 reloc_warn.extent_item_size = found_key.offset;296 reloc_warn.mirror_num = mirror_num;297 reloc_warn.fs_info = fs_info;298 299 iterate_extent_inodes(&ctx, true,300 data_reloc_print_warning_inode, &reloc_warn);301 }302}303 304static void __cold btrfs_print_data_csum_error(struct btrfs_inode *inode,305 u64 logical_start, u8 *csum, u8 *csum_expected, int mirror_num)306{307 struct btrfs_root *root = inode->root;308 const u32 csum_size = root->fs_info->csum_size;309 310 /* For data reloc tree, it's better to do a backref lookup instead. */311 if (btrfs_root_id(root) == BTRFS_DATA_RELOC_TREE_OBJECTID)312 return print_data_reloc_error(inode, logical_start, csum,313 csum_expected, mirror_num);314 315 /* Output without objectid, which is more meaningful */316 if (btrfs_root_id(root) >= BTRFS_LAST_FREE_OBJECTID) {317 btrfs_warn_rl(root->fs_info,318"csum failed root %lld ino %lld off %llu csum " CSUM_FMT " expected csum " CSUM_FMT " mirror %d",319 btrfs_root_id(root), btrfs_ino(inode),320 logical_start,321 CSUM_FMT_VALUE(csum_size, csum),322 CSUM_FMT_VALUE(csum_size, csum_expected),323 mirror_num);324 } else {325 btrfs_warn_rl(root->fs_info,326"csum failed root %llu ino %llu off %llu csum " CSUM_FMT " expected csum " CSUM_FMT " mirror %d",327 btrfs_root_id(root), btrfs_ino(inode),328 logical_start,329 CSUM_FMT_VALUE(csum_size, csum),330 CSUM_FMT_VALUE(csum_size, csum_expected),331 mirror_num);332 }333}334 335/*336 * Lock inode i_rwsem based on arguments passed.337 *338 * ilock_flags can have the following bit set:339 *340 * BTRFS_ILOCK_SHARED - acquire a shared lock on the inode341 * BTRFS_ILOCK_TRY - try to acquire the lock, if fails on first attempt342 * return -EAGAIN343 * BTRFS_ILOCK_MMAP - acquire a write lock on the i_mmap_lock344 */345int btrfs_inode_lock(struct btrfs_inode *inode, unsigned int ilock_flags)346{347 if (ilock_flags & BTRFS_ILOCK_SHARED) {348 if (ilock_flags & BTRFS_ILOCK_TRY) {349 if (!inode_trylock_shared(&inode->vfs_inode))350 return -EAGAIN;351 else352 return 0;353 }354 inode_lock_shared(&inode->vfs_inode);355 } else {356 if (ilock_flags & BTRFS_ILOCK_TRY) {357 if (!inode_trylock(&inode->vfs_inode))358 return -EAGAIN;359 else360 return 0;361 }362 inode_lock(&inode->vfs_inode);363 }364 if (ilock_flags & BTRFS_ILOCK_MMAP)365 down_write(&inode->i_mmap_lock);366 return 0;367}368 369/*370 * Unock inode i_rwsem.371 *372 * ilock_flags should contain the same bits set as passed to btrfs_inode_lock()373 * to decide whether the lock acquired is shared or exclusive.374 */375void btrfs_inode_unlock(struct btrfs_inode *inode, unsigned int ilock_flags)376{377 if (ilock_flags & BTRFS_ILOCK_MMAP)378 up_write(&inode->i_mmap_lock);379 if (ilock_flags & BTRFS_ILOCK_SHARED)380 inode_unlock_shared(&inode->vfs_inode);381 else382 inode_unlock(&inode->vfs_inode);383}384 385/*386 * Cleanup all submitted ordered extents in specified range to handle errors387 * from the btrfs_run_delalloc_range() callback.388 *389 * NOTE: caller must ensure that when an error happens, it can not call390 * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING391 * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata392 * to be released, which we want to happen only when finishing the ordered393 * extent (btrfs_finish_ordered_io()).394 */395static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode,396 struct folio *locked_folio,397 u64 offset, u64 bytes)398{399 unsigned long index = offset >> PAGE_SHIFT;400 unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT;401 u64 page_start = 0, page_end = 0;402 struct folio *folio;403 404 if (locked_folio) {405 page_start = folio_pos(locked_folio);406 page_end = page_start + folio_size(locked_folio) - 1;407 }408 409 while (index <= end_index) {410 /*411 * For locked page, we will call btrfs_mark_ordered_io_finished412 * through btrfs_mark_ordered_io_finished() on it413 * in run_delalloc_range() for the error handling, which will414 * clear page Ordered and run the ordered extent accounting.415 *416 * Here we can't just clear the Ordered bit, or417 * btrfs_mark_ordered_io_finished() would skip the accounting418 * for the page range, and the ordered extent will never finish.419 */420 if (locked_folio && index == (page_start >> PAGE_SHIFT)) {421 index++;422 continue;423 }424 folio = __filemap_get_folio(inode->vfs_inode.i_mapping, index, 0, 0);425 index++;426 if (IS_ERR(folio))427 continue;428 429 /*430 * Here we just clear all Ordered bits for every page in the431 * range, then btrfs_mark_ordered_io_finished() will handle432 * the ordered extent accounting for the range.433 */434 btrfs_folio_clamp_clear_ordered(inode->root->fs_info, folio,435 offset, bytes);436 folio_put(folio);437 }438 439 if (locked_folio) {440 /* The locked page covers the full range, nothing needs to be done */441 if (bytes + offset <= page_start + folio_size(locked_folio))442 return;443 /*444 * In case this page belongs to the delalloc range being445 * instantiated then skip it, since the first page of a range is446 * going to be properly cleaned up by the caller of447 * run_delalloc_range448 */449 if (page_start >= offset && page_end <= (offset + bytes - 1)) {450 bytes = offset + bytes - folio_pos(locked_folio) -451 folio_size(locked_folio);452 offset = folio_pos(locked_folio) + folio_size(locked_folio);453 }454 }455 456 return btrfs_mark_ordered_io_finished(inode, NULL, offset, bytes, false);457}458 459static int btrfs_dirty_inode(struct btrfs_inode *inode);460 461static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,462 struct btrfs_new_inode_args *args)463{464 int err;465 466 if (args->default_acl) {467 err = __btrfs_set_acl(trans, args->inode, args->default_acl,468 ACL_TYPE_DEFAULT);469 if (err)470 return err;471 }472 if (args->acl) {473 err = __btrfs_set_acl(trans, args->inode, args->acl, ACL_TYPE_ACCESS);474 if (err)475 return err;476 }477 if (!args->default_acl && !args->acl)478 cache_no_acl(args->inode);479 return btrfs_xattr_security_init(trans, args->inode, args->dir,480 &args->dentry->d_name);481}482 483/*484 * this does all the hard work for inserting an inline extent into485 * the btree. The caller should have done a btrfs_drop_extents so that486 * no overlapping inline items exist in the btree487 */488static int insert_inline_extent(struct btrfs_trans_handle *trans,489 struct btrfs_path *path,490 struct btrfs_inode *inode, bool extent_inserted,491 size_t size, size_t compressed_size,492 int compress_type,493 struct folio *compressed_folio,494 bool update_i_size)495{496 struct btrfs_root *root = inode->root;497 struct extent_buffer *leaf;498 const u32 sectorsize = trans->fs_info->sectorsize;499 char *kaddr;500 unsigned long ptr;501 struct btrfs_file_extent_item *ei;502 int ret;503 size_t cur_size = size;504 u64 i_size;505 506 /*507 * The decompressed size must still be no larger than a sector. Under508 * heavy race, we can have size == 0 passed in, but that shouldn't be a509 * big deal and we can continue the insertion.510 */511 ASSERT(size <= sectorsize);512 513 /*514 * The compressed size also needs to be no larger than a sector.515 * That's also why we only need one page as the parameter.516 */517 if (compressed_folio)518 ASSERT(compressed_size <= sectorsize);519 else520 ASSERT(compressed_size == 0);521 522 if (compressed_size && compressed_folio)523 cur_size = compressed_size;524 525 if (!extent_inserted) {526 struct btrfs_key key;527 size_t datasize;528 529 key.objectid = btrfs_ino(inode);530 key.offset = 0;531 key.type = BTRFS_EXTENT_DATA_KEY;532 533 datasize = btrfs_file_extent_calc_inline_size(cur_size);534 ret = btrfs_insert_empty_item(trans, root, path, &key,535 datasize);536 if (ret)537 goto fail;538 }539 leaf = path->nodes[0];540 ei = btrfs_item_ptr(leaf, path->slots[0],541 struct btrfs_file_extent_item);542 btrfs_set_file_extent_generation(leaf, ei, trans->transid);543 btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);544 btrfs_set_file_extent_encryption(leaf, ei, 0);545 btrfs_set_file_extent_other_encoding(leaf, ei, 0);546 btrfs_set_file_extent_ram_bytes(leaf, ei, size);547 ptr = btrfs_file_extent_inline_start(ei);548 549 if (compress_type != BTRFS_COMPRESS_NONE) {550 kaddr = kmap_local_folio(compressed_folio, 0);551 write_extent_buffer(leaf, kaddr, ptr, compressed_size);552 kunmap_local(kaddr);553 554 btrfs_set_file_extent_compression(leaf, ei,555 compress_type);556 } else {557 struct folio *folio;558 559 folio = __filemap_get_folio(inode->vfs_inode.i_mapping,560 0, 0, 0);561 ASSERT(!IS_ERR(folio));562 btrfs_set_file_extent_compression(leaf, ei, 0);563 kaddr = kmap_local_folio(folio, 0);564 write_extent_buffer(leaf, kaddr, ptr, size);565 kunmap_local(kaddr);566 folio_put(folio);567 }568 btrfs_mark_buffer_dirty(trans, leaf);569 btrfs_release_path(path);570 571 /*572 * We align size to sectorsize for inline extents just for simplicity573 * sake.574 */575 ret = btrfs_inode_set_file_extent_range(inode, 0,576 ALIGN(size, root->fs_info->sectorsize));577 if (ret)578 goto fail;579 580 /*581 * We're an inline extent, so nobody can extend the file past i_size582 * without locking a page we already have locked.583 *584 * We must do any i_size and inode updates before we unlock the pages.585 * Otherwise we could end up racing with unlink.586 */587 i_size = i_size_read(&inode->vfs_inode);588 if (update_i_size && size > i_size) {589 i_size_write(&inode->vfs_inode, size);590 i_size = size;591 }592 inode->disk_i_size = i_size;593 594fail:595 return ret;596}597 598static bool can_cow_file_range_inline(struct btrfs_inode *inode,599 u64 offset, u64 size,600 size_t compressed_size)601{602 struct btrfs_fs_info *fs_info = inode->root->fs_info;603 u64 data_len = (compressed_size ?: size);604 605 /* Inline extents must start at offset 0. */606 if (offset != 0)607 return false;608 609 /*610 * Due to the page size limit, for subpage we can only trigger the611 * writeback for the dirty sectors of page, that means data writeback612 * is doing more writeback than what we want.613 *614 * This is especially unexpected for some call sites like fallocate,615 * where we only increase i_size after everything is done.616 * This means we can trigger inline extent even if we didn't want to.617 * So here we skip inline extent creation completely.618 */619 if (fs_info->sectorsize != PAGE_SIZE)620 return false;621 622 /* Inline extents are limited to sectorsize. */623 if (size > fs_info->sectorsize)624 return false;625 626 /* We cannot exceed the maximum inline data size. */627 if (data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info))628 return false;629 630 /* We cannot exceed the user specified max_inline size. */631 if (data_len > fs_info->max_inline)632 return false;633 634 /* Inline extents must be the entirety of the file. */635 if (size < i_size_read(&inode->vfs_inode))636 return false;637 638 return true;639}640 641/*642 * conditionally insert an inline extent into the file. This643 * does the checks required to make sure the data is small enough644 * to fit as an inline extent.645 *646 * If being used directly, you must have already checked we're allowed to cow647 * the range by getting true from can_cow_file_range_inline().648 */649static noinline int __cow_file_range_inline(struct btrfs_inode *inode, u64 offset,650 u64 size, size_t compressed_size,651 int compress_type,652 struct folio *compressed_folio,653 bool update_i_size)654{655 struct btrfs_drop_extents_args drop_args = { 0 };656 struct btrfs_root *root = inode->root;657 struct btrfs_fs_info *fs_info = root->fs_info;658 struct btrfs_trans_handle *trans;659 u64 data_len = (compressed_size ?: size);660 int ret;661 struct btrfs_path *path;662 663 path = btrfs_alloc_path();664 if (!path)665 return -ENOMEM;666 667 trans = btrfs_join_transaction(root);668 if (IS_ERR(trans)) {669 btrfs_free_path(path);670 return PTR_ERR(trans);671 }672 trans->block_rsv = &inode->block_rsv;673 674 drop_args.path = path;675 drop_args.start = 0;676 drop_args.end = fs_info->sectorsize;677 drop_args.drop_cache = true;678 drop_args.replace_extent = true;679 drop_args.extent_item_size = btrfs_file_extent_calc_inline_size(data_len);680 ret = btrfs_drop_extents(trans, root, inode, &drop_args);681 if (ret) {682 btrfs_abort_transaction(trans, ret);683 goto out;684 }685 686 ret = insert_inline_extent(trans, path, inode, drop_args.extent_inserted,687 size, compressed_size, compress_type,688 compressed_folio, update_i_size);689 if (ret && ret != -ENOSPC) {690 btrfs_abort_transaction(trans, ret);691 goto out;692 } else if (ret == -ENOSPC) {693 ret = 1;694 goto out;695 }696 697 btrfs_update_inode_bytes(inode, size, drop_args.bytes_found);698 ret = btrfs_update_inode(trans, inode);699 if (ret && ret != -ENOSPC) {700 btrfs_abort_transaction(trans, ret);701 goto out;702 } else if (ret == -ENOSPC) {703 ret = 1;704 goto out;705 }706 707 btrfs_set_inode_full_sync(inode);708out:709 /*710 * Don't forget to free the reserved space, as for inlined extent711 * it won't count as data extent, free them directly here.712 * And at reserve time, it's always aligned to page size, so713 * just free one page here.714 */715 btrfs_qgroup_free_data(inode, NULL, 0, PAGE_SIZE, NULL);716 btrfs_free_path(path);717 btrfs_end_transaction(trans);718 return ret;719}720 721static noinline int cow_file_range_inline(struct btrfs_inode *inode,722 struct folio *locked_folio,723 u64 offset, u64 end,724 size_t compressed_size,725 int compress_type,726 struct folio *compressed_folio,727 bool update_i_size)728{729 struct extent_state *cached = NULL;730 unsigned long clear_flags = EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |731 EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING | EXTENT_LOCKED;732 u64 size = min_t(u64, i_size_read(&inode->vfs_inode), end + 1);733 int ret;734 735 if (!can_cow_file_range_inline(inode, offset, size, compressed_size))736 return 1;737 738 lock_extent(&inode->io_tree, offset, end, &cached);739 ret = __cow_file_range_inline(inode, offset, size, compressed_size,740 compress_type, compressed_folio,741 update_i_size);742 if (ret > 0) {743 unlock_extent(&inode->io_tree, offset, end, &cached);744 return ret;745 }746 747 /*748 * In the successful case (ret == 0 here), cow_file_range will return 1.749 *750 * Quite a bit further up the callstack in extent_writepage(), ret == 1751 * is treated as a short circuited success and does not unlock the folio,752 * so we must do it here.753 *754 * In the failure case, the locked_folio does get unlocked by755 * btrfs_folio_end_all_writers, which asserts that it is still locked756 * at that point, so we must *not* unlock it here.757 *758 * The other two callsites in compress_file_range do not have a759 * locked_folio, so they are not relevant to this logic.760 */761 if (ret == 0)762 locked_folio = NULL;763 764 extent_clear_unlock_delalloc(inode, offset, end, locked_folio, &cached,765 clear_flags, PAGE_UNLOCK |766 PAGE_START_WRITEBACK | PAGE_END_WRITEBACK);767 return ret;768}769 770struct async_extent {771 u64 start;772 u64 ram_size;773 u64 compressed_size;774 struct folio **folios;775 unsigned long nr_folios;776 int compress_type;777 struct list_head list;778};779 780struct async_chunk {781 struct btrfs_inode *inode;782 struct folio *locked_folio;783 u64 start;784 u64 end;785 blk_opf_t write_flags;786 struct list_head extents;787 struct cgroup_subsys_state *blkcg_css;788 struct btrfs_work work;789 struct async_cow *async_cow;790};791 792struct async_cow {793 atomic_t num_chunks;794 struct async_chunk chunks[];795};796 797static noinline int add_async_extent(struct async_chunk *cow,798 u64 start, u64 ram_size,799 u64 compressed_size,800 struct folio **folios,801 unsigned long nr_folios,802 int compress_type)803{804 struct async_extent *async_extent;805 806 async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS);807 if (!async_extent)808 return -ENOMEM;809 async_extent->start = start;810 async_extent->ram_size = ram_size;811 async_extent->compressed_size = compressed_size;812 async_extent->folios = folios;813 async_extent->nr_folios = nr_folios;814 async_extent->compress_type = compress_type;815 list_add_tail(&async_extent->list, &cow->extents);816 return 0;817}818 819/*820 * Check if the inode needs to be submitted to compression, based on mount821 * options, defragmentation, properties or heuristics.822 */823static inline int inode_need_compress(struct btrfs_inode *inode, u64 start,824 u64 end)825{826 struct btrfs_fs_info *fs_info = inode->root->fs_info;827 828 if (!btrfs_inode_can_compress(inode)) {829 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG),830 KERN_ERR "BTRFS: unexpected compression for ino %llu\n",831 btrfs_ino(inode));832 return 0;833 }834 /*835 * Special check for subpage.836 *837 * We lock the full page then run each delalloc range in the page, thus838 * for the following case, we will hit some subpage specific corner case:839 *840 * 0 32K 64K841 * | |///////| |///////|842 * \- A \- B843 *844 * In above case, both range A and range B will try to unlock the full845 * page [0, 64K), causing the one finished later will have page846 * unlocked already, triggering various page lock requirement BUG_ON()s.847 *848 * So here we add an artificial limit that subpage compression can only849 * if the range is fully page aligned.850 *851 * In theory we only need to ensure the first page is fully covered, but852 * the tailing partial page will be locked until the full compression853 * finishes, delaying the write of other range.854 *855 * TODO: Make btrfs_run_delalloc_range() to lock all delalloc range856 * first to prevent any submitted async extent to unlock the full page.857 * By this, we can ensure for subpage case that only the last async_cow858 * will unlock the full page.859 */860 if (fs_info->sectorsize < PAGE_SIZE) {861 if (!PAGE_ALIGNED(start) ||862 !PAGE_ALIGNED(end + 1))863 return 0;864 }865 866 /* force compress */867 if (btrfs_test_opt(fs_info, FORCE_COMPRESS))868 return 1;869 /* defrag ioctl */870 if (inode->defrag_compress)871 return 1;872 /* bad compression ratios */873 if (inode->flags & BTRFS_INODE_NOCOMPRESS)874 return 0;875 if (btrfs_test_opt(fs_info, COMPRESS) ||876 inode->flags & BTRFS_INODE_COMPRESS ||877 inode->prop_compress)878 return btrfs_compress_heuristic(inode, start, end);879 return 0;880}881 882static inline void inode_should_defrag(struct btrfs_inode *inode,883 u64 start, u64 end, u64 num_bytes, u32 small_write)884{885 /* If this is a small write inside eof, kick off a defrag */886 if (num_bytes < small_write &&887 (start > 0 || end + 1 < inode->disk_i_size))888 btrfs_add_inode_defrag(inode, small_write);889}890 891static int extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)892{893 unsigned long end_index = end >> PAGE_SHIFT;894 struct folio *folio;895 int ret = 0;896 897 for (unsigned long index = start >> PAGE_SHIFT;898 index <= end_index; index++) {899 folio = __filemap_get_folio(inode->i_mapping, index, 0, 0);900 if (IS_ERR(folio)) {901 if (!ret)902 ret = PTR_ERR(folio);903 continue;904 }905 folio_clear_dirty_for_io(folio);906 folio_put(folio);907 }908 return ret;909}910 911/*912 * Work queue call back to started compression on a file and pages.913 *914 * This is done inside an ordered work queue, and the compression is spread915 * across many cpus. The actual IO submission is step two, and the ordered work916 * queue takes care of making sure that happens in the same order things were917 * put onto the queue by writepages and friends.918 *919 * If this code finds it can't get good compression, it puts an entry onto the920 * work queue to write the uncompressed bytes. This makes sure that both921 * compressed inodes and uncompressed inodes are written in the same order that922 * the flusher thread sent them down.923 */924static void compress_file_range(struct btrfs_work *work)925{926 struct async_chunk *async_chunk =927 container_of(work, struct async_chunk, work);928 struct btrfs_inode *inode = async_chunk->inode;929 struct btrfs_fs_info *fs_info = inode->root->fs_info;930 struct address_space *mapping = inode->vfs_inode.i_mapping;931 u64 blocksize = fs_info->sectorsize;932 u64 start = async_chunk->start;933 u64 end = async_chunk->end;934 u64 actual_end;935 u64 i_size;936 int ret = 0;937 struct folio **folios;938 unsigned long nr_folios;939 unsigned long total_compressed = 0;940 unsigned long total_in = 0;941 unsigned int poff;942 int i;943 int compress_type = fs_info->compress_type;944 945 inode_should_defrag(inode, start, end, end - start + 1, SZ_16K);946 947 /*948 * We need to call clear_page_dirty_for_io on each page in the range.949 * Otherwise applications with the file mmap'd can wander in and change950 * the page contents while we are compressing them.951 */952 ret = extent_range_clear_dirty_for_io(&inode->vfs_inode, start, end);953 954 /*955 * All the folios should have been locked thus no failure.956 *957 * And even if some folios are missing, btrfs_compress_folios()958 * would handle them correctly, so here just do an ASSERT() check for959 * early logic errors.960 */961 ASSERT(ret == 0);962 963 /*964 * We need to save i_size before now because it could change in between965 * us evaluating the size and assigning it. This is because we lock and966 * unlock the page in truncate and fallocate, and then modify the i_size967 * later on.968 *969 * The barriers are to emulate READ_ONCE, remove that once i_size_read970 * does that for us.971 */972 barrier();973 i_size = i_size_read(&inode->vfs_inode);974 barrier();975 actual_end = min_t(u64, i_size, end + 1);976again:977 folios = NULL;978 nr_folios = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;979 nr_folios = min_t(unsigned long, nr_folios, BTRFS_MAX_COMPRESSED_PAGES);980 981 /*982 * we don't want to send crud past the end of i_size through983 * compression, that's just a waste of CPU time. So, if the984 * end of the file is before the start of our current985 * requested range of bytes, we bail out to the uncompressed986 * cleanup code that can deal with all of this.987 *988 * It isn't really the fastest way to fix things, but this is a989 * very uncommon corner.990 */991 if (actual_end <= start)992 goto cleanup_and_bail_uncompressed;993 994 total_compressed = actual_end - start;995 996 /*997 * Skip compression for a small file range(<=blocksize) that998 * isn't an inline extent, since it doesn't save disk space at all.999 */1000 if (total_compressed <= blocksize &&1001 (start > 0 || end + 1 < inode->disk_i_size))1002 goto cleanup_and_bail_uncompressed;1003 1004 /*1005 * For subpage case, we require full page alignment for the sector1006 * aligned range.1007 * Thus we must also check against @actual_end, not just @end.1008 */1009 if (blocksize < PAGE_SIZE) {1010 if (!PAGE_ALIGNED(start) ||1011 !PAGE_ALIGNED(round_up(actual_end, blocksize)))1012 goto cleanup_and_bail_uncompressed;1013 }1014 1015 total_compressed = min_t(unsigned long, total_compressed,1016 BTRFS_MAX_UNCOMPRESSED);1017 total_in = 0;1018 ret = 0;1019 1020 /*1021 * We do compression for mount -o compress and when the inode has not1022 * been flagged as NOCOMPRESS. This flag can change at any time if we1023 * discover bad compression ratios.1024 */1025 if (!inode_need_compress(inode, start, end))1026 goto cleanup_and_bail_uncompressed;1027 1028 folios = kcalloc(nr_folios, sizeof(struct folio *), GFP_NOFS);1029 if (!folios) {1030 /*1031 * Memory allocation failure is not a fatal error, we can fall1032 * back to uncompressed code.1033 */1034 goto cleanup_and_bail_uncompressed;1035 }1036 1037 if (inode->defrag_compress)1038 compress_type = inode->defrag_compress;1039 else if (inode->prop_compress)1040 compress_type = inode->prop_compress;1041 1042 /* Compression level is applied here. */1043 ret = btrfs_compress_folios(compress_type | (fs_info->compress_level << 4),1044 mapping, start, folios, &nr_folios, &total_in,1045 &total_compressed);1046 if (ret)1047 goto mark_incompressible;1048 1049 /*1050 * Zero the tail end of the last page, as we might be sending it down1051 * to disk.1052 */1053 poff = offset_in_page(total_compressed);1054 if (poff)1055 folio_zero_range(folios[nr_folios - 1], poff, PAGE_SIZE - poff);1056 1057 /*1058 * Try to create an inline extent.1059 *1060 * If we didn't compress the entire range, try to create an uncompressed1061 * inline extent, else a compressed one.1062 *1063 * Check cow_file_range() for why we don't even try to create inline1064 * extent for the subpage case.1065 */1066 if (total_in < actual_end)1067 ret = cow_file_range_inline(inode, NULL, start, end, 0,1068 BTRFS_COMPRESS_NONE, NULL, false);1069 else1070 ret = cow_file_range_inline(inode, NULL, start, end, total_compressed,1071 compress_type, folios[0], false);1072 if (ret <= 0) {1073 if (ret < 0)1074 mapping_set_error(mapping, -EIO);1075 goto free_pages;1076 }1077 1078 /*1079 * We aren't doing an inline extent. Round the compressed size up to a1080 * block size boundary so the allocator does sane things.1081 */1082 total_compressed = ALIGN(total_compressed, blocksize);1083 1084 /*1085 * One last check to make sure the compression is really a win, compare1086 * the page count read with the blocks on disk, compression must free at1087 * least one sector.1088 */1089 total_in = round_up(total_in, fs_info->sectorsize);1090 if (total_compressed + blocksize > total_in)1091 goto mark_incompressible;1092 1093 /*1094 * The async work queues will take care of doing actual allocation on1095 * disk for these compressed pages, and will submit the bios.1096 */1097 ret = add_async_extent(async_chunk, start, total_in, total_compressed, folios,1098 nr_folios, compress_type);1099 BUG_ON(ret);1100 if (start + total_in < end) {1101 start += total_in;1102 cond_resched();1103 goto again;1104 }1105 return;1106 1107mark_incompressible:1108 if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) && !inode->prop_compress)1109 inode->flags |= BTRFS_INODE_NOCOMPRESS;1110cleanup_and_bail_uncompressed:1111 ret = add_async_extent(async_chunk, start, end - start + 1, 0, NULL, 0,1112 BTRFS_COMPRESS_NONE);1113 BUG_ON(ret);1114free_pages:1115 if (folios) {1116 for (i = 0; i < nr_folios; i++) {1117 WARN_ON(folios[i]->mapping);1118 btrfs_free_compr_folio(folios[i]);1119 }1120 kfree(folios);1121 }1122}1123 1124static void free_async_extent_pages(struct async_extent *async_extent)1125{1126 int i;1127 1128 if (!async_extent->folios)1129 return;1130 1131 for (i = 0; i < async_extent->nr_folios; i++) {1132 WARN_ON(async_extent->folios[i]->mapping);1133 btrfs_free_compr_folio(async_extent->folios[i]);1134 }1135 kfree(async_extent->folios);1136 async_extent->nr_folios = 0;1137 async_extent->folios = NULL;1138}1139 1140static void submit_uncompressed_range(struct btrfs_inode *inode,1141 struct async_extent *async_extent,1142 struct folio *locked_folio)1143{1144 u64 start = async_extent->start;1145 u64 end = async_extent->start + async_extent->ram_size - 1;1146 int ret;1147 struct writeback_control wbc = {1148 .sync_mode = WB_SYNC_ALL,1149 .range_start = start,1150 .range_end = end,1151 .no_cgroup_owner = 1,1152 };1153 1154 wbc_attach_fdatawrite_inode(&wbc, &inode->vfs_inode);1155 ret = run_delalloc_cow(inode, locked_folio, start, end,1156 &wbc, false);1157 wbc_detach_inode(&wbc);1158 if (ret < 0) {1159 btrfs_cleanup_ordered_extents(inode, locked_folio,1160 start, end - start + 1);1161 if (locked_folio) {1162 const u64 page_start = folio_pos(locked_folio);1163 1164 folio_start_writeback(locked_folio);1165 folio_end_writeback(locked_folio);1166 btrfs_mark_ordered_io_finished(inode, locked_folio,1167 page_start, PAGE_SIZE,1168 !ret);1169 mapping_set_error(locked_folio->mapping, ret);1170 folio_unlock(locked_folio);1171 }1172 }1173}1174 1175static void submit_one_async_extent(struct async_chunk *async_chunk,1176 struct async_extent *async_extent,1177 u64 *alloc_hint)1178{1179 struct btrfs_inode *inode = async_chunk->inode;1180 struct extent_io_tree *io_tree = &inode->io_tree;1181 struct btrfs_root *root = inode->root;1182 struct btrfs_fs_info *fs_info = root->fs_info;1183 struct btrfs_ordered_extent *ordered;1184 struct btrfs_file_extent file_extent;1185 struct btrfs_key ins;1186 struct folio *locked_folio = NULL;1187 struct extent_state *cached = NULL;1188 struct extent_map *em;1189 int ret = 0;1190 u64 start = async_extent->start;1191 u64 end = async_extent->start + async_extent->ram_size - 1;1192 1193 if (async_chunk->blkcg_css)1194 kthread_associate_blkcg(async_chunk->blkcg_css);1195 1196 /*1197 * If async_chunk->locked_folio is in the async_extent range, we need to1198 * handle it.1199 */1200 if (async_chunk->locked_folio) {1201 u64 locked_folio_start = folio_pos(async_chunk->locked_folio);1202 u64 locked_folio_end = locked_folio_start +1203 folio_size(async_chunk->locked_folio) - 1;1204 1205 if (!(start >= locked_folio_end || end <= locked_folio_start))1206 locked_folio = async_chunk->locked_folio;1207 }1208 1209 if (async_extent->compress_type == BTRFS_COMPRESS_NONE) {1210 submit_uncompressed_range(inode, async_extent, locked_folio);1211 goto done;1212 }1213 1214 ret = btrfs_reserve_extent(root, async_extent->ram_size,1215 async_extent->compressed_size,1216 async_extent->compressed_size,1217 0, *alloc_hint, &ins, 1, 1);1218 if (ret) {1219 /*1220 * We can't reserve contiguous space for the compressed size.1221 * Unlikely, but it's possible that we could have enough1222 * non-contiguous space for the uncompressed size instead. So1223 * fall back to uncompressed.1224 */1225 submit_uncompressed_range(inode, async_extent, locked_folio);1226 goto done;1227 }1228 1229 lock_extent(io_tree, start, end, &cached);1230 1231 /* Here we're doing allocation and writeback of the compressed pages */1232 file_extent.disk_bytenr = ins.objectid;1233 file_extent.disk_num_bytes = ins.offset;1234 file_extent.ram_bytes = async_extent->ram_size;1235 file_extent.num_bytes = async_extent->ram_size;1236 file_extent.offset = 0;1237 file_extent.compression = async_extent->compress_type;1238 1239 em = btrfs_create_io_em(inode, start, &file_extent, BTRFS_ORDERED_COMPRESSED);1240 if (IS_ERR(em)) {1241 ret = PTR_ERR(em);1242 goto out_free_reserve;1243 }1244 free_extent_map(em);1245 1246 ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent,1247 1 << BTRFS_ORDERED_COMPRESSED);1248 if (IS_ERR(ordered)) {1249 btrfs_drop_extent_map_range(inode, start, end, false);1250 ret = PTR_ERR(ordered);1251 goto out_free_reserve;1252 }1253 btrfs_dec_block_group_reservations(fs_info, ins.objectid);1254 1255 /* Clear dirty, set writeback and unlock the pages. */1256 extent_clear_unlock_delalloc(inode, start, end,1257 NULL, &cached, EXTENT_LOCKED | EXTENT_DELALLOC,1258 PAGE_UNLOCK | PAGE_START_WRITEBACK);1259 btrfs_submit_compressed_write(ordered,1260 async_extent->folios, /* compressed_folios */1261 async_extent->nr_folios,1262 async_chunk->write_flags, true);1263 *alloc_hint = ins.objectid + ins.offset;1264done:1265 if (async_chunk->blkcg_css)1266 kthread_associate_blkcg(NULL);1267 kfree(async_extent);1268 return;1269 1270out_free_reserve:1271 btrfs_dec_block_group_reservations(fs_info, ins.objectid);1272 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);1273 mapping_set_error(inode->vfs_inode.i_mapping, -EIO);1274 extent_clear_unlock_delalloc(inode, start, end,1275 NULL, &cached,1276 EXTENT_LOCKED | EXTENT_DELALLOC |1277 EXTENT_DELALLOC_NEW |1278 EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING,1279 PAGE_UNLOCK | PAGE_START_WRITEBACK |1280 PAGE_END_WRITEBACK);1281 free_async_extent_pages(async_extent);1282 if (async_chunk->blkcg_css)1283 kthread_associate_blkcg(NULL);1284 btrfs_debug(fs_info,1285"async extent submission failed root=%lld inode=%llu start=%llu len=%llu ret=%d",1286 btrfs_root_id(root), btrfs_ino(inode), start,1287 async_extent->ram_size, ret);1288 kfree(async_extent);1289}1290 1291u64 btrfs_get_extent_allocation_hint(struct btrfs_inode *inode, u64 start,1292 u64 num_bytes)1293{1294 struct extent_map_tree *em_tree = &inode->extent_tree;1295 struct extent_map *em;1296 u64 alloc_hint = 0;1297 1298 read_lock(&em_tree->lock);1299 em = search_extent_mapping(em_tree, start, num_bytes);1300 if (em) {1301 /*1302 * if block start isn't an actual block number then find the1303 * first block in this inode and use that as a hint. If that1304 * block is also bogus then just don't worry about it.1305 */1306 if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) {1307 free_extent_map(em);1308 em = search_extent_mapping(em_tree, 0, 0);1309 if (em && em->disk_bytenr < EXTENT_MAP_LAST_BYTE)1310 alloc_hint = extent_map_block_start(em);1311 if (em)1312 free_extent_map(em);1313 } else {1314 alloc_hint = extent_map_block_start(em);1315 free_extent_map(em);1316 }1317 }1318 read_unlock(&em_tree->lock);1319 1320 return alloc_hint;1321}1322 1323/*1324 * when extent_io.c finds a delayed allocation range in the file,1325 * the call backs end up in this code. The basic idea is to1326 * allocate extents on disk for the range, and create ordered data structs1327 * in ram to track those extents.1328 *1329 * locked_folio is the folio that writepage had locked already. We use1330 * it to make sure we don't do extra locks or unlocks.1331 *1332 * When this function fails, it unlocks all pages except @locked_folio.1333 *1334 * When this function successfully creates an inline extent, it returns 1 and1335 * unlocks all pages including locked_folio and starts I/O on them.1336 * (In reality inline extents are limited to a single page, so locked_folio is1337 * the only page handled anyway).1338 *1339 * When this function succeed and creates a normal extent, the page locking1340 * status depends on the passed in flags:1341 *1342 * - If @keep_locked is set, all pages are kept locked.1343 * - Else all pages except for @locked_folio are unlocked.1344 *1345 * When a failure happens in the second or later iteration of the1346 * while-loop, the ordered extents created in previous iterations are kept1347 * intact. So, the caller must clean them up by calling1348 * btrfs_cleanup_ordered_extents(). See btrfs_run_delalloc_range() for1349 * example.1350 */1351static noinline int cow_file_range(struct btrfs_inode *inode,1352 struct folio *locked_folio, u64 start,1353 u64 end, u64 *done_offset,1354 bool keep_locked, bool no_inline)1355{1356 struct btrfs_root *root = inode->root;1357 struct btrfs_fs_info *fs_info = root->fs_info;1358 struct extent_state *cached = NULL;1359 u64 alloc_hint = 0;1360 u64 orig_start = start;1361 u64 num_bytes;1362 unsigned long ram_size;1363 u64 cur_alloc_size = 0;1364 u64 min_alloc_size;1365 u64 blocksize = fs_info->sectorsize;1366 struct btrfs_key ins;1367 struct extent_map *em;1368 unsigned clear_bits;1369 unsigned long page_ops;1370 bool extent_reserved = false;1371 int ret = 0;1372 1373 if (btrfs_is_free_space_inode(inode)) {1374 ret = -EINVAL;1375 goto out_unlock;1376 }1377 1378 num_bytes = ALIGN(end - start + 1, blocksize);1379 num_bytes = max(blocksize, num_bytes);1380 ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy));1381 1382 inode_should_defrag(inode, start, end, num_bytes, SZ_64K);1383 1384 if (!no_inline) {1385 /* lets try to make an inline extent */1386 ret = cow_file_range_inline(inode, locked_folio, start, end, 0,1387 BTRFS_COMPRESS_NONE, NULL, false);1388 if (ret <= 0) {1389 /*1390 * We succeeded, return 1 so the caller knows we're done1391 * with this page and already handled the IO.1392 *1393 * If there was an error then cow_file_range_inline() has1394 * already done the cleanup.1395 */1396 if (ret == 0)1397 ret = 1;1398 goto done;1399 }1400 }1401 1402 alloc_hint = btrfs_get_extent_allocation_hint(inode, start, num_bytes);1403 1404 /*1405 * Relocation relies on the relocated extents to have exactly the same1406 * size as the original extents. Normally writeback for relocation data1407 * extents follows a NOCOW path because relocation preallocates the1408 * extents. However, due to an operation such as scrub turning a block1409 * group to RO mode, it may fallback to COW mode, so we must make sure1410 * an extent allocated during COW has exactly the requested size and can1411 * not be split into smaller extents, otherwise relocation breaks and1412 * fails during the stage where it updates the bytenr of file extent1413 * items.1414 */1415 if (btrfs_is_data_reloc_root(root))1416 min_alloc_size = num_bytes;1417 else1418 min_alloc_size = fs_info->sectorsize;1419 1420 while (num_bytes > 0) {1421 struct btrfs_ordered_extent *ordered;1422 struct btrfs_file_extent file_extent;1423 1424 cur_alloc_size = num_bytes;1425 ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size,1426 min_alloc_size, 0, alloc_hint,1427 &ins, 1, 1);1428 if (ret == -EAGAIN) {1429 /*1430 * btrfs_reserve_extent only returns -EAGAIN for zoned1431 * file systems, which is an indication that there are1432 * no active zones to allocate from at the moment.1433 *1434 * If this is the first loop iteration, wait for at1435 * least one zone to finish before retrying the1436 * allocation. Otherwise ask the caller to write out1437 * the already allocated blocks before coming back to1438 * us, or return -ENOSPC if it can't handle retries.1439 */1440 ASSERT(btrfs_is_zoned(fs_info));1441 if (start == orig_start) {1442 wait_on_bit_io(&inode->root->fs_info->flags,1443 BTRFS_FS_NEED_ZONE_FINISH,1444 TASK_UNINTERRUPTIBLE);1445 continue;1446 }1447 if (done_offset) {1448 *done_offset = start - 1;1449 return 0;1450 }1451 ret = -ENOSPC;1452 }1453 if (ret < 0)1454 goto out_unlock;1455 cur_alloc_size = ins.offset;1456 extent_reserved = true;1457 1458 ram_size = ins.offset;1459 file_extent.disk_bytenr = ins.objectid;1460 file_extent.disk_num_bytes = ins.offset;1461 file_extent.num_bytes = ins.offset;1462 file_extent.ram_bytes = ins.offset;1463 file_extent.offset = 0;1464 file_extent.compression = BTRFS_COMPRESS_NONE;1465 1466 lock_extent(&inode->io_tree, start, start + ram_size - 1,1467 &cached);1468 1469 em = btrfs_create_io_em(inode, start, &file_extent,1470 BTRFS_ORDERED_REGULAR);1471 if (IS_ERR(em)) {1472 unlock_extent(&inode->io_tree, start,1473 start + ram_size - 1, &cached);1474 ret = PTR_ERR(em);1475 goto out_reserve;1476 }1477 free_extent_map(em);1478 1479 ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent,1480 1 << BTRFS_ORDERED_REGULAR);1481 if (IS_ERR(ordered)) {1482 unlock_extent(&inode->io_tree, start,1483 start + ram_size - 1, &cached);1484 ret = PTR_ERR(ordered);1485 goto out_drop_extent_cache;1486 }1487 1488 if (btrfs_is_data_reloc_root(root)) {1489 ret = btrfs_reloc_clone_csums(ordered);1490 1491 /*1492 * Only drop cache here, and process as normal.1493 *1494 * We must not allow extent_clear_unlock_delalloc()1495 * at out_unlock label to free meta of this ordered1496 * extent, as its meta should be freed by1497 * btrfs_finish_ordered_io().1498 *1499 * So we must continue until @start is increased to1500 * skip current ordered extent.1501 */1502 if (ret)1503 btrfs_drop_extent_map_range(inode, start,1504 start + ram_size - 1,1505 false);1506 }1507 btrfs_put_ordered_extent(ordered);1508 1509 btrfs_dec_block_group_reservations(fs_info, ins.objectid);1510 1511 /*1512 * We're not doing compressed IO, don't unlock the first page1513 * (which the caller expects to stay locked), don't clear any1514 * dirty bits and don't set any writeback bits1515 *1516 * Do set the Ordered (Private2) bit so we know this page was1517 * properly setup for writepage.1518 */1519 page_ops = (keep_locked ? 0 : PAGE_UNLOCK);1520 page_ops |= PAGE_SET_ORDERED;1521 1522 extent_clear_unlock_delalloc(inode, start, start + ram_size - 1,1523 locked_folio, &cached,1524 EXTENT_LOCKED | EXTENT_DELALLOC,1525 page_ops);1526 if (num_bytes < cur_alloc_size)1527 num_bytes = 0;1528 else1529 num_bytes -= cur_alloc_size;1530 alloc_hint = ins.objectid + ins.offset;1531 start += cur_alloc_size;1532 extent_reserved = false;1533 1534 /*1535 * btrfs_reloc_clone_csums() error, since start is increased1536 * extent_clear_unlock_delalloc() at out_unlock label won't1537 * free metadata of current ordered extent, we're OK to exit.1538 */1539 if (ret)1540 goto out_unlock;1541 }1542done:1543 if (done_offset)1544 *done_offset = end;1545 return ret;1546 1547out_drop_extent_cache:1548 btrfs_drop_extent_map_range(inode, start, start + ram_size - 1, false);1549out_reserve:1550 btrfs_dec_block_group_reservations(fs_info, ins.objectid);1551 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);1552out_unlock:1553 /*1554 * Now, we have three regions to clean up:1555 *1556 * |-------(1)----|---(2)---|-------------(3)----------|1557 * `- orig_start `- start `- start + cur_alloc_size `- end1558 *1559 * We process each region below.1560 */1561 1562 clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |1563 EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV;1564 page_ops = PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK;1565 1566 /*1567 * For the range (1). We have already instantiated the ordered extents1568 * for this region. They are cleaned up by1569 * btrfs_cleanup_ordered_extents() in e.g,1570 * btrfs_run_delalloc_range(). EXTENT_LOCKED | EXTENT_DELALLOC are1571 * already cleared in the above loop. And, EXTENT_DELALLOC_NEW |1572 * EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV are handled by the cleanup1573 * function.1574 *1575 * However, in case of @keep_locked, we still need to unlock the pages1576 * (except @locked_folio) to ensure all the pages are unlocked.1577 */1578 if (keep_locked && orig_start < start) {1579 if (!locked_folio)1580 mapping_set_error(inode->vfs_inode.i_mapping, ret);1581 extent_clear_unlock_delalloc(inode, orig_start, start - 1,1582 locked_folio, NULL, 0, page_ops);1583 }1584 1585 /*1586 * At this point we're unlocked, we want to make sure we're only1587 * clearing these flags under the extent lock, so lock the rest of the1588 * range and clear everything up.1589 */1590 lock_extent(&inode->io_tree, start, end, NULL);1591 1592 /*1593 * For the range (2). If we reserved an extent for our delalloc range1594 * (or a subrange) and failed to create the respective ordered extent,1595 * then it means that when we reserved the extent we decremented the1596 * extent's size from the data space_info's bytes_may_use counter and1597 * incremented the space_info's bytes_reserved counter by the same1598 * amount. We must make sure extent_clear_unlock_delalloc() does not try1599 * to decrement again the data space_info's bytes_may_use counter,1600 * therefore we do not pass it the flag EXTENT_CLEAR_DATA_RESV.1601 */1602 if (extent_reserved) {1603 extent_clear_unlock_delalloc(inode, start,1604 start + cur_alloc_size - 1,1605 locked_folio, &cached, clear_bits,1606 page_ops);1607 btrfs_qgroup_free_data(inode, NULL, start, cur_alloc_size, NULL);1608 start += cur_alloc_size;1609 }1610 1611 /*1612 * For the range (3). We never touched the region. In addition to the1613 * clear_bits above, we add EXTENT_CLEAR_DATA_RESV to release the data1614 * space_info's bytes_may_use counter, reserved in1615 * btrfs_check_data_free_space().1616 */1617 if (start < end) {1618 clear_bits |= EXTENT_CLEAR_DATA_RESV;1619 extent_clear_unlock_delalloc(inode, start, end, locked_folio,1620 &cached, clear_bits, page_ops);1621 btrfs_qgroup_free_data(inode, NULL, start, end - start + 1, NULL);1622 }1623 return ret;1624}1625 1626/*1627 * Phase two of compressed writeback. This is the ordered portion of the code,1628 * which only gets called in the order the work was queued. We walk all the1629 * async extents created by compress_file_range and send them down to the disk.1630 *1631 * If called with @do_free == true then it'll try to finish the work and free1632 * the work struct eventually.1633 */1634static noinline void submit_compressed_extents(struct btrfs_work *work, bool do_free)1635{1636 struct async_chunk *async_chunk = container_of(work, struct async_chunk,1637 work);1638 struct btrfs_fs_info *fs_info = btrfs_work_owner(work);1639 struct async_extent *async_extent;1640 unsigned long nr_pages;1641 u64 alloc_hint = 0;1642 1643 if (do_free) {1644 struct async_cow *async_cow;1645 1646 btrfs_add_delayed_iput(async_chunk->inode);1647 if (async_chunk->blkcg_css)1648 css_put(async_chunk->blkcg_css);1649 1650 async_cow = async_chunk->async_cow;1651 if (atomic_dec_and_test(&async_cow->num_chunks))1652 kvfree(async_cow);1653 return;1654 }1655 1656 nr_pages = (async_chunk->end - async_chunk->start + PAGE_SIZE) >>1657 PAGE_SHIFT;1658 1659 while (!list_empty(&async_chunk->extents)) {1660 async_extent = list_entry(async_chunk->extents.next,1661 struct async_extent, list);1662 list_del(&async_extent->list);1663 submit_one_async_extent(async_chunk, async_extent, &alloc_hint);1664 }1665 1666 /* atomic_sub_return implies a barrier */1667 if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) <1668 5 * SZ_1M)1669 cond_wake_up_nomb(&fs_info->async_submit_wait);1670}1671 1672static bool run_delalloc_compressed(struct btrfs_inode *inode,1673 struct folio *locked_folio, u64 start,1674 u64 end, struct writeback_control *wbc)1675{1676 struct btrfs_fs_info *fs_info = inode->root->fs_info;1677 struct cgroup_subsys_state *blkcg_css = wbc_blkcg_css(wbc);1678 struct async_cow *ctx;1679 struct async_chunk *async_chunk;1680 unsigned long nr_pages;1681 u64 num_chunks = DIV_ROUND_UP(end - start, SZ_512K);1682 int i;1683 unsigned nofs_flag;1684 const blk_opf_t write_flags = wbc_to_write_flags(wbc);1685 1686 nofs_flag = memalloc_nofs_save();1687 ctx = kvmalloc(struct_size(ctx, chunks, num_chunks), GFP_KERNEL);1688 memalloc_nofs_restore(nofs_flag);1689 if (!ctx)1690 return false;1691 1692 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &inode->runtime_flags);1693 1694 async_chunk = ctx->chunks;1695 atomic_set(&ctx->num_chunks, num_chunks);1696 1697 for (i = 0; i < num_chunks; i++) {1698 u64 cur_end = min(end, start + SZ_512K - 1);1699 1700 /*1701 * igrab is called higher up in the call chain, take only the1702 * lightweight reference for the callback lifetime1703 */1704 ihold(&inode->vfs_inode);1705 async_chunk[i].async_cow = ctx;1706 async_chunk[i].inode = inode;1707 async_chunk[i].start = start;1708 async_chunk[i].end = cur_end;1709 async_chunk[i].write_flags = write_flags;1710 INIT_LIST_HEAD(&async_chunk[i].extents);1711 1712 /*1713 * The locked_folio comes all the way from writepage and its1714 * the original folio we were actually given. As we spread1715 * this large delalloc region across multiple async_chunk1716 * structs, only the first struct needs a pointer to1717 * locked_folio.1718 *1719 * This way we don't need racey decisions about who is supposed1720 * to unlock it.1721 */1722 if (locked_folio) {1723 /*1724 * Depending on the compressibility, the pages might or1725 * might not go through async. We want all of them to1726 * be accounted against wbc once. Let's do it here1727 * before the paths diverge. wbc accounting is used1728 * only for foreign writeback detection and doesn't1729 * need full accuracy. Just account the whole thing1730 * against the first page.1731 */1732 wbc_account_cgroup_owner(wbc, &locked_folio->page,1733 cur_end - start);1734 async_chunk[i].locked_folio = locked_folio;1735 locked_folio = NULL;1736 } else {1737 async_chunk[i].locked_folio = NULL;1738 }1739 1740 if (blkcg_css != blkcg_root_css) {1741 css_get(blkcg_css);1742 async_chunk[i].blkcg_css = blkcg_css;1743 async_chunk[i].write_flags |= REQ_BTRFS_CGROUP_PUNT;1744 } else {1745 async_chunk[i].blkcg_css = NULL;1746 }1747 1748 btrfs_init_work(&async_chunk[i].work, compress_file_range,1749 submit_compressed_extents);1750 1751 nr_pages = DIV_ROUND_UP(cur_end - start, PAGE_SIZE);1752 atomic_add(nr_pages, &fs_info->async_delalloc_pages);1753 1754 btrfs_queue_work(fs_info->delalloc_workers, &async_chunk[i].work);1755 1756 start = cur_end + 1;1757 }1758 return true;1759}1760 1761/*1762 * Run the delalloc range from start to end, and write back any dirty pages1763 * covered by the range.1764 */1765static noinline int run_delalloc_cow(struct btrfs_inode *inode,1766 struct folio *locked_folio, u64 start,1767 u64 end, struct writeback_control *wbc,1768 bool pages_dirty)1769{1770 u64 done_offset = end;1771 int ret;1772 1773 while (start <= end) {1774 ret = cow_file_range(inode, locked_folio, start, end,1775 &done_offset, true, false);1776 if (ret)1777 return ret;1778 extent_write_locked_range(&inode->vfs_inode, locked_folio,1779 start, done_offset, wbc, pages_dirty);1780 start = done_offset + 1;1781 }1782 1783 return 1;1784}1785 1786static int fallback_to_cow(struct btrfs_inode *inode,1787 struct folio *locked_folio, const u64 start,1788 const u64 end)1789{1790 const bool is_space_ino = btrfs_is_free_space_inode(inode);1791 const bool is_reloc_ino = btrfs_is_data_reloc_root(inode->root);1792 const u64 range_bytes = end + 1 - start;1793 struct extent_io_tree *io_tree = &inode->io_tree;1794 struct extent_state *cached_state = NULL;1795 u64 range_start = start;1796 u64 count;1797 int ret;1798 1799 /*1800 * If EXTENT_NORESERVE is set it means that when the buffered write was1801 * made we had not enough available data space and therefore we did not1802 * reserve data space for it, since we though we could do NOCOW for the1803 * respective file range (either there is prealloc extent or the inode1804 * has the NOCOW bit set).1805 *1806 * However when we need to fallback to COW mode (because for example the1807 * block group for the corresponding extent was turned to RO mode by a1808 * scrub or relocation) we need to do the following:1809 *1810 * 1) We increment the bytes_may_use counter of the data space info.1811 * If COW succeeds, it allocates a new data extent and after doing1812 * that it decrements the space info's bytes_may_use counter and1813 * increments its bytes_reserved counter by the same amount (we do1814 * this at btrfs_add_reserved_bytes()). So we need to increment the1815 * bytes_may_use counter to compensate (when space is reserved at1816 * buffered write time, the bytes_may_use counter is incremented);1817 *1818 * 2) We clear the EXTENT_NORESERVE bit from the range. We do this so1819 * that if the COW path fails for any reason, it decrements (through1820 * extent_clear_unlock_delalloc()) the bytes_may_use counter of the1821 * data space info, which we incremented in the step above.1822 *1823 * If we need to fallback to cow and the inode corresponds to a free1824 * space cache inode or an inode of the data relocation tree, we must1825 * also increment bytes_may_use of the data space_info for the same1826 * reason. Space caches and relocated data extents always get a prealloc1827 * extent for them, however scrub or balance may have set the block1828 * group that contains that extent to RO mode and therefore force COW1829 * when starting writeback.1830 */1831 lock_extent(io_tree, start, end, &cached_state);1832 count = count_range_bits(io_tree, &range_start, end, range_bytes,1833 EXTENT_NORESERVE, 0, NULL);1834 if (count > 0 || is_space_ino || is_reloc_ino) {1835 u64 bytes = count;1836 struct btrfs_fs_info *fs_info = inode->root->fs_info;1837 struct btrfs_space_info *sinfo = fs_info->data_sinfo;1838 1839 if (is_space_ino || is_reloc_ino)1840 bytes = range_bytes;1841 1842 spin_lock(&sinfo->lock);1843 btrfs_space_info_update_bytes_may_use(fs_info, sinfo, bytes);1844 spin_unlock(&sinfo->lock);1845 1846 if (count > 0)1847 clear_extent_bit(io_tree, start, end, EXTENT_NORESERVE,1848 NULL);1849 }1850 unlock_extent(io_tree, start, end, &cached_state);1851 1852 /*1853 * Don't try to create inline extents, as a mix of inline extent that1854 * is written out and unlocked directly and a normal NOCOW extent1855 * doesn't work.1856 */1857 ret = cow_file_range(inode, locked_folio, start, end, NULL, false,1858 true);1859 ASSERT(ret != 1);1860 return ret;1861}1862 1863struct can_nocow_file_extent_args {1864 /* Input fields. */1865 1866 /* Start file offset of the range we want to NOCOW. */1867 u64 start;1868 /* End file offset (inclusive) of the range we want to NOCOW. */1869 u64 end;1870 bool writeback_path;1871 bool strict;1872 /*1873 * Free the path passed to can_nocow_file_extent() once it's not needed1874 * anymore.1875 */1876 bool free_path;1877 1878 /*1879 * Output fields. Only set when can_nocow_file_extent() returns 1.1880 * The expected file extent for the NOCOW write.1881 */1882 struct btrfs_file_extent file_extent;1883};1884 1885/*1886 * Check if we can NOCOW the file extent that the path points to.1887 * This function may return with the path released, so the caller should check1888 * if path->nodes[0] is NULL or not if it needs to use the path afterwards.1889 *1890 * Returns: < 0 on error1891 * 0 if we can not NOCOW1892 * 1 if we can NOCOW1893 */1894static int can_nocow_file_extent(struct btrfs_path *path,1895 struct btrfs_key *key,1896 struct btrfs_inode *inode,1897 struct can_nocow_file_extent_args *args)1898{1899 const bool is_freespace_inode = btrfs_is_free_space_inode(inode);1900 struct extent_buffer *leaf = path->nodes[0];1901 struct btrfs_root *root = inode->root;1902 struct btrfs_file_extent_item *fi;1903 struct btrfs_root *csum_root;1904 u64 io_start;1905 u64 extent_end;1906 u8 extent_type;1907 int can_nocow = 0;1908 int ret = 0;1909 bool nowait = path->nowait;1910 1911 fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);1912 extent_type = btrfs_file_extent_type(leaf, fi);1913 1914 if (extent_type == BTRFS_FILE_EXTENT_INLINE)1915 goto out;1916 1917 if (!(inode->flags & BTRFS_INODE_NODATACOW) &&1918 extent_type == BTRFS_FILE_EXTENT_REG)1919 goto out;1920 1921 /*1922 * If the extent was created before the generation where the last snapshot1923 * for its subvolume was created, then this implies the extent is shared,1924 * hence we must COW.1925 */1926 if (!args->strict &&1927 btrfs_file_extent_generation(leaf, fi) <=1928 btrfs_root_last_snapshot(&root->root_item))1929 goto out;1930 1931 /* An explicit hole, must COW. */1932 if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0)1933 goto out;1934 1935 /* Compressed/encrypted/encoded extents must be COWed. */1936 if (btrfs_file_extent_compression(leaf, fi) ||1937 btrfs_file_extent_encryption(leaf, fi) ||1938 btrfs_file_extent_other_encoding(leaf, fi))1939 goto out;1940 1941 extent_end = btrfs_file_extent_end(path);1942 1943 args->file_extent.disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);1944 args->file_extent.disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);1945 args->file_extent.ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);1946 args->file_extent.offset = btrfs_file_extent_offset(leaf, fi);1947 args->file_extent.compression = btrfs_file_extent_compression(leaf, fi);1948 1949 /*1950 * The following checks can be expensive, as they need to take other1951 * locks and do btree or rbtree searches, so release the path to avoid1952 * blocking other tasks for too long.1953 */1954 btrfs_release_path(path);1955 1956 ret = btrfs_cross_ref_exist(root, btrfs_ino(inode),1957 key->offset - args->file_extent.offset,1958 args->file_extent.disk_bytenr, args->strict, path);1959 WARN_ON_ONCE(ret > 0 && is_freespace_inode);1960 if (ret != 0)1961 goto out;1962 1963 if (args->free_path) {1964 /*1965 * We don't need the path anymore, plus through the1966 * btrfs_lookup_csums_list() call below we will end up allocating1967 * another path. So free the path to avoid unnecessary extra1968 * memory usage.1969 */1970 btrfs_free_path(path);1971 path = NULL;1972 }1973 1974 /* If there are pending snapshots for this root, we must COW. */1975 if (args->writeback_path && !is_freespace_inode &&1976 atomic_read(&root->snapshot_force_cow))1977 goto out;1978 1979 args->file_extent.num_bytes = min(args->end + 1, extent_end) - args->start;1980 args->file_extent.offset += args->start - key->offset;1981 io_start = args->file_extent.disk_bytenr + args->file_extent.offset;1982 1983 /*1984 * Force COW if csums exist in the range. This ensures that csums for a1985 * given extent are either valid or do not exist.1986 */1987 1988 csum_root = btrfs_csum_root(root->fs_info, io_start);1989 ret = btrfs_lookup_csums_list(csum_root, io_start,1990 io_start + args->file_extent.num_bytes - 1,1991 NULL, nowait);1992 WARN_ON_ONCE(ret > 0 && is_freespace_inode);1993 if (ret != 0)1994 goto out;1995 1996 can_nocow = 1;1997 out:1998 if (args->free_path && path)1999 btrfs_free_path(path);2000 2001 return ret < 0 ? ret : can_nocow;2002}2003 2004/*2005 * when nowcow writeback call back. This checks for snapshots or COW copies2006 * of the extents that exist in the file, and COWs the file as required.2007 *2008 * If no cow copies or snapshots exist, we write directly to the existing2009 * blocks on disk2010 */2011static noinline int run_delalloc_nocow(struct btrfs_inode *inode,2012 struct folio *locked_folio,2013 const u64 start, const u64 end)2014{2015 struct btrfs_fs_info *fs_info = inode->root->fs_info;2016 struct btrfs_root *root = inode->root;2017 struct btrfs_path *path;2018 u64 cow_start = (u64)-1;2019 u64 cur_offset = start;2020 int ret;2021 bool check_prev = true;2022 u64 ino = btrfs_ino(inode);2023 struct can_nocow_file_extent_args nocow_args = { 0 };2024 2025 /*2026 * Normally on a zoned device we're only doing COW writes, but in case2027 * of relocation on a zoned filesystem serializes I/O so that we're only2028 * writing sequentially and can end up here as well.2029 */2030 ASSERT(!btrfs_is_zoned(fs_info) || btrfs_is_data_reloc_root(root));2031 2032 path = btrfs_alloc_path();2033 if (!path) {2034 ret = -ENOMEM;2035 goto error;2036 }2037 2038 nocow_args.end = end;2039 nocow_args.writeback_path = true;2040 2041 while (cur_offset <= end) {2042 struct btrfs_block_group *nocow_bg = NULL;2043 struct btrfs_ordered_extent *ordered;2044 struct btrfs_key found_key;2045 struct btrfs_file_extent_item *fi;2046 struct extent_buffer *leaf;2047 struct extent_state *cached_state = NULL;2048 u64 extent_end;2049 u64 nocow_end;2050 int extent_type;2051 bool is_prealloc;2052 2053 ret = btrfs_lookup_file_extent(NULL, root, path, ino,2054 cur_offset, 0);2055 if (ret < 0)2056 goto error;2057 2058 /*2059 * If there is no extent for our range when doing the initial2060 * search, then go back to the previous slot as it will be the2061 * one containing the search offset2062 */2063 if (ret > 0 && path->slots[0] > 0 && check_prev) {2064 leaf = path->nodes[0];2065 btrfs_item_key_to_cpu(leaf, &found_key,2066 path->slots[0] - 1);2067 if (found_key.objectid == ino &&2068 found_key.type == BTRFS_EXTENT_DATA_KEY)2069 path->slots[0]--;2070 }2071 check_prev = false;2072next_slot:2073 /* Go to next leaf if we have exhausted the current one */2074 leaf = path->nodes[0];2075 if (path->slots[0] >= btrfs_header_nritems(leaf)) {2076 ret = btrfs_next_leaf(root, path);2077 if (ret < 0)2078 goto error;2079 if (ret > 0)2080 break;2081 leaf = path->nodes[0];2082 }2083 2084 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);2085 2086 /* Didn't find anything for our INO */2087 if (found_key.objectid > ino)2088 break;2089 /*2090 * Keep searching until we find an EXTENT_ITEM or there are no2091 * more extents for this inode2092 */2093 if (WARN_ON_ONCE(found_key.objectid < ino) ||2094 found_key.type < BTRFS_EXTENT_DATA_KEY) {2095 path->slots[0]++;2096 goto next_slot;2097 }2098 2099 /* Found key is not EXTENT_DATA_KEY or starts after req range */2100 if (found_key.type > BTRFS_EXTENT_DATA_KEY ||2101 found_key.offset > end)2102 break;2103 2104 /*2105 * If the found extent starts after requested offset, then2106 * adjust extent_end to be right before this extent begins2107 */2108 if (found_key.offset > cur_offset) {2109 extent_end = found_key.offset;2110 extent_type = 0;2111 goto must_cow;2112 }2113 2114 /*2115 * Found extent which begins before our range and potentially2116 * intersect it2117 */2118 fi = btrfs_item_ptr(leaf, path->slots[0],2119 struct btrfs_file_extent_item);2120 extent_type = btrfs_file_extent_type(leaf, fi);2121 /* If this is triggered then we have a memory corruption. */2122 ASSERT(extent_type < BTRFS_NR_FILE_EXTENT_TYPES);2123 if (WARN_ON(extent_type >= BTRFS_NR_FILE_EXTENT_TYPES)) {2124 ret = -EUCLEAN;2125 goto error;2126 }2127 extent_end = btrfs_file_extent_end(path);2128 2129 /*2130 * If the extent we got ends before our current offset, skip to2131 * the next extent.2132 */2133 if (extent_end <= cur_offset) {2134 path->slots[0]++;2135 goto next_slot;2136 }2137 2138 nocow_args.start = cur_offset;2139 ret = can_nocow_file_extent(path, &found_key, inode, &nocow_args);2140 if (ret < 0)2141 goto error;2142 if (ret == 0)2143 goto must_cow;2144 2145 ret = 0;2146 nocow_bg = btrfs_inc_nocow_writers(fs_info,2147 nocow_args.file_extent.disk_bytenr +2148 nocow_args.file_extent.offset);2149 if (!nocow_bg) {2150must_cow:2151 /*2152 * If we can't perform NOCOW writeback for the range,2153 * then record the beginning of the range that needs to2154 * be COWed. It will be written out before the next2155 * NOCOW range if we find one, or when exiting this2156 * loop.2157 */2158 if (cow_start == (u64)-1)2159 cow_start = cur_offset;2160 cur_offset = extent_end;2161 if (cur_offset > end)2162 break;2163 if (!path->nodes[0])2164 continue;2165 path->slots[0]++;2166 goto next_slot;2167 }2168 2169 /*2170 * COW range from cow_start to found_key.offset - 1. As the key2171 * will contain the beginning of the first extent that can be2172 * NOCOW, following one which needs to be COW'ed2173 */2174 if (cow_start != (u64)-1) {2175 ret = fallback_to_cow(inode, locked_folio, cow_start,2176 found_key.offset - 1);2177 cow_start = (u64)-1;2178 if (ret) {2179 btrfs_dec_nocow_writers(nocow_bg);2180 goto error;2181 }2182 }2183 2184 nocow_end = cur_offset + nocow_args.file_extent.num_bytes - 1;2185 lock_extent(&inode->io_tree, cur_offset, nocow_end, &cached_state);2186 2187 is_prealloc = extent_type == BTRFS_FILE_EXTENT_PREALLOC;2188 if (is_prealloc) {2189 struct extent_map *em;2190 2191 em = btrfs_create_io_em(inode, cur_offset,2192 &nocow_args.file_extent,2193 BTRFS_ORDERED_PREALLOC);2194 if (IS_ERR(em)) {2195 unlock_extent(&inode->io_tree, cur_offset,2196 nocow_end, &cached_state);2197 btrfs_dec_nocow_writers(nocow_bg);2198 ret = PTR_ERR(em);2199 goto error;2200 }2201 free_extent_map(em);2202 }2203 2204 ordered = btrfs_alloc_ordered_extent(inode, cur_offset,2205 &nocow_args.file_extent,2206 is_prealloc2207 ? (1 << BTRFS_ORDERED_PREALLOC)2208 : (1 << BTRFS_ORDERED_NOCOW));2209 btrfs_dec_nocow_writers(nocow_bg);2210 if (IS_ERR(ordered)) {2211 if (is_prealloc) {2212 btrfs_drop_extent_map_range(inode, cur_offset,2213 nocow_end, false);2214 }2215 unlock_extent(&inode->io_tree, cur_offset,2216 nocow_end, &cached_state);2217 ret = PTR_ERR(ordered);2218 goto error;2219 }2220 2221 if (btrfs_is_data_reloc_root(root))2222 /*2223 * Error handled later, as we must prevent2224 * extent_clear_unlock_delalloc() in error handler2225 * from freeing metadata of created ordered extent.2226 */2227 ret = btrfs_reloc_clone_csums(ordered);2228 btrfs_put_ordered_extent(ordered);2229 2230 extent_clear_unlock_delalloc(inode, cur_offset, nocow_end,2231 locked_folio, &cached_state,2232 EXTENT_LOCKED | EXTENT_DELALLOC |2233 EXTENT_CLEAR_DATA_RESV,2234 PAGE_UNLOCK | PAGE_SET_ORDERED);2235 2236 cur_offset = extent_end;2237 2238 /*2239 * btrfs_reloc_clone_csums() error, now we're OK to call error2240 * handler, as metadata for created ordered extent will only2241 * be freed by btrfs_finish_ordered_io().2242 */2243 if (ret)2244 goto error;2245 }2246 btrfs_release_path(path);2247 2248 if (cur_offset <= end && cow_start == (u64)-1)2249 cow_start = cur_offset;2250 2251 if (cow_start != (u64)-1) {2252 cur_offset = end;2253 ret = fallback_to_cow(inode, locked_folio, cow_start, end);2254 cow_start = (u64)-1;2255 if (ret)2256 goto error;2257 }2258 2259 btrfs_free_path(path);2260 return 0;2261 2262error:2263 /*2264 * If an error happened while a COW region is outstanding, cur_offset2265 * needs to be reset to cow_start to ensure the COW region is unlocked2266 * as well.2267 */2268 if (cow_start != (u64)-1)2269 cur_offset = cow_start;2270 2271 /*2272 * We need to lock the extent here because we're clearing DELALLOC and2273 * we're not locked at this point.2274 */2275 if (cur_offset < end) {2276 struct extent_state *cached = NULL;2277 2278 lock_extent(&inode->io_tree, cur_offset, end, &cached);2279 extent_clear_unlock_delalloc(inode, cur_offset, end,2280 locked_folio, &cached,2281 EXTENT_LOCKED | EXTENT_DELALLOC |2282 EXTENT_DEFRAG |2283 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |2284 PAGE_START_WRITEBACK |2285 PAGE_END_WRITEBACK);2286 btrfs_qgroup_free_data(inode, NULL, cur_offset, end - cur_offset + 1, NULL);2287 }2288 btrfs_free_path(path);2289 return ret;2290}2291 2292static bool should_nocow(struct btrfs_inode *inode, u64 start, u64 end)2293{2294 if (inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)) {2295 if (inode->defrag_bytes &&2296 test_range_bit_exists(&inode->io_tree, start, end, EXTENT_DEFRAG))2297 return false;2298 return true;2299 }2300 return false;2301}2302 2303/*2304 * Function to process delayed allocation (create CoW) for ranges which are2305 * being touched for the first time.2306 */2307int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_folio,2308 u64 start, u64 end, struct writeback_control *wbc)2309{2310 const bool zoned = btrfs_is_zoned(inode->root->fs_info);2311 int ret;2312 2313 /*2314 * The range must cover part of the @locked_folio, or a return of 12315 * can confuse the caller.2316 */2317 ASSERT(!(end <= folio_pos(locked_folio) ||2318 start >= folio_pos(locked_folio) + folio_size(locked_folio)));2319 2320 if (should_nocow(inode, start, end)) {2321 ret = run_delalloc_nocow(inode, locked_folio, start, end);2322 goto out;2323 }2324 2325 if (btrfs_inode_can_compress(inode) &&2326 inode_need_compress(inode, start, end) &&2327 run_delalloc_compressed(inode, locked_folio, start, end, wbc))2328 return 1;2329 2330 if (zoned)2331 ret = run_delalloc_cow(inode, locked_folio, start, end, wbc,2332 true);2333 else2334 ret = cow_file_range(inode, locked_folio, start, end, NULL,2335 false, false);2336 2337out:2338 if (ret < 0)2339 btrfs_cleanup_ordered_extents(inode, locked_folio, start,2340 end - start + 1);2341 return ret;2342}2343 2344void btrfs_split_delalloc_extent(struct btrfs_inode *inode,2345 struct extent_state *orig, u64 split)2346{2347 struct btrfs_fs_info *fs_info = inode->root->fs_info;2348 u64 size;2349 2350 lockdep_assert_held(&inode->io_tree.lock);2351 2352 /* not delalloc, ignore it */2353 if (!(orig->state & EXTENT_DELALLOC))2354 return;2355 2356 size = orig->end - orig->start + 1;2357 if (size > fs_info->max_extent_size) {2358 u32 num_extents;2359 u64 new_size;2360 2361 /*2362 * See the explanation in btrfs_merge_delalloc_extent, the same2363 * applies here, just in reverse.2364 */2365 new_size = orig->end - split + 1;2366 num_extents = count_max_extents(fs_info, new_size);2367 new_size = split - orig->start;2368 num_extents += count_max_extents(fs_info, new_size);2369 if (count_max_extents(fs_info, size) >= num_extents)2370 return;2371 }2372 2373 spin_lock(&inode->lock);2374 btrfs_mod_outstanding_extents(inode, 1);2375 spin_unlock(&inode->lock);2376}2377 2378/*2379 * Handle merged delayed allocation extents so we can keep track of new extents2380 * that are just merged onto old extents, such as when we are doing sequential2381 * writes, so we can properly account for the metadata space we'll need.2382 */2383void btrfs_merge_delalloc_extent(struct btrfs_inode *inode, struct extent_state *new,2384 struct extent_state *other)2385{2386 struct btrfs_fs_info *fs_info = inode->root->fs_info;2387 u64 new_size, old_size;2388 u32 num_extents;2389 2390 lockdep_assert_held(&inode->io_tree.lock);2391 2392 /* not delalloc, ignore it */2393 if (!(other->state & EXTENT_DELALLOC))2394 return;2395 2396 if (new->start > other->start)2397 new_size = new->end - other->start + 1;2398 else2399 new_size = other->end - new->start + 1;2400 2401 /* we're not bigger than the max, unreserve the space and go */2402 if (new_size <= fs_info->max_extent_size) {2403 spin_lock(&inode->lock);2404 btrfs_mod_outstanding_extents(inode, -1);2405 spin_unlock(&inode->lock);2406 return;2407 }2408 2409 /*2410 * We have to add up either side to figure out how many extents were2411 * accounted for before we merged into one big extent. If the number of2412 * extents we accounted for is <= the amount we need for the new range2413 * then we can return, otherwise drop. Think of it like this2414 *2415 * [ 4k][MAX_SIZE]2416 *2417 * So we've grown the extent by a MAX_SIZE extent, this would mean we2418 * need 2 outstanding extents, on one side we have 1 and the other side2419 * we have 1 so they are == and we can return. But in this case2420 *2421 * [MAX_SIZE+4k][MAX_SIZE+4k]2422 *2423 * Each range on their own accounts for 2 extents, but merged together2424 * they are only 3 extents worth of accounting, so we need to drop in2425 * this case.2426 */2427 old_size = other->end - other->start + 1;2428 num_extents = count_max_extents(fs_info, old_size);2429 old_size = new->end - new->start + 1;2430 num_extents += count_max_extents(fs_info, old_size);2431 if (count_max_extents(fs_info, new_size) >= num_extents)2432 return;2433 2434 spin_lock(&inode->lock);2435 btrfs_mod_outstanding_extents(inode, -1);2436 spin_unlock(&inode->lock);2437}2438 2439static void btrfs_add_delalloc_inode(struct btrfs_inode *inode)2440{2441 struct btrfs_root *root = inode->root;2442 struct btrfs_fs_info *fs_info = root->fs_info;2443 2444 spin_lock(&root->delalloc_lock);2445 ASSERT(list_empty(&inode->delalloc_inodes));2446 list_add_tail(&inode->delalloc_inodes, &root->delalloc_inodes);2447 root->nr_delalloc_inodes++;2448 if (root->nr_delalloc_inodes == 1) {2449 spin_lock(&fs_info->delalloc_root_lock);2450 ASSERT(list_empty(&root->delalloc_root));2451 list_add_tail(&root->delalloc_root, &fs_info->delalloc_roots);2452 spin_unlock(&fs_info->delalloc_root_lock);2453 }2454 spin_unlock(&root->delalloc_lock);2455}2456 2457void btrfs_del_delalloc_inode(struct btrfs_inode *inode)2458{2459 struct btrfs_root *root = inode->root;2460 struct btrfs_fs_info *fs_info = root->fs_info;2461 2462 lockdep_assert_held(&root->delalloc_lock);2463 2464 /*2465 * We may be called after the inode was already deleted from the list,2466 * namely in the transaction abort path btrfs_destroy_delalloc_inodes(),2467 * and then later through btrfs_clear_delalloc_extent() while the inode2468 * still has ->delalloc_bytes > 0.2469 */2470 if (!list_empty(&inode->delalloc_inodes)) {2471 list_del_init(&inode->delalloc_inodes);2472 root->nr_delalloc_inodes--;2473 if (!root->nr_delalloc_inodes) {2474 ASSERT(list_empty(&root->delalloc_inodes));2475 spin_lock(&fs_info->delalloc_root_lock);2476 ASSERT(!list_empty(&root->delalloc_root));2477 list_del_init(&root->delalloc_root);2478 spin_unlock(&fs_info->delalloc_root_lock);2479 }2480 }2481}2482 2483/*2484 * Properly track delayed allocation bytes in the inode and to maintain the2485 * list of inodes that have pending delalloc work to be done.2486 */2487void btrfs_set_delalloc_extent(struct btrfs_inode *inode, struct extent_state *state,2488 u32 bits)2489{2490 struct btrfs_fs_info *fs_info = inode->root->fs_info;2491 2492 lockdep_assert_held(&inode->io_tree.lock);2493 2494 if ((bits & EXTENT_DEFRAG) && !(bits & EXTENT_DELALLOC))2495 WARN_ON(1);2496 /*2497 * set_bit and clear bit hooks normally require _irqsave/restore2498 * but in this case, we are only testing for the DELALLOC2499 * bit, which is only set or cleared with irqs on2500 */2501 if (!(state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) {2502 u64 len = state->end + 1 - state->start;2503 u64 prev_delalloc_bytes;2504 u32 num_extents = count_max_extents(fs_info, len);2505 2506 spin_lock(&inode->lock);2507 btrfs_mod_outstanding_extents(inode, num_extents);2508 spin_unlock(&inode->lock);2509 2510 /* For sanity tests */2511 if (btrfs_is_testing(fs_info))2512 return;2513 2514 percpu_counter_add_batch(&fs_info->delalloc_bytes, len,2515 fs_info->delalloc_batch);2516 spin_lock(&inode->lock);2517 prev_delalloc_bytes = inode->delalloc_bytes;2518 inode->delalloc_bytes += len;2519 if (bits & EXTENT_DEFRAG)2520 inode->defrag_bytes += len;2521 spin_unlock(&inode->lock);2522 2523 /*2524 * We don't need to be under the protection of the inode's lock,2525 * because we are called while holding the inode's io_tree lock2526 * and are therefore protected against concurrent calls of this2527 * function and btrfs_clear_delalloc_extent().2528 */2529 if (!btrfs_is_free_space_inode(inode) && prev_delalloc_bytes == 0)2530 btrfs_add_delalloc_inode(inode);2531 }2532 2533 if (!(state->state & EXTENT_DELALLOC_NEW) &&2534 (bits & EXTENT_DELALLOC_NEW)) {2535 spin_lock(&inode->lock);2536 inode->new_delalloc_bytes += state->end + 1 - state->start;2537 spin_unlock(&inode->lock);2538 }2539}2540 2541/*2542 * Once a range is no longer delalloc this function ensures that proper2543 * accounting happens.2544 */2545void btrfs_clear_delalloc_extent(struct btrfs_inode *inode,2546 struct extent_state *state, u32 bits)2547{2548 struct btrfs_fs_info *fs_info = inode->root->fs_info;2549 u64 len = state->end + 1 - state->start;2550 u32 num_extents = count_max_extents(fs_info, len);2551 2552 lockdep_assert_held(&inode->io_tree.lock);2553 2554 if ((state->state & EXTENT_DEFRAG) && (bits & EXTENT_DEFRAG)) {2555 spin_lock(&inode->lock);2556 inode->defrag_bytes -= len;2557 spin_unlock(&inode->lock);2558 }2559 2560 /*2561 * set_bit and clear bit hooks normally require _irqsave/restore2562 * but in this case, we are only testing for the DELALLOC2563 * bit, which is only set or cleared with irqs on2564 */2565 if ((state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) {2566 struct btrfs_root *root = inode->root;2567 u64 new_delalloc_bytes;2568 2569 spin_lock(&inode->lock);2570 btrfs_mod_outstanding_extents(inode, -num_extents);2571 spin_unlock(&inode->lock);2572 2573 /*2574 * We don't reserve metadata space for space cache inodes so we2575 * don't need to call delalloc_release_metadata if there is an2576 * error.2577 */2578 if (bits & EXTENT_CLEAR_META_RESV &&2579 root != fs_info->tree_root)2580 btrfs_delalloc_release_metadata(inode, len, true);2581 2582 /* For sanity tests. */2583 if (btrfs_is_testing(fs_info))2584 return;2585 2586 if (!btrfs_is_data_reloc_root(root) &&2587 !btrfs_is_free_space_inode(inode) &&2588 !(state->state & EXTENT_NORESERVE) &&2589 (bits & EXTENT_CLEAR_DATA_RESV))2590 btrfs_free_reserved_data_space_noquota(fs_info, len);2591 2592 percpu_counter_add_batch(&fs_info->delalloc_bytes, -len,2593 fs_info->delalloc_batch);2594 spin_lock(&inode->lock);2595 inode->delalloc_bytes -= len;2596 new_delalloc_bytes = inode->delalloc_bytes;2597 spin_unlock(&inode->lock);2598 2599 /*2600 * We don't need to be under the protection of the inode's lock,2601 * because we are called while holding the inode's io_tree lock2602 * and are therefore protected against concurrent calls of this2603 * function and btrfs_set_delalloc_extent().2604 */2605 if (!btrfs_is_free_space_inode(inode) && new_delalloc_bytes == 0) {2606 spin_lock(&root->delalloc_lock);2607 btrfs_del_delalloc_inode(inode);2608 spin_unlock(&root->delalloc_lock);2609 }2610 }2611 2612 if ((state->state & EXTENT_DELALLOC_NEW) &&2613 (bits & EXTENT_DELALLOC_NEW)) {2614 spin_lock(&inode->lock);2615 ASSERT(inode->new_delalloc_bytes >= len);2616 inode->new_delalloc_bytes -= len;2617 if (bits & EXTENT_ADD_INODE_BYTES)2618 inode_add_bytes(&inode->vfs_inode, len);2619 spin_unlock(&inode->lock);2620 }2621}2622 2623/*2624 * given a list of ordered sums record them in the inode. This happens2625 * at IO completion time based on sums calculated at bio submission time.2626 */2627static int add_pending_csums(struct btrfs_trans_handle *trans,2628 struct list_head *list)2629{2630 struct btrfs_ordered_sum *sum;2631 struct btrfs_root *csum_root = NULL;2632 int ret;2633 2634 list_for_each_entry(sum, list, list) {2635 trans->adding_csums = true;2636 if (!csum_root)2637 csum_root = btrfs_csum_root(trans->fs_info,2638 sum->logical);2639 ret = btrfs_csum_file_blocks(trans, csum_root, sum);2640 trans->adding_csums = false;2641 if (ret)2642 return ret;2643 }2644 return 0;2645}2646 2647static int btrfs_find_new_delalloc_bytes(struct btrfs_inode *inode,2648 const u64 start,2649 const u64 len,2650 struct extent_state **cached_state)2651{2652 u64 search_start = start;2653 const u64 end = start + len - 1;2654 2655 while (search_start < end) {2656 const u64 search_len = end - search_start + 1;2657 struct extent_map *em;2658 u64 em_len;2659 int ret = 0;2660 2661 em = btrfs_get_extent(inode, NULL, search_start, search_len);2662 if (IS_ERR(em))2663 return PTR_ERR(em);2664 2665 if (em->disk_bytenr != EXTENT_MAP_HOLE)2666 goto next;2667 2668 em_len = em->len;2669 if (em->start < search_start)2670 em_len -= search_start - em->start;2671 if (em_len > search_len)2672 em_len = search_len;2673 2674 ret = set_extent_bit(&inode->io_tree, search_start,2675 search_start + em_len - 1,2676 EXTENT_DELALLOC_NEW, cached_state);2677next:2678 search_start = extent_map_end(em);2679 free_extent_map(em);2680 if (ret)2681 return ret;2682 }2683 return 0;2684}2685 2686int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end,2687 unsigned int extra_bits,2688 struct extent_state **cached_state)2689{2690 WARN_ON(PAGE_ALIGNED(end));2691 2692 if (start >= i_size_read(&inode->vfs_inode) &&2693 !(inode->flags & BTRFS_INODE_PREALLOC)) {2694 /*2695 * There can't be any extents following eof in this case so just2696 * set the delalloc new bit for the range directly.2697 */2698 extra_bits |= EXTENT_DELALLOC_NEW;2699 } else {2700 int ret;2701 2702 ret = btrfs_find_new_delalloc_bytes(inode, start,2703 end + 1 - start,2704 cached_state);2705 if (ret)2706 return ret;2707 }2708 2709 return set_extent_bit(&inode->io_tree, start, end,2710 EXTENT_DELALLOC | extra_bits, cached_state);2711}2712 2713/* see btrfs_writepage_start_hook for details on why this is required */2714struct btrfs_writepage_fixup {2715 struct folio *folio;2716 struct btrfs_inode *inode;2717 struct btrfs_work work;2718};2719 2720static void btrfs_writepage_fixup_worker(struct btrfs_work *work)2721{2722 struct btrfs_writepage_fixup *fixup =2723 container_of(work, struct btrfs_writepage_fixup, work);2724 struct btrfs_ordered_extent *ordered;2725 struct extent_state *cached_state = NULL;2726 struct extent_changeset *data_reserved = NULL;2727 struct folio *folio = fixup->folio;2728 struct btrfs_inode *inode = fixup->inode;2729 struct btrfs_fs_info *fs_info = inode->root->fs_info;2730 u64 page_start = folio_pos(folio);2731 u64 page_end = folio_pos(folio) + folio_size(folio) - 1;2732 int ret = 0;2733 bool free_delalloc_space = true;2734 2735 /*2736 * This is similar to page_mkwrite, we need to reserve the space before2737 * we take the folio lock.2738 */2739 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start,2740 folio_size(folio));2741again:2742 folio_lock(folio);2743 2744 /*2745 * Before we queued this fixup, we took a reference on the folio.2746 * folio->mapping may go NULL, but it shouldn't be moved to a different2747 * address space.2748 */2749 if (!folio->mapping || !folio_test_dirty(folio) ||2750 !folio_test_checked(folio)) {2751 /*2752 * Unfortunately this is a little tricky, either2753 *2754 * 1) We got here and our folio had already been dealt with and2755 * we reserved our space, thus ret == 0, so we need to just2756 * drop our space reservation and bail. This can happen the2757 * first time we come into the fixup worker, or could happen2758 * while waiting for the ordered extent.2759 * 2) Our folio was already dealt with, but we happened to get an2760 * ENOSPC above from the btrfs_delalloc_reserve_space. In2761 * this case we obviously don't have anything to release, but2762 * because the folio was already dealt with we don't want to2763 * mark the folio with an error, so make sure we're resetting2764 * ret to 0. This is why we have this check _before_ the ret2765 * check, because we do not want to have a surprise ENOSPC2766 * when the folio was already properly dealt with.2767 */2768 if (!ret) {2769 btrfs_delalloc_release_extents(inode, folio_size(folio));2770 btrfs_delalloc_release_space(inode, data_reserved,2771 page_start, folio_size(folio),2772 true);2773 }2774 ret = 0;2775 goto out_page;2776 }2777 2778 /*2779 * We can't mess with the folio state unless it is locked, so now that2780 * it is locked bail if we failed to make our space reservation.2781 */2782 if (ret)2783 goto out_page;2784 2785 lock_extent(&inode->io_tree, page_start, page_end, &cached_state);2786 2787 /* already ordered? We're done */2788 if (folio_test_ordered(folio))2789 goto out_reserved;2790 2791 ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE);2792 if (ordered) {2793 unlock_extent(&inode->io_tree, page_start, page_end,2794 &cached_state);2795 folio_unlock(folio);2796 btrfs_start_ordered_extent(ordered);2797 btrfs_put_ordered_extent(ordered);2798 goto again;2799 }2800 2801 ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 0,2802 &cached_state);2803 if (ret)2804 goto out_reserved;2805 2806 /*2807 * Everything went as planned, we're now the owner of a dirty page with2808 * delayed allocation bits set and space reserved for our COW2809 * destination.2810 *2811 * The page was dirty when we started, nothing should have cleaned it.2812 */2813 BUG_ON(!folio_test_dirty(folio));2814 free_delalloc_space = false;2815out_reserved:2816 btrfs_delalloc_release_extents(inode, PAGE_SIZE);2817 if (free_delalloc_space)2818 btrfs_delalloc_release_space(inode, data_reserved, page_start,2819 PAGE_SIZE, true);2820 unlock_extent(&inode->io_tree, page_start, page_end, &cached_state);2821out_page:2822 if (ret) {2823 /*2824 * We hit ENOSPC or other errors. Update the mapping and page2825 * to reflect the errors and clean the page.2826 */2827 mapping_set_error(folio->mapping, ret);2828 btrfs_mark_ordered_io_finished(inode, folio, page_start,2829 folio_size(folio), !ret);2830 folio_clear_dirty_for_io(folio);2831 }2832 btrfs_folio_clear_checked(fs_info, folio, page_start, PAGE_SIZE);2833 folio_unlock(folio);2834 folio_put(folio);2835 kfree(fixup);2836 extent_changeset_free(data_reserved);2837 /*2838 * As a precaution, do a delayed iput in case it would be the last iput2839 * that could need flushing space. Recursing back to fixup worker would2840 * deadlock.2841 */2842 btrfs_add_delayed_iput(inode);2843}2844 2845/*2846 * There are a few paths in the higher layers of the kernel that directly2847 * set the folio dirty bit without asking the filesystem if it is a2848 * good idea. This causes problems because we want to make sure COW2849 * properly happens and the data=ordered rules are followed.2850 *2851 * In our case any range that doesn't have the ORDERED bit set2852 * hasn't been properly setup for IO. We kick off an async process2853 * to fix it up. The async helper will wait for ordered extents, set2854 * the delalloc bit and make it safe to write the folio.2855 */2856int btrfs_writepage_cow_fixup(struct folio *folio)2857{2858 struct inode *inode = folio->mapping->host;2859 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);2860 struct btrfs_writepage_fixup *fixup;2861 2862 /* This folio has ordered extent covering it already */2863 if (folio_test_ordered(folio))2864 return 0;2865 2866 /*2867 * folio_checked is set below when we create a fixup worker for this2868 * folio, don't try to create another one if we're already2869 * folio_test_checked.2870 *2871 * The extent_io writepage code will redirty the foio if we send back2872 * EAGAIN.2873 */2874 if (folio_test_checked(folio))2875 return -EAGAIN;2876 2877 fixup = kzalloc(sizeof(*fixup), GFP_NOFS);2878 if (!fixup)2879 return -EAGAIN;2880 2881 /*2882 * We are already holding a reference to this inode from2883 * write_cache_pages. We need to hold it because the space reservation2884 * takes place outside of the folio lock, and we can't trust2885 * page->mapping outside of the folio lock.2886 */2887 ihold(inode);2888 btrfs_folio_set_checked(fs_info, folio, folio_pos(folio), folio_size(folio));2889 folio_get(folio);2890 btrfs_init_work(&fixup->work, btrfs_writepage_fixup_worker, NULL);2891 fixup->folio = folio;2892 fixup->inode = BTRFS_I(inode);2893 btrfs_queue_work(fs_info->fixup_workers, &fixup->work);2894 2895 return -EAGAIN;2896}2897 2898static int insert_reserved_file_extent(struct btrfs_trans_handle *trans,2899 struct btrfs_inode *inode, u64 file_pos,2900 struct btrfs_file_extent_item *stack_fi,2901 const bool update_inode_bytes,2902 u64 qgroup_reserved)2903{2904 struct btrfs_root *root = inode->root;2905 const u64 sectorsize = root->fs_info->sectorsize;2906 struct btrfs_path *path;2907 struct extent_buffer *leaf;2908 struct btrfs_key ins;2909 u64 disk_num_bytes = btrfs_stack_file_extent_disk_num_bytes(stack_fi);2910 u64 disk_bytenr = btrfs_stack_file_extent_disk_bytenr(stack_fi);2911 u64 offset = btrfs_stack_file_extent_offset(stack_fi);2912 u64 num_bytes = btrfs_stack_file_extent_num_bytes(stack_fi);2913 u64 ram_bytes = btrfs_stack_file_extent_ram_bytes(stack_fi);2914 struct btrfs_drop_extents_args drop_args = { 0 };2915 int ret;2916 2917 path = btrfs_alloc_path();2918 if (!path)2919 return -ENOMEM;2920 2921 /*2922 * we may be replacing one extent in the tree with another.2923 * The new extent is pinned in the extent map, and we don't want2924 * to drop it from the cache until it is completely in the btree.2925 *2926 * So, tell btrfs_drop_extents to leave this extent in the cache.2927 * the caller is expected to unpin it and allow it to be merged2928 * with the others.2929 */2930 drop_args.path = path;2931 drop_args.start = file_pos;2932 drop_args.end = file_pos + num_bytes;2933 drop_args.replace_extent = true;2934 drop_args.extent_item_size = sizeof(*stack_fi);2935 ret = btrfs_drop_extents(trans, root, inode, &drop_args);2936 if (ret)2937 goto out;2938 2939 if (!drop_args.extent_inserted) {2940 ins.objectid = btrfs_ino(inode);2941 ins.offset = file_pos;2942 ins.type = BTRFS_EXTENT_DATA_KEY;2943 2944 ret = btrfs_insert_empty_item(trans, root, path, &ins,2945 sizeof(*stack_fi));2946 if (ret)2947 goto out;2948 }2949 leaf = path->nodes[0];2950 btrfs_set_stack_file_extent_generation(stack_fi, trans->transid);2951 write_extent_buffer(leaf, stack_fi,2952 btrfs_item_ptr_offset(leaf, path->slots[0]),2953 sizeof(struct btrfs_file_extent_item));2954 2955 btrfs_mark_buffer_dirty(trans, leaf);2956 btrfs_release_path(path);2957 2958 /*2959 * If we dropped an inline extent here, we know the range where it is2960 * was not marked with the EXTENT_DELALLOC_NEW bit, so we update the2961 * number of bytes only for that range containing the inline extent.2962 * The remaining of the range will be processed when clearning the2963 * EXTENT_DELALLOC_BIT bit through the ordered extent completion.2964 */2965 if (file_pos == 0 && !IS_ALIGNED(drop_args.bytes_found, sectorsize)) {2966 u64 inline_size = round_down(drop_args.bytes_found, sectorsize);2967 2968 inline_size = drop_args.bytes_found - inline_size;2969 btrfs_update_inode_bytes(inode, sectorsize, inline_size);2970 drop_args.bytes_found -= inline_size;2971 num_bytes -= sectorsize;2972 }2973 2974 if (update_inode_bytes)2975 btrfs_update_inode_bytes(inode, num_bytes, drop_args.bytes_found);2976 2977 ins.objectid = disk_bytenr;2978 ins.offset = disk_num_bytes;2979 ins.type = BTRFS_EXTENT_ITEM_KEY;2980 2981 ret = btrfs_inode_set_file_extent_range(inode, file_pos, ram_bytes);2982 if (ret)2983 goto out;2984 2985 ret = btrfs_alloc_reserved_file_extent(trans, root, btrfs_ino(inode),2986 file_pos - offset,2987 qgroup_reserved, &ins);2988out:2989 btrfs_free_path(path);2990 2991 return ret;2992}2993 2994static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info,2995 u64 start, u64 len)2996{2997 struct btrfs_block_group *cache;2998 2999 cache = btrfs_lookup_block_group(fs_info, start);3000 ASSERT(cache);3001 3002 spin_lock(&cache->lock);3003 cache->delalloc_bytes -= len;3004 spin_unlock(&cache->lock);3005 3006 btrfs_put_block_group(cache);3007}3008 3009static int insert_ordered_extent_file_extent(struct btrfs_trans_handle *trans,3010 struct btrfs_ordered_extent *oe)3011{3012 struct btrfs_file_extent_item stack_fi;3013 bool update_inode_bytes;3014 u64 num_bytes = oe->num_bytes;3015 u64 ram_bytes = oe->ram_bytes;3016 3017 memset(&stack_fi, 0, sizeof(stack_fi));3018 btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_REG);3019 btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, oe->disk_bytenr);3020 btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi,3021 oe->disk_num_bytes);3022 btrfs_set_stack_file_extent_offset(&stack_fi, oe->offset);3023 if (test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags))3024 num_bytes = oe->truncated_len;3025 btrfs_set_stack_file_extent_num_bytes(&stack_fi, num_bytes);3026 btrfs_set_stack_file_extent_ram_bytes(&stack_fi, ram_bytes);3027 btrfs_set_stack_file_extent_compression(&stack_fi, oe->compress_type);3028 /* Encryption and other encoding is reserved and all 0 */3029 3030 /*3031 * For delalloc, when completing an ordered extent we update the inode's3032 * bytes when clearing the range in the inode's io tree, so pass false3033 * as the argument 'update_inode_bytes' to insert_reserved_file_extent(),3034 * except if the ordered extent was truncated.3035 */3036 update_inode_bytes = test_bit(BTRFS_ORDERED_DIRECT, &oe->flags) ||3037 test_bit(BTRFS_ORDERED_ENCODED, &oe->flags) ||3038 test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags);3039 3040 return insert_reserved_file_extent(trans, oe->inode,3041 oe->file_offset, &stack_fi,3042 update_inode_bytes, oe->qgroup_rsv);3043}3044 3045/*3046 * As ordered data IO finishes, this gets called so we can finish3047 * an ordered extent if the range of bytes in the file it covers are3048 * fully written.3049 */3050int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent)3051{3052 struct btrfs_inode *inode = ordered_extent->inode;3053 struct btrfs_root *root = inode->root;3054 struct btrfs_fs_info *fs_info = root->fs_info;3055 struct btrfs_trans_handle *trans = NULL;3056 struct extent_io_tree *io_tree = &inode->io_tree;3057 struct extent_state *cached_state = NULL;3058 u64 start, end;3059 int compress_type = 0;3060 int ret = 0;3061 u64 logical_len = ordered_extent->num_bytes;3062 bool freespace_inode;3063 bool truncated = false;3064 bool clear_reserved_extent = true;3065 unsigned int clear_bits = EXTENT_DEFRAG;3066 3067 start = ordered_extent->file_offset;3068 end = start + ordered_extent->num_bytes - 1;3069 3070 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&3071 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) &&3072 !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags) &&3073 !test_bit(BTRFS_ORDERED_ENCODED, &ordered_extent->flags))3074 clear_bits |= EXTENT_DELALLOC_NEW;3075 3076 freespace_inode = btrfs_is_free_space_inode(inode);3077 if (!freespace_inode)3078 btrfs_lockdep_acquire(fs_info, btrfs_ordered_extent);3079 3080 if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) {3081 ret = -EIO;3082 goto out;3083 }3084 3085 if (btrfs_is_zoned(fs_info))3086 btrfs_zone_finish_endio(fs_info, ordered_extent->disk_bytenr,3087 ordered_extent->disk_num_bytes);3088 3089 if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) {3090 truncated = true;3091 logical_len = ordered_extent->truncated_len;3092 /* Truncated the entire extent, don't bother adding */3093 if (!logical_len)3094 goto out;3095 }3096 3097 if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {3098 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */3099 3100 btrfs_inode_safe_disk_i_size_write(inode, 0);3101 if (freespace_inode)3102 trans = btrfs_join_transaction_spacecache(root);3103 else3104 trans = btrfs_join_transaction(root);3105 if (IS_ERR(trans)) {3106 ret = PTR_ERR(trans);3107 trans = NULL;3108 goto out;3109 }3110 trans->block_rsv = &inode->block_rsv;3111 ret = btrfs_update_inode_fallback(trans, inode);3112 if (ret) /* -ENOMEM or corruption */3113 btrfs_abort_transaction(trans, ret);3114 3115 ret = btrfs_insert_raid_extent(trans, ordered_extent);3116 if (ret)3117 btrfs_abort_transaction(trans, ret);3118 3119 goto out;3120 }3121 3122 clear_bits |= EXTENT_LOCKED;3123 lock_extent(io_tree, start, end, &cached_state);3124 3125 if (freespace_inode)3126 trans = btrfs_join_transaction_spacecache(root);3127 else3128 trans = btrfs_join_transaction(root);3129 if (IS_ERR(trans)) {3130 ret = PTR_ERR(trans);3131 trans = NULL;3132 goto out;3133 }3134 3135 trans->block_rsv = &inode->block_rsv;3136 3137 ret = btrfs_insert_raid_extent(trans, ordered_extent);3138 if (ret)3139 goto out;3140 3141 if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags))3142 compress_type = ordered_extent->compress_type;3143 if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {3144 BUG_ON(compress_type);3145 ret = btrfs_mark_extent_written(trans, inode,3146 ordered_extent->file_offset,3147 ordered_extent->file_offset +3148 logical_len);3149 btrfs_zoned_release_data_reloc_bg(fs_info, ordered_extent->disk_bytenr,3150 ordered_extent->disk_num_bytes);3151 } else {3152 BUG_ON(root == fs_info->tree_root);3153 ret = insert_ordered_extent_file_extent(trans, ordered_extent);3154 if (!ret) {3155 clear_reserved_extent = false;3156 btrfs_release_delalloc_bytes(fs_info,3157 ordered_extent->disk_bytenr,3158 ordered_extent->disk_num_bytes);3159 }3160 }3161 if (ret < 0) {3162 btrfs_abort_transaction(trans, ret);3163 goto out;3164 }3165 3166 ret = unpin_extent_cache(inode, ordered_extent->file_offset,3167 ordered_extent->num_bytes, trans->transid);3168 if (ret < 0) {3169 btrfs_abort_transaction(trans, ret);3170 goto out;3171 }3172 3173 ret = add_pending_csums(trans, &ordered_extent->list);3174 if (ret) {3175 btrfs_abort_transaction(trans, ret);3176 goto out;3177 }3178 3179 /*3180 * If this is a new delalloc range, clear its new delalloc flag to3181 * update the inode's number of bytes. This needs to be done first3182 * before updating the inode item.3183 */3184 if ((clear_bits & EXTENT_DELALLOC_NEW) &&3185 !test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags))3186 clear_extent_bit(&inode->io_tree, start, end,3187 EXTENT_DELALLOC_NEW | EXTENT_ADD_INODE_BYTES,3188 &cached_state);3189 3190 btrfs_inode_safe_disk_i_size_write(inode, 0);3191 ret = btrfs_update_inode_fallback(trans, inode);3192 if (ret) { /* -ENOMEM or corruption */3193 btrfs_abort_transaction(trans, ret);3194 goto out;3195 }3196out:3197 clear_extent_bit(&inode->io_tree, start, end, clear_bits,3198 &cached_state);3199 3200 if (trans)3201 btrfs_end_transaction(trans);3202 3203 if (ret || truncated) {3204 u64 unwritten_start = start;3205 3206 /*3207 * If we failed to finish this ordered extent for any reason we3208 * need to make sure BTRFS_ORDERED_IOERR is set on the ordered3209 * extent, and mark the inode with the error if it wasn't3210 * already set. Any error during writeback would have already3211 * set the mapping error, so we need to set it if we're the ones3212 * marking this ordered extent as failed.3213 */3214 if (ret)3215 btrfs_mark_ordered_extent_error(ordered_extent);3216 3217 if (truncated)3218 unwritten_start += logical_len;3219 clear_extent_uptodate(io_tree, unwritten_start, end, NULL);3220 3221 /*3222 * Drop extent maps for the part of the extent we didn't write.3223 *3224 * We have an exception here for the free_space_inode, this is3225 * because when we do btrfs_get_extent() on the free space inode3226 * we will search the commit root. If this is a new block group3227 * we won't find anything, and we will trip over the assert in3228 * writepage where we do ASSERT(em->block_start !=3229 * EXTENT_MAP_HOLE).3230 *3231 * Theoretically we could also skip this for any NOCOW extent as3232 * we don't mess with the extent map tree in the NOCOW case, but3233 * for now simply skip this if we are the free space inode.3234 */3235 if (!btrfs_is_free_space_inode(inode))3236 btrfs_drop_extent_map_range(inode, unwritten_start,3237 end, false);3238 3239 /*3240 * If the ordered extent had an IOERR or something else went3241 * wrong we need to return the space for this ordered extent3242 * back to the allocator. We only free the extent in the3243 * truncated case if we didn't write out the extent at all.3244 *3245 * If we made it past insert_reserved_file_extent before we3246 * errored out then we don't need to do this as the accounting3247 * has already been done.3248 */3249 if ((ret || !logical_len) &&3250 clear_reserved_extent &&3251 !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&3252 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {3253 /*3254 * Discard the range before returning it back to the3255 * free space pool3256 */3257 if (ret && btrfs_test_opt(fs_info, DISCARD_SYNC))3258 btrfs_discard_extent(fs_info,3259 ordered_extent->disk_bytenr,3260 ordered_extent->disk_num_bytes,3261 NULL);3262 btrfs_free_reserved_extent(fs_info,3263 ordered_extent->disk_bytenr,3264 ordered_extent->disk_num_bytes, 1);3265 /*3266 * Actually free the qgroup rsv which was released when3267 * the ordered extent was created.3268 */3269 btrfs_qgroup_free_refroot(fs_info, btrfs_root_id(inode->root),3270 ordered_extent->qgroup_rsv,3271 BTRFS_QGROUP_RSV_DATA);3272 }3273 }3274 3275 /*3276 * This needs to be done to make sure anybody waiting knows we are done3277 * updating everything for this ordered extent.3278 */3279 btrfs_remove_ordered_extent(inode, ordered_extent);3280 3281 /* once for us */3282 btrfs_put_ordered_extent(ordered_extent);3283 /* once for the tree */3284 btrfs_put_ordered_extent(ordered_extent);3285 3286 return ret;3287}3288 3289int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered)3290{3291 if (btrfs_is_zoned(ordered->inode->root->fs_info) &&3292 !test_bit(BTRFS_ORDERED_IOERR, &ordered->flags) &&3293 list_empty(&ordered->bioc_list))3294 btrfs_finish_ordered_zoned(ordered);3295 return btrfs_finish_one_ordered(ordered);3296}3297 3298/*3299 * Verify the checksum for a single sector without any extra action that depend3300 * on the type of I/O.3301 */3302int btrfs_check_sector_csum(struct btrfs_fs_info *fs_info, struct page *page,3303 u32 pgoff, u8 *csum, const u8 * const csum_expected)3304{3305 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);3306 char *kaddr;3307 3308 ASSERT(pgoff + fs_info->sectorsize <= PAGE_SIZE);3309 3310 shash->tfm = fs_info->csum_shash;3311 3312 kaddr = kmap_local_page(page) + pgoff;3313 crypto_shash_digest(shash, kaddr, fs_info->sectorsize, csum);3314 kunmap_local(kaddr);3315 3316 if (memcmp(csum, csum_expected, fs_info->csum_size))3317 return -EIO;3318 return 0;3319}3320 3321/*3322 * Verify the checksum of a single data sector.3323 *3324 * @bbio: btrfs_io_bio which contains the csum3325 * @dev: device the sector is on3326 * @bio_offset: offset to the beginning of the bio (in bytes)3327 * @bv: bio_vec to check3328 *3329 * Check if the checksum on a data block is valid. When a checksum mismatch is3330 * detected, report the error and fill the corrupted range with zero.3331 *3332 * Return %true if the sector is ok or had no checksum to start with, else %false.3333 */3334bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev,3335 u32 bio_offset, struct bio_vec *bv)3336{3337 struct btrfs_inode *inode = bbio->inode;3338 struct btrfs_fs_info *fs_info = inode->root->fs_info;3339 u64 file_offset = bbio->file_offset + bio_offset;3340 u64 end = file_offset + bv->bv_len - 1;3341 u8 *csum_expected;3342 u8 csum[BTRFS_CSUM_SIZE];3343 3344 ASSERT(bv->bv_len == fs_info->sectorsize);3345 3346 if (!bbio->csum)3347 return true;3348 3349 if (btrfs_is_data_reloc_root(inode->root) &&3350 test_range_bit(&inode->io_tree, file_offset, end, EXTENT_NODATASUM,3351 NULL)) {3352 /* Skip the range without csum for data reloc inode */3353 clear_extent_bits(&inode->io_tree, file_offset, end,3354 EXTENT_NODATASUM);3355 return true;3356 }3357 3358 csum_expected = bbio->csum + (bio_offset >> fs_info->sectorsize_bits) *3359 fs_info->csum_size;3360 if (btrfs_check_sector_csum(fs_info, bv->bv_page, bv->bv_offset, csum,3361 csum_expected))3362 goto zeroit;3363 return true;3364 3365zeroit:3366 btrfs_print_data_csum_error(inode, file_offset, csum, csum_expected,3367 bbio->mirror_num);3368 if (dev)3369 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS);3370 memzero_bvec(bv);3371 return false;3372}3373 3374/*3375 * Perform a delayed iput on @inode.3376 *3377 * @inode: The inode we want to perform iput on3378 *3379 * This function uses the generic vfs_inode::i_count to track whether we should3380 * just decrement it (in case it's > 1) or if this is the last iput then link3381 * the inode to the delayed iput machinery. Delayed iputs are processed at3382 * transaction commit time/superblock commit/cleaner kthread.3383 */3384void btrfs_add_delayed_iput(struct btrfs_inode *inode)3385{3386 struct btrfs_fs_info *fs_info = inode->root->fs_info;3387 unsigned long flags;3388 3389 if (atomic_add_unless(&inode->vfs_inode.i_count, -1, 1))3390 return;3391 3392 atomic_inc(&fs_info->nr_delayed_iputs);3393 /*3394 * Need to be irq safe here because we can be called from either an irq3395 * context (see bio.c and btrfs_put_ordered_extent()) or a non-irq3396 * context.3397 */3398 spin_lock_irqsave(&fs_info->delayed_iput_lock, flags);3399 ASSERT(list_empty(&inode->delayed_iput));3400 list_add_tail(&inode->delayed_iput, &fs_info->delayed_iputs);3401 spin_unlock_irqrestore(&fs_info->delayed_iput_lock, flags);3402 if (!test_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags))3403 wake_up_process(fs_info->cleaner_kthread);3404}3405 3406static void run_delayed_iput_locked(struct btrfs_fs_info *fs_info,3407 struct btrfs_inode *inode)3408{3409 list_del_init(&inode->delayed_iput);3410 spin_unlock_irq(&fs_info->delayed_iput_lock);3411 iput(&inode->vfs_inode);3412 if (atomic_dec_and_test(&fs_info->nr_delayed_iputs))3413 wake_up(&fs_info->delayed_iputs_wait);3414 spin_lock_irq(&fs_info->delayed_iput_lock);3415}3416 3417static void btrfs_run_delayed_iput(struct btrfs_fs_info *fs_info,3418 struct btrfs_inode *inode)3419{3420 if (!list_empty(&inode->delayed_iput)) {3421 spin_lock_irq(&fs_info->delayed_iput_lock);3422 if (!list_empty(&inode->delayed_iput))3423 run_delayed_iput_locked(fs_info, inode);3424 spin_unlock_irq(&fs_info->delayed_iput_lock);3425 }3426}3427 3428void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info)3429{3430 /*3431 * btrfs_put_ordered_extent() can run in irq context (see bio.c), which3432 * calls btrfs_add_delayed_iput() and that needs to lock3433 * fs_info->delayed_iput_lock. So we need to disable irqs here to3434 * prevent a deadlock.3435 */3436 spin_lock_irq(&fs_info->delayed_iput_lock);3437 while (!list_empty(&fs_info->delayed_iputs)) {3438 struct btrfs_inode *inode;3439 3440 inode = list_first_entry(&fs_info->delayed_iputs,3441 struct btrfs_inode, delayed_iput);3442 run_delayed_iput_locked(fs_info, inode);3443 if (need_resched()) {3444 spin_unlock_irq(&fs_info->delayed_iput_lock);3445 cond_resched();3446 spin_lock_irq(&fs_info->delayed_iput_lock);3447 }3448 }3449 spin_unlock_irq(&fs_info->delayed_iput_lock);3450}3451 3452/*3453 * Wait for flushing all delayed iputs3454 *3455 * @fs_info: the filesystem3456 *3457 * This will wait on any delayed iputs that are currently running with KILLABLE3458 * set. Once they are all done running we will return, unless we are killed in3459 * which case we return EINTR. This helps in user operations like fallocate etc3460 * that might get blocked on the iputs.3461 *3462 * Return EINTR if we were killed, 0 if nothing's pending3463 */3464int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info)3465{3466 int ret = wait_event_killable(fs_info->delayed_iputs_wait,3467 atomic_read(&fs_info->nr_delayed_iputs) == 0);3468 if (ret)3469 return -EINTR;3470 return 0;3471}3472 3473/*3474 * This creates an orphan entry for the given inode in case something goes wrong3475 * in the middle of an unlink.3476 */3477int btrfs_orphan_add(struct btrfs_trans_handle *trans,3478 struct btrfs_inode *inode)3479{3480 int ret;3481 3482 ret = btrfs_insert_orphan_item(trans, inode->root, btrfs_ino(inode));3483 if (ret && ret != -EEXIST) {3484 btrfs_abort_transaction(trans, ret);3485 return ret;3486 }3487 3488 return 0;3489}3490 3491/*3492 * We have done the delete so we can go ahead and remove the orphan item for3493 * this particular inode.3494 */3495static int btrfs_orphan_del(struct btrfs_trans_handle *trans,3496 struct btrfs_inode *inode)3497{3498 return btrfs_del_orphan_item(trans, inode->root, btrfs_ino(inode));3499}3500 3501/*3502 * this cleans up any orphans that may be left on the list from the last use3503 * of this root.3504 */3505int btrfs_orphan_cleanup(struct btrfs_root *root)3506{3507 struct btrfs_fs_info *fs_info = root->fs_info;3508 struct btrfs_path *path;3509 struct extent_buffer *leaf;3510 struct btrfs_key key, found_key;3511 struct btrfs_trans_handle *trans;3512 struct inode *inode;3513 u64 last_objectid = 0;3514 int ret = 0, nr_unlink = 0;3515 3516 if (test_and_set_bit(BTRFS_ROOT_ORPHAN_CLEANUP, &root->state))3517 return 0;3518 3519 path = btrfs_alloc_path();3520 if (!path) {3521 ret = -ENOMEM;3522 goto out;3523 }3524 path->reada = READA_BACK;3525 3526 key.objectid = BTRFS_ORPHAN_OBJECTID;3527 key.type = BTRFS_ORPHAN_ITEM_KEY;3528 key.offset = (u64)-1;3529 3530 while (1) {3531 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);3532 if (ret < 0)3533 goto out;3534 3535 /*3536 * if ret == 0 means we found what we were searching for, which3537 * is weird, but possible, so only screw with path if we didn't3538 * find the key and see if we have stuff that matches3539 */3540 if (ret > 0) {3541 ret = 0;3542 if (path->slots[0] == 0)3543 break;3544 path->slots[0]--;3545 }3546 3547 /* pull out the item */3548 leaf = path->nodes[0];3549 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);3550 3551 /* make sure the item matches what we want */3552 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID)3553 break;3554 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY)3555 break;3556 3557 /* release the path since we're done with it */3558 btrfs_release_path(path);3559 3560 /*3561 * this is where we are basically btrfs_lookup, without the3562 * crossing root thing. we store the inode number in the3563 * offset of the orphan item.3564 */3565 3566 if (found_key.offset == last_objectid) {3567 /*3568 * We found the same inode as before. This means we were3569 * not able to remove its items via eviction triggered3570 * by an iput(). A transaction abort may have happened,3571 * due to -ENOSPC for example, so try to grab the error3572 * that lead to a transaction abort, if any.3573 */3574 btrfs_err(fs_info,3575 "Error removing orphan entry, stopping orphan cleanup");3576 ret = BTRFS_FS_ERROR(fs_info) ?: -EINVAL;3577 goto out;3578 }3579 3580 last_objectid = found_key.offset;3581 3582 found_key.objectid = found_key.offset;3583 found_key.type = BTRFS_INODE_ITEM_KEY;3584 found_key.offset = 0;3585 inode = btrfs_iget(last_objectid, root);3586 if (IS_ERR(inode)) {3587 ret = PTR_ERR(inode);3588 inode = NULL;3589 if (ret != -ENOENT)3590 goto out;3591 }3592 3593 if (!inode && root == fs_info->tree_root) {3594 struct btrfs_root *dead_root;3595 int is_dead_root = 0;3596 3597 /*3598 * This is an orphan in the tree root. Currently these3599 * could come from 2 sources:3600 * a) a root (snapshot/subvolume) deletion in progress3601 * b) a free space cache inode3602 * We need to distinguish those two, as the orphan item3603 * for a root must not get deleted before the deletion3604 * of the snapshot/subvolume's tree completes.3605 *3606 * btrfs_find_orphan_roots() ran before us, which has3607 * found all deleted roots and loaded them into3608 * fs_info->fs_roots_radix. So here we can find if an3609 * orphan item corresponds to a deleted root by looking3610 * up the root from that radix tree.3611 */3612 3613 spin_lock(&fs_info->fs_roots_radix_lock);3614 dead_root = radix_tree_lookup(&fs_info->fs_roots_radix,3615 (unsigned long)found_key.objectid);3616 if (dead_root && btrfs_root_refs(&dead_root->root_item) == 0)3617 is_dead_root = 1;3618 spin_unlock(&fs_info->fs_roots_radix_lock);3619 3620 if (is_dead_root) {3621 /* prevent this orphan from being found again */3622 key.offset = found_key.objectid - 1;3623 continue;3624 }3625 3626 }3627 3628 /*3629 * If we have an inode with links, there are a couple of3630 * possibilities:3631 *3632 * 1. We were halfway through creating fsverity metadata for the3633 * file. In that case, the orphan item represents incomplete3634 * fsverity metadata which must be cleaned up with3635 * btrfs_drop_verity_items and deleting the orphan item.3636 3637 * 2. Old kernels (before v3.12) used to create an3638 * orphan item for truncate indicating that there were possibly3639 * extent items past i_size that needed to be deleted. In v3.12,3640 * truncate was changed to update i_size in sync with the extent3641 * items, but the (useless) orphan item was still created. Since3642 * v4.18, we don't create the orphan item for truncate at all.3643 *3644 * So, this item could mean that we need to do a truncate, but3645 * only if this filesystem was last used on a pre-v3.12 kernel3646 * and was not cleanly unmounted. The odds of that are quite3647 * slim, and it's a pain to do the truncate now, so just delete3648 * the orphan item.3649 *3650 * It's also possible that this orphan item was supposed to be3651 * deleted but wasn't. The inode number may have been reused,3652 * but either way, we can delete the orphan item.3653 */3654 if (!inode || inode->i_nlink) {3655 if (inode) {3656 ret = btrfs_drop_verity_items(BTRFS_I(inode));3657 iput(inode);3658 inode = NULL;3659 if (ret)3660 goto out;3661 }3662 trans = btrfs_start_transaction(root, 1);3663 if (IS_ERR(trans)) {3664 ret = PTR_ERR(trans);3665 goto out;3666 }3667 btrfs_debug(fs_info, "auto deleting %Lu",3668 found_key.objectid);3669 ret = btrfs_del_orphan_item(trans, root,3670 found_key.objectid);3671 btrfs_end_transaction(trans);3672 if (ret)3673 goto out;3674 continue;3675 }3676 3677 nr_unlink++;3678 3679 /* this will do delete_inode and everything for us */3680 iput(inode);3681 }3682 /* release the path since we're done with it */3683 btrfs_release_path(path);3684 3685 if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) {3686 trans = btrfs_join_transaction(root);3687 if (!IS_ERR(trans))3688 btrfs_end_transaction(trans);3689 }3690 3691 if (nr_unlink)3692 btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink);3693 3694out:3695 if (ret)3696 btrfs_err(fs_info, "could not do orphan cleanup %d", ret);3697 btrfs_free_path(path);3698 return ret;3699}3700 3701/*3702 * very simple check to peek ahead in the leaf looking for xattrs. If we3703 * don't find any xattrs, we know there can't be any acls.3704 *3705 * slot is the slot the inode is in, objectid is the objectid of the inode3706 */3707static noinline int acls_after_inode_item(struct extent_buffer *leaf,3708 int slot, u64 objectid,3709 int *first_xattr_slot)3710{3711 u32 nritems = btrfs_header_nritems(leaf);3712 struct btrfs_key found_key;3713 static u64 xattr_access = 0;3714 static u64 xattr_default = 0;3715 int scanned = 0;3716 3717 if (!xattr_access) {3718 xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS,3719 strlen(XATTR_NAME_POSIX_ACL_ACCESS));3720 xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT,3721 strlen(XATTR_NAME_POSIX_ACL_DEFAULT));3722 }3723 3724 slot++;3725 *first_xattr_slot = -1;3726 while (slot < nritems) {3727 btrfs_item_key_to_cpu(leaf, &found_key, slot);3728 3729 /* we found a different objectid, there must not be acls */3730 if (found_key.objectid != objectid)3731 return 0;3732 3733 /* we found an xattr, assume we've got an acl */3734 if (found_key.type == BTRFS_XATTR_ITEM_KEY) {3735 if (*first_xattr_slot == -1)3736 *first_xattr_slot = slot;3737 if (found_key.offset == xattr_access ||3738 found_key.offset == xattr_default)3739 return 1;3740 }3741 3742 /*3743 * we found a key greater than an xattr key, there can't3744 * be any acls later on3745 */3746 if (found_key.type > BTRFS_XATTR_ITEM_KEY)3747 return 0;3748 3749 slot++;3750 scanned++;3751 3752 /*3753 * it goes inode, inode backrefs, xattrs, extents,3754 * so if there are a ton of hard links to an inode there can3755 * be a lot of backrefs. Don't waste time searching too hard,3756 * this is just an optimization3757 */3758 if (scanned >= 8)3759 break;3760 }3761 /* we hit the end of the leaf before we found an xattr or3762 * something larger than an xattr. We have to assume the inode3763 * has acls3764 */3765 if (*first_xattr_slot == -1)3766 *first_xattr_slot = slot;3767 return 1;3768}3769 3770static int btrfs_init_file_extent_tree(struct btrfs_inode *inode)3771{3772 struct btrfs_fs_info *fs_info = inode->root->fs_info;3773 3774 if (WARN_ON_ONCE(inode->file_extent_tree))3775 return 0;3776 if (btrfs_fs_incompat(fs_info, NO_HOLES))3777 return 0;3778 if (!S_ISREG(inode->vfs_inode.i_mode))3779 return 0;3780 if (btrfs_is_free_space_inode(inode))3781 return 0;3782 3783 inode->file_extent_tree = kmalloc(sizeof(struct extent_io_tree), GFP_KERNEL);3784 if (!inode->file_extent_tree)3785 return -ENOMEM;3786 3787 extent_io_tree_init(fs_info, inode->file_extent_tree, IO_TREE_INODE_FILE_EXTENT);3788 /* Lockdep class is set only for the file extent tree. */3789 lockdep_set_class(&inode->file_extent_tree->lock, &file_extent_tree_class);3790 3791 return 0;3792}3793 3794/*3795 * read an inode from the btree into the in-memory inode3796 */3797static int btrfs_read_locked_inode(struct inode *inode,3798 struct btrfs_path *in_path)3799{3800 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);3801 struct btrfs_path *path = in_path;3802 struct extent_buffer *leaf;3803 struct btrfs_inode_item *inode_item;3804 struct btrfs_root *root = BTRFS_I(inode)->root;3805 struct btrfs_key location;3806 unsigned long ptr;3807 int maybe_acls;3808 u32 rdev;3809 int ret;3810 bool filled = false;3811 int first_xattr_slot;3812 3813 ret = btrfs_init_file_extent_tree(BTRFS_I(inode));3814 if (ret)3815 return ret;3816 3817 ret = btrfs_fill_inode(inode, &rdev);3818 if (!ret)3819 filled = true;3820 3821 if (!path) {3822 path = btrfs_alloc_path();3823 if (!path)3824 return -ENOMEM;3825 }3826 3827 btrfs_get_inode_key(BTRFS_I(inode), &location);3828 3829 ret = btrfs_lookup_inode(NULL, root, path, &location, 0);3830 if (ret) {3831 if (path != in_path)3832 btrfs_free_path(path);3833 return ret;3834 }3835 3836 leaf = path->nodes[0];3837 3838 if (filled)3839 goto cache_index;3840 3841 inode_item = btrfs_item_ptr(leaf, path->slots[0],3842 struct btrfs_inode_item);3843 inode->i_mode = btrfs_inode_mode(leaf, inode_item);3844 set_nlink(inode, btrfs_inode_nlink(leaf, inode_item));3845 i_uid_write(inode, btrfs_inode_uid(leaf, inode_item));3846 i_gid_write(inode, btrfs_inode_gid(leaf, inode_item));3847 btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item));3848 btrfs_inode_set_file_extent_range(BTRFS_I(inode), 0,3849 round_up(i_size_read(inode), fs_info->sectorsize));3850 3851 inode_set_atime(inode, btrfs_timespec_sec(leaf, &inode_item->atime),3852 btrfs_timespec_nsec(leaf, &inode_item->atime));3853 3854 inode_set_mtime(inode, btrfs_timespec_sec(leaf, &inode_item->mtime),3855 btrfs_timespec_nsec(leaf, &inode_item->mtime));3856 3857 inode_set_ctime(inode, btrfs_timespec_sec(leaf, &inode_item->ctime),3858 btrfs_timespec_nsec(leaf, &inode_item->ctime));3859 3860 BTRFS_I(inode)->i_otime_sec = btrfs_timespec_sec(leaf, &inode_item->otime);3861 BTRFS_I(inode)->i_otime_nsec = btrfs_timespec_nsec(leaf, &inode_item->otime);3862 3863 inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item));3864 BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item);3865 BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item);3866 3867 inode_set_iversion_queried(inode,3868 btrfs_inode_sequence(leaf, inode_item));3869 inode->i_generation = BTRFS_I(inode)->generation;3870 inode->i_rdev = 0;3871 rdev = btrfs_inode_rdev(leaf, inode_item);3872 3873 if (S_ISDIR(inode->i_mode))3874 BTRFS_I(inode)->index_cnt = (u64)-1;3875 3876 btrfs_inode_split_flags(btrfs_inode_flags(leaf, inode_item),3877 &BTRFS_I(inode)->flags, &BTRFS_I(inode)->ro_flags);3878 3879cache_index:3880 /*3881 * If we were modified in the current generation and evicted from memory3882 * and then re-read we need to do a full sync since we don't have any3883 * idea about which extents were modified before we were evicted from3884 * cache.3885 *3886 * This is required for both inode re-read from disk and delayed inode3887 * in the delayed_nodes xarray.3888 */3889 if (BTRFS_I(inode)->last_trans == btrfs_get_fs_generation(fs_info))3890 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,3891 &BTRFS_I(inode)->runtime_flags);3892 3893 /*3894 * We don't persist the id of the transaction where an unlink operation3895 * against the inode was last made. So here we assume the inode might3896 * have been evicted, and therefore the exact value of last_unlink_trans3897 * lost, and set it to last_trans to avoid metadata inconsistencies3898 * between the inode and its parent if the inode is fsync'ed and the log3899 * replayed. For example, in the scenario:3900 *3901 * touch mydir/foo3902 * ln mydir/foo mydir/bar3903 * sync3904 * unlink mydir/bar3905 * echo 2 > /proc/sys/vm/drop_caches # evicts inode3906 * xfs_io -c fsync mydir/foo3907 * <power failure>3908 * mount fs, triggers fsync log replay3909 *3910 * We must make sure that when we fsync our inode foo we also log its3911 * parent inode, otherwise after log replay the parent still has the3912 * dentry with the "bar" name but our inode foo has a link count of 13913 * and doesn't have an inode ref with the name "bar" anymore.3914 *3915 * Setting last_unlink_trans to last_trans is a pessimistic approach,3916 * but it guarantees correctness at the expense of occasional full3917 * transaction commits on fsync if our inode is a directory, or if our3918 * inode is not a directory, logging its parent unnecessarily.3919 */3920 BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans;3921 3922 /*3923 * Same logic as for last_unlink_trans. We don't persist the generation3924 * of the last transaction where this inode was used for a reflink3925 * operation, so after eviction and reloading the inode we must be3926 * pessimistic and assume the last transaction that modified the inode.3927 */3928 BTRFS_I(inode)->last_reflink_trans = BTRFS_I(inode)->last_trans;3929 3930 path->slots[0]++;3931 if (inode->i_nlink != 1 ||3932 path->slots[0] >= btrfs_header_nritems(leaf))3933 goto cache_acl;3934 3935 btrfs_item_key_to_cpu(leaf, &location, path->slots[0]);3936 if (location.objectid != btrfs_ino(BTRFS_I(inode)))3937 goto cache_acl;3938 3939 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);3940 if (location.type == BTRFS_INODE_REF_KEY) {3941 struct btrfs_inode_ref *ref;3942 3943 ref = (struct btrfs_inode_ref *)ptr;3944 BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref);3945 } else if (location.type == BTRFS_INODE_EXTREF_KEY) {3946 struct btrfs_inode_extref *extref;3947 3948 extref = (struct btrfs_inode_extref *)ptr;3949 BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf,3950 extref);3951 }3952cache_acl:3953 /*3954 * try to precache a NULL acl entry for files that don't have3955 * any xattrs or acls3956 */3957 maybe_acls = acls_after_inode_item(leaf, path->slots[0],3958 btrfs_ino(BTRFS_I(inode)), &first_xattr_slot);3959 if (first_xattr_slot != -1) {3960 path->slots[0] = first_xattr_slot;3961 ret = btrfs_load_inode_props(inode, path);3962 if (ret)3963 btrfs_err(fs_info,3964 "error loading props for ino %llu (root %llu): %d",3965 btrfs_ino(BTRFS_I(inode)),3966 btrfs_root_id(root), ret);3967 }3968 if (path != in_path)3969 btrfs_free_path(path);3970 3971 if (!maybe_acls)3972 cache_no_acl(inode);3973 3974 switch (inode->i_mode & S_IFMT) {3975 case S_IFREG:3976 inode->i_mapping->a_ops = &btrfs_aops;3977 inode->i_fop = &btrfs_file_operations;3978 inode->i_op = &btrfs_file_inode_operations;3979 break;3980 case S_IFDIR:3981 inode->i_fop = &btrfs_dir_file_operations;3982 inode->i_op = &btrfs_dir_inode_operations;3983 break;3984 case S_IFLNK:3985 inode->i_op = &btrfs_symlink_inode_operations;3986 inode_nohighmem(inode);3987 inode->i_mapping->a_ops = &btrfs_aops;3988 break;3989 default:3990 inode->i_op = &btrfs_special_inode_operations;3991 init_special_inode(inode, inode->i_mode, rdev);3992 break;3993 }3994 3995 btrfs_sync_inode_flags_to_i_flags(inode);3996 return 0;3997}3998 3999/*4000 * given a leaf and an inode, copy the inode fields into the leaf4001 */4002static void fill_inode_item(struct btrfs_trans_handle *trans,4003 struct extent_buffer *leaf,4004 struct btrfs_inode_item *item,4005 struct inode *inode)4006{4007 struct btrfs_map_token token;4008 u64 flags;4009 4010 btrfs_init_map_token(&token, leaf);4011 4012 btrfs_set_token_inode_uid(&token, item, i_uid_read(inode));4013 btrfs_set_token_inode_gid(&token, item, i_gid_read(inode));4014 btrfs_set_token_inode_size(&token, item, BTRFS_I(inode)->disk_i_size);4015 btrfs_set_token_inode_mode(&token, item, inode->i_mode);4016 btrfs_set_token_inode_nlink(&token, item, inode->i_nlink);4017 4018 btrfs_set_token_timespec_sec(&token, &item->atime,4019 inode_get_atime_sec(inode));4020 btrfs_set_token_timespec_nsec(&token, &item->atime,4021 inode_get_atime_nsec(inode));4022 4023 btrfs_set_token_timespec_sec(&token, &item->mtime,4024 inode_get_mtime_sec(inode));4025 btrfs_set_token_timespec_nsec(&token, &item->mtime,4026 inode_get_mtime_nsec(inode));4027 4028 btrfs_set_token_timespec_sec(&token, &item->ctime,4029 inode_get_ctime_sec(inode));4030 btrfs_set_token_timespec_nsec(&token, &item->ctime,4031 inode_get_ctime_nsec(inode));4032 4033 btrfs_set_token_timespec_sec(&token, &item->otime, BTRFS_I(inode)->i_otime_sec);4034 btrfs_set_token_timespec_nsec(&token, &item->otime, BTRFS_I(inode)->i_otime_nsec);4035 4036 btrfs_set_token_inode_nbytes(&token, item, inode_get_bytes(inode));4037 btrfs_set_token_inode_generation(&token, item,4038 BTRFS_I(inode)->generation);4039 btrfs_set_token_inode_sequence(&token, item, inode_peek_iversion(inode));4040 btrfs_set_token_inode_transid(&token, item, trans->transid);4041 btrfs_set_token_inode_rdev(&token, item, inode->i_rdev);4042 flags = btrfs_inode_combine_flags(BTRFS_I(inode)->flags,4043 BTRFS_I(inode)->ro_flags);4044 btrfs_set_token_inode_flags(&token, item, flags);4045 btrfs_set_token_inode_block_group(&token, item, 0);4046}4047 4048/*4049 * copy everything in the in-memory inode into the btree.4050 */4051static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans,4052 struct btrfs_inode *inode)4053{4054 struct btrfs_inode_item *inode_item;4055 struct btrfs_path *path;4056 struct extent_buffer *leaf;4057 struct btrfs_key key;4058 int ret;4059 4060 path = btrfs_alloc_path();4061 if (!path)4062 return -ENOMEM;4063 4064 btrfs_get_inode_key(inode, &key);4065 ret = btrfs_lookup_inode(trans, inode->root, path, &key, 1);4066 if (ret) {4067 if (ret > 0)4068 ret = -ENOENT;4069 goto failed;4070 }4071 4072 leaf = path->nodes[0];4073 inode_item = btrfs_item_ptr(leaf, path->slots[0],4074 struct btrfs_inode_item);4075 4076 fill_inode_item(trans, leaf, inode_item, &inode->vfs_inode);4077 btrfs_mark_buffer_dirty(trans, leaf);4078 btrfs_set_inode_last_trans(trans, inode);4079 ret = 0;4080failed:4081 btrfs_free_path(path);4082 return ret;4083}4084 4085/*4086 * copy everything in the in-memory inode into the btree.4087 */4088int btrfs_update_inode(struct btrfs_trans_handle *trans,4089 struct btrfs_inode *inode)4090{4091 struct btrfs_root *root = inode->root;4092 struct btrfs_fs_info *fs_info = root->fs_info;4093 int ret;4094 4095 /*4096 * If the inode is a free space inode, we can deadlock during commit4097 * if we put it into the delayed code.4098 *4099 * The data relocation inode should also be directly updated4100 * without delay4101 */4102 if (!btrfs_is_free_space_inode(inode)4103 && !btrfs_is_data_reloc_root(root)4104 && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {4105 btrfs_update_root_times(trans, root);4106 4107 ret = btrfs_delayed_update_inode(trans, inode);4108 if (!ret)4109 btrfs_set_inode_last_trans(trans, inode);4110 return ret;4111 }4112 4113 return btrfs_update_inode_item(trans, inode);4114}4115 4116int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans,4117 struct btrfs_inode *inode)4118{4119 int ret;4120 4121 ret = btrfs_update_inode(trans, inode);4122 if (ret == -ENOSPC)4123 return btrfs_update_inode_item(trans, inode);4124 return ret;4125}4126 4127/*4128 * unlink helper that gets used here in inode.c and in the tree logging4129 * recovery code. It remove a link in a directory with a given name, and4130 * also drops the back refs in the inode to the directory4131 */4132static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans,4133 struct btrfs_inode *dir,4134 struct btrfs_inode *inode,4135 const struct fscrypt_str *name,4136 struct btrfs_rename_ctx *rename_ctx)4137{4138 struct btrfs_root *root = dir->root;4139 struct btrfs_fs_info *fs_info = root->fs_info;4140 struct btrfs_path *path;4141 int ret = 0;4142 struct btrfs_dir_item *di;4143 u64 index;4144 u64 ino = btrfs_ino(inode);4145 u64 dir_ino = btrfs_ino(dir);4146 4147 path = btrfs_alloc_path();4148 if (!path) {4149 ret = -ENOMEM;4150 goto out;4151 }4152 4153 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, name, -1);4154 if (IS_ERR_OR_NULL(di)) {4155 ret = di ? PTR_ERR(di) : -ENOENT;4156 goto err;4157 }4158 ret = btrfs_delete_one_dir_name(trans, root, path, di);4159 if (ret)4160 goto err;4161 btrfs_release_path(path);4162 4163 /*4164 * If we don't have dir index, we have to get it by looking up4165 * the inode ref, since we get the inode ref, remove it directly,4166 * it is unnecessary to do delayed deletion.4167 *4168 * But if we have dir index, needn't search inode ref to get it.4169 * Since the inode ref is close to the inode item, it is better4170 * that we delay to delete it, and just do this deletion when4171 * we update the inode item.4172 */4173 if (inode->dir_index) {4174 ret = btrfs_delayed_delete_inode_ref(inode);4175 if (!ret) {4176 index = inode->dir_index;4177 goto skip_backref;4178 }4179 }4180 4181 ret = btrfs_del_inode_ref(trans, root, name, ino, dir_ino, &index);4182 if (ret) {4183 btrfs_info(fs_info,4184 "failed to delete reference to %.*s, inode %llu parent %llu",4185 name->len, name->name, ino, dir_ino);4186 btrfs_abort_transaction(trans, ret);4187 goto err;4188 }4189skip_backref:4190 if (rename_ctx)4191 rename_ctx->index = index;4192 4193 ret = btrfs_delete_delayed_dir_index(trans, dir, index);4194 if (ret) {4195 btrfs_abort_transaction(trans, ret);4196 goto err;4197 }4198 4199 /*4200 * If we are in a rename context, we don't need to update anything in the4201 * log. That will be done later during the rename by btrfs_log_new_name().4202 * Besides that, doing it here would only cause extra unnecessary btree4203 * operations on the log tree, increasing latency for applications.4204 */4205 if (!rename_ctx) {4206 btrfs_del_inode_ref_in_log(trans, root, name, inode, dir_ino);4207 btrfs_del_dir_entries_in_log(trans, root, name, dir, index);4208 }4209 4210 /*4211 * If we have a pending delayed iput we could end up with the final iput4212 * being run in btrfs-cleaner context. If we have enough of these built4213 * up we can end up burning a lot of time in btrfs-cleaner without any4214 * way to throttle the unlinks. Since we're currently holding a ref on4215 * the inode we can run the delayed iput here without any issues as the4216 * final iput won't be done until after we drop the ref we're currently4217 * holding.4218 */4219 btrfs_run_delayed_iput(fs_info, inode);4220err:4221 btrfs_free_path(path);4222 if (ret)4223 goto out;4224 4225 btrfs_i_size_write(dir, dir->vfs_inode.i_size - name->len * 2);4226 inode_inc_iversion(&inode->vfs_inode);4227 inode_set_ctime_current(&inode->vfs_inode);4228 inode_inc_iversion(&dir->vfs_inode);4229 inode_set_mtime_to_ts(&dir->vfs_inode, inode_set_ctime_current(&dir->vfs_inode));4230 ret = btrfs_update_inode(trans, dir);4231out:4232 return ret;4233}4234 4235int btrfs_unlink_inode(struct btrfs_trans_handle *trans,4236 struct btrfs_inode *dir, struct btrfs_inode *inode,4237 const struct fscrypt_str *name)4238{4239 int ret;4240 4241 ret = __btrfs_unlink_inode(trans, dir, inode, name, NULL);4242 if (!ret) {4243 drop_nlink(&inode->vfs_inode);4244 ret = btrfs_update_inode(trans, inode);4245 }4246 return ret;4247}4248 4249/*4250 * helper to start transaction for unlink and rmdir.4251 *4252 * unlink and rmdir are special in btrfs, they do not always free space, so4253 * if we cannot make our reservations the normal way try and see if there is4254 * plenty of slack room in the global reserve to migrate, otherwise we cannot4255 * allow the unlink to occur.4256 */4257static struct btrfs_trans_handle *__unlink_start_trans(struct btrfs_inode *dir)4258{4259 struct btrfs_root *root = dir->root;4260 4261 return btrfs_start_transaction_fallback_global_rsv(root,4262 BTRFS_UNLINK_METADATA_UNITS);4263}4264 4265static int btrfs_unlink(struct inode *dir, struct dentry *dentry)4266{4267 struct btrfs_trans_handle *trans;4268 struct inode *inode = d_inode(dentry);4269 int ret;4270 struct fscrypt_name fname;4271 4272 ret = fscrypt_setup_filename(dir, &dentry->d_name, 1, &fname);4273 if (ret)4274 return ret;4275 4276 /* This needs to handle no-key deletions later on */4277 4278 trans = __unlink_start_trans(BTRFS_I(dir));4279 if (IS_ERR(trans)) {4280 ret = PTR_ERR(trans);4281 goto fscrypt_free;4282 }4283 4284 btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),4285 false);4286 4287 ret = btrfs_unlink_inode(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),4288 &fname.disk_name);4289 if (ret)4290 goto end_trans;4291 4292 if (inode->i_nlink == 0) {4293 ret = btrfs_orphan_add(trans, BTRFS_I(inode));4294 if (ret)4295 goto end_trans;4296 }4297 4298end_trans:4299 btrfs_end_transaction(trans);4300 btrfs_btree_balance_dirty(BTRFS_I(dir)->root->fs_info);4301fscrypt_free:4302 fscrypt_free_filename(&fname);4303 return ret;4304}4305 4306static int btrfs_unlink_subvol(struct btrfs_trans_handle *trans,4307 struct btrfs_inode *dir, struct dentry *dentry)4308{4309 struct btrfs_root *root = dir->root;4310 struct btrfs_inode *inode = BTRFS_I(d_inode(dentry));4311 struct btrfs_path *path;4312 struct extent_buffer *leaf;4313 struct btrfs_dir_item *di;4314 struct btrfs_key key;4315 u64 index;4316 int ret;4317 u64 objectid;4318 u64 dir_ino = btrfs_ino(dir);4319 struct fscrypt_name fname;4320 4321 ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 1, &fname);4322 if (ret)4323 return ret;4324 4325 /* This needs to handle no-key deletions later on */4326 4327 if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) {4328 objectid = btrfs_root_id(inode->root);4329 } else if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) {4330 objectid = inode->ref_root_id;4331 } else {4332 WARN_ON(1);4333 fscrypt_free_filename(&fname);4334 return -EINVAL;4335 }4336 4337 path = btrfs_alloc_path();4338 if (!path) {4339 ret = -ENOMEM;4340 goto out;4341 }4342 4343 di = btrfs_lookup_dir_item(trans, root, path, dir_ino,4344 &fname.disk_name, -1);4345 if (IS_ERR_OR_NULL(di)) {4346 ret = di ? PTR_ERR(di) : -ENOENT;4347 goto out;4348 }4349 4350 leaf = path->nodes[0];4351 btrfs_dir_item_key_to_cpu(leaf, di, &key);4352 WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid);4353 ret = btrfs_delete_one_dir_name(trans, root, path, di);4354 if (ret) {4355 btrfs_abort_transaction(trans, ret);4356 goto out;4357 }4358 btrfs_release_path(path);4359 4360 /*4361 * This is a placeholder inode for a subvolume we didn't have a4362 * reference to at the time of the snapshot creation. In the meantime4363 * we could have renamed the real subvol link into our snapshot, so4364 * depending on btrfs_del_root_ref to return -ENOENT here is incorrect.4365 * Instead simply lookup the dir_index_item for this entry so we can4366 * remove it. Otherwise we know we have a ref to the root and we can4367 * call btrfs_del_root_ref, and it _shouldn't_ fail.4368 */4369 if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) {4370 di = btrfs_search_dir_index_item(root, path, dir_ino, &fname.disk_name);4371 if (IS_ERR(di)) {4372 ret = PTR_ERR(di);4373 btrfs_abort_transaction(trans, ret);4374 goto out;4375 }4376 4377 leaf = path->nodes[0];4378 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);4379 index = key.offset;4380 btrfs_release_path(path);4381 } else {4382 ret = btrfs_del_root_ref(trans, objectid,4383 btrfs_root_id(root), dir_ino,4384 &index, &fname.disk_name);4385 if (ret) {4386 btrfs_abort_transaction(trans, ret);4387 goto out;4388 }4389 }4390 4391 ret = btrfs_delete_delayed_dir_index(trans, dir, index);4392 if (ret) {4393 btrfs_abort_transaction(trans, ret);4394 goto out;4395 }4396 4397 btrfs_i_size_write(dir, dir->vfs_inode.i_size - fname.disk_name.len * 2);4398 inode_inc_iversion(&dir->vfs_inode);4399 inode_set_mtime_to_ts(&dir->vfs_inode, inode_set_ctime_current(&dir->vfs_inode));4400 ret = btrfs_update_inode_fallback(trans, dir);4401 if (ret)4402 btrfs_abort_transaction(trans, ret);4403out:4404 btrfs_free_path(path);4405 fscrypt_free_filename(&fname);4406 return ret;4407}4408 4409/*4410 * Helper to check if the subvolume references other subvolumes or if it's4411 * default.4412 */4413static noinline int may_destroy_subvol(struct btrfs_root *root)4414{4415 struct btrfs_fs_info *fs_info = root->fs_info;4416 struct btrfs_path *path;4417 struct btrfs_dir_item *di;4418 struct btrfs_key key;4419 struct fscrypt_str name = FSTR_INIT("default", 7);4420 u64 dir_id;4421 int ret;4422 4423 path = btrfs_alloc_path();4424 if (!path)4425 return -ENOMEM;4426 4427 /* Make sure this root isn't set as the default subvol */4428 dir_id = btrfs_super_root_dir(fs_info->super_copy);4429 di = btrfs_lookup_dir_item(NULL, fs_info->tree_root, path,4430 dir_id, &name, 0);4431 if (di && !IS_ERR(di)) {4432 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key);4433 if (key.objectid == btrfs_root_id(root)) {4434 ret = -EPERM;4435 btrfs_err(fs_info,4436 "deleting default subvolume %llu is not allowed",4437 key.objectid);4438 goto out;4439 }4440 btrfs_release_path(path);4441 }4442 4443 key.objectid = btrfs_root_id(root);4444 key.type = BTRFS_ROOT_REF_KEY;4445 key.offset = (u64)-1;4446 4447 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);4448 if (ret < 0)4449 goto out;4450 if (ret == 0) {4451 /*4452 * Key with offset -1 found, there would have to exist a root4453 * with such id, but this is out of valid range.4454 */4455 ret = -EUCLEAN;4456 goto out;4457 }4458 4459 ret = 0;4460 if (path->slots[0] > 0) {4461 path->slots[0]--;4462 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);4463 if (key.objectid == btrfs_root_id(root) && key.type == BTRFS_ROOT_REF_KEY)4464 ret = -ENOTEMPTY;4465 }4466out:4467 btrfs_free_path(path);4468 return ret;4469}4470 4471/* Delete all dentries for inodes belonging to the root */4472static void btrfs_prune_dentries(struct btrfs_root *root)4473{4474 struct btrfs_fs_info *fs_info = root->fs_info;4475 struct btrfs_inode *inode;4476 u64 min_ino = 0;4477 4478 if (!BTRFS_FS_ERROR(fs_info))4479 WARN_ON(btrfs_root_refs(&root->root_item) != 0);4480 4481 inode = btrfs_find_first_inode(root, min_ino);4482 while (inode) {4483 if (atomic_read(&inode->vfs_inode.i_count) > 1)4484 d_prune_aliases(&inode->vfs_inode);4485 4486 min_ino = btrfs_ino(inode) + 1;4487 /*4488 * btrfs_drop_inode() will have it removed from the inode4489 * cache when its usage count hits zero.4490 */4491 iput(&inode->vfs_inode);4492 cond_resched();4493 inode = btrfs_find_first_inode(root, min_ino);4494 }4495}4496 4497int btrfs_delete_subvolume(struct btrfs_inode *dir, struct dentry *dentry)4498{4499 struct btrfs_root *root = dir->root;4500 struct btrfs_fs_info *fs_info = root->fs_info;4501 struct inode *inode = d_inode(dentry);4502 struct btrfs_root *dest = BTRFS_I(inode)->root;4503 struct btrfs_trans_handle *trans;4504 struct btrfs_block_rsv block_rsv;4505 u64 root_flags;4506 u64 qgroup_reserved = 0;4507 int ret;4508 4509 down_write(&fs_info->subvol_sem);4510 4511 /*4512 * Don't allow to delete a subvolume with send in progress. This is4513 * inside the inode lock so the error handling that has to drop the bit4514 * again is not run concurrently.4515 */4516 spin_lock(&dest->root_item_lock);4517 if (dest->send_in_progress) {4518 spin_unlock(&dest->root_item_lock);4519 btrfs_warn(fs_info,4520 "attempt to delete subvolume %llu during send",4521 btrfs_root_id(dest));4522 ret = -EPERM;4523 goto out_up_write;4524 }4525 if (atomic_read(&dest->nr_swapfiles)) {4526 spin_unlock(&dest->root_item_lock);4527 btrfs_warn(fs_info,4528 "attempt to delete subvolume %llu with active swapfile",4529 btrfs_root_id(root));4530 ret = -EPERM;4531 goto out_up_write;4532 }4533 root_flags = btrfs_root_flags(&dest->root_item);4534 btrfs_set_root_flags(&dest->root_item,4535 root_flags | BTRFS_ROOT_SUBVOL_DEAD);4536 spin_unlock(&dest->root_item_lock);4537 4538 ret = may_destroy_subvol(dest);4539 if (ret)4540 goto out_undead;4541 4542 btrfs_init_block_rsv(&block_rsv, BTRFS_BLOCK_RSV_TEMP);4543 /*4544 * One for dir inode,4545 * two for dir entries,4546 * two for root ref/backref.4547 */4548 ret = btrfs_subvolume_reserve_metadata(root, &block_rsv, 5, true);4549 if (ret)4550 goto out_undead;4551 qgroup_reserved = block_rsv.qgroup_rsv_reserved;4552 4553 trans = btrfs_start_transaction(root, 0);4554 if (IS_ERR(trans)) {4555 ret = PTR_ERR(trans);4556 goto out_release;4557 }4558 btrfs_qgroup_convert_reserved_meta(root, qgroup_reserved);4559 qgroup_reserved = 0;4560 trans->block_rsv = &block_rsv;4561 trans->bytes_reserved = block_rsv.size;4562 4563 btrfs_record_snapshot_destroy(trans, dir);4564 4565 ret = btrfs_unlink_subvol(trans, dir, dentry);4566 if (ret) {4567 btrfs_abort_transaction(trans, ret);4568 goto out_end_trans;4569 }4570 4571 ret = btrfs_record_root_in_trans(trans, dest);4572 if (ret) {4573 btrfs_abort_transaction(trans, ret);4574 goto out_end_trans;4575 }4576 4577 memset(&dest->root_item.drop_progress, 0,4578 sizeof(dest->root_item.drop_progress));4579 btrfs_set_root_drop_level(&dest->root_item, 0);4580 btrfs_set_root_refs(&dest->root_item, 0);4581 4582 if (!test_and_set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &dest->state)) {4583 ret = btrfs_insert_orphan_item(trans,4584 fs_info->tree_root,4585 btrfs_root_id(dest));4586 if (ret) {4587 btrfs_abort_transaction(trans, ret);4588 goto out_end_trans;4589 }4590 }4591 4592 ret = btrfs_uuid_tree_remove(trans, dest->root_item.uuid,4593 BTRFS_UUID_KEY_SUBVOL, btrfs_root_id(dest));4594 if (ret && ret != -ENOENT) {4595 btrfs_abort_transaction(trans, ret);4596 goto out_end_trans;4597 }4598 if (!btrfs_is_empty_uuid(dest->root_item.received_uuid)) {4599 ret = btrfs_uuid_tree_remove(trans,4600 dest->root_item.received_uuid,4601 BTRFS_UUID_KEY_RECEIVED_SUBVOL,4602 btrfs_root_id(dest));4603 if (ret && ret != -ENOENT) {4604 btrfs_abort_transaction(trans, ret);4605 goto out_end_trans;4606 }4607 }4608 4609 free_anon_bdev(dest->anon_dev);4610 dest->anon_dev = 0;4611out_end_trans:4612 trans->block_rsv = NULL;4613 trans->bytes_reserved = 0;4614 ret = btrfs_end_transaction(trans);4615 inode->i_flags |= S_DEAD;4616out_release:4617 btrfs_block_rsv_release(fs_info, &block_rsv, (u64)-1, NULL);4618 if (qgroup_reserved)4619 btrfs_qgroup_free_meta_prealloc(root, qgroup_reserved);4620out_undead:4621 if (ret) {4622 spin_lock(&dest->root_item_lock);4623 root_flags = btrfs_root_flags(&dest->root_item);4624 btrfs_set_root_flags(&dest->root_item,4625 root_flags & ~BTRFS_ROOT_SUBVOL_DEAD);4626 spin_unlock(&dest->root_item_lock);4627 }4628out_up_write:4629 up_write(&fs_info->subvol_sem);4630 if (!ret) {4631 d_invalidate(dentry);4632 btrfs_prune_dentries(dest);4633 ASSERT(dest->send_in_progress == 0);4634 }4635 4636 return ret;4637}4638 4639static int btrfs_rmdir(struct inode *dir, struct dentry *dentry)4640{4641 struct inode *inode = d_inode(dentry);4642 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;4643 int ret = 0;4644 struct btrfs_trans_handle *trans;4645 u64 last_unlink_trans;4646 struct fscrypt_name fname;4647 4648 if (inode->i_size > BTRFS_EMPTY_DIR_SIZE)4649 return -ENOTEMPTY;4650 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_FIRST_FREE_OBJECTID) {4651 if (unlikely(btrfs_fs_incompat(fs_info, EXTENT_TREE_V2))) {4652 btrfs_err(fs_info,4653 "extent tree v2 doesn't support snapshot deletion yet");4654 return -EOPNOTSUPP;4655 }4656 return btrfs_delete_subvolume(BTRFS_I(dir), dentry);4657 }4658 4659 ret = fscrypt_setup_filename(dir, &dentry->d_name, 1, &fname);4660 if (ret)4661 return ret;4662 4663 /* This needs to handle no-key deletions later on */4664 4665 trans = __unlink_start_trans(BTRFS_I(dir));4666 if (IS_ERR(trans)) {4667 ret = PTR_ERR(trans);4668 goto out_notrans;4669 }4670 4671 if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {4672 ret = btrfs_unlink_subvol(trans, BTRFS_I(dir), dentry);4673 goto out;4674 }4675 4676 ret = btrfs_orphan_add(trans, BTRFS_I(inode));4677 if (ret)4678 goto out;4679 4680 last_unlink_trans = BTRFS_I(inode)->last_unlink_trans;4681 4682 /* now the directory is empty */4683 ret = btrfs_unlink_inode(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),4684 &fname.disk_name);4685 if (!ret) {4686 btrfs_i_size_write(BTRFS_I(inode), 0);4687 /*4688 * Propagate the last_unlink_trans value of the deleted dir to4689 * its parent directory. This is to prevent an unrecoverable4690 * log tree in the case we do something like this:4691 * 1) create dir foo4692 * 2) create snapshot under dir foo4693 * 3) delete the snapshot4694 * 4) rmdir foo4695 * 5) mkdir foo4696 * 6) fsync foo or some file inside foo4697 */4698 if (last_unlink_trans >= trans->transid)4699 BTRFS_I(dir)->last_unlink_trans = last_unlink_trans;4700 }4701out:4702 btrfs_end_transaction(trans);4703out_notrans:4704 btrfs_btree_balance_dirty(fs_info);4705 fscrypt_free_filename(&fname);4706 4707 return ret;4708}4709 4710/*4711 * Read, zero a chunk and write a block.4712 *4713 * @inode - inode that we're zeroing4714 * @from - the offset to start zeroing4715 * @len - the length to zero, 0 to zero the entire range respective to the4716 * offset4717 * @front - zero up to the offset instead of from the offset on4718 *4719 * This will find the block for the "from" offset and cow the block and zero the4720 * part we want to zero. This is used with truncate and hole punching.4721 */4722int btrfs_truncate_block(struct btrfs_inode *inode, loff_t from, loff_t len,4723 int front)4724{4725 struct btrfs_fs_info *fs_info = inode->root->fs_info;4726 struct address_space *mapping = inode->vfs_inode.i_mapping;4727 struct extent_io_tree *io_tree = &inode->io_tree;4728 struct btrfs_ordered_extent *ordered;4729 struct extent_state *cached_state = NULL;4730 struct extent_changeset *data_reserved = NULL;4731 bool only_release_metadata = false;4732 u32 blocksize = fs_info->sectorsize;4733 pgoff_t index = from >> PAGE_SHIFT;4734 unsigned offset = from & (blocksize - 1);4735 struct folio *folio;4736 gfp_t mask = btrfs_alloc_write_mask(mapping);4737 size_t write_bytes = blocksize;4738 int ret = 0;4739 u64 block_start;4740 u64 block_end;4741 4742 if (IS_ALIGNED(offset, blocksize) &&4743 (!len || IS_ALIGNED(len, blocksize)))4744 goto out;4745 4746 block_start = round_down(from, blocksize);4747 block_end = block_start + blocksize - 1;4748 4749 ret = btrfs_check_data_free_space(inode, &data_reserved, block_start,4750 blocksize, false);4751 if (ret < 0) {4752 if (btrfs_check_nocow_lock(inode, block_start, &write_bytes, false) > 0) {4753 /* For nocow case, no need to reserve data space */4754 only_release_metadata = true;4755 } else {4756 goto out;4757 }4758 }4759 ret = btrfs_delalloc_reserve_metadata(inode, blocksize, blocksize, false);4760 if (ret < 0) {4761 if (!only_release_metadata)4762 btrfs_free_reserved_data_space(inode, data_reserved,4763 block_start, blocksize);4764 goto out;4765 }4766again:4767 folio = __filemap_get_folio(mapping, index,4768 FGP_LOCK | FGP_ACCESSED | FGP_CREAT, mask);4769 if (IS_ERR(folio)) {4770 btrfs_delalloc_release_space(inode, data_reserved, block_start,4771 blocksize, true);4772 btrfs_delalloc_release_extents(inode, blocksize);4773 ret = -ENOMEM;4774 goto out;4775 }4776 4777 if (!folio_test_uptodate(folio)) {4778 ret = btrfs_read_folio(NULL, folio);4779 folio_lock(folio);4780 if (folio->mapping != mapping) {4781 folio_unlock(folio);4782 folio_put(folio);4783 goto again;4784 }4785 if (!folio_test_uptodate(folio)) {4786 ret = -EIO;4787 goto out_unlock;4788 }4789 }4790 4791 /*4792 * We unlock the page after the io is completed and then re-lock it4793 * above. release_folio() could have come in between that and cleared4794 * folio private, but left the page in the mapping. Set the page mapped4795 * here to make sure it's properly set for the subpage stuff.4796 */4797 ret = set_folio_extent_mapped(folio);4798 if (ret < 0)4799 goto out_unlock;4800 4801 folio_wait_writeback(folio);4802 4803 lock_extent(io_tree, block_start, block_end, &cached_state);4804 4805 ordered = btrfs_lookup_ordered_extent(inode, block_start);4806 if (ordered) {4807 unlock_extent(io_tree, block_start, block_end, &cached_state);4808 folio_unlock(folio);4809 folio_put(folio);4810 btrfs_start_ordered_extent(ordered);4811 btrfs_put_ordered_extent(ordered);4812 goto again;4813 }4814 4815 clear_extent_bit(&inode->io_tree, block_start, block_end,4816 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,4817 &cached_state);4818 4819 ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0,4820 &cached_state);4821 if (ret) {4822 unlock_extent(io_tree, block_start, block_end, &cached_state);4823 goto out_unlock;4824 }4825 4826 if (offset != blocksize) {4827 if (!len)4828 len = blocksize - offset;4829 if (front)4830 folio_zero_range(folio, block_start - folio_pos(folio),4831 offset);4832 else4833 folio_zero_range(folio,4834 (block_start - folio_pos(folio)) + offset,4835 len);4836 }4837 btrfs_folio_clear_checked(fs_info, folio, block_start,4838 block_end + 1 - block_start);4839 btrfs_folio_set_dirty(fs_info, folio, block_start,4840 block_end + 1 - block_start);4841 unlock_extent(io_tree, block_start, block_end, &cached_state);4842 4843 if (only_release_metadata)4844 set_extent_bit(&inode->io_tree, block_start, block_end,4845 EXTENT_NORESERVE, NULL);4846 4847out_unlock:4848 if (ret) {4849 if (only_release_metadata)4850 btrfs_delalloc_release_metadata(inode, blocksize, true);4851 else4852 btrfs_delalloc_release_space(inode, data_reserved,4853 block_start, blocksize, true);4854 }4855 btrfs_delalloc_release_extents(inode, blocksize);4856 folio_unlock(folio);4857 folio_put(folio);4858out:4859 if (only_release_metadata)4860 btrfs_check_nocow_unlock(inode);4861 extent_changeset_free(data_reserved);4862 return ret;4863}4864 4865static int maybe_insert_hole(struct btrfs_inode *inode, u64 offset, u64 len)4866{4867 struct btrfs_root *root = inode->root;4868 struct btrfs_fs_info *fs_info = root->fs_info;4869 struct btrfs_trans_handle *trans;4870 struct btrfs_drop_extents_args drop_args = { 0 };4871 int ret;4872 4873 /*4874 * If NO_HOLES is enabled, we don't need to do anything.4875 * Later, up in the call chain, either btrfs_set_inode_last_sub_trans()4876 * or btrfs_update_inode() will be called, which guarantee that the next4877 * fsync will know this inode was changed and needs to be logged.4878 */4879 if (btrfs_fs_incompat(fs_info, NO_HOLES))4880 return 0;4881 4882 /*4883 * 1 - for the one we're dropping4884 * 1 - for the one we're adding4885 * 1 - for updating the inode.4886 */4887 trans = btrfs_start_transaction(root, 3);4888 if (IS_ERR(trans))4889 return PTR_ERR(trans);4890 4891 drop_args.start = offset;4892 drop_args.end = offset + len;4893 drop_args.drop_cache = true;4894 4895 ret = btrfs_drop_extents(trans, root, inode, &drop_args);4896 if (ret) {4897 btrfs_abort_transaction(trans, ret);4898 btrfs_end_transaction(trans);4899 return ret;4900 }4901 4902 ret = btrfs_insert_hole_extent(trans, root, btrfs_ino(inode), offset, len);4903 if (ret) {4904 btrfs_abort_transaction(trans, ret);4905 } else {4906 btrfs_update_inode_bytes(inode, 0, drop_args.bytes_found);4907 btrfs_update_inode(trans, inode);4908 }4909 btrfs_end_transaction(trans);4910 return ret;4911}4912 4913/*4914 * This function puts in dummy file extents for the area we're creating a hole4915 * for. So if we are truncating this file to a larger size we need to insert4916 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for4917 * the range between oldsize and size4918 */4919int btrfs_cont_expand(struct btrfs_inode *inode, loff_t oldsize, loff_t size)4920{4921 struct btrfs_root *root = inode->root;4922 struct btrfs_fs_info *fs_info = root->fs_info;4923 struct extent_io_tree *io_tree = &inode->io_tree;4924 struct extent_map *em = NULL;4925 struct extent_state *cached_state = NULL;4926 u64 hole_start = ALIGN(oldsize, fs_info->sectorsize);4927 u64 block_end = ALIGN(size, fs_info->sectorsize);4928 u64 last_byte;4929 u64 cur_offset;4930 u64 hole_size;4931 int ret = 0;4932 4933 /*4934 * If our size started in the middle of a block we need to zero out the4935 * rest of the block before we expand the i_size, otherwise we could4936 * expose stale data.4937 */4938 ret = btrfs_truncate_block(inode, oldsize, 0, 0);4939 if (ret)4940 return ret;4941 4942 if (size <= hole_start)4943 return 0;4944 4945 btrfs_lock_and_flush_ordered_range(inode, hole_start, block_end - 1,4946 &cached_state);4947 cur_offset = hole_start;4948 while (1) {4949 em = btrfs_get_extent(inode, NULL, cur_offset, block_end - cur_offset);4950 if (IS_ERR(em)) {4951 ret = PTR_ERR(em);4952 em = NULL;4953 break;4954 }4955 last_byte = min(extent_map_end(em), block_end);4956 last_byte = ALIGN(last_byte, fs_info->sectorsize);4957 hole_size = last_byte - cur_offset;4958 4959 if (!(em->flags & EXTENT_FLAG_PREALLOC)) {4960 struct extent_map *hole_em;4961 4962 ret = maybe_insert_hole(inode, cur_offset, hole_size);4963 if (ret)4964 break;4965 4966 ret = btrfs_inode_set_file_extent_range(inode,4967 cur_offset, hole_size);4968 if (ret)4969 break;4970 4971 hole_em = alloc_extent_map();4972 if (!hole_em) {4973 btrfs_drop_extent_map_range(inode, cur_offset,4974 cur_offset + hole_size - 1,4975 false);4976 btrfs_set_inode_full_sync(inode);4977 goto next;4978 }4979 hole_em->start = cur_offset;4980 hole_em->len = hole_size;4981 4982 hole_em->disk_bytenr = EXTENT_MAP_HOLE;4983 hole_em->disk_num_bytes = 0;4984 hole_em->ram_bytes = hole_size;4985 hole_em->generation = btrfs_get_fs_generation(fs_info);4986 4987 ret = btrfs_replace_extent_map_range(inode, hole_em, true);4988 free_extent_map(hole_em);4989 } else {4990 ret = btrfs_inode_set_file_extent_range(inode,4991 cur_offset, hole_size);4992 if (ret)4993 break;4994 }4995next:4996 free_extent_map(em);4997 em = NULL;4998 cur_offset = last_byte;4999 if (cur_offset >= block_end)5000 break;5001 }5002 free_extent_map(em);5003 unlock_extent(io_tree, hole_start, block_end - 1, &cached_state);5004 return ret;5005}5006 5007static int btrfs_setsize(struct inode *inode, struct iattr *attr)5008{5009 struct btrfs_root *root = BTRFS_I(inode)->root;5010 struct btrfs_trans_handle *trans;5011 loff_t oldsize = i_size_read(inode);5012 loff_t newsize = attr->ia_size;5013 int mask = attr->ia_valid;5014 int ret;5015 5016 /*5017 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a5018 * special case where we need to update the times despite not having5019 * these flags set. For all other operations the VFS set these flags5020 * explicitly if it wants a timestamp update.5021 */5022 if (newsize != oldsize) {5023 inode_inc_iversion(inode);5024 if (!(mask & (ATTR_CTIME | ATTR_MTIME))) {5025 inode_set_mtime_to_ts(inode,5026 inode_set_ctime_current(inode));5027 }5028 }5029 5030 if (newsize > oldsize) {5031 /*5032 * Don't do an expanding truncate while snapshotting is ongoing.5033 * This is to ensure the snapshot captures a fully consistent5034 * state of this file - if the snapshot captures this expanding5035 * truncation, it must capture all writes that happened before5036 * this truncation.5037 */5038 btrfs_drew_write_lock(&root->snapshot_lock);5039 ret = btrfs_cont_expand(BTRFS_I(inode), oldsize, newsize);5040 if (ret) {5041 btrfs_drew_write_unlock(&root->snapshot_lock);5042 return ret;5043 }5044 5045 trans = btrfs_start_transaction(root, 1);5046 if (IS_ERR(trans)) {5047 btrfs_drew_write_unlock(&root->snapshot_lock);5048 return PTR_ERR(trans);5049 }5050 5051 i_size_write(inode, newsize);5052 btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0);5053 pagecache_isize_extended(inode, oldsize, newsize);5054 ret = btrfs_update_inode(trans, BTRFS_I(inode));5055 btrfs_drew_write_unlock(&root->snapshot_lock);5056 btrfs_end_transaction(trans);5057 } else {5058 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);5059 5060 if (btrfs_is_zoned(fs_info)) {5061 ret = btrfs_wait_ordered_range(BTRFS_I(inode),5062 ALIGN(newsize, fs_info->sectorsize),5063 (u64)-1);5064 if (ret)5065 return ret;5066 }5067 5068 /*5069 * We're truncating a file that used to have good data down to5070 * zero. Make sure any new writes to the file get on disk5071 * on close.5072 */5073 if (newsize == 0)5074 set_bit(BTRFS_INODE_FLUSH_ON_CLOSE,5075 &BTRFS_I(inode)->runtime_flags);5076 5077 truncate_setsize(inode, newsize);5078 5079 inode_dio_wait(inode);5080 5081 ret = btrfs_truncate(BTRFS_I(inode), newsize == oldsize);5082 if (ret && inode->i_nlink) {5083 int err;5084 5085 /*5086 * Truncate failed, so fix up the in-memory size. We5087 * adjusted disk_i_size down as we removed extents, so5088 * wait for disk_i_size to be stable and then update the5089 * in-memory size to match.5090 */5091 err = btrfs_wait_ordered_range(BTRFS_I(inode), 0, (u64)-1);5092 if (err)5093 return err;5094 i_size_write(inode, BTRFS_I(inode)->disk_i_size);5095 }5096 }5097 5098 return ret;5099}5100 5101static int btrfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,5102 struct iattr *attr)5103{5104 struct inode *inode = d_inode(dentry);5105 struct btrfs_root *root = BTRFS_I(inode)->root;5106 int err;5107 5108 if (btrfs_root_readonly(root))5109 return -EROFS;5110 5111 err = setattr_prepare(idmap, dentry, attr);5112 if (err)5113 return err;5114 5115 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {5116 err = btrfs_setsize(inode, attr);5117 if (err)5118 return err;5119 }5120 5121 if (attr->ia_valid) {5122 setattr_copy(idmap, inode, attr);5123 inode_inc_iversion(inode);5124 err = btrfs_dirty_inode(BTRFS_I(inode));5125 5126 if (!err && attr->ia_valid & ATTR_MODE)5127 err = posix_acl_chmod(idmap, dentry, inode->i_mode);5128 }5129 5130 return err;5131}5132 5133/*5134 * While truncating the inode pages during eviction, we get the VFS5135 * calling btrfs_invalidate_folio() against each folio of the inode. This5136 * is slow because the calls to btrfs_invalidate_folio() result in a5137 * huge amount of calls to lock_extent() and clear_extent_bit(),5138 * which keep merging and splitting extent_state structures over and over,5139 * wasting lots of time.5140 *5141 * Therefore if the inode is being evicted, let btrfs_invalidate_folio()5142 * skip all those expensive operations on a per folio basis and do only5143 * the ordered io finishing, while we release here the extent_map and5144 * extent_state structures, without the excessive merging and splitting.5145 */5146static void evict_inode_truncate_pages(struct inode *inode)5147{5148 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;5149 struct rb_node *node;5150 5151 ASSERT(inode->i_state & I_FREEING);5152 truncate_inode_pages_final(&inode->i_data);5153 5154 btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false);5155 5156 /*5157 * Keep looping until we have no more ranges in the io tree.5158 * We can have ongoing bios started by readahead that have5159 * their endio callback (extent_io.c:end_bio_extent_readpage)5160 * still in progress (unlocked the pages in the bio but did not yet5161 * unlocked the ranges in the io tree). Therefore this means some5162 * ranges can still be locked and eviction started because before5163 * submitting those bios, which are executed by a separate task (work5164 * queue kthread), inode references (inode->i_count) were not taken5165 * (which would be dropped in the end io callback of each bio).5166 * Therefore here we effectively end up waiting for those bios and5167 * anyone else holding locked ranges without having bumped the inode's5168 * reference count - if we don't do it, when they access the inode's5169 * io_tree to unlock a range it may be too late, leading to an5170 * use-after-free issue.5171 */5172 spin_lock(&io_tree->lock);5173 while (!RB_EMPTY_ROOT(&io_tree->state)) {5174 struct extent_state *state;5175 struct extent_state *cached_state = NULL;5176 u64 start;5177 u64 end;5178 unsigned state_flags;5179 5180 node = rb_first(&io_tree->state);5181 state = rb_entry(node, struct extent_state, rb_node);5182 start = state->start;5183 end = state->end;5184 state_flags = state->state;5185 spin_unlock(&io_tree->lock);5186 5187 lock_extent(io_tree, start, end, &cached_state);5188 5189 /*5190 * If still has DELALLOC flag, the extent didn't reach disk,5191 * and its reserved space won't be freed by delayed_ref.5192 * So we need to free its reserved space here.5193 * (Refer to comment in btrfs_invalidate_folio, case 2)5194 *5195 * Note, end is the bytenr of last byte, so we need + 1 here.5196 */5197 if (state_flags & EXTENT_DELALLOC)5198 btrfs_qgroup_free_data(BTRFS_I(inode), NULL, start,5199 end - start + 1, NULL);5200 5201 clear_extent_bit(io_tree, start, end,5202 EXTENT_CLEAR_ALL_BITS | EXTENT_DO_ACCOUNTING,5203 &cached_state);5204 5205 cond_resched();5206 spin_lock(&io_tree->lock);5207 }5208 spin_unlock(&io_tree->lock);5209}5210 5211static struct btrfs_trans_handle *evict_refill_and_join(struct btrfs_root *root,5212 struct btrfs_block_rsv *rsv)5213{5214 struct btrfs_fs_info *fs_info = root->fs_info;5215 struct btrfs_trans_handle *trans;5216 u64 delayed_refs_extra = btrfs_calc_delayed_ref_bytes(fs_info, 1);5217 int ret;5218 5219 /*5220 * Eviction should be taking place at some place safe because of our5221 * delayed iputs. However the normal flushing code will run delayed5222 * iputs, so we cannot use FLUSH_ALL otherwise we'll deadlock.5223 *5224 * We reserve the delayed_refs_extra here again because we can't use5225 * btrfs_start_transaction(root, 0) for the same deadlocky reason as5226 * above. We reserve our extra bit here because we generate a ton of5227 * delayed refs activity by truncating.5228 *5229 * BTRFS_RESERVE_FLUSH_EVICT will steal from the global_rsv if it can,5230 * if we fail to make this reservation we can re-try without the5231 * delayed_refs_extra so we can make some forward progress.5232 */5233 ret = btrfs_block_rsv_refill(fs_info, rsv, rsv->size + delayed_refs_extra,5234 BTRFS_RESERVE_FLUSH_EVICT);5235 if (ret) {5236 ret = btrfs_block_rsv_refill(fs_info, rsv, rsv->size,5237 BTRFS_RESERVE_FLUSH_EVICT);5238 if (ret) {5239 btrfs_warn(fs_info,5240 "could not allocate space for delete; will truncate on mount");5241 return ERR_PTR(-ENOSPC);5242 }5243 delayed_refs_extra = 0;5244 }5245 5246 trans = btrfs_join_transaction(root);5247 if (IS_ERR(trans))5248 return trans;5249 5250 if (delayed_refs_extra) {5251 trans->block_rsv = &fs_info->trans_block_rsv;5252 trans->bytes_reserved = delayed_refs_extra;5253 btrfs_block_rsv_migrate(rsv, trans->block_rsv,5254 delayed_refs_extra, true);5255 }5256 return trans;5257}5258 5259void btrfs_evict_inode(struct inode *inode)5260{5261 struct btrfs_fs_info *fs_info;5262 struct btrfs_trans_handle *trans;5263 struct btrfs_root *root = BTRFS_I(inode)->root;5264 struct btrfs_block_rsv *rsv = NULL;5265 int ret;5266 5267 trace_btrfs_inode_evict(inode);5268 5269 if (!root) {5270 fsverity_cleanup_inode(inode);5271 clear_inode(inode);5272 return;5273 }5274 5275 fs_info = inode_to_fs_info(inode);5276 evict_inode_truncate_pages(inode);5277 5278 if (inode->i_nlink &&5279 ((btrfs_root_refs(&root->root_item) != 0 &&5280 btrfs_root_id(root) != BTRFS_ROOT_TREE_OBJECTID) ||5281 btrfs_is_free_space_inode(BTRFS_I(inode))))5282 goto out;5283 5284 if (is_bad_inode(inode))5285 goto out;5286 5287 if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags))5288 goto out;5289 5290 if (inode->i_nlink > 0) {5291 BUG_ON(btrfs_root_refs(&root->root_item) != 0 &&5292 btrfs_root_id(root) != BTRFS_ROOT_TREE_OBJECTID);5293 goto out;5294 }5295 5296 /*5297 * This makes sure the inode item in tree is uptodate and the space for5298 * the inode update is released.5299 */5300 ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode));5301 if (ret)5302 goto out;5303 5304 /*5305 * This drops any pending insert or delete operations we have for this5306 * inode. We could have a delayed dir index deletion queued up, but5307 * we're removing the inode completely so that'll be taken care of in5308 * the truncate.5309 */5310 btrfs_kill_delayed_inode_items(BTRFS_I(inode));5311 5312 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);5313 if (!rsv)5314 goto out;5315 rsv->size = btrfs_calc_metadata_size(fs_info, 1);5316 rsv->failfast = true;5317 5318 btrfs_i_size_write(BTRFS_I(inode), 0);5319 5320 while (1) {5321 struct btrfs_truncate_control control = {5322 .inode = BTRFS_I(inode),5323 .ino = btrfs_ino(BTRFS_I(inode)),5324 .new_size = 0,5325 .min_type = 0,5326 };5327 5328 trans = evict_refill_and_join(root, rsv);5329 if (IS_ERR(trans))5330 goto out;5331 5332 trans->block_rsv = rsv;5333 5334 ret = btrfs_truncate_inode_items(trans, root, &control);5335 trans->block_rsv = &fs_info->trans_block_rsv;5336 btrfs_end_transaction(trans);5337 /*5338 * We have not added new delayed items for our inode after we5339 * have flushed its delayed items, so no need to throttle on5340 * delayed items. However we have modified extent buffers.5341 */5342 btrfs_btree_balance_dirty_nodelay(fs_info);5343 if (ret && ret != -ENOSPC && ret != -EAGAIN)5344 goto out;5345 else if (!ret)5346 break;5347 }5348 5349 /*5350 * Errors here aren't a big deal, it just means we leave orphan items in5351 * the tree. They will be cleaned up on the next mount. If the inode5352 * number gets reused, cleanup deletes the orphan item without doing5353 * anything, and unlink reuses the existing orphan item.5354 *5355 * If it turns out that we are dropping too many of these, we might want5356 * to add a mechanism for retrying these after a commit.5357 */5358 trans = evict_refill_and_join(root, rsv);5359 if (!IS_ERR(trans)) {5360 trans->block_rsv = rsv;5361 btrfs_orphan_del(trans, BTRFS_I(inode));5362 trans->block_rsv = &fs_info->trans_block_rsv;5363 btrfs_end_transaction(trans);5364 }5365 5366out:5367 btrfs_free_block_rsv(fs_info, rsv);5368 /*5369 * If we didn't successfully delete, the orphan item will still be in5370 * the tree and we'll retry on the next mount. Again, we might also want5371 * to retry these periodically in the future.5372 */5373 btrfs_remove_delayed_node(BTRFS_I(inode));5374 fsverity_cleanup_inode(inode);5375 clear_inode(inode);5376}5377 5378/*5379 * Return the key found in the dir entry in the location pointer, fill @type5380 * with BTRFS_FT_*, and return 0.5381 *5382 * If no dir entries were found, returns -ENOENT.5383 * If found a corrupted location in dir entry, returns -EUCLEAN.5384 */5385static int btrfs_inode_by_name(struct btrfs_inode *dir, struct dentry *dentry,5386 struct btrfs_key *location, u8 *type)5387{5388 struct btrfs_dir_item *di;5389 struct btrfs_path *path;5390 struct btrfs_root *root = dir->root;5391 int ret = 0;5392 struct fscrypt_name fname;5393 5394 path = btrfs_alloc_path();5395 if (!path)5396 return -ENOMEM;5397 5398 ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 1, &fname);5399 if (ret < 0)5400 goto out;5401 /*5402 * fscrypt_setup_filename() should never return a positive value, but5403 * gcc on sparc/parisc thinks it can, so assert that doesn't happen.5404 */5405 ASSERT(ret == 0);5406 5407 /* This needs to handle no-key deletions later on */5408 5409 di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(dir),5410 &fname.disk_name, 0);5411 if (IS_ERR_OR_NULL(di)) {5412 ret = di ? PTR_ERR(di) : -ENOENT;5413 goto out;5414 }5415 5416 btrfs_dir_item_key_to_cpu(path->nodes[0], di, location);5417 if (location->type != BTRFS_INODE_ITEM_KEY &&5418 location->type != BTRFS_ROOT_ITEM_KEY) {5419 ret = -EUCLEAN;5420 btrfs_warn(root->fs_info,5421"%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location(%llu %u %llu))",5422 __func__, fname.disk_name.name, btrfs_ino(dir),5423 location->objectid, location->type, location->offset);5424 }5425 if (!ret)5426 *type = btrfs_dir_ftype(path->nodes[0], di);5427out:5428 fscrypt_free_filename(&fname);5429 btrfs_free_path(path);5430 return ret;5431}5432 5433/*5434 * when we hit a tree root in a directory, the btrfs part of the inode5435 * needs to be changed to reflect the root directory of the tree root. This5436 * is kind of like crossing a mount point.5437 */5438static int fixup_tree_root_location(struct btrfs_fs_info *fs_info,5439 struct btrfs_inode *dir,5440 struct dentry *dentry,5441 struct btrfs_key *location,5442 struct btrfs_root **sub_root)5443{5444 struct btrfs_path *path;5445 struct btrfs_root *new_root;5446 struct btrfs_root_ref *ref;5447 struct extent_buffer *leaf;5448 struct btrfs_key key;5449 int ret;5450 int err = 0;5451 struct fscrypt_name fname;5452 5453 ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 0, &fname);5454 if (ret)5455 return ret;5456 5457 path = btrfs_alloc_path();5458 if (!path) {5459 err = -ENOMEM;5460 goto out;5461 }5462 5463 err = -ENOENT;5464 key.objectid = btrfs_root_id(dir->root);5465 key.type = BTRFS_ROOT_REF_KEY;5466 key.offset = location->objectid;5467 5468 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);5469 if (ret) {5470 if (ret < 0)5471 err = ret;5472 goto out;5473 }5474 5475 leaf = path->nodes[0];5476 ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);5477 if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(dir) ||5478 btrfs_root_ref_name_len(leaf, ref) != fname.disk_name.len)5479 goto out;5480 5481 ret = memcmp_extent_buffer(leaf, fname.disk_name.name,5482 (unsigned long)(ref + 1), fname.disk_name.len);5483 if (ret)5484 goto out;5485 5486 btrfs_release_path(path);5487 5488 new_root = btrfs_get_fs_root(fs_info, location->objectid, true);5489 if (IS_ERR(new_root)) {5490 err = PTR_ERR(new_root);5491 goto out;5492 }5493 5494 *sub_root = new_root;5495 location->objectid = btrfs_root_dirid(&new_root->root_item);5496 location->type = BTRFS_INODE_ITEM_KEY;5497 location->offset = 0;5498 err = 0;5499out:5500 btrfs_free_path(path);5501 fscrypt_free_filename(&fname);5502 return err;5503}5504 5505static int btrfs_add_inode_to_root(struct btrfs_inode *inode, bool prealloc)5506{5507 struct btrfs_root *root = inode->root;5508 struct btrfs_inode *existing;5509 const u64 ino = btrfs_ino(inode);5510 int ret;5511 5512 if (inode_unhashed(&inode->vfs_inode))5513 return 0;5514 5515 if (prealloc) {5516 ret = xa_reserve(&root->inodes, ino, GFP_NOFS);5517 if (ret)5518 return ret;5519 }5520 5521 existing = xa_store(&root->inodes, ino, inode, GFP_ATOMIC);5522 5523 if (xa_is_err(existing)) {5524 ret = xa_err(existing);5525 ASSERT(ret != -EINVAL);5526 ASSERT(ret != -ENOMEM);5527 return ret;5528 } else if (existing) {5529 WARN_ON(!(existing->vfs_inode.i_state & (I_WILL_FREE | I_FREEING)));5530 }5531 5532 return 0;5533}5534 5535static void btrfs_del_inode_from_root(struct btrfs_inode *inode)5536{5537 struct btrfs_root *root = inode->root;5538 struct btrfs_inode *entry;5539 bool empty = false;5540 5541 xa_lock(&root->inodes);5542 entry = __xa_erase(&root->inodes, btrfs_ino(inode));5543 if (entry == inode)5544 empty = xa_empty(&root->inodes);5545 xa_unlock(&root->inodes);5546 5547 if (empty && btrfs_root_refs(&root->root_item) == 0) {5548 xa_lock(&root->inodes);5549 empty = xa_empty(&root->inodes);5550 xa_unlock(&root->inodes);5551 if (empty)5552 btrfs_add_dead_root(root);5553 }5554}5555 5556 5557static int btrfs_init_locked_inode(struct inode *inode, void *p)5558{5559 struct btrfs_iget_args *args = p;5560 5561 btrfs_set_inode_number(BTRFS_I(inode), args->ino);5562 BTRFS_I(inode)->root = btrfs_grab_root(args->root);5563 5564 if (args->root && args->root == args->root->fs_info->tree_root &&5565 args->ino != BTRFS_BTREE_INODE_OBJECTID)5566 set_bit(BTRFS_INODE_FREE_SPACE_INODE,5567 &BTRFS_I(inode)->runtime_flags);5568 return 0;5569}5570 5571static int btrfs_find_actor(struct inode *inode, void *opaque)5572{5573 struct btrfs_iget_args *args = opaque;5574 5575 return args->ino == btrfs_ino(BTRFS_I(inode)) &&5576 args->root == BTRFS_I(inode)->root;5577}5578 5579static struct inode *btrfs_iget_locked(u64 ino, struct btrfs_root *root)5580{5581 struct inode *inode;5582 struct btrfs_iget_args args;5583 unsigned long hashval = btrfs_inode_hash(ino, root);5584 5585 args.ino = ino;5586 args.root = root;5587 5588 inode = iget5_locked_rcu(root->fs_info->sb, hashval, btrfs_find_actor,5589 btrfs_init_locked_inode,5590 (void *)&args);5591 return inode;5592}5593 5594/*5595 * Get an inode object given its inode number and corresponding root.5596 * Path can be preallocated to prevent recursing back to iget through5597 * allocator. NULL is also valid but may require an additional allocation5598 * later.5599 */5600struct inode *btrfs_iget_path(u64 ino, struct btrfs_root *root,5601 struct btrfs_path *path)5602{5603 struct inode *inode;5604 int ret;5605 5606 inode = btrfs_iget_locked(ino, root);5607 if (!inode)5608 return ERR_PTR(-ENOMEM);5609 5610 if (!(inode->i_state & I_NEW))5611 return inode;5612 5613 ret = btrfs_read_locked_inode(inode, path);5614 /*5615 * ret > 0 can come from btrfs_search_slot called by5616 * btrfs_read_locked_inode(), this means the inode item was not found.5617 */5618 if (ret > 0)5619 ret = -ENOENT;5620 if (ret < 0)5621 goto error;5622 5623 ret = btrfs_add_inode_to_root(BTRFS_I(inode), true);5624 if (ret < 0)5625 goto error;5626 5627 unlock_new_inode(inode);5628 5629 return inode;5630error:5631 iget_failed(inode);5632 return ERR_PTR(ret);5633}5634 5635struct inode *btrfs_iget(u64 ino, struct btrfs_root *root)5636{5637 return btrfs_iget_path(ino, root, NULL);5638}5639 5640static struct inode *new_simple_dir(struct inode *dir,5641 struct btrfs_key *key,5642 struct btrfs_root *root)5643{5644 struct timespec64 ts;5645 struct inode *inode = new_inode(dir->i_sb);5646 5647 if (!inode)5648 return ERR_PTR(-ENOMEM);5649 5650 BTRFS_I(inode)->root = btrfs_grab_root(root);5651 BTRFS_I(inode)->ref_root_id = key->objectid;5652 set_bit(BTRFS_INODE_ROOT_STUB, &BTRFS_I(inode)->runtime_flags);5653 set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);5654 5655 btrfs_set_inode_number(BTRFS_I(inode), BTRFS_EMPTY_SUBVOL_DIR_OBJECTID);5656 /*5657 * We only need lookup, the rest is read-only and there's no inode5658 * associated with the dentry5659 */5660 inode->i_op = &simple_dir_inode_operations;5661 inode->i_opflags &= ~IOP_XATTR;5662 inode->i_fop = &simple_dir_operations;5663 inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO;5664 5665 ts = inode_set_ctime_current(inode);5666 inode_set_mtime_to_ts(inode, ts);5667 inode_set_atime_to_ts(inode, inode_get_atime(dir));5668 BTRFS_I(inode)->i_otime_sec = ts.tv_sec;5669 BTRFS_I(inode)->i_otime_nsec = ts.tv_nsec;5670 5671 inode->i_uid = dir->i_uid;5672 inode->i_gid = dir->i_gid;5673 5674 return inode;5675}5676 5677static_assert(BTRFS_FT_UNKNOWN == FT_UNKNOWN);5678static_assert(BTRFS_FT_REG_FILE == FT_REG_FILE);5679static_assert(BTRFS_FT_DIR == FT_DIR);5680static_assert(BTRFS_FT_CHRDEV == FT_CHRDEV);5681static_assert(BTRFS_FT_BLKDEV == FT_BLKDEV);5682static_assert(BTRFS_FT_FIFO == FT_FIFO);5683static_assert(BTRFS_FT_SOCK == FT_SOCK);5684static_assert(BTRFS_FT_SYMLINK == FT_SYMLINK);5685 5686static inline u8 btrfs_inode_type(struct inode *inode)5687{5688 return fs_umode_to_ftype(inode->i_mode);5689}5690 5691struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry)5692{5693 struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);5694 struct inode *inode;5695 struct btrfs_root *root = BTRFS_I(dir)->root;5696 struct btrfs_root *sub_root = root;5697 struct btrfs_key location = { 0 };5698 u8 di_type = 0;5699 int ret = 0;5700 5701 if (dentry->d_name.len > BTRFS_NAME_LEN)5702 return ERR_PTR(-ENAMETOOLONG);5703 5704 ret = btrfs_inode_by_name(BTRFS_I(dir), dentry, &location, &di_type);5705 if (ret < 0)5706 return ERR_PTR(ret);5707 5708 if (location.type == BTRFS_INODE_ITEM_KEY) {5709 inode = btrfs_iget(location.objectid, root);5710 if (IS_ERR(inode))5711 return inode;5712 5713 /* Do extra check against inode mode with di_type */5714 if (btrfs_inode_type(inode) != di_type) {5715 btrfs_crit(fs_info,5716"inode mode mismatch with dir: inode mode=0%o btrfs type=%u dir type=%u",5717 inode->i_mode, btrfs_inode_type(inode),5718 di_type);5719 iput(inode);5720 return ERR_PTR(-EUCLEAN);5721 }5722 return inode;5723 }5724 5725 ret = fixup_tree_root_location(fs_info, BTRFS_I(dir), dentry,5726 &location, &sub_root);5727 if (ret < 0) {5728 if (ret != -ENOENT)5729 inode = ERR_PTR(ret);5730 else5731 inode = new_simple_dir(dir, &location, root);5732 } else {5733 inode = btrfs_iget(location.objectid, sub_root);5734 btrfs_put_root(sub_root);5735 5736 if (IS_ERR(inode))5737 return inode;5738 5739 down_read(&fs_info->cleanup_work_sem);5740 if (!sb_rdonly(inode->i_sb))5741 ret = btrfs_orphan_cleanup(sub_root);5742 up_read(&fs_info->cleanup_work_sem);5743 if (ret) {5744 iput(inode);5745 inode = ERR_PTR(ret);5746 }5747 }5748 5749 return inode;5750}5751 5752static int btrfs_dentry_delete(const struct dentry *dentry)5753{5754 struct btrfs_root *root;5755 struct inode *inode = d_inode(dentry);5756 5757 if (!inode && !IS_ROOT(dentry))5758 inode = d_inode(dentry->d_parent);5759 5760 if (inode) {5761 root = BTRFS_I(inode)->root;5762 if (btrfs_root_refs(&root->root_item) == 0)5763 return 1;5764 5765 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)5766 return 1;5767 }5768 return 0;5769}5770 5771static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry,5772 unsigned int flags)5773{5774 struct inode *inode = btrfs_lookup_dentry(dir, dentry);5775 5776 if (inode == ERR_PTR(-ENOENT))5777 inode = NULL;5778 return d_splice_alias(inode, dentry);5779}5780 5781/*5782 * Find the highest existing sequence number in a directory and then set the5783 * in-memory index_cnt variable to the first free sequence number.5784 */5785static int btrfs_set_inode_index_count(struct btrfs_inode *inode)5786{5787 struct btrfs_root *root = inode->root;5788 struct btrfs_key key, found_key;5789 struct btrfs_path *path;5790 struct extent_buffer *leaf;5791 int ret;5792 5793 key.objectid = btrfs_ino(inode);5794 key.type = BTRFS_DIR_INDEX_KEY;5795 key.offset = (u64)-1;5796 5797 path = btrfs_alloc_path();5798 if (!path)5799 return -ENOMEM;5800 5801 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);5802 if (ret < 0)5803 goto out;5804 /* FIXME: we should be able to handle this */5805 if (ret == 0)5806 goto out;5807 ret = 0;5808 5809 if (path->slots[0] == 0) {5810 inode->index_cnt = BTRFS_DIR_START_INDEX;5811 goto out;5812 }5813 5814 path->slots[0]--;5815 5816 leaf = path->nodes[0];5817 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);5818 5819 if (found_key.objectid != btrfs_ino(inode) ||5820 found_key.type != BTRFS_DIR_INDEX_KEY) {5821 inode->index_cnt = BTRFS_DIR_START_INDEX;5822 goto out;5823 }5824 5825 inode->index_cnt = found_key.offset + 1;5826out:5827 btrfs_free_path(path);5828 return ret;5829}5830 5831static int btrfs_get_dir_last_index(struct btrfs_inode *dir, u64 *index)5832{5833 int ret = 0;5834 5835 btrfs_inode_lock(dir, 0);5836 if (dir->index_cnt == (u64)-1) {5837 ret = btrfs_inode_delayed_dir_index_count(dir);5838 if (ret) {5839 ret = btrfs_set_inode_index_count(dir);5840 if (ret)5841 goto out;5842 }5843 }5844 5845 /* index_cnt is the index number of next new entry, so decrement it. */5846 *index = dir->index_cnt - 1;5847out:5848 btrfs_inode_unlock(dir, 0);5849 5850 return ret;5851}5852 5853/*5854 * All this infrastructure exists because dir_emit can fault, and we are holding5855 * the tree lock when doing readdir. For now just allocate a buffer and copy5856 * our information into that, and then dir_emit from the buffer. This is5857 * similar to what NFS does, only we don't keep the buffer around in pagecache5858 * because I'm afraid I'll mess that up. Long term we need to make filldir do5859 * copy_to_user_inatomic so we don't have to worry about page faulting under the5860 * tree lock.5861 */5862static int btrfs_opendir(struct inode *inode, struct file *file)5863{5864 struct btrfs_file_private *private;5865 u64 last_index;5866 int ret;5867 5868 ret = btrfs_get_dir_last_index(BTRFS_I(inode), &last_index);5869 if (ret)5870 return ret;5871 5872 private = kzalloc(sizeof(struct btrfs_file_private), GFP_KERNEL);5873 if (!private)5874 return -ENOMEM;5875 private->last_index = last_index;5876 private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL);5877 if (!private->filldir_buf) {5878 kfree(private);5879 return -ENOMEM;5880 }5881 file->private_data = private;5882 return 0;5883}5884 5885static loff_t btrfs_dir_llseek(struct file *file, loff_t offset, int whence)5886{5887 struct btrfs_file_private *private = file->private_data;5888 int ret;5889 5890 ret = btrfs_get_dir_last_index(BTRFS_I(file_inode(file)),5891 &private->last_index);5892 if (ret)5893 return ret;5894 5895 return generic_file_llseek(file, offset, whence);5896}5897 5898struct dir_entry {5899 u64 ino;5900 u64 offset;5901 unsigned type;5902 int name_len;5903};5904 5905static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx)5906{5907 while (entries--) {5908 struct dir_entry *entry = addr;5909 char *name = (char *)(entry + 1);5910 5911 ctx->pos = get_unaligned(&entry->offset);5912 if (!dir_emit(ctx, name, get_unaligned(&entry->name_len),5913 get_unaligned(&entry->ino),5914 get_unaligned(&entry->type)))5915 return 1;5916 addr += sizeof(struct dir_entry) +5917 get_unaligned(&entry->name_len);5918 ctx->pos++;5919 }5920 return 0;5921}5922 5923static int btrfs_real_readdir(struct file *file, struct dir_context *ctx)5924{5925 struct inode *inode = file_inode(file);5926 struct btrfs_root *root = BTRFS_I(inode)->root;5927 struct btrfs_file_private *private = file->private_data;5928 struct btrfs_dir_item *di;5929 struct btrfs_key key;5930 struct btrfs_key found_key;5931 struct btrfs_path *path;5932 void *addr;5933 LIST_HEAD(ins_list);5934 LIST_HEAD(del_list);5935 int ret;5936 char *name_ptr;5937 int name_len;5938 int entries = 0;5939 int total_len = 0;5940 bool put = false;5941 struct btrfs_key location;5942 5943 if (!dir_emit_dots(file, ctx))5944 return 0;5945 5946 path = btrfs_alloc_path();5947 if (!path)5948 return -ENOMEM;5949 5950 addr = private->filldir_buf;5951 path->reada = READA_FORWARD;5952 5953 put = btrfs_readdir_get_delayed_items(BTRFS_I(inode), private->last_index,5954 &ins_list, &del_list);5955 5956again:5957 key.type = BTRFS_DIR_INDEX_KEY;5958 key.offset = ctx->pos;5959 key.objectid = btrfs_ino(BTRFS_I(inode));5960 5961 btrfs_for_each_slot(root, &key, &found_key, path, ret) {5962 struct dir_entry *entry;5963 struct extent_buffer *leaf = path->nodes[0];5964 u8 ftype;5965 5966 if (found_key.objectid != key.objectid)5967 break;5968 if (found_key.type != BTRFS_DIR_INDEX_KEY)5969 break;5970 if (found_key.offset < ctx->pos)5971 continue;5972 if (found_key.offset > private->last_index)5973 break;5974 if (btrfs_should_delete_dir_index(&del_list, found_key.offset))5975 continue;5976 di = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dir_item);5977 name_len = btrfs_dir_name_len(leaf, di);5978 if ((total_len + sizeof(struct dir_entry) + name_len) >=5979 PAGE_SIZE) {5980 btrfs_release_path(path);5981 ret = btrfs_filldir(private->filldir_buf, entries, ctx);5982 if (ret)5983 goto nopos;5984 addr = private->filldir_buf;5985 entries = 0;5986 total_len = 0;5987 goto again;5988 }5989 5990 ftype = btrfs_dir_flags_to_ftype(btrfs_dir_flags(leaf, di));5991 entry = addr;5992 name_ptr = (char *)(entry + 1);5993 read_extent_buffer(leaf, name_ptr,5994 (unsigned long)(di + 1), name_len);5995 put_unaligned(name_len, &entry->name_len);5996 put_unaligned(fs_ftype_to_dtype(ftype), &entry->type);5997 btrfs_dir_item_key_to_cpu(leaf, di, &location);5998 put_unaligned(location.objectid, &entry->ino);5999 put_unaligned(found_key.offset, &entry->offset);6000 entries++;6001 addr += sizeof(struct dir_entry) + name_len;6002 total_len += sizeof(struct dir_entry) + name_len;6003 }6004 /* Catch error encountered during iteration */6005 if (ret < 0)6006 goto err;6007 6008 btrfs_release_path(path);6009 6010 ret = btrfs_filldir(private->filldir_buf, entries, ctx);6011 if (ret)6012 goto nopos;6013 6014 ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list);6015 if (ret)6016 goto nopos;6017 6018 /*6019 * Stop new entries from being returned after we return the last6020 * entry.6021 *6022 * New directory entries are assigned a strictly increasing6023 * offset. This means that new entries created during readdir6024 * are *guaranteed* to be seen in the future by that readdir.6025 * This has broken buggy programs which operate on names as6026 * they're returned by readdir. Until we re-use freed offsets6027 * we have this hack to stop new entries from being returned6028 * under the assumption that they'll never reach this huge6029 * offset.6030 *6031 * This is being careful not to overflow 32bit loff_t unless the6032 * last entry requires it because doing so has broken 32bit apps6033 * in the past.6034 */6035 if (ctx->pos >= INT_MAX)6036 ctx->pos = LLONG_MAX;6037 else6038 ctx->pos = INT_MAX;6039nopos:6040 ret = 0;6041err:6042 if (put)6043 btrfs_readdir_put_delayed_items(BTRFS_I(inode), &ins_list, &del_list);6044 btrfs_free_path(path);6045 return ret;6046}6047 6048/*6049 * This is somewhat expensive, updating the tree every time the6050 * inode changes. But, it is most likely to find the inode in cache.6051 * FIXME, needs more benchmarking...there are no reasons other than performance6052 * to keep or drop this code.6053 */6054static int btrfs_dirty_inode(struct btrfs_inode *inode)6055{6056 struct btrfs_root *root = inode->root;6057 struct btrfs_fs_info *fs_info = root->fs_info;6058 struct btrfs_trans_handle *trans;6059 int ret;6060 6061 if (test_bit(BTRFS_INODE_DUMMY, &inode->runtime_flags))6062 return 0;6063 6064 trans = btrfs_join_transaction(root);6065 if (IS_ERR(trans))6066 return PTR_ERR(trans);6067 6068 ret = btrfs_update_inode(trans, inode);6069 if (ret == -ENOSPC || ret == -EDQUOT) {6070 /* whoops, lets try again with the full transaction */6071 btrfs_end_transaction(trans);6072 trans = btrfs_start_transaction(root, 1);6073 if (IS_ERR(trans))6074 return PTR_ERR(trans);6075 6076 ret = btrfs_update_inode(trans, inode);6077 }6078 btrfs_end_transaction(trans);6079 if (inode->delayed_node)6080 btrfs_balance_delayed_items(fs_info);6081 6082 return ret;6083}6084 6085/*6086 * This is a copy of file_update_time. We need this so we can return error on6087 * ENOSPC for updating the inode in the case of file write and mmap writes.6088 */6089static int btrfs_update_time(struct inode *inode, int flags)6090{6091 struct btrfs_root *root = BTRFS_I(inode)->root;6092 bool dirty;6093 6094 if (btrfs_root_readonly(root))6095 return -EROFS;6096 6097 dirty = inode_update_timestamps(inode, flags);6098 return dirty ? btrfs_dirty_inode(BTRFS_I(inode)) : 0;6099}6100 6101/*6102 * helper to find a free sequence number in a given directory. This current6103 * code is very simple, later versions will do smarter things in the btree6104 */6105int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index)6106{6107 int ret = 0;6108 6109 if (dir->index_cnt == (u64)-1) {6110 ret = btrfs_inode_delayed_dir_index_count(dir);6111 if (ret) {6112 ret = btrfs_set_inode_index_count(dir);6113 if (ret)6114 return ret;6115 }6116 }6117 6118 *index = dir->index_cnt;6119 dir->index_cnt++;6120 6121 return ret;6122}6123 6124static int btrfs_insert_inode_locked(struct inode *inode)6125{6126 struct btrfs_iget_args args;6127 6128 args.ino = btrfs_ino(BTRFS_I(inode));6129 args.root = BTRFS_I(inode)->root;6130 6131 return insert_inode_locked4(inode,6132 btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root),6133 btrfs_find_actor, &args);6134}6135 6136int btrfs_new_inode_prepare(struct btrfs_new_inode_args *args,6137 unsigned int *trans_num_items)6138{6139 struct inode *dir = args->dir;6140 struct inode *inode = args->inode;6141 int ret;6142 6143 if (!args->orphan) {6144 ret = fscrypt_setup_filename(dir, &args->dentry->d_name, 0,6145 &args->fname);6146 if (ret)6147 return ret;6148 }6149 6150 ret = posix_acl_create(dir, &inode->i_mode, &args->default_acl, &args->acl);6151 if (ret) {6152 fscrypt_free_filename(&args->fname);6153 return ret;6154 }6155 6156 /* 1 to add inode item */6157 *trans_num_items = 1;6158 /* 1 to add compression property */6159 if (BTRFS_I(dir)->prop_compress)6160 (*trans_num_items)++;6161 /* 1 to add default ACL xattr */6162 if (args->default_acl)6163 (*trans_num_items)++;6164 /* 1 to add access ACL xattr */6165 if (args->acl)6166 (*trans_num_items)++;6167#ifdef CONFIG_SECURITY6168 /* 1 to add LSM xattr */6169 if (dir->i_security)6170 (*trans_num_items)++;6171#endif6172 if (args->orphan) {6173 /* 1 to add orphan item */6174 (*trans_num_items)++;6175 } else {6176 /*6177 * 1 to add dir item6178 * 1 to add dir index6179 * 1 to update parent inode item6180 *6181 * No need for 1 unit for the inode ref item because it is6182 * inserted in a batch together with the inode item at6183 * btrfs_create_new_inode().6184 */6185 *trans_num_items += 3;6186 }6187 return 0;6188}6189 6190void btrfs_new_inode_args_destroy(struct btrfs_new_inode_args *args)6191{6192 posix_acl_release(args->acl);6193 posix_acl_release(args->default_acl);6194 fscrypt_free_filename(&args->fname);6195}6196 6197/*6198 * Inherit flags from the parent inode.6199 *6200 * Currently only the compression flags and the cow flags are inherited.6201 */6202static void btrfs_inherit_iflags(struct btrfs_inode *inode, struct btrfs_inode *dir)6203{6204 unsigned int flags;6205 6206 flags = dir->flags;6207 6208 if (flags & BTRFS_INODE_NOCOMPRESS) {6209 inode->flags &= ~BTRFS_INODE_COMPRESS;6210 inode->flags |= BTRFS_INODE_NOCOMPRESS;6211 } else if (flags & BTRFS_INODE_COMPRESS) {6212 inode->flags &= ~BTRFS_INODE_NOCOMPRESS;6213 inode->flags |= BTRFS_INODE_COMPRESS;6214 }6215 6216 if (flags & BTRFS_INODE_NODATACOW) {6217 inode->flags |= BTRFS_INODE_NODATACOW;6218 if (S_ISREG(inode->vfs_inode.i_mode))6219 inode->flags |= BTRFS_INODE_NODATASUM;6220 }6221 6222 btrfs_sync_inode_flags_to_i_flags(&inode->vfs_inode);6223}6224 6225int btrfs_create_new_inode(struct btrfs_trans_handle *trans,6226 struct btrfs_new_inode_args *args)6227{6228 struct timespec64 ts;6229 struct inode *dir = args->dir;6230 struct inode *inode = args->inode;6231 const struct fscrypt_str *name = args->orphan ? NULL : &args->fname.disk_name;6232 struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);6233 struct btrfs_root *root;6234 struct btrfs_inode_item *inode_item;6235 struct btrfs_path *path;6236 u64 objectid;6237 struct btrfs_inode_ref *ref;6238 struct btrfs_key key[2];6239 u32 sizes[2];6240 struct btrfs_item_batch batch;6241 unsigned long ptr;6242 int ret;6243 bool xa_reserved = false;6244 6245 path = btrfs_alloc_path();6246 if (!path)6247 return -ENOMEM;6248 6249 if (!args->subvol)6250 BTRFS_I(inode)->root = btrfs_grab_root(BTRFS_I(dir)->root);6251 root = BTRFS_I(inode)->root;6252 6253 ret = btrfs_init_file_extent_tree(BTRFS_I(inode));6254 if (ret)6255 goto out;6256 6257 ret = btrfs_get_free_objectid(root, &objectid);6258 if (ret)6259 goto out;6260 btrfs_set_inode_number(BTRFS_I(inode), objectid);6261 6262 ret = xa_reserve(&root->inodes, objectid, GFP_NOFS);6263 if (ret)6264 goto out;6265 xa_reserved = true;6266 6267 if (args->orphan) {6268 /*6269 * O_TMPFILE, set link count to 0, so that after this point, we6270 * fill in an inode item with the correct link count.6271 */6272 set_nlink(inode, 0);6273 } else {6274 trace_btrfs_inode_request(dir);6275 6276 ret = btrfs_set_inode_index(BTRFS_I(dir), &BTRFS_I(inode)->dir_index);6277 if (ret)6278 goto out;6279 }6280 6281 if (S_ISDIR(inode->i_mode))6282 BTRFS_I(inode)->index_cnt = BTRFS_DIR_START_INDEX;6283 6284 BTRFS_I(inode)->generation = trans->transid;6285 inode->i_generation = BTRFS_I(inode)->generation;6286 6287 /*6288 * We don't have any capability xattrs set here yet, shortcut any6289 * queries for the xattrs here. If we add them later via the inode6290 * security init path or any other path this flag will be cleared.6291 */6292 set_bit(BTRFS_INODE_NO_CAP_XATTR, &BTRFS_I(inode)->runtime_flags);6293 6294 /*6295 * Subvolumes don't inherit flags from their parent directory.6296 * Originally this was probably by accident, but we probably can't6297 * change it now without compatibility issues.6298 */6299 if (!args->subvol)6300 btrfs_inherit_iflags(BTRFS_I(inode), BTRFS_I(dir));6301 6302 if (S_ISREG(inode->i_mode)) {6303 if (btrfs_test_opt(fs_info, NODATASUM))6304 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;6305 if (btrfs_test_opt(fs_info, NODATACOW))6306 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW |6307 BTRFS_INODE_NODATASUM;6308 }6309 6310 ret = btrfs_insert_inode_locked(inode);6311 if (ret < 0) {6312 if (!args->orphan)6313 BTRFS_I(dir)->index_cnt--;6314 goto out;6315 }6316 6317 /*6318 * We could have gotten an inode number from somebody who was fsynced6319 * and then removed in this same transaction, so let's just set full6320 * sync since it will be a full sync anyway and this will blow away the6321 * old info in the log.6322 */6323 btrfs_set_inode_full_sync(BTRFS_I(inode));6324 6325 key[0].objectid = objectid;6326 key[0].type = BTRFS_INODE_ITEM_KEY;6327 key[0].offset = 0;6328 6329 sizes[0] = sizeof(struct btrfs_inode_item);6330 6331 if (!args->orphan) {6332 /*6333 * Start new inodes with an inode_ref. This is slightly more6334 * efficient for small numbers of hard links since they will6335 * be packed into one item. Extended refs will kick in if we6336 * add more hard links than can fit in the ref item.6337 */6338 key[1].objectid = objectid;6339 key[1].type = BTRFS_INODE_REF_KEY;6340 if (args->subvol) {6341 key[1].offset = objectid;6342 sizes[1] = 2 + sizeof(*ref);6343 } else {6344 key[1].offset = btrfs_ino(BTRFS_I(dir));6345 sizes[1] = name->len + sizeof(*ref);6346 }6347 }6348 6349 batch.keys = &key[0];6350 batch.data_sizes = &sizes[0];6351 batch.total_data_size = sizes[0] + (args->orphan ? 0 : sizes[1]);6352 batch.nr = args->orphan ? 1 : 2;6353 ret = btrfs_insert_empty_items(trans, root, path, &batch);6354 if (ret != 0) {6355 btrfs_abort_transaction(trans, ret);6356 goto discard;6357 }6358 6359 ts = simple_inode_init_ts(inode);6360 BTRFS_I(inode)->i_otime_sec = ts.tv_sec;6361 BTRFS_I(inode)->i_otime_nsec = ts.tv_nsec;6362 6363 /*6364 * We're going to fill the inode item now, so at this point the inode6365 * must be fully initialized.6366 */6367 6368 inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0],6369 struct btrfs_inode_item);6370 memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item,6371 sizeof(*inode_item));6372 fill_inode_item(trans, path->nodes[0], inode_item, inode);6373 6374 if (!args->orphan) {6375 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1,6376 struct btrfs_inode_ref);6377 ptr = (unsigned long)(ref + 1);6378 if (args->subvol) {6379 btrfs_set_inode_ref_name_len(path->nodes[0], ref, 2);6380 btrfs_set_inode_ref_index(path->nodes[0], ref, 0);6381 write_extent_buffer(path->nodes[0], "..", ptr, 2);6382 } else {6383 btrfs_set_inode_ref_name_len(path->nodes[0], ref,6384 name->len);6385 btrfs_set_inode_ref_index(path->nodes[0], ref,6386 BTRFS_I(inode)->dir_index);6387 write_extent_buffer(path->nodes[0], name->name, ptr,6388 name->len);6389 }6390 }6391 6392 btrfs_mark_buffer_dirty(trans, path->nodes[0]);6393 /*6394 * We don't need the path anymore, plus inheriting properties, adding6395 * ACLs, security xattrs, orphan item or adding the link, will result in6396 * allocating yet another path. So just free our path.6397 */6398 btrfs_free_path(path);6399 path = NULL;6400 6401 if (args->subvol) {6402 struct inode *parent;6403 6404 /*6405 * Subvolumes inherit properties from their parent subvolume,6406 * not the directory they were created in.6407 */6408 parent = btrfs_iget(BTRFS_FIRST_FREE_OBJECTID, BTRFS_I(dir)->root);6409 if (IS_ERR(parent)) {6410 ret = PTR_ERR(parent);6411 } else {6412 ret = btrfs_inode_inherit_props(trans, inode, parent);6413 iput(parent);6414 }6415 } else {6416 ret = btrfs_inode_inherit_props(trans, inode, dir);6417 }6418 if (ret) {6419 btrfs_err(fs_info,6420 "error inheriting props for ino %llu (root %llu): %d",6421 btrfs_ino(BTRFS_I(inode)), btrfs_root_id(root), ret);6422 }6423 6424 /*6425 * Subvolumes don't inherit ACLs or get passed to the LSM. This is6426 * probably a bug.6427 */6428 if (!args->subvol) {6429 ret = btrfs_init_inode_security(trans, args);6430 if (ret) {6431 btrfs_abort_transaction(trans, ret);6432 goto discard;6433 }6434 }6435 6436 ret = btrfs_add_inode_to_root(BTRFS_I(inode), false);6437 if (WARN_ON(ret)) {6438 /* Shouldn't happen, we used xa_reserve() before. */6439 btrfs_abort_transaction(trans, ret);6440 goto discard;6441 }6442 6443 trace_btrfs_inode_new(inode);6444 btrfs_set_inode_last_trans(trans, BTRFS_I(inode));6445 6446 btrfs_update_root_times(trans, root);6447 6448 if (args->orphan) {6449 ret = btrfs_orphan_add(trans, BTRFS_I(inode));6450 } else {6451 ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), name,6452 0, BTRFS_I(inode)->dir_index);6453 }6454 if (ret) {6455 btrfs_abort_transaction(trans, ret);6456 goto discard;6457 }6458 6459 return 0;6460 6461discard:6462 /*6463 * discard_new_inode() calls iput(), but the caller owns the reference6464 * to the inode.6465 */6466 ihold(inode);6467 discard_new_inode(inode);6468out:6469 if (xa_reserved)6470 xa_release(&root->inodes, objectid);6471 6472 btrfs_free_path(path);6473 return ret;6474}6475 6476/*6477 * utility function to add 'inode' into 'parent_inode' with6478 * a give name and a given sequence number.6479 * if 'add_backref' is true, also insert a backref from the6480 * inode to the parent directory.6481 */6482int btrfs_add_link(struct btrfs_trans_handle *trans,6483 struct btrfs_inode *parent_inode, struct btrfs_inode *inode,6484 const struct fscrypt_str *name, int add_backref, u64 index)6485{6486 int ret = 0;6487 struct btrfs_key key;6488 struct btrfs_root *root = parent_inode->root;6489 u64 ino = btrfs_ino(inode);6490 u64 parent_ino = btrfs_ino(parent_inode);6491 6492 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {6493 memcpy(&key, &inode->root->root_key, sizeof(key));6494 } else {6495 key.objectid = ino;6496 key.type = BTRFS_INODE_ITEM_KEY;6497 key.offset = 0;6498 }6499 6500 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {6501 ret = btrfs_add_root_ref(trans, key.objectid,6502 btrfs_root_id(root), parent_ino,6503 index, name);6504 } else if (add_backref) {6505 ret = btrfs_insert_inode_ref(trans, root, name,6506 ino, parent_ino, index);6507 }6508 6509 /* Nothing to clean up yet */6510 if (ret)6511 return ret;6512 6513 ret = btrfs_insert_dir_item(trans, name, parent_inode, &key,6514 btrfs_inode_type(&inode->vfs_inode), index);6515 if (ret == -EEXIST || ret == -EOVERFLOW)6516 goto fail_dir_item;6517 else if (ret) {6518 btrfs_abort_transaction(trans, ret);6519 return ret;6520 }6521 6522 btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size +6523 name->len * 2);6524 inode_inc_iversion(&parent_inode->vfs_inode);6525 /*6526 * If we are replaying a log tree, we do not want to update the mtime6527 * and ctime of the parent directory with the current time, since the6528 * log replay procedure is responsible for setting them to their correct6529 * values (the ones it had when the fsync was done).6530 */6531 if (!test_bit(BTRFS_FS_LOG_RECOVERING, &root->fs_info->flags))6532 inode_set_mtime_to_ts(&parent_inode->vfs_inode,6533 inode_set_ctime_current(&parent_inode->vfs_inode));6534 6535 ret = btrfs_update_inode(trans, parent_inode);6536 if (ret)6537 btrfs_abort_transaction(trans, ret);6538 return ret;6539 6540fail_dir_item:6541 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {6542 u64 local_index;6543 int err;6544 err = btrfs_del_root_ref(trans, key.objectid,6545 btrfs_root_id(root), parent_ino,6546 &local_index, name);6547 if (err)6548 btrfs_abort_transaction(trans, err);6549 } else if (add_backref) {6550 u64 local_index;6551 int err;6552 6553 err = btrfs_del_inode_ref(trans, root, name, ino, parent_ino,6554 &local_index);6555 if (err)6556 btrfs_abort_transaction(trans, err);6557 }6558 6559 /* Return the original error code */6560 return ret;6561}6562 6563static int btrfs_create_common(struct inode *dir, struct dentry *dentry,6564 struct inode *inode)6565{6566 struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);6567 struct btrfs_root *root = BTRFS_I(dir)->root;6568 struct btrfs_new_inode_args new_inode_args = {6569 .dir = dir,6570 .dentry = dentry,6571 .inode = inode,6572 };6573 unsigned int trans_num_items;6574 struct btrfs_trans_handle *trans;6575 int err;6576 6577 err = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items);6578 if (err)6579 goto out_inode;6580 6581 trans = btrfs_start_transaction(root, trans_num_items);6582 if (IS_ERR(trans)) {6583 err = PTR_ERR(trans);6584 goto out_new_inode_args;6585 }6586 6587 err = btrfs_create_new_inode(trans, &new_inode_args);6588 if (!err)6589 d_instantiate_new(dentry, inode);6590 6591 btrfs_end_transaction(trans);6592 btrfs_btree_balance_dirty(fs_info);6593out_new_inode_args:6594 btrfs_new_inode_args_destroy(&new_inode_args);6595out_inode:6596 if (err)6597 iput(inode);6598 return err;6599}6600 6601static int btrfs_mknod(struct mnt_idmap *idmap, struct inode *dir,6602 struct dentry *dentry, umode_t mode, dev_t rdev)6603{6604 struct inode *inode;6605 6606 inode = new_inode(dir->i_sb);6607 if (!inode)6608 return -ENOMEM;6609 inode_init_owner(idmap, inode, dir, mode);6610 inode->i_op = &btrfs_special_inode_operations;6611 init_special_inode(inode, inode->i_mode, rdev);6612 return btrfs_create_common(dir, dentry, inode);6613}6614 6615static int btrfs_create(struct mnt_idmap *idmap, struct inode *dir,6616 struct dentry *dentry, umode_t mode, bool excl)6617{6618 struct inode *inode;6619 6620 inode = new_inode(dir->i_sb);6621 if (!inode)6622 return -ENOMEM;6623 inode_init_owner(idmap, inode, dir, mode);6624 inode->i_fop = &btrfs_file_operations;6625 inode->i_op = &btrfs_file_inode_operations;6626 inode->i_mapping->a_ops = &btrfs_aops;6627 return btrfs_create_common(dir, dentry, inode);6628}6629 6630static int btrfs_link(struct dentry *old_dentry, struct inode *dir,6631 struct dentry *dentry)6632{6633 struct btrfs_trans_handle *trans = NULL;6634 struct btrfs_root *root = BTRFS_I(dir)->root;6635 struct inode *inode = d_inode(old_dentry);6636 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);6637 struct fscrypt_name fname;6638 u64 index;6639 int err;6640 int drop_inode = 0;6641 6642 /* do not allow sys_link's with other subvols of the same device */6643 if (btrfs_root_id(root) != btrfs_root_id(BTRFS_I(inode)->root))6644 return -EXDEV;6645 6646 if (inode->i_nlink >= BTRFS_LINK_MAX)6647 return -EMLINK;6648 6649 err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &fname);6650 if (err)6651 goto fail;6652 6653 err = btrfs_set_inode_index(BTRFS_I(dir), &index);6654 if (err)6655 goto fail;6656 6657 /*6658 * 2 items for inode and inode ref6659 * 2 items for dir items6660 * 1 item for parent inode6661 * 1 item for orphan item deletion if O_TMPFILE6662 */6663 trans = btrfs_start_transaction(root, inode->i_nlink ? 5 : 6);6664 if (IS_ERR(trans)) {6665 err = PTR_ERR(trans);6666 trans = NULL;6667 goto fail;6668 }6669 6670 /* There are several dir indexes for this inode, clear the cache. */6671 BTRFS_I(inode)->dir_index = 0ULL;6672 inc_nlink(inode);6673 inode_inc_iversion(inode);6674 inode_set_ctime_current(inode);6675 ihold(inode);6676 set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags);6677 6678 err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode),6679 &fname.disk_name, 1, index);6680 6681 if (err) {6682 drop_inode = 1;6683 } else {6684 struct dentry *parent = dentry->d_parent;6685 6686 err = btrfs_update_inode(trans, BTRFS_I(inode));6687 if (err)6688 goto fail;6689 if (inode->i_nlink == 1) {6690 /*6691 * If new hard link count is 1, it's a file created6692 * with open(2) O_TMPFILE flag.6693 */6694 err = btrfs_orphan_del(trans, BTRFS_I(inode));6695 if (err)6696 goto fail;6697 }6698 d_instantiate(dentry, inode);6699 btrfs_log_new_name(trans, old_dentry, NULL, 0, parent);6700 }6701 6702fail:6703 fscrypt_free_filename(&fname);6704 if (trans)6705 btrfs_end_transaction(trans);6706 if (drop_inode) {6707 inode_dec_link_count(inode);6708 iput(inode);6709 }6710 btrfs_btree_balance_dirty(fs_info);6711 return err;6712}6713 6714static int btrfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,6715 struct dentry *dentry, umode_t mode)6716{6717 struct inode *inode;6718 6719 inode = new_inode(dir->i_sb);6720 if (!inode)6721 return -ENOMEM;6722 inode_init_owner(idmap, inode, dir, S_IFDIR | mode);6723 inode->i_op = &btrfs_dir_inode_operations;6724 inode->i_fop = &btrfs_dir_file_operations;6725 return btrfs_create_common(dir, dentry, inode);6726}6727 6728static noinline int uncompress_inline(struct btrfs_path *path,6729 struct folio *folio,6730 struct btrfs_file_extent_item *item)6731{6732 int ret;6733 struct extent_buffer *leaf = path->nodes[0];6734 char *tmp;6735 size_t max_size;6736 unsigned long inline_size;6737 unsigned long ptr;6738 int compress_type;6739 6740 compress_type = btrfs_file_extent_compression(leaf, item);6741 max_size = btrfs_file_extent_ram_bytes(leaf, item);6742 inline_size = btrfs_file_extent_inline_item_len(leaf, path->slots[0]);6743 tmp = kmalloc(inline_size, GFP_NOFS);6744 if (!tmp)6745 return -ENOMEM;6746 ptr = btrfs_file_extent_inline_start(item);6747 6748 read_extent_buffer(leaf, tmp, ptr, inline_size);6749 6750 max_size = min_t(unsigned long, PAGE_SIZE, max_size);6751 ret = btrfs_decompress(compress_type, tmp, folio, 0, inline_size,6752 max_size);6753 6754 /*6755 * decompression code contains a memset to fill in any space between the end6756 * of the uncompressed data and the end of max_size in case the decompressed6757 * data ends up shorter than ram_bytes. That doesn't cover the hole between6758 * the end of an inline extent and the beginning of the next block, so we6759 * cover that region here.6760 */6761 6762 if (max_size < PAGE_SIZE)6763 folio_zero_range(folio, max_size, PAGE_SIZE - max_size);6764 kfree(tmp);6765 return ret;6766}6767 6768static int read_inline_extent(struct btrfs_inode *inode, struct btrfs_path *path,6769 struct folio *folio)6770{6771 struct btrfs_file_extent_item *fi;6772 void *kaddr;6773 size_t copy_size;6774 6775 if (!folio || folio_test_uptodate(folio))6776 return 0;6777 6778 ASSERT(folio_pos(folio) == 0);6779 6780 fi = btrfs_item_ptr(path->nodes[0], path->slots[0],6781 struct btrfs_file_extent_item);6782 if (btrfs_file_extent_compression(path->nodes[0], fi) != BTRFS_COMPRESS_NONE)6783 return uncompress_inline(path, folio, fi);6784 6785 copy_size = min_t(u64, PAGE_SIZE,6786 btrfs_file_extent_ram_bytes(path->nodes[0], fi));6787 kaddr = kmap_local_folio(folio, 0);6788 read_extent_buffer(path->nodes[0], kaddr,6789 btrfs_file_extent_inline_start(fi), copy_size);6790 kunmap_local(kaddr);6791 if (copy_size < PAGE_SIZE)6792 folio_zero_range(folio, copy_size, PAGE_SIZE - copy_size);6793 return 0;6794}6795 6796/*6797 * Lookup the first extent overlapping a range in a file.6798 *6799 * @inode: file to search in6800 * @page: page to read extent data into if the extent is inline6801 * @start: file offset6802 * @len: length of range starting at @start6803 *6804 * Return the first &struct extent_map which overlaps the given range, reading6805 * it from the B-tree and caching it if necessary. Note that there may be more6806 * extents which overlap the given range after the returned extent_map.6807 *6808 * If @page is not NULL and the extent is inline, this also reads the extent6809 * data directly into the page and marks the extent up to date in the io_tree.6810 *6811 * Return: ERR_PTR on error, non-NULL extent_map on success.6812 */6813struct extent_map *btrfs_get_extent(struct btrfs_inode *inode,6814 struct folio *folio, u64 start, u64 len)6815{6816 struct btrfs_fs_info *fs_info = inode->root->fs_info;6817 int ret = 0;6818 u64 extent_start = 0;6819 u64 extent_end = 0;6820 u64 objectid = btrfs_ino(inode);6821 int extent_type = -1;6822 struct btrfs_path *path = NULL;6823 struct btrfs_root *root = inode->root;6824 struct btrfs_file_extent_item *item;6825 struct extent_buffer *leaf;6826 struct btrfs_key found_key;6827 struct extent_map *em = NULL;6828 struct extent_map_tree *em_tree = &inode->extent_tree;6829 6830 read_lock(&em_tree->lock);6831 em = lookup_extent_mapping(em_tree, start, len);6832 read_unlock(&em_tree->lock);6833 6834 if (em) {6835 if (em->start > start || em->start + em->len <= start)6836 free_extent_map(em);6837 else if (em->disk_bytenr == EXTENT_MAP_INLINE && folio)6838 free_extent_map(em);6839 else6840 goto out;6841 }6842 em = alloc_extent_map();6843 if (!em) {6844 ret = -ENOMEM;6845 goto out;6846 }6847 em->start = EXTENT_MAP_HOLE;6848 em->disk_bytenr = EXTENT_MAP_HOLE;6849 em->len = (u64)-1;6850 6851 path = btrfs_alloc_path();6852 if (!path) {6853 ret = -ENOMEM;6854 goto out;6855 }6856 6857 /* Chances are we'll be called again, so go ahead and do readahead */6858 path->reada = READA_FORWARD;6859 6860 /*6861 * The same explanation in load_free_space_cache applies here as well,6862 * we only read when we're loading the free space cache, and at that6863 * point the commit_root has everything we need.6864 */6865 if (btrfs_is_free_space_inode(inode)) {6866 path->search_commit_root = 1;6867 path->skip_locking = 1;6868 }6869 6870 ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0);6871 if (ret < 0) {6872 goto out;6873 } else if (ret > 0) {6874 if (path->slots[0] == 0)6875 goto not_found;6876 path->slots[0]--;6877 ret = 0;6878 }6879 6880 leaf = path->nodes[0];6881 item = btrfs_item_ptr(leaf, path->slots[0],6882 struct btrfs_file_extent_item);6883 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);6884 if (found_key.objectid != objectid ||6885 found_key.type != BTRFS_EXTENT_DATA_KEY) {6886 /*6887 * If we backup past the first extent we want to move forward6888 * and see if there is an extent in front of us, otherwise we'll6889 * say there is a hole for our whole search range which can6890 * cause problems.6891 */6892 extent_end = start;6893 goto next;6894 }6895 6896 extent_type = btrfs_file_extent_type(leaf, item);6897 extent_start = found_key.offset;6898 extent_end = btrfs_file_extent_end(path);6899 if (extent_type == BTRFS_FILE_EXTENT_REG ||6900 extent_type == BTRFS_FILE_EXTENT_PREALLOC) {6901 /* Only regular file could have regular/prealloc extent */6902 if (!S_ISREG(inode->vfs_inode.i_mode)) {6903 ret = -EUCLEAN;6904 btrfs_crit(fs_info,6905 "regular/prealloc extent found for non-regular inode %llu",6906 btrfs_ino(inode));6907 goto out;6908 }6909 trace_btrfs_get_extent_show_fi_regular(inode, leaf, item,6910 extent_start);6911 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {6912 trace_btrfs_get_extent_show_fi_inline(inode, leaf, item,6913 path->slots[0],6914 extent_start);6915 }6916next:6917 if (start >= extent_end) {6918 path->slots[0]++;6919 if (path->slots[0] >= btrfs_header_nritems(leaf)) {6920 ret = btrfs_next_leaf(root, path);6921 if (ret < 0)6922 goto out;6923 else if (ret > 0)6924 goto not_found;6925 6926 leaf = path->nodes[0];6927 }6928 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);6929 if (found_key.objectid != objectid ||6930 found_key.type != BTRFS_EXTENT_DATA_KEY)6931 goto not_found;6932 if (start + len <= found_key.offset)6933 goto not_found;6934 if (start > found_key.offset)6935 goto next;6936 6937 /* New extent overlaps with existing one */6938 em->start = start;6939 em->len = found_key.offset - start;6940 em->disk_bytenr = EXTENT_MAP_HOLE;6941 goto insert;6942 }6943 6944 btrfs_extent_item_to_extent_map(inode, path, item, em);6945 6946 if (extent_type == BTRFS_FILE_EXTENT_REG ||6947 extent_type == BTRFS_FILE_EXTENT_PREALLOC) {6948 goto insert;6949 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {6950 /*6951 * Inline extent can only exist at file offset 0. This is6952 * ensured by tree-checker and inline extent creation path.6953 * Thus all members representing file offsets should be zero.6954 */6955 ASSERT(extent_start == 0);6956 ASSERT(em->start == 0);6957 6958 /*6959 * btrfs_extent_item_to_extent_map() should have properly6960 * initialized em members already.6961 *6962 * Other members are not utilized for inline extents.6963 */6964 ASSERT(em->disk_bytenr == EXTENT_MAP_INLINE);6965 ASSERT(em->len == fs_info->sectorsize);6966 6967 ret = read_inline_extent(inode, path, folio);6968 if (ret < 0)6969 goto out;6970 goto insert;6971 }6972not_found:6973 em->start = start;6974 em->len = len;6975 em->disk_bytenr = EXTENT_MAP_HOLE;6976insert:6977 ret = 0;6978 btrfs_release_path(path);6979 if (em->start > start || extent_map_end(em) <= start) {6980 btrfs_err(fs_info,6981 "bad extent! em: [%llu %llu] passed [%llu %llu]",6982 em->start, em->len, start, len);6983 ret = -EIO;6984 goto out;6985 }6986 6987 write_lock(&em_tree->lock);6988 ret = btrfs_add_extent_mapping(inode, &em, start, len);6989 write_unlock(&em_tree->lock);6990out:6991 btrfs_free_path(path);6992 6993 trace_btrfs_get_extent(root, inode, em);6994 6995 if (ret) {6996 free_extent_map(em);6997 return ERR_PTR(ret);6998 }6999 return em;7000}7001 7002static bool btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr)7003{7004 struct btrfs_block_group *block_group;7005 bool readonly = false;7006 7007 block_group = btrfs_lookup_block_group(fs_info, bytenr);7008 if (!block_group || block_group->ro)7009 readonly = true;7010 if (block_group)7011 btrfs_put_block_group(block_group);7012 return readonly;7013}7014 7015/*7016 * Check if we can do nocow write into the range [@offset, @offset + @len)7017 *7018 * @offset: File offset7019 * @len: The length to write, will be updated to the nocow writeable7020 * range7021 * @orig_start: (optional) Return the original file offset of the file extent7022 * @orig_len: (optional) Return the original on-disk length of the file extent7023 * @ram_bytes: (optional) Return the ram_bytes of the file extent7024 * @strict: if true, omit optimizations that might force us into unnecessary7025 * cow. e.g., don't trust generation number.7026 *7027 * Return:7028 * >0 and update @len if we can do nocow write7029 * 0 if we can't do nocow write7030 * <0 if error happened7031 *7032 * NOTE: This only checks the file extents, caller is responsible to wait for7033 * any ordered extents.7034 */7035noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,7036 struct btrfs_file_extent *file_extent,7037 bool nowait, bool strict)7038{7039 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);7040 struct can_nocow_file_extent_args nocow_args = { 0 };7041 struct btrfs_path *path;7042 int ret;7043 struct extent_buffer *leaf;7044 struct btrfs_root *root = BTRFS_I(inode)->root;7045 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;7046 struct btrfs_file_extent_item *fi;7047 struct btrfs_key key;7048 int found_type;7049 7050 path = btrfs_alloc_path();7051 if (!path)7052 return -ENOMEM;7053 path->nowait = nowait;7054 7055 ret = btrfs_lookup_file_extent(NULL, root, path,7056 btrfs_ino(BTRFS_I(inode)), offset, 0);7057 if (ret < 0)7058 goto out;7059 7060 if (ret == 1) {7061 if (path->slots[0] == 0) {7062 /* can't find the item, must cow */7063 ret = 0;7064 goto out;7065 }7066 path->slots[0]--;7067 }7068 ret = 0;7069 leaf = path->nodes[0];7070 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);7071 if (key.objectid != btrfs_ino(BTRFS_I(inode)) ||7072 key.type != BTRFS_EXTENT_DATA_KEY) {7073 /* not our file or wrong item type, must cow */7074 goto out;7075 }7076 7077 if (key.offset > offset) {7078 /* Wrong offset, must cow */7079 goto out;7080 }7081 7082 if (btrfs_file_extent_end(path) <= offset)7083 goto out;7084 7085 fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);7086 found_type = btrfs_file_extent_type(leaf, fi);7087 7088 nocow_args.start = offset;7089 nocow_args.end = offset + *len - 1;7090 nocow_args.strict = strict;7091 nocow_args.free_path = true;7092 7093 ret = can_nocow_file_extent(path, &key, BTRFS_I(inode), &nocow_args);7094 /* can_nocow_file_extent() has freed the path. */7095 path = NULL;7096 7097 if (ret != 1) {7098 /* Treat errors as not being able to NOCOW. */7099 ret = 0;7100 goto out;7101 }7102 7103 ret = 0;7104 if (btrfs_extent_readonly(fs_info,7105 nocow_args.file_extent.disk_bytenr +7106 nocow_args.file_extent.offset))7107 goto out;7108 7109 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&7110 found_type == BTRFS_FILE_EXTENT_PREALLOC) {7111 u64 range_end;7112 7113 range_end = round_up(offset + nocow_args.file_extent.num_bytes,7114 root->fs_info->sectorsize) - 1;7115 ret = test_range_bit_exists(io_tree, offset, range_end, EXTENT_DELALLOC);7116 if (ret) {7117 ret = -EAGAIN;7118 goto out;7119 }7120 }7121 7122 if (file_extent)7123 memcpy(file_extent, &nocow_args.file_extent, sizeof(*file_extent));7124 7125 *len = nocow_args.file_extent.num_bytes;7126 ret = 1;7127out:7128 btrfs_free_path(path);7129 return ret;7130}7131 7132/* The callers of this must take lock_extent() */7133struct extent_map *btrfs_create_io_em(struct btrfs_inode *inode, u64 start,7134 const struct btrfs_file_extent *file_extent,7135 int type)7136{7137 struct extent_map *em;7138 int ret;7139 7140 /*7141 * Note the missing NOCOW type.7142 *7143 * For pure NOCOW writes, we should not create an io extent map, but7144 * just reusing the existing one.7145 * Only PREALLOC writes (NOCOW write into preallocated range) can7146 * create an io extent map.7147 */7148 ASSERT(type == BTRFS_ORDERED_PREALLOC ||7149 type == BTRFS_ORDERED_COMPRESSED ||7150 type == BTRFS_ORDERED_REGULAR);7151 7152 switch (type) {7153 case BTRFS_ORDERED_PREALLOC:7154 /* We're only referring part of a larger preallocated extent. */7155 ASSERT(file_extent->num_bytes <= file_extent->ram_bytes);7156 break;7157 case BTRFS_ORDERED_REGULAR:7158 /* COW results a new extent matching our file extent size. */7159 ASSERT(file_extent->disk_num_bytes == file_extent->num_bytes);7160 ASSERT(file_extent->ram_bytes == file_extent->num_bytes);7161 7162 /* Since it's a new extent, we should not have any offset. */7163 ASSERT(file_extent->offset == 0);7164 break;7165 case BTRFS_ORDERED_COMPRESSED:7166 /* Must be compressed. */7167 ASSERT(file_extent->compression != BTRFS_COMPRESS_NONE);7168 7169 /*7170 * Encoded write can make us to refer to part of the7171 * uncompressed extent.7172 */7173 ASSERT(file_extent->num_bytes <= file_extent->ram_bytes);7174 break;7175 }7176 7177 em = alloc_extent_map();7178 if (!em)7179 return ERR_PTR(-ENOMEM);7180 7181 em->start = start;7182 em->len = file_extent->num_bytes;7183 em->disk_bytenr = file_extent->disk_bytenr;7184 em->disk_num_bytes = file_extent->disk_num_bytes;7185 em->ram_bytes = file_extent->ram_bytes;7186 em->generation = -1;7187 em->offset = file_extent->offset;7188 em->flags |= EXTENT_FLAG_PINNED;7189 if (type == BTRFS_ORDERED_COMPRESSED)7190 extent_map_set_compression(em, file_extent->compression);7191 7192 ret = btrfs_replace_extent_map_range(inode, em, true);7193 if (ret) {7194 free_extent_map(em);7195 return ERR_PTR(ret);7196 }7197 7198 /* em got 2 refs now, callers needs to do free_extent_map once. */7199 return em;7200}7201 7202/*7203 * For release_folio() and invalidate_folio() we have a race window where7204 * folio_end_writeback() is called but the subpage spinlock is not yet released.7205 * If we continue to release/invalidate the page, we could cause use-after-free7206 * for subpage spinlock. So this function is to spin and wait for subpage7207 * spinlock.7208 */7209static void wait_subpage_spinlock(struct folio *folio)7210{7211 struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);7212 struct btrfs_subpage *subpage;7213 7214 if (!btrfs_is_subpage(fs_info, folio->mapping))7215 return;7216 7217 ASSERT(folio_test_private(folio) && folio_get_private(folio));7218 subpage = folio_get_private(folio);7219 7220 /*7221 * This may look insane as we just acquire the spinlock and release it,7222 * without doing anything. But we just want to make sure no one is7223 * still holding the subpage spinlock.7224 * And since the page is not dirty nor writeback, and we have page7225 * locked, the only possible way to hold a spinlock is from the endio7226 * function to clear page writeback.7227 *7228 * Here we just acquire the spinlock so that all existing callers7229 * should exit and we're safe to release/invalidate the page.7230 */7231 spin_lock_irq(&subpage->lock);7232 spin_unlock_irq(&subpage->lock);7233}7234 7235static int btrfs_launder_folio(struct folio *folio)7236{7237 return btrfs_qgroup_free_data(folio_to_inode(folio), NULL, folio_pos(folio),7238 PAGE_SIZE, NULL);7239}7240 7241static bool __btrfs_release_folio(struct folio *folio, gfp_t gfp_flags)7242{7243 if (try_release_extent_mapping(folio, gfp_flags)) {7244 wait_subpage_spinlock(folio);7245 clear_folio_extent_mapped(folio);7246 return true;7247 }7248 return false;7249}7250 7251static bool btrfs_release_folio(struct folio *folio, gfp_t gfp_flags)7252{7253 if (folio_test_writeback(folio) || folio_test_dirty(folio))7254 return false;7255 return __btrfs_release_folio(folio, gfp_flags);7256}7257 7258#ifdef CONFIG_MIGRATION7259static int btrfs_migrate_folio(struct address_space *mapping,7260 struct folio *dst, struct folio *src,7261 enum migrate_mode mode)7262{7263 int ret = filemap_migrate_folio(mapping, dst, src, mode);7264 7265 if (ret != MIGRATEPAGE_SUCCESS)7266 return ret;7267 7268 if (folio_test_ordered(src)) {7269 folio_clear_ordered(src);7270 folio_set_ordered(dst);7271 }7272 7273 return MIGRATEPAGE_SUCCESS;7274}7275#else7276#define btrfs_migrate_folio NULL7277#endif7278 7279static void btrfs_invalidate_folio(struct folio *folio, size_t offset,7280 size_t length)7281{7282 struct btrfs_inode *inode = folio_to_inode(folio);7283 struct btrfs_fs_info *fs_info = inode->root->fs_info;7284 struct extent_io_tree *tree = &inode->io_tree;7285 struct extent_state *cached_state = NULL;7286 u64 page_start = folio_pos(folio);7287 u64 page_end = page_start + folio_size(folio) - 1;7288 u64 cur;7289 int inode_evicting = inode->vfs_inode.i_state & I_FREEING;7290 7291 /*7292 * We have folio locked so no new ordered extent can be created on this7293 * page, nor bio can be submitted for this folio.7294 *7295 * But already submitted bio can still be finished on this folio.7296 * Furthermore, endio function won't skip folio which has Ordered7297 * (Private2) already cleared, so it's possible for endio and7298 * invalidate_folio to do the same ordered extent accounting twice7299 * on one folio.7300 *7301 * So here we wait for any submitted bios to finish, so that we won't7302 * do double ordered extent accounting on the same folio.7303 */7304 folio_wait_writeback(folio);7305 wait_subpage_spinlock(folio);7306 7307 /*7308 * For subpage case, we have call sites like7309 * btrfs_punch_hole_lock_range() which passes range not aligned to7310 * sectorsize.7311 * If the range doesn't cover the full folio, we don't need to and7312 * shouldn't clear page extent mapped, as folio->private can still7313 * record subpage dirty bits for other part of the range.7314 *7315 * For cases that invalidate the full folio even the range doesn't7316 * cover the full folio, like invalidating the last folio, we're7317 * still safe to wait for ordered extent to finish.7318 */7319 if (!(offset == 0 && length == folio_size(folio))) {7320 btrfs_release_folio(folio, GFP_NOFS);7321 return;7322 }7323 7324 if (!inode_evicting)7325 lock_extent(tree, page_start, page_end, &cached_state);7326 7327 cur = page_start;7328 while (cur < page_end) {7329 struct btrfs_ordered_extent *ordered;7330 u64 range_end;7331 u32 range_len;7332 u32 extra_flags = 0;7333 7334 ordered = btrfs_lookup_first_ordered_range(inode, cur,7335 page_end + 1 - cur);7336 if (!ordered) {7337 range_end = page_end;7338 /*7339 * No ordered extent covering this range, we are safe7340 * to delete all extent states in the range.7341 */7342 extra_flags = EXTENT_CLEAR_ALL_BITS;7343 goto next;7344 }7345 if (ordered->file_offset > cur) {7346 /*7347 * There is a range between [cur, oe->file_offset) not7348 * covered by any ordered extent.7349 * We are safe to delete all extent states, and handle7350 * the ordered extent in the next iteration.7351 */7352 range_end = ordered->file_offset - 1;7353 extra_flags = EXTENT_CLEAR_ALL_BITS;7354 goto next;7355 }7356 7357 range_end = min(ordered->file_offset + ordered->num_bytes - 1,7358 page_end);7359 ASSERT(range_end + 1 - cur < U32_MAX);7360 range_len = range_end + 1 - cur;7361 if (!btrfs_folio_test_ordered(fs_info, folio, cur, range_len)) {7362 /*7363 * If Ordered (Private2) is cleared, it means endio has7364 * already been executed for the range.7365 * We can't delete the extent states as7366 * btrfs_finish_ordered_io() may still use some of them.7367 */7368 goto next;7369 }7370 btrfs_folio_clear_ordered(fs_info, folio, cur, range_len);7371 7372 /*7373 * IO on this page will never be started, so we need to account7374 * for any ordered extents now. Don't clear EXTENT_DELALLOC_NEW7375 * here, must leave that up for the ordered extent completion.7376 *7377 * This will also unlock the range for incoming7378 * btrfs_finish_ordered_io().7379 */7380 if (!inode_evicting)7381 clear_extent_bit(tree, cur, range_end,7382 EXTENT_DELALLOC |7383 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING |7384 EXTENT_DEFRAG, &cached_state);7385 7386 spin_lock_irq(&inode->ordered_tree_lock);7387 set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);7388 ordered->truncated_len = min(ordered->truncated_len,7389 cur - ordered->file_offset);7390 spin_unlock_irq(&inode->ordered_tree_lock);7391 7392 /*7393 * If the ordered extent has finished, we're safe to delete all7394 * the extent states of the range, otherwise7395 * btrfs_finish_ordered_io() will get executed by endio for7396 * other pages, so we can't delete extent states.7397 */7398 if (btrfs_dec_test_ordered_pending(inode, &ordered,7399 cur, range_end + 1 - cur)) {7400 btrfs_finish_ordered_io(ordered);7401 /*7402 * The ordered extent has finished, now we're again7403 * safe to delete all extent states of the range.7404 */7405 extra_flags = EXTENT_CLEAR_ALL_BITS;7406 }7407next:7408 if (ordered)7409 btrfs_put_ordered_extent(ordered);7410 /*7411 * Qgroup reserved space handler7412 * Sector(s) here will be either:7413 *7414 * 1) Already written to disk or bio already finished7415 * Then its QGROUP_RESERVED bit in io_tree is already cleared.7416 * Qgroup will be handled by its qgroup_record then.7417 * btrfs_qgroup_free_data() call will do nothing here.7418 *7419 * 2) Not written to disk yet7420 * Then btrfs_qgroup_free_data() call will clear the7421 * QGROUP_RESERVED bit of its io_tree, and free the qgroup7422 * reserved data space.7423 * Since the IO will never happen for this page.7424 */7425 btrfs_qgroup_free_data(inode, NULL, cur, range_end + 1 - cur, NULL);7426 if (!inode_evicting) {7427 clear_extent_bit(tree, cur, range_end, EXTENT_LOCKED |7428 EXTENT_DELALLOC | EXTENT_UPTODATE |7429 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG |7430 extra_flags, &cached_state);7431 }7432 cur = range_end + 1;7433 }7434 /*7435 * We have iterated through all ordered extents of the page, the page7436 * should not have Ordered (Private2) anymore, or the above iteration7437 * did something wrong.7438 */7439 ASSERT(!folio_test_ordered(folio));7440 btrfs_folio_clear_checked(fs_info, folio, folio_pos(folio), folio_size(folio));7441 if (!inode_evicting)7442 __btrfs_release_folio(folio, GFP_NOFS);7443 clear_folio_extent_mapped(folio);7444}7445 7446static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback)7447{7448 struct btrfs_truncate_control control = {7449 .inode = inode,7450 .ino = btrfs_ino(inode),7451 .min_type = BTRFS_EXTENT_DATA_KEY,7452 .clear_extent_range = true,7453 };7454 struct btrfs_root *root = inode->root;7455 struct btrfs_fs_info *fs_info = root->fs_info;7456 struct btrfs_block_rsv *rsv;7457 int ret;7458 struct btrfs_trans_handle *trans;7459 u64 mask = fs_info->sectorsize - 1;7460 const u64 min_size = btrfs_calc_metadata_size(fs_info, 1);7461 7462 if (!skip_writeback) {7463 ret = btrfs_wait_ordered_range(inode,7464 inode->vfs_inode.i_size & (~mask),7465 (u64)-1);7466 if (ret)7467 return ret;7468 }7469 7470 /*7471 * Yes ladies and gentlemen, this is indeed ugly. We have a couple of7472 * things going on here:7473 *7474 * 1) We need to reserve space to update our inode.7475 *7476 * 2) We need to have something to cache all the space that is going to7477 * be free'd up by the truncate operation, but also have some slack7478 * space reserved in case it uses space during the truncate (thank you7479 * very much snapshotting).7480 *7481 * And we need these to be separate. The fact is we can use a lot of7482 * space doing the truncate, and we have no earthly idea how much space7483 * we will use, so we need the truncate reservation to be separate so it7484 * doesn't end up using space reserved for updating the inode. We also7485 * need to be able to stop the transaction and start a new one, which7486 * means we need to be able to update the inode several times, and we7487 * have no idea of knowing how many times that will be, so we can't just7488 * reserve 1 item for the entirety of the operation, so that has to be7489 * done separately as well.7490 *7491 * So that leaves us with7492 *7493 * 1) rsv - for the truncate reservation, which we will steal from the7494 * transaction reservation.7495 * 2) fs_info->trans_block_rsv - this will have 1 items worth left for7496 * updating the inode.7497 */7498 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);7499 if (!rsv)7500 return -ENOMEM;7501 rsv->size = min_size;7502 rsv->failfast = true;7503 7504 /*7505 * 1 for the truncate slack space7506 * 1 for updating the inode.7507 */7508 trans = btrfs_start_transaction(root, 2);7509 if (IS_ERR(trans)) {7510 ret = PTR_ERR(trans);7511 goto out;7512 }7513 7514 /* Migrate the slack space for the truncate to our reserve */7515 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv,7516 min_size, false);7517 /*7518 * We have reserved 2 metadata units when we started the transaction and7519 * min_size matches 1 unit, so this should never fail, but if it does,7520 * it's not critical we just fail truncation.7521 */7522 if (WARN_ON(ret)) {7523 btrfs_end_transaction(trans);7524 goto out;7525 }7526 7527 trans->block_rsv = rsv;7528 7529 while (1) {7530 struct extent_state *cached_state = NULL;7531 const u64 new_size = inode->vfs_inode.i_size;7532 const u64 lock_start = ALIGN_DOWN(new_size, fs_info->sectorsize);7533 7534 control.new_size = new_size;7535 lock_extent(&inode->io_tree, lock_start, (u64)-1, &cached_state);7536 /*7537 * We want to drop from the next block forward in case this new7538 * size is not block aligned since we will be keeping the last7539 * block of the extent just the way it is.7540 */7541 btrfs_drop_extent_map_range(inode,7542 ALIGN(new_size, fs_info->sectorsize),7543 (u64)-1, false);7544 7545 ret = btrfs_truncate_inode_items(trans, root, &control);7546 7547 inode_sub_bytes(&inode->vfs_inode, control.sub_bytes);7548 btrfs_inode_safe_disk_i_size_write(inode, control.last_size);7549 7550 unlock_extent(&inode->io_tree, lock_start, (u64)-1, &cached_state);7551 7552 trans->block_rsv = &fs_info->trans_block_rsv;7553 if (ret != -ENOSPC && ret != -EAGAIN)7554 break;7555 7556 ret = btrfs_update_inode(trans, inode);7557 if (ret)7558 break;7559 7560 btrfs_end_transaction(trans);7561 btrfs_btree_balance_dirty(fs_info);7562 7563 trans = btrfs_start_transaction(root, 2);7564 if (IS_ERR(trans)) {7565 ret = PTR_ERR(trans);7566 trans = NULL;7567 break;7568 }7569 7570 btrfs_block_rsv_release(fs_info, rsv, -1, NULL);7571 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,7572 rsv, min_size, false);7573 /*7574 * We have reserved 2 metadata units when we started the7575 * transaction and min_size matches 1 unit, so this should never7576 * fail, but if it does, it's not critical we just fail truncation.7577 */7578 if (WARN_ON(ret))7579 break;7580 7581 trans->block_rsv = rsv;7582 }7583 7584 /*7585 * We can't call btrfs_truncate_block inside a trans handle as we could7586 * deadlock with freeze, if we got BTRFS_NEED_TRUNCATE_BLOCK then we7587 * know we've truncated everything except the last little bit, and can7588 * do btrfs_truncate_block and then update the disk_i_size.7589 */7590 if (ret == BTRFS_NEED_TRUNCATE_BLOCK) {7591 btrfs_end_transaction(trans);7592 btrfs_btree_balance_dirty(fs_info);7593 7594 ret = btrfs_truncate_block(inode, inode->vfs_inode.i_size, 0, 0);7595 if (ret)7596 goto out;7597 trans = btrfs_start_transaction(root, 1);7598 if (IS_ERR(trans)) {7599 ret = PTR_ERR(trans);7600 goto out;7601 }7602 btrfs_inode_safe_disk_i_size_write(inode, 0);7603 }7604 7605 if (trans) {7606 int ret2;7607 7608 trans->block_rsv = &fs_info->trans_block_rsv;7609 ret2 = btrfs_update_inode(trans, inode);7610 if (ret2 && !ret)7611 ret = ret2;7612 7613 ret2 = btrfs_end_transaction(trans);7614 if (ret2 && !ret)7615 ret = ret2;7616 btrfs_btree_balance_dirty(fs_info);7617 }7618out:7619 btrfs_free_block_rsv(fs_info, rsv);7620 /*7621 * So if we truncate and then write and fsync we normally would just7622 * write the extents that changed, which is a problem if we need to7623 * first truncate that entire inode. So set this flag so we write out7624 * all of the extents in the inode to the sync log so we're completely7625 * safe.7626 *7627 * If no extents were dropped or trimmed we don't need to force the next7628 * fsync to truncate all the inode's items from the log and re-log them7629 * all. This means the truncate operation did not change the file size,7630 * or changed it to a smaller size but there was only an implicit hole7631 * between the old i_size and the new i_size, and there were no prealloc7632 * extents beyond i_size to drop.7633 */7634 if (control.extents_found > 0)7635 btrfs_set_inode_full_sync(inode);7636 7637 return ret;7638}7639 7640struct inode *btrfs_new_subvol_inode(struct mnt_idmap *idmap,7641 struct inode *dir)7642{7643 struct inode *inode;7644 7645 inode = new_inode(dir->i_sb);7646 if (inode) {7647 /*7648 * Subvolumes don't inherit the sgid bit or the parent's gid if7649 * the parent's sgid bit is set. This is probably a bug.7650 */7651 inode_init_owner(idmap, inode, NULL,7652 S_IFDIR | (~current_umask() & S_IRWXUGO));7653 inode->i_op = &btrfs_dir_inode_operations;7654 inode->i_fop = &btrfs_dir_file_operations;7655 }7656 return inode;7657}7658 7659struct inode *btrfs_alloc_inode(struct super_block *sb)7660{7661 struct btrfs_fs_info *fs_info = btrfs_sb(sb);7662 struct btrfs_inode *ei;7663 struct inode *inode;7664 7665 ei = alloc_inode_sb(sb, btrfs_inode_cachep, GFP_KERNEL);7666 if (!ei)7667 return NULL;7668 7669 ei->root = NULL;7670 ei->generation = 0;7671 ei->last_trans = 0;7672 ei->last_sub_trans = 0;7673 ei->logged_trans = 0;7674 ei->delalloc_bytes = 0;7675 ei->new_delalloc_bytes = 0;7676 ei->defrag_bytes = 0;7677 ei->disk_i_size = 0;7678 ei->flags = 0;7679 ei->ro_flags = 0;7680 /*7681 * ->index_cnt will be properly initialized later when creating a new7682 * inode (btrfs_create_new_inode()) or when reading an existing inode7683 * from disk (btrfs_read_locked_inode()).7684 */7685 ei->csum_bytes = 0;7686 ei->dir_index = 0;7687 ei->last_unlink_trans = 0;7688 ei->last_reflink_trans = 0;7689 ei->last_log_commit = 0;7690 7691 spin_lock_init(&ei->lock);7692 ei->outstanding_extents = 0;7693 if (sb->s_magic != BTRFS_TEST_MAGIC)7694 btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv,7695 BTRFS_BLOCK_RSV_DELALLOC);7696 ei->runtime_flags = 0;7697 ei->prop_compress = BTRFS_COMPRESS_NONE;7698 ei->defrag_compress = BTRFS_COMPRESS_NONE;7699 7700 ei->delayed_node = NULL;7701 7702 ei->i_otime_sec = 0;7703 ei->i_otime_nsec = 0;7704 7705 inode = &ei->vfs_inode;7706 extent_map_tree_init(&ei->extent_tree);7707 7708 /* This io tree sets the valid inode. */7709 extent_io_tree_init(fs_info, &ei->io_tree, IO_TREE_INODE_IO);7710 ei->io_tree.inode = ei;7711 7712 ei->file_extent_tree = NULL;7713 7714 mutex_init(&ei->log_mutex);7715 spin_lock_init(&ei->ordered_tree_lock);7716 ei->ordered_tree = RB_ROOT;7717 ei->ordered_tree_last = NULL;7718 INIT_LIST_HEAD(&ei->delalloc_inodes);7719 INIT_LIST_HEAD(&ei->delayed_iput);7720 init_rwsem(&ei->i_mmap_lock);7721 7722 return inode;7723}7724 7725#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS7726void btrfs_test_destroy_inode(struct inode *inode)7727{7728 btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false);7729 kfree(BTRFS_I(inode)->file_extent_tree);7730 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));7731}7732#endif7733 7734void btrfs_free_inode(struct inode *inode)7735{7736 kfree(BTRFS_I(inode)->file_extent_tree);7737 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));7738}7739 7740void btrfs_destroy_inode(struct inode *vfs_inode)7741{7742 struct btrfs_ordered_extent *ordered;7743 struct btrfs_inode *inode = BTRFS_I(vfs_inode);7744 struct btrfs_root *root = inode->root;7745 bool freespace_inode;7746 7747 WARN_ON(!hlist_empty(&vfs_inode->i_dentry));7748 WARN_ON(vfs_inode->i_data.nrpages);7749 WARN_ON(inode->block_rsv.reserved);7750 WARN_ON(inode->block_rsv.size);7751 WARN_ON(inode->outstanding_extents);7752 if (!S_ISDIR(vfs_inode->i_mode)) {7753 WARN_ON(inode->delalloc_bytes);7754 WARN_ON(inode->new_delalloc_bytes);7755 WARN_ON(inode->csum_bytes);7756 }7757 if (!root || !btrfs_is_data_reloc_root(root))7758 WARN_ON(inode->defrag_bytes);7759 7760 /*7761 * This can happen where we create an inode, but somebody else also7762 * created the same inode and we need to destroy the one we already7763 * created.7764 */7765 if (!root)7766 return;7767 7768 /*7769 * If this is a free space inode do not take the ordered extents lockdep7770 * map.7771 */7772 freespace_inode = btrfs_is_free_space_inode(inode);7773 7774 while (1) {7775 ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);7776 if (!ordered)7777 break;7778 else {7779 btrfs_err(root->fs_info,7780 "found ordered extent %llu %llu on inode cleanup",7781 ordered->file_offset, ordered->num_bytes);7782 7783 if (!freespace_inode)7784 btrfs_lockdep_acquire(root->fs_info, btrfs_ordered_extent);7785 7786 btrfs_remove_ordered_extent(inode, ordered);7787 btrfs_put_ordered_extent(ordered);7788 btrfs_put_ordered_extent(ordered);7789 }7790 }7791 btrfs_qgroup_check_reserved_leak(inode);7792 btrfs_del_inode_from_root(inode);7793 btrfs_drop_extent_map_range(inode, 0, (u64)-1, false);7794 btrfs_inode_clear_file_extent_range(inode, 0, (u64)-1);7795 btrfs_put_root(inode->root);7796}7797 7798int btrfs_drop_inode(struct inode *inode)7799{7800 struct btrfs_root *root = BTRFS_I(inode)->root;7801 7802 if (root == NULL)7803 return 1;7804 7805 /* the snap/subvol tree is on deleting */7806 if (btrfs_root_refs(&root->root_item) == 0)7807 return 1;7808 else7809 return generic_drop_inode(inode);7810}7811 7812static void init_once(void *foo)7813{7814 struct btrfs_inode *ei = foo;7815 7816 inode_init_once(&ei->vfs_inode);7817}7818 7819void __cold btrfs_destroy_cachep(void)7820{7821 /*7822 * Make sure all delayed rcu free inodes are flushed before we7823 * destroy cache.7824 */7825 rcu_barrier();7826 kmem_cache_destroy(btrfs_inode_cachep);7827}7828 7829int __init btrfs_init_cachep(void)7830{7831 btrfs_inode_cachep = kmem_cache_create("btrfs_inode",7832 sizeof(struct btrfs_inode), 0,7833 SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT,7834 init_once);7835 if (!btrfs_inode_cachep)7836 return -ENOMEM;7837 7838 return 0;7839}7840 7841static int btrfs_getattr(struct mnt_idmap *idmap,7842 const struct path *path, struct kstat *stat,7843 u32 request_mask, unsigned int flags)7844{7845 u64 delalloc_bytes;7846 u64 inode_bytes;7847 struct inode *inode = d_inode(path->dentry);7848 u32 blocksize = btrfs_sb(inode->i_sb)->sectorsize;7849 u32 bi_flags = BTRFS_I(inode)->flags;7850 u32 bi_ro_flags = BTRFS_I(inode)->ro_flags;7851 7852 stat->result_mask |= STATX_BTIME;7853 stat->btime.tv_sec = BTRFS_I(inode)->i_otime_sec;7854 stat->btime.tv_nsec = BTRFS_I(inode)->i_otime_nsec;7855 if (bi_flags & BTRFS_INODE_APPEND)7856 stat->attributes |= STATX_ATTR_APPEND;7857 if (bi_flags & BTRFS_INODE_COMPRESS)7858 stat->attributes |= STATX_ATTR_COMPRESSED;7859 if (bi_flags & BTRFS_INODE_IMMUTABLE)7860 stat->attributes |= STATX_ATTR_IMMUTABLE;7861 if (bi_flags & BTRFS_INODE_NODUMP)7862 stat->attributes |= STATX_ATTR_NODUMP;7863 if (bi_ro_flags & BTRFS_INODE_RO_VERITY)7864 stat->attributes |= STATX_ATTR_VERITY;7865 7866 stat->attributes_mask |= (STATX_ATTR_APPEND |7867 STATX_ATTR_COMPRESSED |7868 STATX_ATTR_IMMUTABLE |7869 STATX_ATTR_NODUMP);7870 7871 generic_fillattr(idmap, request_mask, inode, stat);7872 stat->dev = BTRFS_I(inode)->root->anon_dev;7873 7874 stat->subvol = BTRFS_I(inode)->root->root_key.objectid;7875 stat->result_mask |= STATX_SUBVOL;7876 7877 spin_lock(&BTRFS_I(inode)->lock);7878 delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes;7879 inode_bytes = inode_get_bytes(inode);7880 spin_unlock(&BTRFS_I(inode)->lock);7881 stat->blocks = (ALIGN(inode_bytes, blocksize) +7882 ALIGN(delalloc_bytes, blocksize)) >> SECTOR_SHIFT;7883 return 0;7884}7885 7886static int btrfs_rename_exchange(struct inode *old_dir,7887 struct dentry *old_dentry,7888 struct inode *new_dir,7889 struct dentry *new_dentry)7890{7891 struct btrfs_fs_info *fs_info = inode_to_fs_info(old_dir);7892 struct btrfs_trans_handle *trans;7893 unsigned int trans_num_items;7894 struct btrfs_root *root = BTRFS_I(old_dir)->root;7895 struct btrfs_root *dest = BTRFS_I(new_dir)->root;7896 struct inode *new_inode = new_dentry->d_inode;7897 struct inode *old_inode = old_dentry->d_inode;7898 struct btrfs_rename_ctx old_rename_ctx;7899 struct btrfs_rename_ctx new_rename_ctx;7900 u64 old_ino = btrfs_ino(BTRFS_I(old_inode));7901 u64 new_ino = btrfs_ino(BTRFS_I(new_inode));7902 u64 old_idx = 0;7903 u64 new_idx = 0;7904 int ret;7905 int ret2;7906 bool need_abort = false;7907 struct fscrypt_name old_fname, new_fname;7908 struct fscrypt_str *old_name, *new_name;7909 7910 /*7911 * For non-subvolumes allow exchange only within one subvolume, in the7912 * same inode namespace. Two subvolumes (represented as directory) can7913 * be exchanged as they're a logical link and have a fixed inode number.7914 */7915 if (root != dest &&7916 (old_ino != BTRFS_FIRST_FREE_OBJECTID ||7917 new_ino != BTRFS_FIRST_FREE_OBJECTID))7918 return -EXDEV;7919 7920 ret = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &old_fname);7921 if (ret)7922 return ret;7923 7924 ret = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &new_fname);7925 if (ret) {7926 fscrypt_free_filename(&old_fname);7927 return ret;7928 }7929 7930 old_name = &old_fname.disk_name;7931 new_name = &new_fname.disk_name;7932 7933 /* close the race window with snapshot create/destroy ioctl */7934 if (old_ino == BTRFS_FIRST_FREE_OBJECTID ||7935 new_ino == BTRFS_FIRST_FREE_OBJECTID)7936 down_read(&fs_info->subvol_sem);7937 7938 /*7939 * For each inode:7940 * 1 to remove old dir item7941 * 1 to remove old dir index7942 * 1 to add new dir item7943 * 1 to add new dir index7944 * 1 to update parent inode7945 *7946 * If the parents are the same, we only need to account for one7947 */7948 trans_num_items = (old_dir == new_dir ? 9 : 10);7949 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {7950 /*7951 * 1 to remove old root ref7952 * 1 to remove old root backref7953 * 1 to add new root ref7954 * 1 to add new root backref7955 */7956 trans_num_items += 4;7957 } else {7958 /*7959 * 1 to update inode item7960 * 1 to remove old inode ref7961 * 1 to add new inode ref7962 */7963 trans_num_items += 3;7964 }7965 if (new_ino == BTRFS_FIRST_FREE_OBJECTID)7966 trans_num_items += 4;7967 else7968 trans_num_items += 3;7969 trans = btrfs_start_transaction(root, trans_num_items);7970 if (IS_ERR(trans)) {7971 ret = PTR_ERR(trans);7972 goto out_notrans;7973 }7974 7975 if (dest != root) {7976 ret = btrfs_record_root_in_trans(trans, dest);7977 if (ret)7978 goto out_fail;7979 }7980 7981 /*7982 * We need to find a free sequence number both in the source and7983 * in the destination directory for the exchange.7984 */7985 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx);7986 if (ret)7987 goto out_fail;7988 ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx);7989 if (ret)7990 goto out_fail;7991 7992 BTRFS_I(old_inode)->dir_index = 0ULL;7993 BTRFS_I(new_inode)->dir_index = 0ULL;7994 7995 /* Reference for the source. */7996 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {7997 /* force full log commit if subvolume involved. */7998 btrfs_set_log_full_commit(trans);7999 } else {8000 ret = btrfs_insert_inode_ref(trans, dest, new_name, old_ino,8001 btrfs_ino(BTRFS_I(new_dir)),8002 old_idx);8003 if (ret)8004 goto out_fail;8005 need_abort = true;8006 }8007 8008 /* And now for the dest. */8009 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {8010 /* force full log commit if subvolume involved. */8011 btrfs_set_log_full_commit(trans);8012 } else {8013 ret = btrfs_insert_inode_ref(trans, root, old_name, new_ino,8014 btrfs_ino(BTRFS_I(old_dir)),8015 new_idx);8016 if (ret) {8017 if (need_abort)8018 btrfs_abort_transaction(trans, ret);8019 goto out_fail;8020 }8021 }8022 8023 /* Update inode version and ctime/mtime. */8024 inode_inc_iversion(old_dir);8025 inode_inc_iversion(new_dir);8026 inode_inc_iversion(old_inode);8027 inode_inc_iversion(new_inode);8028 simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);8029 8030 if (old_dentry->d_parent != new_dentry->d_parent) {8031 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),8032 BTRFS_I(old_inode), true);8033 btrfs_record_unlink_dir(trans, BTRFS_I(new_dir),8034 BTRFS_I(new_inode), true);8035 }8036 8037 /* src is a subvolume */8038 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {8039 ret = btrfs_unlink_subvol(trans, BTRFS_I(old_dir), old_dentry);8040 } else { /* src is an inode */8041 ret = __btrfs_unlink_inode(trans, BTRFS_I(old_dir),8042 BTRFS_I(old_dentry->d_inode),8043 old_name, &old_rename_ctx);8044 if (!ret)8045 ret = btrfs_update_inode(trans, BTRFS_I(old_inode));8046 }8047 if (ret) {8048 btrfs_abort_transaction(trans, ret);8049 goto out_fail;8050 }8051 8052 /* dest is a subvolume */8053 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {8054 ret = btrfs_unlink_subvol(trans, BTRFS_I(new_dir), new_dentry);8055 } else { /* dest is an inode */8056 ret = __btrfs_unlink_inode(trans, BTRFS_I(new_dir),8057 BTRFS_I(new_dentry->d_inode),8058 new_name, &new_rename_ctx);8059 if (!ret)8060 ret = btrfs_update_inode(trans, BTRFS_I(new_inode));8061 }8062 if (ret) {8063 btrfs_abort_transaction(trans, ret);8064 goto out_fail;8065 }8066 8067 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),8068 new_name, 0, old_idx);8069 if (ret) {8070 btrfs_abort_transaction(trans, ret);8071 goto out_fail;8072 }8073 8074 ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode),8075 old_name, 0, new_idx);8076 if (ret) {8077 btrfs_abort_transaction(trans, ret);8078 goto out_fail;8079 }8080 8081 if (old_inode->i_nlink == 1)8082 BTRFS_I(old_inode)->dir_index = old_idx;8083 if (new_inode->i_nlink == 1)8084 BTRFS_I(new_inode)->dir_index = new_idx;8085 8086 /*8087 * Now pin the logs of the roots. We do it to ensure that no other task8088 * can sync the logs while we are in progress with the rename, because8089 * that could result in an inconsistency in case any of the inodes that8090 * are part of this rename operation were logged before.8091 */8092 if (old_ino != BTRFS_FIRST_FREE_OBJECTID)8093 btrfs_pin_log_trans(root);8094 if (new_ino != BTRFS_FIRST_FREE_OBJECTID)8095 btrfs_pin_log_trans(dest);8096 8097 /* Do the log updates for all inodes. */8098 if (old_ino != BTRFS_FIRST_FREE_OBJECTID)8099 btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir),8100 old_rename_ctx.index, new_dentry->d_parent);8101 if (new_ino != BTRFS_FIRST_FREE_OBJECTID)8102 btrfs_log_new_name(trans, new_dentry, BTRFS_I(new_dir),8103 new_rename_ctx.index, old_dentry->d_parent);8104 8105 /* Now unpin the logs. */8106 if (old_ino != BTRFS_FIRST_FREE_OBJECTID)8107 btrfs_end_log_trans(root);8108 if (new_ino != BTRFS_FIRST_FREE_OBJECTID)8109 btrfs_end_log_trans(dest);8110out_fail:8111 ret2 = btrfs_end_transaction(trans);8112 ret = ret ? ret : ret2;8113out_notrans:8114 if (new_ino == BTRFS_FIRST_FREE_OBJECTID ||8115 old_ino == BTRFS_FIRST_FREE_OBJECTID)8116 up_read(&fs_info->subvol_sem);8117 8118 fscrypt_free_filename(&new_fname);8119 fscrypt_free_filename(&old_fname);8120 return ret;8121}8122 8123static struct inode *new_whiteout_inode(struct mnt_idmap *idmap,8124 struct inode *dir)8125{8126 struct inode *inode;8127 8128 inode = new_inode(dir->i_sb);8129 if (inode) {8130 inode_init_owner(idmap, inode, dir,8131 S_IFCHR | WHITEOUT_MODE);8132 inode->i_op = &btrfs_special_inode_operations;8133 init_special_inode(inode, inode->i_mode, WHITEOUT_DEV);8134 }8135 return inode;8136}8137 8138static int btrfs_rename(struct mnt_idmap *idmap,8139 struct inode *old_dir, struct dentry *old_dentry,8140 struct inode *new_dir, struct dentry *new_dentry,8141 unsigned int flags)8142{8143 struct btrfs_fs_info *fs_info = inode_to_fs_info(old_dir);8144 struct btrfs_new_inode_args whiteout_args = {8145 .dir = old_dir,8146 .dentry = old_dentry,8147 };8148 struct btrfs_trans_handle *trans;8149 unsigned int trans_num_items;8150 struct btrfs_root *root = BTRFS_I(old_dir)->root;8151 struct btrfs_root *dest = BTRFS_I(new_dir)->root;8152 struct inode *new_inode = d_inode(new_dentry);8153 struct inode *old_inode = d_inode(old_dentry);8154 struct btrfs_rename_ctx rename_ctx;8155 u64 index = 0;8156 int ret;8157 int ret2;8158 u64 old_ino = btrfs_ino(BTRFS_I(old_inode));8159 struct fscrypt_name old_fname, new_fname;8160 8161 if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)8162 return -EPERM;8163 8164 /* we only allow rename subvolume link between subvolumes */8165 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)8166 return -EXDEV;8167 8168 if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID ||8169 (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID))8170 return -ENOTEMPTY;8171 8172 if (S_ISDIR(old_inode->i_mode) && new_inode &&8173 new_inode->i_size > BTRFS_EMPTY_DIR_SIZE)8174 return -ENOTEMPTY;8175 8176 ret = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &old_fname);8177 if (ret)8178 return ret;8179 8180 ret = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &new_fname);8181 if (ret) {8182 fscrypt_free_filename(&old_fname);8183 return ret;8184 }8185 8186 /* check for collisions, even if the name isn't there */8187 ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino, &new_fname.disk_name);8188 if (ret) {8189 if (ret == -EEXIST) {8190 /* we shouldn't get8191 * eexist without a new_inode */8192 if (WARN_ON(!new_inode)) {8193 goto out_fscrypt_names;8194 }8195 } else {8196 /* maybe -EOVERFLOW */8197 goto out_fscrypt_names;8198 }8199 }8200 ret = 0;8201 8202 /*8203 * we're using rename to replace one file with another. Start IO on it8204 * now so we don't add too much work to the end of the transaction8205 */8206 if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size)8207 filemap_flush(old_inode->i_mapping);8208 8209 if (flags & RENAME_WHITEOUT) {8210 whiteout_args.inode = new_whiteout_inode(idmap, old_dir);8211 if (!whiteout_args.inode) {8212 ret = -ENOMEM;8213 goto out_fscrypt_names;8214 }8215 ret = btrfs_new_inode_prepare(&whiteout_args, &trans_num_items);8216 if (ret)8217 goto out_whiteout_inode;8218 } else {8219 /* 1 to update the old parent inode. */8220 trans_num_items = 1;8221 }8222 8223 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {8224 /* Close the race window with snapshot create/destroy ioctl */8225 down_read(&fs_info->subvol_sem);8226 /*8227 * 1 to remove old root ref8228 * 1 to remove old root backref8229 * 1 to add new root ref8230 * 1 to add new root backref8231 */8232 trans_num_items += 4;8233 } else {8234 /*8235 * 1 to update inode8236 * 1 to remove old inode ref8237 * 1 to add new inode ref8238 */8239 trans_num_items += 3;8240 }8241 /*8242 * 1 to remove old dir item8243 * 1 to remove old dir index8244 * 1 to add new dir item8245 * 1 to add new dir index8246 */8247 trans_num_items += 4;8248 /* 1 to update new parent inode if it's not the same as the old parent */8249 if (new_dir != old_dir)8250 trans_num_items++;8251 if (new_inode) {8252 /*8253 * 1 to update inode8254 * 1 to remove inode ref8255 * 1 to remove dir item8256 * 1 to remove dir index8257 * 1 to possibly add orphan item8258 */8259 trans_num_items += 5;8260 }8261 trans = btrfs_start_transaction(root, trans_num_items);8262 if (IS_ERR(trans)) {8263 ret = PTR_ERR(trans);8264 goto out_notrans;8265 }8266 8267 if (dest != root) {8268 ret = btrfs_record_root_in_trans(trans, dest);8269 if (ret)8270 goto out_fail;8271 }8272 8273 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index);8274 if (ret)8275 goto out_fail;8276 8277 BTRFS_I(old_inode)->dir_index = 0ULL;8278 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {8279 /* force full log commit if subvolume involved. */8280 btrfs_set_log_full_commit(trans);8281 } else {8282 ret = btrfs_insert_inode_ref(trans, dest, &new_fname.disk_name,8283 old_ino, btrfs_ino(BTRFS_I(new_dir)),8284 index);8285 if (ret)8286 goto out_fail;8287 }8288 8289 inode_inc_iversion(old_dir);8290 inode_inc_iversion(new_dir);8291 inode_inc_iversion(old_inode);8292 simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);8293 8294 if (old_dentry->d_parent != new_dentry->d_parent)8295 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),8296 BTRFS_I(old_inode), true);8297 8298 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {8299 ret = btrfs_unlink_subvol(trans, BTRFS_I(old_dir), old_dentry);8300 } else {8301 ret = __btrfs_unlink_inode(trans, BTRFS_I(old_dir),8302 BTRFS_I(d_inode(old_dentry)),8303 &old_fname.disk_name, &rename_ctx);8304 if (!ret)8305 ret = btrfs_update_inode(trans, BTRFS_I(old_inode));8306 }8307 if (ret) {8308 btrfs_abort_transaction(trans, ret);8309 goto out_fail;8310 }8311 8312 if (new_inode) {8313 inode_inc_iversion(new_inode);8314 if (unlikely(btrfs_ino(BTRFS_I(new_inode)) ==8315 BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {8316 ret = btrfs_unlink_subvol(trans, BTRFS_I(new_dir), new_dentry);8317 BUG_ON(new_inode->i_nlink == 0);8318 } else {8319 ret = btrfs_unlink_inode(trans, BTRFS_I(new_dir),8320 BTRFS_I(d_inode(new_dentry)),8321 &new_fname.disk_name);8322 }8323 if (!ret && new_inode->i_nlink == 0)8324 ret = btrfs_orphan_add(trans,8325 BTRFS_I(d_inode(new_dentry)));8326 if (ret) {8327 btrfs_abort_transaction(trans, ret);8328 goto out_fail;8329 }8330 }8331 8332 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),8333 &new_fname.disk_name, 0, index);8334 if (ret) {8335 btrfs_abort_transaction(trans, ret);8336 goto out_fail;8337 }8338 8339 if (old_inode->i_nlink == 1)8340 BTRFS_I(old_inode)->dir_index = index;8341 8342 if (old_ino != BTRFS_FIRST_FREE_OBJECTID)8343 btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir),8344 rename_ctx.index, new_dentry->d_parent);8345 8346 if (flags & RENAME_WHITEOUT) {8347 ret = btrfs_create_new_inode(trans, &whiteout_args);8348 if (ret) {8349 btrfs_abort_transaction(trans, ret);8350 goto out_fail;8351 } else {8352 unlock_new_inode(whiteout_args.inode);8353 iput(whiteout_args.inode);8354 whiteout_args.inode = NULL;8355 }8356 }8357out_fail:8358 ret2 = btrfs_end_transaction(trans);8359 ret = ret ? ret : ret2;8360out_notrans:8361 if (old_ino == BTRFS_FIRST_FREE_OBJECTID)8362 up_read(&fs_info->subvol_sem);8363 if (flags & RENAME_WHITEOUT)8364 btrfs_new_inode_args_destroy(&whiteout_args);8365out_whiteout_inode:8366 if (flags & RENAME_WHITEOUT)8367 iput(whiteout_args.inode);8368out_fscrypt_names:8369 fscrypt_free_filename(&old_fname);8370 fscrypt_free_filename(&new_fname);8371 return ret;8372}8373 8374static int btrfs_rename2(struct mnt_idmap *idmap, struct inode *old_dir,8375 struct dentry *old_dentry, struct inode *new_dir,8376 struct dentry *new_dentry, unsigned int flags)8377{8378 int ret;8379 8380 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))8381 return -EINVAL;8382 8383 if (flags & RENAME_EXCHANGE)8384 ret = btrfs_rename_exchange(old_dir, old_dentry, new_dir,8385 new_dentry);8386 else8387 ret = btrfs_rename(idmap, old_dir, old_dentry, new_dir,8388 new_dentry, flags);8389 8390 btrfs_btree_balance_dirty(BTRFS_I(new_dir)->root->fs_info);8391 8392 return ret;8393}8394 8395struct btrfs_delalloc_work {8396 struct inode *inode;8397 struct completion completion;8398 struct list_head list;8399 struct btrfs_work work;8400};8401 8402static void btrfs_run_delalloc_work(struct btrfs_work *work)8403{8404 struct btrfs_delalloc_work *delalloc_work;8405 struct inode *inode;8406 8407 delalloc_work = container_of(work, struct btrfs_delalloc_work,8408 work);8409 inode = delalloc_work->inode;8410 filemap_flush(inode->i_mapping);8411 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,8412 &BTRFS_I(inode)->runtime_flags))8413 filemap_flush(inode->i_mapping);8414 8415 iput(inode);8416 complete(&delalloc_work->completion);8417}8418 8419static struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode)8420{8421 struct btrfs_delalloc_work *work;8422 8423 work = kmalloc(sizeof(*work), GFP_NOFS);8424 if (!work)8425 return NULL;8426 8427 init_completion(&work->completion);8428 INIT_LIST_HEAD(&work->list);8429 work->inode = inode;8430 btrfs_init_work(&work->work, btrfs_run_delalloc_work, NULL);8431 8432 return work;8433}8434 8435/*8436 * some fairly slow code that needs optimization. This walks the list8437 * of all the inodes with pending delalloc and forces them to disk.8438 */8439static int start_delalloc_inodes(struct btrfs_root *root,8440 struct writeback_control *wbc, bool snapshot,8441 bool in_reclaim_context)8442{8443 struct btrfs_inode *binode;8444 struct inode *inode;8445 struct btrfs_delalloc_work *work, *next;8446 LIST_HEAD(works);8447 LIST_HEAD(splice);8448 int ret = 0;8449 bool full_flush = wbc->nr_to_write == LONG_MAX;8450 8451 mutex_lock(&root->delalloc_mutex);8452 spin_lock(&root->delalloc_lock);8453 list_splice_init(&root->delalloc_inodes, &splice);8454 while (!list_empty(&splice)) {8455 binode = list_entry(splice.next, struct btrfs_inode,8456 delalloc_inodes);8457 8458 list_move_tail(&binode->delalloc_inodes,8459 &root->delalloc_inodes);8460 8461 if (in_reclaim_context &&8462 test_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &binode->runtime_flags))8463 continue;8464 8465 inode = igrab(&binode->vfs_inode);8466 if (!inode) {8467 cond_resched_lock(&root->delalloc_lock);8468 continue;8469 }8470 spin_unlock(&root->delalloc_lock);8471 8472 if (snapshot)8473 set_bit(BTRFS_INODE_SNAPSHOT_FLUSH,8474 &binode->runtime_flags);8475 if (full_flush) {8476 work = btrfs_alloc_delalloc_work(inode);8477 if (!work) {8478 iput(inode);8479 ret = -ENOMEM;8480 goto out;8481 }8482 list_add_tail(&work->list, &works);8483 btrfs_queue_work(root->fs_info->flush_workers,8484 &work->work);8485 } else {8486 ret = filemap_fdatawrite_wbc(inode->i_mapping, wbc);8487 btrfs_add_delayed_iput(BTRFS_I(inode));8488 if (ret || wbc->nr_to_write <= 0)8489 goto out;8490 }8491 cond_resched();8492 spin_lock(&root->delalloc_lock);8493 }8494 spin_unlock(&root->delalloc_lock);8495 8496out:8497 list_for_each_entry_safe(work, next, &works, list) {8498 list_del_init(&work->list);8499 wait_for_completion(&work->completion);8500 kfree(work);8501 }8502 8503 if (!list_empty(&splice)) {8504 spin_lock(&root->delalloc_lock);8505 list_splice_tail(&splice, &root->delalloc_inodes);8506 spin_unlock(&root->delalloc_lock);8507 }8508 mutex_unlock(&root->delalloc_mutex);8509 return ret;8510}8511 8512int btrfs_start_delalloc_snapshot(struct btrfs_root *root, bool in_reclaim_context)8513{8514 struct writeback_control wbc = {8515 .nr_to_write = LONG_MAX,8516 .sync_mode = WB_SYNC_NONE,8517 .range_start = 0,8518 .range_end = LLONG_MAX,8519 };8520 struct btrfs_fs_info *fs_info = root->fs_info;8521 8522 if (BTRFS_FS_ERROR(fs_info))8523 return -EROFS;8524 8525 return start_delalloc_inodes(root, &wbc, true, in_reclaim_context);8526}8527 8528int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr,8529 bool in_reclaim_context)8530{8531 struct writeback_control wbc = {8532 .nr_to_write = nr,8533 .sync_mode = WB_SYNC_NONE,8534 .range_start = 0,8535 .range_end = LLONG_MAX,8536 };8537 struct btrfs_root *root;8538 LIST_HEAD(splice);8539 int ret;8540 8541 if (BTRFS_FS_ERROR(fs_info))8542 return -EROFS;8543 8544 mutex_lock(&fs_info->delalloc_root_mutex);8545 spin_lock(&fs_info->delalloc_root_lock);8546 list_splice_init(&fs_info->delalloc_roots, &splice);8547 while (!list_empty(&splice)) {8548 /*8549 * Reset nr_to_write here so we know that we're doing a full8550 * flush.8551 */8552 if (nr == LONG_MAX)8553 wbc.nr_to_write = LONG_MAX;8554 8555 root = list_first_entry(&splice, struct btrfs_root,8556 delalloc_root);8557 root = btrfs_grab_root(root);8558 BUG_ON(!root);8559 list_move_tail(&root->delalloc_root,8560 &fs_info->delalloc_roots);8561 spin_unlock(&fs_info->delalloc_root_lock);8562 8563 ret = start_delalloc_inodes(root, &wbc, false, in_reclaim_context);8564 btrfs_put_root(root);8565 if (ret < 0 || wbc.nr_to_write <= 0)8566 goto out;8567 spin_lock(&fs_info->delalloc_root_lock);8568 }8569 spin_unlock(&fs_info->delalloc_root_lock);8570 8571 ret = 0;8572out:8573 if (!list_empty(&splice)) {8574 spin_lock(&fs_info->delalloc_root_lock);8575 list_splice_tail(&splice, &fs_info->delalloc_roots);8576 spin_unlock(&fs_info->delalloc_root_lock);8577 }8578 mutex_unlock(&fs_info->delalloc_root_mutex);8579 return ret;8580}8581 8582static int btrfs_symlink(struct mnt_idmap *idmap, struct inode *dir,8583 struct dentry *dentry, const char *symname)8584{8585 struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);8586 struct btrfs_trans_handle *trans;8587 struct btrfs_root *root = BTRFS_I(dir)->root;8588 struct btrfs_path *path;8589 struct btrfs_key key;8590 struct inode *inode;8591 struct btrfs_new_inode_args new_inode_args = {8592 .dir = dir,8593 .dentry = dentry,8594 };8595 unsigned int trans_num_items;8596 int err;8597 int name_len;8598 int datasize;8599 unsigned long ptr;8600 struct btrfs_file_extent_item *ei;8601 struct extent_buffer *leaf;8602 8603 name_len = strlen(symname);8604 if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info))8605 return -ENAMETOOLONG;8606 8607 inode = new_inode(dir->i_sb);8608 if (!inode)8609 return -ENOMEM;8610 inode_init_owner(idmap, inode, dir, S_IFLNK | S_IRWXUGO);8611 inode->i_op = &btrfs_symlink_inode_operations;8612 inode_nohighmem(inode);8613 inode->i_mapping->a_ops = &btrfs_aops;8614 btrfs_i_size_write(BTRFS_I(inode), name_len);8615 inode_set_bytes(inode, name_len);8616 8617 new_inode_args.inode = inode;8618 err = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items);8619 if (err)8620 goto out_inode;8621 /* 1 additional item for the inline extent */8622 trans_num_items++;8623 8624 trans = btrfs_start_transaction(root, trans_num_items);8625 if (IS_ERR(trans)) {8626 err = PTR_ERR(trans);8627 goto out_new_inode_args;8628 }8629 8630 err = btrfs_create_new_inode(trans, &new_inode_args);8631 if (err)8632 goto out;8633 8634 path = btrfs_alloc_path();8635 if (!path) {8636 err = -ENOMEM;8637 btrfs_abort_transaction(trans, err);8638 discard_new_inode(inode);8639 inode = NULL;8640 goto out;8641 }8642 key.objectid = btrfs_ino(BTRFS_I(inode));8643 key.offset = 0;8644 key.type = BTRFS_EXTENT_DATA_KEY;8645 datasize = btrfs_file_extent_calc_inline_size(name_len);8646 err = btrfs_insert_empty_item(trans, root, path, &key,8647 datasize);8648 if (err) {8649 btrfs_abort_transaction(trans, err);8650 btrfs_free_path(path);8651 discard_new_inode(inode);8652 inode = NULL;8653 goto out;8654 }8655 leaf = path->nodes[0];8656 ei = btrfs_item_ptr(leaf, path->slots[0],8657 struct btrfs_file_extent_item);8658 btrfs_set_file_extent_generation(leaf, ei, trans->transid);8659 btrfs_set_file_extent_type(leaf, ei,8660 BTRFS_FILE_EXTENT_INLINE);8661 btrfs_set_file_extent_encryption(leaf, ei, 0);8662 btrfs_set_file_extent_compression(leaf, ei, 0);8663 btrfs_set_file_extent_other_encoding(leaf, ei, 0);8664 btrfs_set_file_extent_ram_bytes(leaf, ei, name_len);8665 8666 ptr = btrfs_file_extent_inline_start(ei);8667 write_extent_buffer(leaf, symname, ptr, name_len);8668 btrfs_mark_buffer_dirty(trans, leaf);8669 btrfs_free_path(path);8670 8671 d_instantiate_new(dentry, inode);8672 err = 0;8673out:8674 btrfs_end_transaction(trans);8675 btrfs_btree_balance_dirty(fs_info);8676out_new_inode_args:8677 btrfs_new_inode_args_destroy(&new_inode_args);8678out_inode:8679 if (err)8680 iput(inode);8681 return err;8682}8683 8684static struct btrfs_trans_handle *insert_prealloc_file_extent(8685 struct btrfs_trans_handle *trans_in,8686 struct btrfs_inode *inode,8687 struct btrfs_key *ins,8688 u64 file_offset)8689{8690 struct btrfs_file_extent_item stack_fi;8691 struct btrfs_replace_extent_info extent_info;8692 struct btrfs_trans_handle *trans = trans_in;8693 struct btrfs_path *path;8694 u64 start = ins->objectid;8695 u64 len = ins->offset;8696 u64 qgroup_released = 0;8697 int ret;8698 8699 memset(&stack_fi, 0, sizeof(stack_fi));8700 8701 btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_PREALLOC);8702 btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, start);8703 btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi, len);8704 btrfs_set_stack_file_extent_num_bytes(&stack_fi, len);8705 btrfs_set_stack_file_extent_ram_bytes(&stack_fi, len);8706 btrfs_set_stack_file_extent_compression(&stack_fi, BTRFS_COMPRESS_NONE);8707 /* Encryption and other encoding is reserved and all 0 */8708 8709 ret = btrfs_qgroup_release_data(inode, file_offset, len, &qgroup_released);8710 if (ret < 0)8711 return ERR_PTR(ret);8712 8713 if (trans) {8714 ret = insert_reserved_file_extent(trans, inode,8715 file_offset, &stack_fi,8716 true, qgroup_released);8717 if (ret)8718 goto free_qgroup;8719 return trans;8720 }8721 8722 extent_info.disk_offset = start;8723 extent_info.disk_len = len;8724 extent_info.data_offset = 0;8725 extent_info.data_len = len;8726 extent_info.file_offset = file_offset;8727 extent_info.extent_buf = (char *)&stack_fi;8728 extent_info.is_new_extent = true;8729 extent_info.update_times = true;8730 extent_info.qgroup_reserved = qgroup_released;8731 extent_info.insertions = 0;8732 8733 path = btrfs_alloc_path();8734 if (!path) {8735 ret = -ENOMEM;8736 goto free_qgroup;8737 }8738 8739 ret = btrfs_replace_file_extents(inode, path, file_offset,8740 file_offset + len - 1, &extent_info,8741 &trans);8742 btrfs_free_path(path);8743 if (ret)8744 goto free_qgroup;8745 return trans;8746 8747free_qgroup:8748 /*8749 * We have released qgroup data range at the beginning of the function,8750 * and normally qgroup_released bytes will be freed when committing8751 * transaction.8752 * But if we error out early, we have to free what we have released8753 * or we leak qgroup data reservation.8754 */8755 btrfs_qgroup_free_refroot(inode->root->fs_info,8756 btrfs_root_id(inode->root), qgroup_released,8757 BTRFS_QGROUP_RSV_DATA);8758 return ERR_PTR(ret);8759}8760 8761static int __btrfs_prealloc_file_range(struct inode *inode, int mode,8762 u64 start, u64 num_bytes, u64 min_size,8763 loff_t actual_len, u64 *alloc_hint,8764 struct btrfs_trans_handle *trans)8765{8766 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);8767 struct extent_map *em;8768 struct btrfs_root *root = BTRFS_I(inode)->root;8769 struct btrfs_key ins;8770 u64 cur_offset = start;8771 u64 clear_offset = start;8772 u64 i_size;8773 u64 cur_bytes;8774 u64 last_alloc = (u64)-1;8775 int ret = 0;8776 bool own_trans = true;8777 u64 end = start + num_bytes - 1;8778 8779 if (trans)8780 own_trans = false;8781 while (num_bytes > 0) {8782 cur_bytes = min_t(u64, num_bytes, SZ_256M);8783 cur_bytes = max(cur_bytes, min_size);8784 /*8785 * If we are severely fragmented we could end up with really8786 * small allocations, so if the allocator is returning small8787 * chunks lets make its job easier by only searching for those8788 * sized chunks.8789 */8790 cur_bytes = min(cur_bytes, last_alloc);8791 ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes,8792 min_size, 0, *alloc_hint, &ins, 1, 0);8793 if (ret)8794 break;8795 8796 /*8797 * We've reserved this space, and thus converted it from8798 * ->bytes_may_use to ->bytes_reserved. Any error that happens8799 * from here on out we will only need to clear our reservation8800 * for the remaining unreserved area, so advance our8801 * clear_offset by our extent size.8802 */8803 clear_offset += ins.offset;8804 8805 last_alloc = ins.offset;8806 trans = insert_prealloc_file_extent(trans, BTRFS_I(inode),8807 &ins, cur_offset);8808 /*8809 * Now that we inserted the prealloc extent we can finally8810 * decrement the number of reservations in the block group.8811 * If we did it before, we could race with relocation and have8812 * relocation miss the reserved extent, making it fail later.8813 */8814 btrfs_dec_block_group_reservations(fs_info, ins.objectid);8815 if (IS_ERR(trans)) {8816 ret = PTR_ERR(trans);8817 btrfs_free_reserved_extent(fs_info, ins.objectid,8818 ins.offset, 0);8819 break;8820 }8821 8822 em = alloc_extent_map();8823 if (!em) {8824 btrfs_drop_extent_map_range(BTRFS_I(inode), cur_offset,8825 cur_offset + ins.offset - 1, false);8826 btrfs_set_inode_full_sync(BTRFS_I(inode));8827 goto next;8828 }8829 8830 em->start = cur_offset;8831 em->len = ins.offset;8832 em->disk_bytenr = ins.objectid;8833 em->offset = 0;8834 em->disk_num_bytes = ins.offset;8835 em->ram_bytes = ins.offset;8836 em->flags |= EXTENT_FLAG_PREALLOC;8837 em->generation = trans->transid;8838 8839 ret = btrfs_replace_extent_map_range(BTRFS_I(inode), em, true);8840 free_extent_map(em);8841next:8842 num_bytes -= ins.offset;8843 cur_offset += ins.offset;8844 *alloc_hint = ins.objectid + ins.offset;8845 8846 inode_inc_iversion(inode);8847 inode_set_ctime_current(inode);8848 BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC;8849 if (!(mode & FALLOC_FL_KEEP_SIZE) &&8850 (actual_len > inode->i_size) &&8851 (cur_offset > inode->i_size)) {8852 if (cur_offset > actual_len)8853 i_size = actual_len;8854 else8855 i_size = cur_offset;8856 i_size_write(inode, i_size);8857 btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0);8858 }8859 8860 ret = btrfs_update_inode(trans, BTRFS_I(inode));8861 8862 if (ret) {8863 btrfs_abort_transaction(trans, ret);8864 if (own_trans)8865 btrfs_end_transaction(trans);8866 break;8867 }8868 8869 if (own_trans) {8870 btrfs_end_transaction(trans);8871 trans = NULL;8872 }8873 }8874 if (clear_offset < end)8875 btrfs_free_reserved_data_space(BTRFS_I(inode), NULL, clear_offset,8876 end - clear_offset + 1);8877 return ret;8878}8879 8880int btrfs_prealloc_file_range(struct inode *inode, int mode,8881 u64 start, u64 num_bytes, u64 min_size,8882 loff_t actual_len, u64 *alloc_hint)8883{8884 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,8885 min_size, actual_len, alloc_hint,8886 NULL);8887}8888 8889int btrfs_prealloc_file_range_trans(struct inode *inode,8890 struct btrfs_trans_handle *trans, int mode,8891 u64 start, u64 num_bytes, u64 min_size,8892 loff_t actual_len, u64 *alloc_hint)8893{8894 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,8895 min_size, actual_len, alloc_hint, trans);8896}8897 8898static int btrfs_permission(struct mnt_idmap *idmap,8899 struct inode *inode, int mask)8900{8901 struct btrfs_root *root = BTRFS_I(inode)->root;8902 umode_t mode = inode->i_mode;8903 8904 if (mask & MAY_WRITE &&8905 (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) {8906 if (btrfs_root_readonly(root))8907 return -EROFS;8908 if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY)8909 return -EACCES;8910 }8911 return generic_permission(idmap, inode, mask);8912}8913 8914static int btrfs_tmpfile(struct mnt_idmap *idmap, struct inode *dir,8915 struct file *file, umode_t mode)8916{8917 struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);8918 struct btrfs_trans_handle *trans;8919 struct btrfs_root *root = BTRFS_I(dir)->root;8920 struct inode *inode;8921 struct btrfs_new_inode_args new_inode_args = {8922 .dir = dir,8923 .dentry = file->f_path.dentry,8924 .orphan = true,8925 };8926 unsigned int trans_num_items;8927 int ret;8928 8929 inode = new_inode(dir->i_sb);8930 if (!inode)8931 return -ENOMEM;8932 inode_init_owner(idmap, inode, dir, mode);8933 inode->i_fop = &btrfs_file_operations;8934 inode->i_op = &btrfs_file_inode_operations;8935 inode->i_mapping->a_ops = &btrfs_aops;8936 8937 new_inode_args.inode = inode;8938 ret = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items);8939 if (ret)8940 goto out_inode;8941 8942 trans = btrfs_start_transaction(root, trans_num_items);8943 if (IS_ERR(trans)) {8944 ret = PTR_ERR(trans);8945 goto out_new_inode_args;8946 }8947 8948 ret = btrfs_create_new_inode(trans, &new_inode_args);8949 8950 /*8951 * We set number of links to 0 in btrfs_create_new_inode(), and here we8952 * set it to 1 because d_tmpfile() will issue a warning if the count is8953 * 0, through:8954 *8955 * d_tmpfile() -> inode_dec_link_count() -> drop_nlink()8956 */8957 set_nlink(inode, 1);8958 8959 if (!ret) {8960 d_tmpfile(file, inode);8961 unlock_new_inode(inode);8962 mark_inode_dirty(inode);8963 }8964 8965 btrfs_end_transaction(trans);8966 btrfs_btree_balance_dirty(fs_info);8967out_new_inode_args:8968 btrfs_new_inode_args_destroy(&new_inode_args);8969out_inode:8970 if (ret)8971 iput(inode);8972 return finish_open_simple(file, ret);8973}8974 8975void btrfs_set_range_writeback(struct btrfs_inode *inode, u64 start, u64 end)8976{8977 struct btrfs_fs_info *fs_info = inode->root->fs_info;8978 unsigned long index = start >> PAGE_SHIFT;8979 unsigned long end_index = end >> PAGE_SHIFT;8980 struct folio *folio;8981 u32 len;8982 8983 ASSERT(end + 1 - start <= U32_MAX);8984 len = end + 1 - start;8985 while (index <= end_index) {8986 folio = __filemap_get_folio(inode->vfs_inode.i_mapping, index, 0, 0);8987 ASSERT(!IS_ERR(folio)); /* folios should be in the extent_io_tree */8988 8989 /* This is for data, which doesn't yet support larger folio. */8990 ASSERT(folio_order(folio) == 0);8991 btrfs_folio_set_writeback(fs_info, folio, start, len);8992 folio_put(folio);8993 index++;8994 }8995}8996 8997int btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info *fs_info,8998 int compress_type)8999{9000 switch (compress_type) {9001 case BTRFS_COMPRESS_NONE:9002 return BTRFS_ENCODED_IO_COMPRESSION_NONE;9003 case BTRFS_COMPRESS_ZLIB:9004 return BTRFS_ENCODED_IO_COMPRESSION_ZLIB;9005 case BTRFS_COMPRESS_LZO:9006 /*9007 * The LZO format depends on the sector size. 64K is the maximum9008 * sector size that we support.9009 */9010 if (fs_info->sectorsize < SZ_4K || fs_info->sectorsize > SZ_64K)9011 return -EINVAL;9012 return BTRFS_ENCODED_IO_COMPRESSION_LZO_4K +9013 (fs_info->sectorsize_bits - 12);9014 case BTRFS_COMPRESS_ZSTD:9015 return BTRFS_ENCODED_IO_COMPRESSION_ZSTD;9016 default:9017 return -EUCLEAN;9018 }9019}9020 9021static ssize_t btrfs_encoded_read_inline(9022 struct kiocb *iocb,9023 struct iov_iter *iter, u64 start,9024 u64 lockend,9025 struct extent_state **cached_state,9026 u64 extent_start, size_t count,9027 struct btrfs_ioctl_encoded_io_args *encoded,9028 bool *unlocked)9029{9030 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));9031 struct btrfs_root *root = inode->root;9032 struct btrfs_fs_info *fs_info = root->fs_info;9033 struct extent_io_tree *io_tree = &inode->io_tree;9034 struct btrfs_path *path;9035 struct extent_buffer *leaf;9036 struct btrfs_file_extent_item *item;9037 u64 ram_bytes;9038 unsigned long ptr;9039 void *tmp;9040 ssize_t ret;9041 9042 path = btrfs_alloc_path();9043 if (!path) {9044 ret = -ENOMEM;9045 goto out;9046 }9047 ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode),9048 extent_start, 0);9049 if (ret) {9050 if (ret > 0) {9051 /* The extent item disappeared? */9052 ret = -EIO;9053 }9054 goto out;9055 }9056 leaf = path->nodes[0];9057 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);9058 9059 ram_bytes = btrfs_file_extent_ram_bytes(leaf, item);9060 ptr = btrfs_file_extent_inline_start(item);9061 9062 encoded->len = min_t(u64, extent_start + ram_bytes,9063 inode->vfs_inode.i_size) - iocb->ki_pos;9064 ret = btrfs_encoded_io_compression_from_extent(fs_info,9065 btrfs_file_extent_compression(leaf, item));9066 if (ret < 0)9067 goto out;9068 encoded->compression = ret;9069 if (encoded->compression) {9070 size_t inline_size;9071 9072 inline_size = btrfs_file_extent_inline_item_len(leaf,9073 path->slots[0]);9074 if (inline_size > count) {9075 ret = -ENOBUFS;9076 goto out;9077 }9078 count = inline_size;9079 encoded->unencoded_len = ram_bytes;9080 encoded->unencoded_offset = iocb->ki_pos - extent_start;9081 } else {9082 count = min_t(u64, count, encoded->len);9083 encoded->len = count;9084 encoded->unencoded_len = count;9085 ptr += iocb->ki_pos - extent_start;9086 }9087 9088 tmp = kmalloc(count, GFP_NOFS);9089 if (!tmp) {9090 ret = -ENOMEM;9091 goto out;9092 }9093 read_extent_buffer(leaf, tmp, ptr, count);9094 btrfs_release_path(path);9095 unlock_extent(io_tree, start, lockend, cached_state);9096 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);9097 *unlocked = true;9098 9099 ret = copy_to_iter(tmp, count, iter);9100 if (ret != count)9101 ret = -EFAULT;9102 kfree(tmp);9103out:9104 btrfs_free_path(path);9105 return ret;9106}9107 9108struct btrfs_encoded_read_private {9109 wait_queue_head_t wait;9110 atomic_t pending;9111 blk_status_t status;9112};9113 9114static void btrfs_encoded_read_endio(struct btrfs_bio *bbio)9115{9116 struct btrfs_encoded_read_private *priv = bbio->private;9117 9118 if (bbio->bio.bi_status) {9119 /*9120 * The memory barrier implied by the atomic_dec_return() here9121 * pairs with the memory barrier implied by the9122 * atomic_dec_return() or io_wait_event() in9123 * btrfs_encoded_read_regular_fill_pages() to ensure that this9124 * write is observed before the load of status in9125 * btrfs_encoded_read_regular_fill_pages().9126 */9127 WRITE_ONCE(priv->status, bbio->bio.bi_status);9128 }9129 if (!atomic_dec_return(&priv->pending))9130 wake_up(&priv->wait);9131 bio_put(&bbio->bio);9132}9133 9134int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode,9135 u64 file_offset, u64 disk_bytenr,9136 u64 disk_io_size, struct page **pages)9137{9138 struct btrfs_fs_info *fs_info = inode->root->fs_info;9139 struct btrfs_encoded_read_private priv = {9140 .pending = ATOMIC_INIT(1),9141 };9142 unsigned long i = 0;9143 struct btrfs_bio *bbio;9144 9145 init_waitqueue_head(&priv.wait);9146 9147 bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, fs_info,9148 btrfs_encoded_read_endio, &priv);9149 bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;9150 bbio->inode = inode;9151 9152 do {9153 size_t bytes = min_t(u64, disk_io_size, PAGE_SIZE);9154 9155 if (bio_add_page(&bbio->bio, pages[i], bytes, 0) < bytes) {9156 atomic_inc(&priv.pending);9157 btrfs_submit_bbio(bbio, 0);9158 9159 bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, fs_info,9160 btrfs_encoded_read_endio, &priv);9161 bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;9162 bbio->inode = inode;9163 continue;9164 }9165 9166 i++;9167 disk_bytenr += bytes;9168 disk_io_size -= bytes;9169 } while (disk_io_size);9170 9171 atomic_inc(&priv.pending);9172 btrfs_submit_bbio(bbio, 0);9173 9174 if (atomic_dec_return(&priv.pending))9175 io_wait_event(priv.wait, !atomic_read(&priv.pending));9176 /* See btrfs_encoded_read_endio() for ordering. */9177 return blk_status_to_errno(READ_ONCE(priv.status));9178}9179 9180static ssize_t btrfs_encoded_read_regular(struct kiocb *iocb,9181 struct iov_iter *iter,9182 u64 start, u64 lockend,9183 struct extent_state **cached_state,9184 u64 disk_bytenr, u64 disk_io_size,9185 size_t count, bool compressed,9186 bool *unlocked)9187{9188 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));9189 struct extent_io_tree *io_tree = &inode->io_tree;9190 struct page **pages;9191 unsigned long nr_pages, i;9192 u64 cur;9193 size_t page_offset;9194 ssize_t ret;9195 9196 nr_pages = DIV_ROUND_UP(disk_io_size, PAGE_SIZE);9197 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS);9198 if (!pages)9199 return -ENOMEM;9200 ret = btrfs_alloc_page_array(nr_pages, pages, false);9201 if (ret) {9202 ret = -ENOMEM;9203 goto out;9204 }9205 9206 ret = btrfs_encoded_read_regular_fill_pages(inode, start, disk_bytenr,9207 disk_io_size, pages);9208 if (ret)9209 goto out;9210 9211 unlock_extent(io_tree, start, lockend, cached_state);9212 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);9213 *unlocked = true;9214 9215 if (compressed) {9216 i = 0;9217 page_offset = 0;9218 } else {9219 i = (iocb->ki_pos - start) >> PAGE_SHIFT;9220 page_offset = (iocb->ki_pos - start) & (PAGE_SIZE - 1);9221 }9222 cur = 0;9223 while (cur < count) {9224 size_t bytes = min_t(size_t, count - cur,9225 PAGE_SIZE - page_offset);9226 9227 if (copy_page_to_iter(pages[i], page_offset, bytes,9228 iter) != bytes) {9229 ret = -EFAULT;9230 goto out;9231 }9232 i++;9233 cur += bytes;9234 page_offset = 0;9235 }9236 ret = count;9237out:9238 for (i = 0; i < nr_pages; i++) {9239 if (pages[i])9240 __free_page(pages[i]);9241 }9242 kfree(pages);9243 return ret;9244}9245 9246ssize_t btrfs_encoded_read(struct kiocb *iocb, struct iov_iter *iter,9247 struct btrfs_ioctl_encoded_io_args *encoded)9248{9249 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));9250 struct btrfs_fs_info *fs_info = inode->root->fs_info;9251 struct extent_io_tree *io_tree = &inode->io_tree;9252 ssize_t ret;9253 size_t count = iov_iter_count(iter);9254 u64 start, lockend, disk_bytenr, disk_io_size;9255 struct extent_state *cached_state = NULL;9256 struct extent_map *em;9257 bool unlocked = false;9258 9259 file_accessed(iocb->ki_filp);9260 9261 btrfs_inode_lock(inode, BTRFS_ILOCK_SHARED);9262 9263 if (iocb->ki_pos >= inode->vfs_inode.i_size) {9264 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);9265 return 0;9266 }9267 start = ALIGN_DOWN(iocb->ki_pos, fs_info->sectorsize);9268 /*9269 * We don't know how long the extent containing iocb->ki_pos is, but if9270 * it's compressed we know that it won't be longer than this.9271 */9272 lockend = start + BTRFS_MAX_UNCOMPRESSED - 1;9273 9274 for (;;) {9275 struct btrfs_ordered_extent *ordered;9276 9277 ret = btrfs_wait_ordered_range(inode, start,9278 lockend - start + 1);9279 if (ret)9280 goto out_unlock_inode;9281 lock_extent(io_tree, start, lockend, &cached_state);9282 ordered = btrfs_lookup_ordered_range(inode, start,9283 lockend - start + 1);9284 if (!ordered)9285 break;9286 btrfs_put_ordered_extent(ordered);9287 unlock_extent(io_tree, start, lockend, &cached_state);9288 cond_resched();9289 }9290 9291 em = btrfs_get_extent(inode, NULL, start, lockend - start + 1);9292 if (IS_ERR(em)) {9293 ret = PTR_ERR(em);9294 goto out_unlock_extent;9295 }9296 9297 if (em->disk_bytenr == EXTENT_MAP_INLINE) {9298 u64 extent_start = em->start;9299 9300 /*9301 * For inline extents we get everything we need out of the9302 * extent item.9303 */9304 free_extent_map(em);9305 em = NULL;9306 ret = btrfs_encoded_read_inline(iocb, iter, start, lockend,9307 &cached_state, extent_start,9308 count, encoded, &unlocked);9309 goto out;9310 }9311 9312 /*9313 * We only want to return up to EOF even if the extent extends beyond9314 * that.9315 */9316 encoded->len = min_t(u64, extent_map_end(em),9317 inode->vfs_inode.i_size) - iocb->ki_pos;9318 if (em->disk_bytenr == EXTENT_MAP_HOLE ||9319 (em->flags & EXTENT_FLAG_PREALLOC)) {9320 disk_bytenr = EXTENT_MAP_HOLE;9321 count = min_t(u64, count, encoded->len);9322 encoded->len = count;9323 encoded->unencoded_len = count;9324 } else if (extent_map_is_compressed(em)) {9325 disk_bytenr = em->disk_bytenr;9326 /*9327 * Bail if the buffer isn't large enough to return the whole9328 * compressed extent.9329 */9330 if (em->disk_num_bytes > count) {9331 ret = -ENOBUFS;9332 goto out_em;9333 }9334 disk_io_size = em->disk_num_bytes;9335 count = em->disk_num_bytes;9336 encoded->unencoded_len = em->ram_bytes;9337 encoded->unencoded_offset = iocb->ki_pos - (em->start - em->offset);9338 ret = btrfs_encoded_io_compression_from_extent(fs_info,9339 extent_map_compression(em));9340 if (ret < 0)9341 goto out_em;9342 encoded->compression = ret;9343 } else {9344 disk_bytenr = extent_map_block_start(em) + (start - em->start);9345 if (encoded->len > count)9346 encoded->len = count;9347 /*9348 * Don't read beyond what we locked. This also limits the page9349 * allocations that we'll do.9350 */9351 disk_io_size = min(lockend + 1, iocb->ki_pos + encoded->len) - start;9352 count = start + disk_io_size - iocb->ki_pos;9353 encoded->len = count;9354 encoded->unencoded_len = count;9355 disk_io_size = ALIGN(disk_io_size, fs_info->sectorsize);9356 }9357 free_extent_map(em);9358 em = NULL;9359 9360 if (disk_bytenr == EXTENT_MAP_HOLE) {9361 unlock_extent(io_tree, start, lockend, &cached_state);9362 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);9363 unlocked = true;9364 ret = iov_iter_zero(count, iter);9365 if (ret != count)9366 ret = -EFAULT;9367 } else {9368 ret = btrfs_encoded_read_regular(iocb, iter, start, lockend,9369 &cached_state, disk_bytenr,9370 disk_io_size, count,9371 encoded->compression,9372 &unlocked);9373 }9374 9375out:9376 if (ret >= 0)9377 iocb->ki_pos += encoded->len;9378out_em:9379 free_extent_map(em);9380out_unlock_extent:9381 if (!unlocked)9382 unlock_extent(io_tree, start, lockend, &cached_state);9383out_unlock_inode:9384 if (!unlocked)9385 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);9386 return ret;9387}9388 9389ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from,9390 const struct btrfs_ioctl_encoded_io_args *encoded)9391{9392 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));9393 struct btrfs_root *root = inode->root;9394 struct btrfs_fs_info *fs_info = root->fs_info;9395 struct extent_io_tree *io_tree = &inode->io_tree;9396 struct extent_changeset *data_reserved = NULL;9397 struct extent_state *cached_state = NULL;9398 struct btrfs_ordered_extent *ordered;9399 struct btrfs_file_extent file_extent;9400 int compression;9401 size_t orig_count;9402 u64 start, end;9403 u64 num_bytes, ram_bytes, disk_num_bytes;9404 unsigned long nr_folios, i;9405 struct folio **folios;9406 struct btrfs_key ins;9407 bool extent_reserved = false;9408 struct extent_map *em;9409 ssize_t ret;9410 9411 switch (encoded->compression) {9412 case BTRFS_ENCODED_IO_COMPRESSION_ZLIB:9413 compression = BTRFS_COMPRESS_ZLIB;9414 break;9415 case BTRFS_ENCODED_IO_COMPRESSION_ZSTD:9416 compression = BTRFS_COMPRESS_ZSTD;9417 break;9418 case BTRFS_ENCODED_IO_COMPRESSION_LZO_4K:9419 case BTRFS_ENCODED_IO_COMPRESSION_LZO_8K:9420 case BTRFS_ENCODED_IO_COMPRESSION_LZO_16K:9421 case BTRFS_ENCODED_IO_COMPRESSION_LZO_32K:9422 case BTRFS_ENCODED_IO_COMPRESSION_LZO_64K:9423 /* The sector size must match for LZO. */9424 if (encoded->compression -9425 BTRFS_ENCODED_IO_COMPRESSION_LZO_4K + 12 !=9426 fs_info->sectorsize_bits)9427 return -EINVAL;9428 compression = BTRFS_COMPRESS_LZO;9429 break;9430 default:9431 return -EINVAL;9432 }9433 if (encoded->encryption != BTRFS_ENCODED_IO_ENCRYPTION_NONE)9434 return -EINVAL;9435 9436 /*9437 * Compressed extents should always have checksums, so error out if we9438 * have a NOCOW file or inode was created while mounted with NODATASUM.9439 */9440 if (inode->flags & BTRFS_INODE_NODATASUM)9441 return -EINVAL;9442 9443 orig_count = iov_iter_count(from);9444 9445 /* The extent size must be sane. */9446 if (encoded->unencoded_len > BTRFS_MAX_UNCOMPRESSED ||9447 orig_count > BTRFS_MAX_COMPRESSED || orig_count == 0)9448 return -EINVAL;9449 9450 /*9451 * The compressed data must be smaller than the decompressed data.9452 *9453 * It's of course possible for data to compress to larger or the same9454 * size, but the buffered I/O path falls back to no compression for such9455 * data, and we don't want to break any assumptions by creating these9456 * extents.9457 *9458 * Note that this is less strict than the current check we have that the9459 * compressed data must be at least one sector smaller than the9460 * decompressed data. We only want to enforce the weaker requirement9461 * from old kernels that it is at least one byte smaller.9462 */9463 if (orig_count >= encoded->unencoded_len)9464 return -EINVAL;9465 9466 /* The extent must start on a sector boundary. */9467 start = iocb->ki_pos;9468 if (!IS_ALIGNED(start, fs_info->sectorsize))9469 return -EINVAL;9470 9471 /*9472 * The extent must end on a sector boundary. However, we allow a write9473 * which ends at or extends i_size to have an unaligned length; we round9474 * up the extent size and set i_size to the unaligned end.9475 */9476 if (start + encoded->len < inode->vfs_inode.i_size &&9477 !IS_ALIGNED(start + encoded->len, fs_info->sectorsize))9478 return -EINVAL;9479 9480 /* Finally, the offset in the unencoded data must be sector-aligned. */9481 if (!IS_ALIGNED(encoded->unencoded_offset, fs_info->sectorsize))9482 return -EINVAL;9483 9484 num_bytes = ALIGN(encoded->len, fs_info->sectorsize);9485 ram_bytes = ALIGN(encoded->unencoded_len, fs_info->sectorsize);9486 end = start + num_bytes - 1;9487 9488 /*9489 * If the extent cannot be inline, the compressed data on disk must be9490 * sector-aligned. For convenience, we extend it with zeroes if it9491 * isn't.9492 */9493 disk_num_bytes = ALIGN(orig_count, fs_info->sectorsize);9494 nr_folios = DIV_ROUND_UP(disk_num_bytes, PAGE_SIZE);9495 folios = kvcalloc(nr_folios, sizeof(struct page *), GFP_KERNEL_ACCOUNT);9496 if (!folios)9497 return -ENOMEM;9498 for (i = 0; i < nr_folios; i++) {9499 size_t bytes = min_t(size_t, PAGE_SIZE, iov_iter_count(from));9500 char *kaddr;9501 9502 folios[i] = folio_alloc(GFP_KERNEL_ACCOUNT, 0);9503 if (!folios[i]) {9504 ret = -ENOMEM;9505 goto out_folios;9506 }9507 kaddr = kmap_local_folio(folios[i], 0);9508 if (copy_from_iter(kaddr, bytes, from) != bytes) {9509 kunmap_local(kaddr);9510 ret = -EFAULT;9511 goto out_folios;9512 }9513 if (bytes < PAGE_SIZE)9514 memset(kaddr + bytes, 0, PAGE_SIZE - bytes);9515 kunmap_local(kaddr);9516 }9517 9518 for (;;) {9519 struct btrfs_ordered_extent *ordered;9520 9521 ret = btrfs_wait_ordered_range(inode, start, num_bytes);9522 if (ret)9523 goto out_folios;9524 ret = invalidate_inode_pages2_range(inode->vfs_inode.i_mapping,9525 start >> PAGE_SHIFT,9526 end >> PAGE_SHIFT);9527 if (ret)9528 goto out_folios;9529 lock_extent(io_tree, start, end, &cached_state);9530 ordered = btrfs_lookup_ordered_range(inode, start, num_bytes);9531 if (!ordered &&9532 !filemap_range_has_page(inode->vfs_inode.i_mapping, start, end))9533 break;9534 if (ordered)9535 btrfs_put_ordered_extent(ordered);9536 unlock_extent(io_tree, start, end, &cached_state);9537 cond_resched();9538 }9539 9540 /*9541 * We don't use the higher-level delalloc space functions because our9542 * num_bytes and disk_num_bytes are different.9543 */9544 ret = btrfs_alloc_data_chunk_ondemand(inode, disk_num_bytes);9545 if (ret)9546 goto out_unlock;9547 ret = btrfs_qgroup_reserve_data(inode, &data_reserved, start, num_bytes);9548 if (ret)9549 goto out_free_data_space;9550 ret = btrfs_delalloc_reserve_metadata(inode, num_bytes, disk_num_bytes,9551 false);9552 if (ret)9553 goto out_qgroup_free_data;9554 9555 /* Try an inline extent first. */9556 if (encoded->unencoded_len == encoded->len &&9557 encoded->unencoded_offset == 0 &&9558 can_cow_file_range_inline(inode, start, encoded->len, orig_count)) {9559 ret = __cow_file_range_inline(inode, start, encoded->len,9560 orig_count, compression, folios[0],9561 true);9562 if (ret <= 0) {9563 if (ret == 0)9564 ret = orig_count;9565 goto out_delalloc_release;9566 }9567 }9568 9569 ret = btrfs_reserve_extent(root, disk_num_bytes, disk_num_bytes,9570 disk_num_bytes, 0, 0, &ins, 1, 1);9571 if (ret)9572 goto out_delalloc_release;9573 extent_reserved = true;9574 9575 file_extent.disk_bytenr = ins.objectid;9576 file_extent.disk_num_bytes = ins.offset;9577 file_extent.num_bytes = num_bytes;9578 file_extent.ram_bytes = ram_bytes;9579 file_extent.offset = encoded->unencoded_offset;9580 file_extent.compression = compression;9581 em = btrfs_create_io_em(inode, start, &file_extent, BTRFS_ORDERED_COMPRESSED);9582 if (IS_ERR(em)) {9583 ret = PTR_ERR(em);9584 goto out_free_reserved;9585 }9586 free_extent_map(em);9587 9588 ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent,9589 (1 << BTRFS_ORDERED_ENCODED) |9590 (1 << BTRFS_ORDERED_COMPRESSED));9591 if (IS_ERR(ordered)) {9592 btrfs_drop_extent_map_range(inode, start, end, false);9593 ret = PTR_ERR(ordered);9594 goto out_free_reserved;9595 }9596 btrfs_dec_block_group_reservations(fs_info, ins.objectid);9597 9598 if (start + encoded->len > inode->vfs_inode.i_size)9599 i_size_write(&inode->vfs_inode, start + encoded->len);9600 9601 unlock_extent(io_tree, start, end, &cached_state);9602 9603 btrfs_delalloc_release_extents(inode, num_bytes);9604 9605 btrfs_submit_compressed_write(ordered, folios, nr_folios, 0, false);9606 ret = orig_count;9607 goto out;9608 9609out_free_reserved:9610 btrfs_dec_block_group_reservations(fs_info, ins.objectid);9611 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);9612out_delalloc_release:9613 btrfs_delalloc_release_extents(inode, num_bytes);9614 btrfs_delalloc_release_metadata(inode, disk_num_bytes, ret < 0);9615out_qgroup_free_data:9616 if (ret < 0)9617 btrfs_qgroup_free_data(inode, data_reserved, start, num_bytes, NULL);9618out_free_data_space:9619 /*9620 * If btrfs_reserve_extent() succeeded, then we already decremented9621 * bytes_may_use.9622 */9623 if (!extent_reserved)9624 btrfs_free_reserved_data_space_noquota(fs_info, disk_num_bytes);9625out_unlock:9626 unlock_extent(io_tree, start, end, &cached_state);9627out_folios:9628 for (i = 0; i < nr_folios; i++) {9629 if (folios[i])9630 folio_put(folios[i]);9631 }9632 kvfree(folios);9633out:9634 if (ret >= 0)9635 iocb->ki_pos += encoded->len;9636 return ret;9637}9638 9639#ifdef CONFIG_SWAP9640/*9641 * Add an entry indicating a block group or device which is pinned by a9642 * swapfile. Returns 0 on success, 1 if there is already an entry for it, or a9643 * negative errno on failure.9644 */9645static int btrfs_add_swapfile_pin(struct inode *inode, void *ptr,9646 bool is_block_group)9647{9648 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;9649 struct btrfs_swapfile_pin *sp, *entry;9650 struct rb_node **p;9651 struct rb_node *parent = NULL;9652 9653 sp = kmalloc(sizeof(*sp), GFP_NOFS);9654 if (!sp)9655 return -ENOMEM;9656 sp->ptr = ptr;9657 sp->inode = inode;9658 sp->is_block_group = is_block_group;9659 sp->bg_extent_count = 1;9660 9661 spin_lock(&fs_info->swapfile_pins_lock);9662 p = &fs_info->swapfile_pins.rb_node;9663 while (*p) {9664 parent = *p;9665 entry = rb_entry(parent, struct btrfs_swapfile_pin, node);9666 if (sp->ptr < entry->ptr ||9667 (sp->ptr == entry->ptr && sp->inode < entry->inode)) {9668 p = &(*p)->rb_left;9669 } else if (sp->ptr > entry->ptr ||9670 (sp->ptr == entry->ptr && sp->inode > entry->inode)) {9671 p = &(*p)->rb_right;9672 } else {9673 if (is_block_group)9674 entry->bg_extent_count++;9675 spin_unlock(&fs_info->swapfile_pins_lock);9676 kfree(sp);9677 return 1;9678 }9679 }9680 rb_link_node(&sp->node, parent, p);9681 rb_insert_color(&sp->node, &fs_info->swapfile_pins);9682 spin_unlock(&fs_info->swapfile_pins_lock);9683 return 0;9684}9685 9686/* Free all of the entries pinned by this swapfile. */9687static void btrfs_free_swapfile_pins(struct inode *inode)9688{9689 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;9690 struct btrfs_swapfile_pin *sp;9691 struct rb_node *node, *next;9692 9693 spin_lock(&fs_info->swapfile_pins_lock);9694 node = rb_first(&fs_info->swapfile_pins);9695 while (node) {9696 next = rb_next(node);9697 sp = rb_entry(node, struct btrfs_swapfile_pin, node);9698 if (sp->inode == inode) {9699 rb_erase(&sp->node, &fs_info->swapfile_pins);9700 if (sp->is_block_group) {9701 btrfs_dec_block_group_swap_extents(sp->ptr,9702 sp->bg_extent_count);9703 btrfs_put_block_group(sp->ptr);9704 }9705 kfree(sp);9706 }9707 node = next;9708 }9709 spin_unlock(&fs_info->swapfile_pins_lock);9710}9711 9712struct btrfs_swap_info {9713 u64 start;9714 u64 block_start;9715 u64 block_len;9716 u64 lowest_ppage;9717 u64 highest_ppage;9718 unsigned long nr_pages;9719 int nr_extents;9720};9721 9722static int btrfs_add_swap_extent(struct swap_info_struct *sis,9723 struct btrfs_swap_info *bsi)9724{9725 unsigned long nr_pages;9726 unsigned long max_pages;9727 u64 first_ppage, first_ppage_reported, next_ppage;9728 int ret;9729 9730 /*9731 * Our swapfile may have had its size extended after the swap header was9732 * written. In that case activating the swapfile should not go beyond9733 * the max size set in the swap header.9734 */9735 if (bsi->nr_pages >= sis->max)9736 return 0;9737 9738 max_pages = sis->max - bsi->nr_pages;9739 first_ppage = PAGE_ALIGN(bsi->block_start) >> PAGE_SHIFT;9740 next_ppage = PAGE_ALIGN_DOWN(bsi->block_start + bsi->block_len) >> PAGE_SHIFT;9741 9742 if (first_ppage >= next_ppage)9743 return 0;9744 nr_pages = next_ppage - first_ppage;9745 nr_pages = min(nr_pages, max_pages);9746 9747 first_ppage_reported = first_ppage;9748 if (bsi->start == 0)9749 first_ppage_reported++;9750 if (bsi->lowest_ppage > first_ppage_reported)9751 bsi->lowest_ppage = first_ppage_reported;9752 if (bsi->highest_ppage < (next_ppage - 1))9753 bsi->highest_ppage = next_ppage - 1;9754 9755 ret = add_swap_extent(sis, bsi->nr_pages, nr_pages, first_ppage);9756 if (ret < 0)9757 return ret;9758 bsi->nr_extents += ret;9759 bsi->nr_pages += nr_pages;9760 return 0;9761}9762 9763static void btrfs_swap_deactivate(struct file *file)9764{9765 struct inode *inode = file_inode(file);9766 9767 btrfs_free_swapfile_pins(inode);9768 atomic_dec(&BTRFS_I(inode)->root->nr_swapfiles);9769}9770 9771static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file,9772 sector_t *span)9773{9774 struct inode *inode = file_inode(file);9775 struct btrfs_root *root = BTRFS_I(inode)->root;9776 struct btrfs_fs_info *fs_info = root->fs_info;9777 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;9778 struct extent_state *cached_state = NULL;9779 struct extent_map *em = NULL;9780 struct btrfs_chunk_map *map = NULL;9781 struct btrfs_device *device = NULL;9782 struct btrfs_swap_info bsi = {9783 .lowest_ppage = (sector_t)-1ULL,9784 };9785 int ret = 0;9786 u64 isize;9787 u64 start;9788 9789 /*9790 * If the swap file was just created, make sure delalloc is done. If the9791 * file changes again after this, the user is doing something stupid and9792 * we don't really care.9793 */9794 ret = btrfs_wait_ordered_range(BTRFS_I(inode), 0, (u64)-1);9795 if (ret)9796 return ret;9797 9798 /*9799 * The inode is locked, so these flags won't change after we check them.9800 */9801 if (BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS) {9802 btrfs_warn(fs_info, "swapfile must not be compressed");9803 return -EINVAL;9804 }9805 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW)) {9806 btrfs_warn(fs_info, "swapfile must not be copy-on-write");9807 return -EINVAL;9808 }9809 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) {9810 btrfs_warn(fs_info, "swapfile must not be checksummed");9811 return -EINVAL;9812 }9813 9814 /*9815 * Balance or device remove/replace/resize can move stuff around from9816 * under us. The exclop protection makes sure they aren't running/won't9817 * run concurrently while we are mapping the swap extents, and9818 * fs_info->swapfile_pins prevents them from running while the swap9819 * file is active and moving the extents. Note that this also prevents9820 * a concurrent device add which isn't actually necessary, but it's not9821 * really worth the trouble to allow it.9822 */9823 if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_SWAP_ACTIVATE)) {9824 btrfs_warn(fs_info,9825 "cannot activate swapfile while exclusive operation is running");9826 return -EBUSY;9827 }9828 9829 /*9830 * Prevent snapshot creation while we are activating the swap file.9831 * We do not want to race with snapshot creation. If snapshot creation9832 * already started before we bumped nr_swapfiles from 0 to 1 and9833 * completes before the first write into the swap file after it is9834 * activated, than that write would fallback to COW.9835 */9836 if (!btrfs_drew_try_write_lock(&root->snapshot_lock)) {9837 btrfs_exclop_finish(fs_info);9838 btrfs_warn(fs_info,9839 "cannot activate swapfile because snapshot creation is in progress");9840 return -EINVAL;9841 }9842 /*9843 * Snapshots can create extents which require COW even if NODATACOW is9844 * set. We use this counter to prevent snapshots. We must increment it9845 * before walking the extents because we don't want a concurrent9846 * snapshot to run after we've already checked the extents.9847 *9848 * It is possible that subvolume is marked for deletion but still not9849 * removed yet. To prevent this race, we check the root status before9850 * activating the swapfile.9851 */9852 spin_lock(&root->root_item_lock);9853 if (btrfs_root_dead(root)) {9854 spin_unlock(&root->root_item_lock);9855 9856 btrfs_exclop_finish(fs_info);9857 btrfs_warn(fs_info,9858 "cannot activate swapfile because subvolume %llu is being deleted",9859 btrfs_root_id(root));9860 return -EPERM;9861 }9862 atomic_inc(&root->nr_swapfiles);9863 spin_unlock(&root->root_item_lock);9864 9865 isize = ALIGN_DOWN(inode->i_size, fs_info->sectorsize);9866 9867 lock_extent(io_tree, 0, isize - 1, &cached_state);9868 start = 0;9869 while (start < isize) {9870 u64 logical_block_start, physical_block_start;9871 struct btrfs_block_group *bg;9872 u64 len = isize - start;9873 9874 em = btrfs_get_extent(BTRFS_I(inode), NULL, start, len);9875 if (IS_ERR(em)) {9876 ret = PTR_ERR(em);9877 goto out;9878 }9879 9880 if (em->disk_bytenr == EXTENT_MAP_HOLE) {9881 btrfs_warn(fs_info, "swapfile must not have holes");9882 ret = -EINVAL;9883 goto out;9884 }9885 if (em->disk_bytenr == EXTENT_MAP_INLINE) {9886 /*9887 * It's unlikely we'll ever actually find ourselves9888 * here, as a file small enough to fit inline won't be9889 * big enough to store more than the swap header, but in9890 * case something changes in the future, let's catch it9891 * here rather than later.9892 */9893 btrfs_warn(fs_info, "swapfile must not be inline");9894 ret = -EINVAL;9895 goto out;9896 }9897 if (extent_map_is_compressed(em)) {9898 btrfs_warn(fs_info, "swapfile must not be compressed");9899 ret = -EINVAL;9900 goto out;9901 }9902 9903 logical_block_start = extent_map_block_start(em) + (start - em->start);9904 len = min(len, em->len - (start - em->start));9905 free_extent_map(em);9906 em = NULL;9907 9908 ret = can_nocow_extent(inode, start, &len, NULL, false, true);9909 if (ret < 0) {9910 goto out;9911 } else if (ret) {9912 ret = 0;9913 } else {9914 btrfs_warn(fs_info,9915 "swapfile must not be copy-on-write");9916 ret = -EINVAL;9917 goto out;9918 }9919 9920 map = btrfs_get_chunk_map(fs_info, logical_block_start, len);9921 if (IS_ERR(map)) {9922 ret = PTR_ERR(map);9923 goto out;9924 }9925 9926 if (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) {9927 btrfs_warn(fs_info,9928 "swapfile must have single data profile");9929 ret = -EINVAL;9930 goto out;9931 }9932 9933 if (device == NULL) {9934 device = map->stripes[0].dev;9935 ret = btrfs_add_swapfile_pin(inode, device, false);9936 if (ret == 1)9937 ret = 0;9938 else if (ret)9939 goto out;9940 } else if (device != map->stripes[0].dev) {9941 btrfs_warn(fs_info, "swapfile must be on one device");9942 ret = -EINVAL;9943 goto out;9944 }9945 9946 physical_block_start = (map->stripes[0].physical +9947 (logical_block_start - map->start));9948 len = min(len, map->chunk_len - (logical_block_start - map->start));9949 btrfs_free_chunk_map(map);9950 map = NULL;9951 9952 bg = btrfs_lookup_block_group(fs_info, logical_block_start);9953 if (!bg) {9954 btrfs_warn(fs_info,9955 "could not find block group containing swapfile");9956 ret = -EINVAL;9957 goto out;9958 }9959 9960 if (!btrfs_inc_block_group_swap_extents(bg)) {9961 btrfs_warn(fs_info,9962 "block group for swapfile at %llu is read-only%s",9963 bg->start,9964 atomic_read(&fs_info->scrubs_running) ?9965 " (scrub running)" : "");9966 btrfs_put_block_group(bg);9967 ret = -EINVAL;9968 goto out;9969 }9970 9971 ret = btrfs_add_swapfile_pin(inode, bg, true);9972 if (ret) {9973 btrfs_put_block_group(bg);9974 if (ret == 1)9975 ret = 0;9976 else9977 goto out;9978 }9979 9980 if (bsi.block_len &&9981 bsi.block_start + bsi.block_len == physical_block_start) {9982 bsi.block_len += len;9983 } else {9984 if (bsi.block_len) {9985 ret = btrfs_add_swap_extent(sis, &bsi);9986 if (ret)9987 goto out;9988 }9989 bsi.start = start;9990 bsi.block_start = physical_block_start;9991 bsi.block_len = len;9992 }9993 9994 start += len;9995 }9996 9997 if (bsi.block_len)9998 ret = btrfs_add_swap_extent(sis, &bsi);9999 10000out:10001 if (!IS_ERR_OR_NULL(em))10002 free_extent_map(em);10003 if (!IS_ERR_OR_NULL(map))10004 btrfs_free_chunk_map(map);10005 10006 unlock_extent(io_tree, 0, isize - 1, &cached_state);10007 10008 if (ret)10009 btrfs_swap_deactivate(file);10010 10011 btrfs_drew_write_unlock(&root->snapshot_lock);10012 10013 btrfs_exclop_finish(fs_info);10014 10015 if (ret)10016 return ret;10017 10018 if (device)10019 sis->bdev = device->bdev;10020 *span = bsi.highest_ppage - bsi.lowest_ppage + 1;10021 sis->max = bsi.nr_pages;10022 sis->pages = bsi.nr_pages - 1;10023 sis->highest_bit = bsi.nr_pages - 1;10024 return bsi.nr_extents;10025}10026#else10027static void btrfs_swap_deactivate(struct file *file)10028{10029}10030 10031static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file,10032 sector_t *span)10033{10034 return -EOPNOTSUPP;10035}10036#endif10037 10038/*10039 * Update the number of bytes used in the VFS' inode. When we replace extents in10040 * a range (clone, dedupe, fallocate's zero range), we must update the number of10041 * bytes used by the inode in an atomic manner, so that concurrent stat(2) calls10042 * always get a correct value.10043 */10044void btrfs_update_inode_bytes(struct btrfs_inode *inode,10045 const u64 add_bytes,10046 const u64 del_bytes)10047{10048 if (add_bytes == del_bytes)10049 return;10050 10051 spin_lock(&inode->lock);10052 if (del_bytes > 0)10053 inode_sub_bytes(&inode->vfs_inode, del_bytes);10054 if (add_bytes > 0)10055 inode_add_bytes(&inode->vfs_inode, add_bytes);10056 spin_unlock(&inode->lock);10057}10058 10059/*10060 * Verify that there are no ordered extents for a given file range.10061 *10062 * @inode: The target inode.10063 * @start: Start offset of the file range, should be sector size aligned.10064 * @end: End offset (inclusive) of the file range, its value +1 should be10065 * sector size aligned.10066 *10067 * This should typically be used for cases where we locked an inode's VFS lock in10068 * exclusive mode, we have also locked the inode's i_mmap_lock in exclusive mode,10069 * we have flushed all delalloc in the range, we have waited for all ordered10070 * extents in the range to complete and finally we have locked the file range in10071 * the inode's io_tree.10072 */10073void btrfs_assert_inode_range_clean(struct btrfs_inode *inode, u64 start, u64 end)10074{10075 struct btrfs_root *root = inode->root;10076 struct btrfs_ordered_extent *ordered;10077 10078 if (!IS_ENABLED(CONFIG_BTRFS_ASSERT))10079 return;10080 10081 ordered = btrfs_lookup_first_ordered_range(inode, start, end + 1 - start);10082 if (ordered) {10083 btrfs_err(root->fs_info,10084"found unexpected ordered extent in file range [%llu, %llu] for inode %llu root %llu (ordered range [%llu, %llu])",10085 start, end, btrfs_ino(inode), btrfs_root_id(root),10086 ordered->file_offset,10087 ordered->file_offset + ordered->num_bytes - 1);10088 btrfs_put_ordered_extent(ordered);10089 }10090 10091 ASSERT(ordered == NULL);10092}10093 10094/*10095 * Find the first inode with a minimum number.10096 *10097 * @root: The root to search for.10098 * @min_ino: The minimum inode number.10099 *10100 * Find the first inode in the @root with a number >= @min_ino and return it.10101 * Returns NULL if no such inode found.10102 */10103struct btrfs_inode *btrfs_find_first_inode(struct btrfs_root *root, u64 min_ino)10104{10105 struct btrfs_inode *inode;10106 unsigned long from = min_ino;10107 10108 xa_lock(&root->inodes);10109 while (true) {10110 inode = xa_find(&root->inodes, &from, ULONG_MAX, XA_PRESENT);10111 if (!inode)10112 break;10113 if (igrab(&inode->vfs_inode))10114 break;10115 10116 from = btrfs_ino(inode) + 1;10117 cond_resched_lock(&root->inodes.xa_lock);10118 }10119 xa_unlock(&root->inodes);10120 10121 return inode;10122}10123 10124static const struct inode_operations btrfs_dir_inode_operations = {10125 .getattr = btrfs_getattr,10126 .lookup = btrfs_lookup,10127 .create = btrfs_create,10128 .unlink = btrfs_unlink,10129 .link = btrfs_link,10130 .mkdir = btrfs_mkdir,10131 .rmdir = btrfs_rmdir,10132 .rename = btrfs_rename2,10133 .symlink = btrfs_symlink,10134 .setattr = btrfs_setattr,10135 .mknod = btrfs_mknod,10136 .listxattr = btrfs_listxattr,10137 .permission = btrfs_permission,10138 .get_inode_acl = btrfs_get_acl,10139 .set_acl = btrfs_set_acl,10140 .update_time = btrfs_update_time,10141 .tmpfile = btrfs_tmpfile,10142 .fileattr_get = btrfs_fileattr_get,10143 .fileattr_set = btrfs_fileattr_set,10144};10145 10146static const struct file_operations btrfs_dir_file_operations = {10147 .llseek = btrfs_dir_llseek,10148 .read = generic_read_dir,10149 .iterate_shared = btrfs_real_readdir,10150 .open = btrfs_opendir,10151 .unlocked_ioctl = btrfs_ioctl,10152#ifdef CONFIG_COMPAT10153 .compat_ioctl = btrfs_compat_ioctl,10154#endif10155 .release = btrfs_release_file,10156 .fsync = btrfs_sync_file,10157};10158 10159/*10160 * btrfs doesn't support the bmap operation because swapfiles10161 * use bmap to make a mapping of extents in the file. They assume10162 * these extents won't change over the life of the file and they10163 * use the bmap result to do IO directly to the drive.10164 *10165 * the btrfs bmap call would return logical addresses that aren't10166 * suitable for IO and they also will change frequently as COW10167 * operations happen. So, swapfile + btrfs == corruption.10168 *10169 * For now we're avoiding this by dropping bmap.10170 */10171static const struct address_space_operations btrfs_aops = {10172 .read_folio = btrfs_read_folio,10173 .writepages = btrfs_writepages,10174 .readahead = btrfs_readahead,10175 .invalidate_folio = btrfs_invalidate_folio,10176 .launder_folio = btrfs_launder_folio,10177 .release_folio = btrfs_release_folio,10178 .migrate_folio = btrfs_migrate_folio,10179 .dirty_folio = filemap_dirty_folio,10180 .error_remove_folio = generic_error_remove_folio,10181 .swap_activate = btrfs_swap_activate,10182 .swap_deactivate = btrfs_swap_deactivate,10183};10184 10185static const struct inode_operations btrfs_file_inode_operations = {10186 .getattr = btrfs_getattr,10187 .setattr = btrfs_setattr,10188 .listxattr = btrfs_listxattr,10189 .permission = btrfs_permission,10190 .fiemap = btrfs_fiemap,10191 .get_inode_acl = btrfs_get_acl,10192 .set_acl = btrfs_set_acl,10193 .update_time = btrfs_update_time,10194 .fileattr_get = btrfs_fileattr_get,10195 .fileattr_set = btrfs_fileattr_set,10196};10197static const struct inode_operations btrfs_special_inode_operations = {10198 .getattr = btrfs_getattr,10199 .setattr = btrfs_setattr,10200 .permission = btrfs_permission,10201 .listxattr = btrfs_listxattr,10202 .get_inode_acl = btrfs_get_acl,10203 .set_acl = btrfs_set_acl,10204 .update_time = btrfs_update_time,10205};10206static const struct inode_operations btrfs_symlink_inode_operations = {10207 .get_link = page_get_link,10208 .getattr = btrfs_getattr,10209 .setattr = btrfs_setattr,10210 .permission = btrfs_permission,10211 .listxattr = btrfs_listxattr,10212 .update_time = btrfs_update_time,10213};10214 10215const struct dentry_operations btrfs_dentry_operations = {10216 .d_delete = btrfs_dentry_delete,10217};10218