1360 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (C) 2007 Oracle. All rights reserved.4 */5 6#include <linux/bio.h>7#include <linux/slab.h>8#include <linux/pagemap.h>9#include <linux/highmem.h>10#include <linux/sched/mm.h>11#include <crypto/hash.h>12#include "messages.h"13#include "ctree.h"14#include "disk-io.h"15#include "transaction.h"16#include "bio.h"17#include "compression.h"18#include "fs.h"19#include "accessors.h"20#include "file-item.h"21 22#define __MAX_CSUM_ITEMS(r, size) ((unsigned long)(((BTRFS_LEAF_DATA_SIZE(r) - \23 sizeof(struct btrfs_item) * 2) / \24 size) - 1))25 26#define MAX_CSUM_ITEMS(r, size) (min_t(u32, __MAX_CSUM_ITEMS(r, size), \27 PAGE_SIZE))28 29/*30 * Set inode's size according to filesystem options.31 *32 * @inode: inode we want to update the disk_i_size for33 * @new_i_size: i_size we want to set to, 0 if we use i_size34 *35 * With NO_HOLES set this simply sets the disk_is_size to whatever i_size_read()36 * returns as it is perfectly fine with a file that has holes without hole file37 * extent items.38 *39 * However without NO_HOLES we need to only return the area that is contiguous40 * from the 0 offset of the file. Otherwise we could end up adjust i_size up41 * to an extent that has a gap in between.42 *43 * Finally new_i_size should only be set in the case of truncate where we're not44 * ready to use i_size_read() as the limiter yet.45 */46void btrfs_inode_safe_disk_i_size_write(struct btrfs_inode *inode, u64 new_i_size)47{48 u64 start, end, i_size;49 int ret;50 51 spin_lock(&inode->lock);52 i_size = new_i_size ?: i_size_read(&inode->vfs_inode);53 if (!inode->file_extent_tree) {54 inode->disk_i_size = i_size;55 goto out_unlock;56 }57 58 ret = find_contiguous_extent_bit(inode->file_extent_tree, 0, &start,59 &end, EXTENT_DIRTY);60 if (!ret && start == 0)61 i_size = min(i_size, end + 1);62 else63 i_size = 0;64 inode->disk_i_size = i_size;65out_unlock:66 spin_unlock(&inode->lock);67}68 69/*70 * Mark range within a file as having a new extent inserted.71 *72 * @inode: inode being modified73 * @start: start file offset of the file extent we've inserted74 * @len: logical length of the file extent item75 *76 * Call when we are inserting a new file extent where there was none before.77 * Does not need to call this in the case where we're replacing an existing file78 * extent, however if not sure it's fine to call this multiple times.79 *80 * The start and len must match the file extent item, so thus must be sectorsize81 * aligned.82 */83int btrfs_inode_set_file_extent_range(struct btrfs_inode *inode, u64 start,84 u64 len)85{86 if (!inode->file_extent_tree)87 return 0;88 89 if (len == 0)90 return 0;91 92 ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize));93 94 return set_extent_bit(inode->file_extent_tree, start, start + len - 1,95 EXTENT_DIRTY, NULL);96}97 98/*99 * Mark an inode range as not having a backing extent.100 *101 * @inode: inode being modified102 * @start: start file offset of the file extent we've inserted103 * @len: logical length of the file extent item104 *105 * Called when we drop a file extent, for example when we truncate. Doesn't106 * need to be called for cases where we're replacing a file extent, like when107 * we've COWed a file extent.108 *109 * The start and len must match the file extent item, so thus must be sectorsize110 * aligned.111 */112int btrfs_inode_clear_file_extent_range(struct btrfs_inode *inode, u64 start,113 u64 len)114{115 if (!inode->file_extent_tree)116 return 0;117 118 if (len == 0)119 return 0;120 121 ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize) ||122 len == (u64)-1);123 124 return clear_extent_bit(inode->file_extent_tree, start,125 start + len - 1, EXTENT_DIRTY, NULL);126}127 128static size_t bytes_to_csum_size(const struct btrfs_fs_info *fs_info, u32 bytes)129{130 ASSERT(IS_ALIGNED(bytes, fs_info->sectorsize));131 132 return (bytes >> fs_info->sectorsize_bits) * fs_info->csum_size;133}134 135static size_t csum_size_to_bytes(const struct btrfs_fs_info *fs_info, u32 csum_size)136{137 ASSERT(IS_ALIGNED(csum_size, fs_info->csum_size));138 139 return (csum_size / fs_info->csum_size) << fs_info->sectorsize_bits;140}141 142static inline u32 max_ordered_sum_bytes(const struct btrfs_fs_info *fs_info)143{144 u32 max_csum_size = round_down(PAGE_SIZE - sizeof(struct btrfs_ordered_sum),145 fs_info->csum_size);146 147 return csum_size_to_bytes(fs_info, max_csum_size);148}149 150/*151 * Calculate the total size needed to allocate for an ordered sum structure152 * spanning @bytes in the file.153 */154static int btrfs_ordered_sum_size(const struct btrfs_fs_info *fs_info, unsigned long bytes)155{156 return sizeof(struct btrfs_ordered_sum) + bytes_to_csum_size(fs_info, bytes);157}158 159int btrfs_insert_hole_extent(struct btrfs_trans_handle *trans,160 struct btrfs_root *root,161 u64 objectid, u64 pos, u64 num_bytes)162{163 int ret = 0;164 struct btrfs_file_extent_item *item;165 struct btrfs_key file_key;166 struct btrfs_path *path;167 struct extent_buffer *leaf;168 169 path = btrfs_alloc_path();170 if (!path)171 return -ENOMEM;172 file_key.objectid = objectid;173 file_key.offset = pos;174 file_key.type = BTRFS_EXTENT_DATA_KEY;175 176 ret = btrfs_insert_empty_item(trans, root, path, &file_key,177 sizeof(*item));178 if (ret < 0)179 goto out;180 leaf = path->nodes[0];181 item = btrfs_item_ptr(leaf, path->slots[0],182 struct btrfs_file_extent_item);183 btrfs_set_file_extent_disk_bytenr(leaf, item, 0);184 btrfs_set_file_extent_disk_num_bytes(leaf, item, 0);185 btrfs_set_file_extent_offset(leaf, item, 0);186 btrfs_set_file_extent_num_bytes(leaf, item, num_bytes);187 btrfs_set_file_extent_ram_bytes(leaf, item, num_bytes);188 btrfs_set_file_extent_generation(leaf, item, trans->transid);189 btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG);190 btrfs_set_file_extent_compression(leaf, item, 0);191 btrfs_set_file_extent_encryption(leaf, item, 0);192 btrfs_set_file_extent_other_encoding(leaf, item, 0);193 194 btrfs_mark_buffer_dirty(trans, leaf);195out:196 btrfs_free_path(path);197 return ret;198}199 200static struct btrfs_csum_item *201btrfs_lookup_csum(struct btrfs_trans_handle *trans,202 struct btrfs_root *root,203 struct btrfs_path *path,204 u64 bytenr, int cow)205{206 struct btrfs_fs_info *fs_info = root->fs_info;207 int ret;208 struct btrfs_key file_key;209 struct btrfs_key found_key;210 struct btrfs_csum_item *item;211 struct extent_buffer *leaf;212 u64 csum_offset = 0;213 const u32 csum_size = fs_info->csum_size;214 int csums_in_item;215 216 file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;217 file_key.offset = bytenr;218 file_key.type = BTRFS_EXTENT_CSUM_KEY;219 ret = btrfs_search_slot(trans, root, &file_key, path, 0, cow);220 if (ret < 0)221 goto fail;222 leaf = path->nodes[0];223 if (ret > 0) {224 ret = 1;225 if (path->slots[0] == 0)226 goto fail;227 path->slots[0]--;228 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);229 if (found_key.type != BTRFS_EXTENT_CSUM_KEY)230 goto fail;231 232 csum_offset = (bytenr - found_key.offset) >>233 fs_info->sectorsize_bits;234 csums_in_item = btrfs_item_size(leaf, path->slots[0]);235 csums_in_item /= csum_size;236 237 if (csum_offset == csums_in_item) {238 ret = -EFBIG;239 goto fail;240 } else if (csum_offset > csums_in_item) {241 goto fail;242 }243 }244 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item);245 item = (struct btrfs_csum_item *)((unsigned char *)item +246 csum_offset * csum_size);247 return item;248fail:249 if (ret > 0)250 ret = -ENOENT;251 return ERR_PTR(ret);252}253 254int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans,255 struct btrfs_root *root,256 struct btrfs_path *path, u64 objectid,257 u64 offset, int mod)258{259 struct btrfs_key file_key;260 int ins_len = mod < 0 ? -1 : 0;261 int cow = mod != 0;262 263 file_key.objectid = objectid;264 file_key.offset = offset;265 file_key.type = BTRFS_EXTENT_DATA_KEY;266 267 return btrfs_search_slot(trans, root, &file_key, path, ins_len, cow);268}269 270/*271 * Find checksums for logical bytenr range [disk_bytenr, disk_bytenr + len) and272 * store the result to @dst.273 *274 * Return >0 for the number of sectors we found.275 * Return 0 for the range [disk_bytenr, disk_bytenr + sectorsize) has no csum276 * for it. Caller may want to try next sector until one range is hit.277 * Return <0 for fatal error.278 */279static int search_csum_tree(struct btrfs_fs_info *fs_info,280 struct btrfs_path *path, u64 disk_bytenr,281 u64 len, u8 *dst)282{283 struct btrfs_root *csum_root;284 struct btrfs_csum_item *item = NULL;285 struct btrfs_key key;286 const u32 sectorsize = fs_info->sectorsize;287 const u32 csum_size = fs_info->csum_size;288 u32 itemsize;289 int ret;290 u64 csum_start;291 u64 csum_len;292 293 ASSERT(IS_ALIGNED(disk_bytenr, sectorsize) &&294 IS_ALIGNED(len, sectorsize));295 296 /* Check if the current csum item covers disk_bytenr */297 if (path->nodes[0]) {298 item = btrfs_item_ptr(path->nodes[0], path->slots[0],299 struct btrfs_csum_item);300 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);301 itemsize = btrfs_item_size(path->nodes[0], path->slots[0]);302 303 csum_start = key.offset;304 csum_len = (itemsize / csum_size) * sectorsize;305 306 if (in_range(disk_bytenr, csum_start, csum_len))307 goto found;308 }309 310 /* Current item doesn't contain the desired range, search again */311 btrfs_release_path(path);312 csum_root = btrfs_csum_root(fs_info, disk_bytenr);313 item = btrfs_lookup_csum(NULL, csum_root, path, disk_bytenr, 0);314 if (IS_ERR(item)) {315 ret = PTR_ERR(item);316 goto out;317 }318 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);319 itemsize = btrfs_item_size(path->nodes[0], path->slots[0]);320 321 csum_start = key.offset;322 csum_len = (itemsize / csum_size) * sectorsize;323 ASSERT(in_range(disk_bytenr, csum_start, csum_len));324 325found:326 ret = (min(csum_start + csum_len, disk_bytenr + len) -327 disk_bytenr) >> fs_info->sectorsize_bits;328 read_extent_buffer(path->nodes[0], dst, (unsigned long)item,329 ret * csum_size);330out:331 if (ret == -ENOENT || ret == -EFBIG)332 ret = 0;333 return ret;334}335 336/*337 * Lookup the checksum for the read bio in csum tree.338 *339 * Return: BLK_STS_RESOURCE if allocating memory fails, BLK_STS_OK otherwise.340 */341blk_status_t btrfs_lookup_bio_sums(struct btrfs_bio *bbio)342{343 struct btrfs_inode *inode = bbio->inode;344 struct btrfs_fs_info *fs_info = inode->root->fs_info;345 struct bio *bio = &bbio->bio;346 struct btrfs_path *path;347 const u32 sectorsize = fs_info->sectorsize;348 const u32 csum_size = fs_info->csum_size;349 u32 orig_len = bio->bi_iter.bi_size;350 u64 orig_disk_bytenr = bio->bi_iter.bi_sector << SECTOR_SHIFT;351 const unsigned int nblocks = orig_len >> fs_info->sectorsize_bits;352 blk_status_t ret = BLK_STS_OK;353 u32 bio_offset = 0;354 355 if ((inode->flags & BTRFS_INODE_NODATASUM) ||356 test_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state))357 return BLK_STS_OK;358 359 /*360 * This function is only called for read bio.361 *362 * This means two things:363 * - All our csums should only be in csum tree364 * No ordered extents csums, as ordered extents are only for write365 * path.366 * - No need to bother any other info from bvec367 * Since we're looking up csums, the only important info is the368 * disk_bytenr and the length, which can be extracted from bi_iter369 * directly.370 */371 ASSERT(bio_op(bio) == REQ_OP_READ);372 path = btrfs_alloc_path();373 if (!path)374 return BLK_STS_RESOURCE;375 376 if (nblocks * csum_size > BTRFS_BIO_INLINE_CSUM_SIZE) {377 bbio->csum = kmalloc_array(nblocks, csum_size, GFP_NOFS);378 if (!bbio->csum) {379 btrfs_free_path(path);380 return BLK_STS_RESOURCE;381 }382 } else {383 bbio->csum = bbio->csum_inline;384 }385 386 /*387 * If requested number of sectors is larger than one leaf can contain,388 * kick the readahead for csum tree.389 */390 if (nblocks > fs_info->csums_per_leaf)391 path->reada = READA_FORWARD;392 393 /*394 * the free space stuff is only read when it hasn't been395 * updated in the current transaction. So, we can safely396 * read from the commit root and sidestep a nasty deadlock397 * between reading the free space cache and updating the csum tree.398 */399 if (btrfs_is_free_space_inode(inode)) {400 path->search_commit_root = 1;401 path->skip_locking = 1;402 }403 404 while (bio_offset < orig_len) {405 int count;406 u64 cur_disk_bytenr = orig_disk_bytenr + bio_offset;407 u8 *csum_dst = bbio->csum +408 (bio_offset >> fs_info->sectorsize_bits) * csum_size;409 410 count = search_csum_tree(fs_info, path, cur_disk_bytenr,411 orig_len - bio_offset, csum_dst);412 if (count < 0) {413 ret = errno_to_blk_status(count);414 if (bbio->csum != bbio->csum_inline)415 kfree(bbio->csum);416 bbio->csum = NULL;417 break;418 }419 420 /*421 * We didn't find a csum for this range. We need to make sure422 * we complain loudly about this, because we are not NODATASUM.423 *424 * However for the DATA_RELOC inode we could potentially be425 * relocating data extents for a NODATASUM inode, so the inode426 * itself won't be marked with NODATASUM, but the extent we're427 * copying is in fact NODATASUM. If we don't find a csum we428 * assume this is the case.429 */430 if (count == 0) {431 memset(csum_dst, 0, csum_size);432 count = 1;433 434 if (btrfs_root_id(inode->root) == BTRFS_DATA_RELOC_TREE_OBJECTID) {435 u64 file_offset = bbio->file_offset + bio_offset;436 437 set_extent_bit(&inode->io_tree, file_offset,438 file_offset + sectorsize - 1,439 EXTENT_NODATASUM, NULL);440 } else {441 btrfs_warn_rl(fs_info,442 "csum hole found for disk bytenr range [%llu, %llu)",443 cur_disk_bytenr, cur_disk_bytenr + sectorsize);444 }445 }446 bio_offset += count * sectorsize;447 }448 449 btrfs_free_path(path);450 return ret;451}452 453/*454 * Search for checksums for a given logical range.455 *456 * @root: The root where to look for checksums.457 * @start: Logical address of target checksum range.458 * @end: End offset (inclusive) of the target checksum range.459 * @list: List for adding each checksum that was found.460 * Can be NULL in case the caller only wants to check if461 * there any checksums for the range.462 * @nowait: Indicate if the search must be non-blocking or not.463 *464 * Return < 0 on error, 0 if no checksums were found, or 1 if checksums were465 * found.466 */467int btrfs_lookup_csums_list(struct btrfs_root *root, u64 start, u64 end,468 struct list_head *list, bool nowait)469{470 struct btrfs_fs_info *fs_info = root->fs_info;471 struct btrfs_key key;472 struct btrfs_path *path;473 struct extent_buffer *leaf;474 struct btrfs_ordered_sum *sums;475 struct btrfs_csum_item *item;476 int ret;477 bool found_csums = false;478 479 ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&480 IS_ALIGNED(end + 1, fs_info->sectorsize));481 482 path = btrfs_alloc_path();483 if (!path)484 return -ENOMEM;485 486 path->nowait = nowait;487 488 key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;489 key.offset = start;490 key.type = BTRFS_EXTENT_CSUM_KEY;491 492 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);493 if (ret < 0)494 goto out;495 if (ret > 0 && path->slots[0] > 0) {496 leaf = path->nodes[0];497 btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);498 499 /*500 * There are two cases we can hit here for the previous csum501 * item:502 *503 * |<- search range ->|504 * |<- csum item ->|505 *506 * Or507 * |<- search range ->|508 * |<- csum item ->|509 *510 * Check if the previous csum item covers the leading part of511 * the search range. If so we have to start from previous csum512 * item.513 */514 if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID &&515 key.type == BTRFS_EXTENT_CSUM_KEY) {516 if (bytes_to_csum_size(fs_info, start - key.offset) <517 btrfs_item_size(leaf, path->slots[0] - 1))518 path->slots[0]--;519 }520 }521 522 while (start <= end) {523 u64 csum_end;524 525 leaf = path->nodes[0];526 if (path->slots[0] >= btrfs_header_nritems(leaf)) {527 ret = btrfs_next_leaf(root, path);528 if (ret < 0)529 goto out;530 if (ret > 0)531 break;532 leaf = path->nodes[0];533 }534 535 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);536 if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||537 key.type != BTRFS_EXTENT_CSUM_KEY ||538 key.offset > end)539 break;540 541 if (key.offset > start)542 start = key.offset;543 544 csum_end = key.offset + csum_size_to_bytes(fs_info,545 btrfs_item_size(leaf, path->slots[0]));546 if (csum_end <= start) {547 path->slots[0]++;548 continue;549 }550 551 found_csums = true;552 if (!list)553 goto out;554 555 csum_end = min(csum_end, end + 1);556 item = btrfs_item_ptr(path->nodes[0], path->slots[0],557 struct btrfs_csum_item);558 while (start < csum_end) {559 unsigned long offset;560 size_t size;561 562 size = min_t(size_t, csum_end - start,563 max_ordered_sum_bytes(fs_info));564 sums = kzalloc(btrfs_ordered_sum_size(fs_info, size),565 GFP_NOFS);566 if (!sums) {567 ret = -ENOMEM;568 goto out;569 }570 571 sums->logical = start;572 sums->len = size;573 574 offset = bytes_to_csum_size(fs_info, start - key.offset);575 576 read_extent_buffer(path->nodes[0],577 sums->sums,578 ((unsigned long)item) + offset,579 bytes_to_csum_size(fs_info, size));580 581 start += size;582 list_add_tail(&sums->list, list);583 }584 path->slots[0]++;585 }586out:587 btrfs_free_path(path);588 if (ret < 0) {589 if (list) {590 struct btrfs_ordered_sum *tmp_sums;591 592 list_for_each_entry_safe(sums, tmp_sums, list, list)593 kfree(sums);594 }595 596 return ret;597 }598 599 return found_csums ? 1 : 0;600}601 602/*603 * Do the same work as btrfs_lookup_csums_list(), the difference is in how604 * we return the result.605 *606 * This version will set the corresponding bits in @csum_bitmap to represent607 * that there is a csum found.608 * Each bit represents a sector. Thus caller should ensure @csum_buf passed609 * in is large enough to contain all csums.610 */611int btrfs_lookup_csums_bitmap(struct btrfs_root *root, struct btrfs_path *path,612 u64 start, u64 end, u8 *csum_buf,613 unsigned long *csum_bitmap)614{615 struct btrfs_fs_info *fs_info = root->fs_info;616 struct btrfs_key key;617 struct extent_buffer *leaf;618 struct btrfs_csum_item *item;619 const u64 orig_start = start;620 bool free_path = false;621 int ret;622 623 ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&624 IS_ALIGNED(end + 1, fs_info->sectorsize));625 626 if (!path) {627 path = btrfs_alloc_path();628 if (!path)629 return -ENOMEM;630 free_path = true;631 }632 633 /* Check if we can reuse the previous path. */634 if (path->nodes[0]) {635 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);636 637 if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID &&638 key.type == BTRFS_EXTENT_CSUM_KEY &&639 key.offset <= start)640 goto search_forward;641 btrfs_release_path(path);642 }643 644 key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;645 key.type = BTRFS_EXTENT_CSUM_KEY;646 key.offset = start;647 648 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);649 if (ret < 0)650 goto fail;651 if (ret > 0 && path->slots[0] > 0) {652 leaf = path->nodes[0];653 btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);654 655 /*656 * There are two cases we can hit here for the previous csum657 * item:658 *659 * |<- search range ->|660 * |<- csum item ->|661 *662 * Or663 * |<- search range ->|664 * |<- csum item ->|665 *666 * Check if the previous csum item covers the leading part of667 * the search range. If so we have to start from previous csum668 * item.669 */670 if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID &&671 key.type == BTRFS_EXTENT_CSUM_KEY) {672 if (bytes_to_csum_size(fs_info, start - key.offset) <673 btrfs_item_size(leaf, path->slots[0] - 1))674 path->slots[0]--;675 }676 }677 678search_forward:679 while (start <= end) {680 u64 csum_end;681 682 leaf = path->nodes[0];683 if (path->slots[0] >= btrfs_header_nritems(leaf)) {684 ret = btrfs_next_leaf(root, path);685 if (ret < 0)686 goto fail;687 if (ret > 0)688 break;689 leaf = path->nodes[0];690 }691 692 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);693 if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||694 key.type != BTRFS_EXTENT_CSUM_KEY ||695 key.offset > end)696 break;697 698 if (key.offset > start)699 start = key.offset;700 701 csum_end = key.offset + csum_size_to_bytes(fs_info,702 btrfs_item_size(leaf, path->slots[0]));703 if (csum_end <= start) {704 path->slots[0]++;705 continue;706 }707 708 csum_end = min(csum_end, end + 1);709 item = btrfs_item_ptr(path->nodes[0], path->slots[0],710 struct btrfs_csum_item);711 while (start < csum_end) {712 unsigned long offset;713 size_t size;714 u8 *csum_dest = csum_buf + bytes_to_csum_size(fs_info,715 start - orig_start);716 717 size = min_t(size_t, csum_end - start, end + 1 - start);718 719 offset = bytes_to_csum_size(fs_info, start - key.offset);720 721 read_extent_buffer(path->nodes[0], csum_dest,722 ((unsigned long)item) + offset,723 bytes_to_csum_size(fs_info, size));724 725 bitmap_set(csum_bitmap,726 (start - orig_start) >> fs_info->sectorsize_bits,727 size >> fs_info->sectorsize_bits);728 729 start += size;730 }731 path->slots[0]++;732 }733 ret = 0;734fail:735 if (free_path)736 btrfs_free_path(path);737 return ret;738}739 740/*741 * Calculate checksums of the data contained inside a bio.742 */743blk_status_t btrfs_csum_one_bio(struct btrfs_bio *bbio)744{745 struct btrfs_ordered_extent *ordered = bbio->ordered;746 struct btrfs_inode *inode = bbio->inode;747 struct btrfs_fs_info *fs_info = inode->root->fs_info;748 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);749 struct bio *bio = &bbio->bio;750 struct btrfs_ordered_sum *sums;751 char *data;752 struct bvec_iter iter;753 struct bio_vec bvec;754 int index;755 unsigned int blockcount;756 int i;757 unsigned nofs_flag;758 759 nofs_flag = memalloc_nofs_save();760 sums = kvzalloc(btrfs_ordered_sum_size(fs_info, bio->bi_iter.bi_size),761 GFP_KERNEL);762 memalloc_nofs_restore(nofs_flag);763 764 if (!sums)765 return BLK_STS_RESOURCE;766 767 sums->len = bio->bi_iter.bi_size;768 INIT_LIST_HEAD(&sums->list);769 770 sums->logical = bio->bi_iter.bi_sector << SECTOR_SHIFT;771 index = 0;772 773 shash->tfm = fs_info->csum_shash;774 775 bio_for_each_segment(bvec, bio, iter) {776 blockcount = BTRFS_BYTES_TO_BLKS(fs_info,777 bvec.bv_len + fs_info->sectorsize778 - 1);779 780 for (i = 0; i < blockcount; i++) {781 data = bvec_kmap_local(&bvec);782 crypto_shash_digest(shash,783 data + (i * fs_info->sectorsize),784 fs_info->sectorsize,785 sums->sums + index);786 kunmap_local(data);787 index += fs_info->csum_size;788 }789 790 }791 792 bbio->sums = sums;793 btrfs_add_ordered_sum(ordered, sums);794 return 0;795}796 797/*798 * Nodatasum I/O on zoned file systems still requires an btrfs_ordered_sum to799 * record the updated logical address on Zone Append completion.800 * Allocate just the structure with an empty sums array here for that case.801 */802blk_status_t btrfs_alloc_dummy_sum(struct btrfs_bio *bbio)803{804 bbio->sums = kmalloc(sizeof(*bbio->sums), GFP_NOFS);805 if (!bbio->sums)806 return BLK_STS_RESOURCE;807 bbio->sums->len = bbio->bio.bi_iter.bi_size;808 bbio->sums->logical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT;809 btrfs_add_ordered_sum(bbio->ordered, bbio->sums);810 return 0;811}812 813/*814 * Remove one checksum overlapping a range.815 *816 * This expects the key to describe the csum pointed to by the path, and it817 * expects the csum to overlap the range [bytenr, len]818 *819 * The csum should not be entirely contained in the range and the range should820 * not be entirely contained in the csum.821 *822 * This calls btrfs_truncate_item with the correct args based on the overlap,823 * and fixes up the key as required.824 */825static noinline void truncate_one_csum(struct btrfs_trans_handle *trans,826 struct btrfs_path *path,827 struct btrfs_key *key,828 u64 bytenr, u64 len)829{830 struct btrfs_fs_info *fs_info = trans->fs_info;831 struct extent_buffer *leaf;832 const u32 csum_size = fs_info->csum_size;833 u64 csum_end;834 u64 end_byte = bytenr + len;835 u32 blocksize_bits = fs_info->sectorsize_bits;836 837 leaf = path->nodes[0];838 csum_end = btrfs_item_size(leaf, path->slots[0]) / csum_size;839 csum_end <<= blocksize_bits;840 csum_end += key->offset;841 842 if (key->offset < bytenr && csum_end <= end_byte) {843 /*844 * [ bytenr - len ]845 * [ ]846 * [csum ]847 * A simple truncate off the end of the item848 */849 u32 new_size = (bytenr - key->offset) >> blocksize_bits;850 new_size *= csum_size;851 btrfs_truncate_item(trans, path, new_size, 1);852 } else if (key->offset >= bytenr && csum_end > end_byte &&853 end_byte > key->offset) {854 /*855 * [ bytenr - len ]856 * [ ]857 * [csum ]858 * we need to truncate from the beginning of the csum859 */860 u32 new_size = (csum_end - end_byte) >> blocksize_bits;861 new_size *= csum_size;862 863 btrfs_truncate_item(trans, path, new_size, 0);864 865 key->offset = end_byte;866 btrfs_set_item_key_safe(trans, path, key);867 } else {868 BUG();869 }870}871 872/*873 * Delete the csum items from the csum tree for a given range of bytes.874 */875int btrfs_del_csums(struct btrfs_trans_handle *trans,876 struct btrfs_root *root, u64 bytenr, u64 len)877{878 struct btrfs_fs_info *fs_info = trans->fs_info;879 struct btrfs_path *path;880 struct btrfs_key key;881 u64 end_byte = bytenr + len;882 u64 csum_end;883 struct extent_buffer *leaf;884 int ret = 0;885 const u32 csum_size = fs_info->csum_size;886 u32 blocksize_bits = fs_info->sectorsize_bits;887 888 ASSERT(btrfs_root_id(root) == BTRFS_CSUM_TREE_OBJECTID ||889 btrfs_root_id(root) == BTRFS_TREE_LOG_OBJECTID);890 891 path = btrfs_alloc_path();892 if (!path)893 return -ENOMEM;894 895 while (1) {896 key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;897 key.offset = end_byte - 1;898 key.type = BTRFS_EXTENT_CSUM_KEY;899 900 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);901 if (ret > 0) {902 ret = 0;903 if (path->slots[0] == 0)904 break;905 path->slots[0]--;906 } else if (ret < 0) {907 break;908 }909 910 leaf = path->nodes[0];911 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);912 913 if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||914 key.type != BTRFS_EXTENT_CSUM_KEY) {915 break;916 }917 918 if (key.offset >= end_byte)919 break;920 921 csum_end = btrfs_item_size(leaf, path->slots[0]) / csum_size;922 csum_end <<= blocksize_bits;923 csum_end += key.offset;924 925 /* this csum ends before we start, we're done */926 if (csum_end <= bytenr)927 break;928 929 /* delete the entire item, it is inside our range */930 if (key.offset >= bytenr && csum_end <= end_byte) {931 int del_nr = 1;932 933 /*934 * Check how many csum items preceding this one in this935 * leaf correspond to our range and then delete them all936 * at once.937 */938 if (key.offset > bytenr && path->slots[0] > 0) {939 int slot = path->slots[0] - 1;940 941 while (slot >= 0) {942 struct btrfs_key pk;943 944 btrfs_item_key_to_cpu(leaf, &pk, slot);945 if (pk.offset < bytenr ||946 pk.type != BTRFS_EXTENT_CSUM_KEY ||947 pk.objectid !=948 BTRFS_EXTENT_CSUM_OBJECTID)949 break;950 path->slots[0] = slot;951 del_nr++;952 key.offset = pk.offset;953 slot--;954 }955 }956 ret = btrfs_del_items(trans, root, path,957 path->slots[0], del_nr);958 if (ret)959 break;960 if (key.offset == bytenr)961 break;962 } else if (key.offset < bytenr && csum_end > end_byte) {963 unsigned long offset;964 unsigned long shift_len;965 unsigned long item_offset;966 /*967 * [ bytenr - len ]968 * [csum ]969 *970 * Our bytes are in the middle of the csum,971 * we need to split this item and insert a new one.972 *973 * But we can't drop the path because the974 * csum could change, get removed, extended etc.975 *976 * The trick here is the max size of a csum item leaves977 * enough room in the tree block for a single978 * item header. So, we split the item in place,979 * adding a new header pointing to the existing980 * bytes. Then we loop around again and we have981 * a nicely formed csum item that we can neatly982 * truncate.983 */984 offset = (bytenr - key.offset) >> blocksize_bits;985 offset *= csum_size;986 987 shift_len = (len >> blocksize_bits) * csum_size;988 989 item_offset = btrfs_item_ptr_offset(leaf,990 path->slots[0]);991 992 memzero_extent_buffer(leaf, item_offset + offset,993 shift_len);994 key.offset = bytenr;995 996 /*997 * btrfs_split_item returns -EAGAIN when the998 * item changed size or key999 */1000 ret = btrfs_split_item(trans, root, path, &key, offset);1001 if (ret && ret != -EAGAIN) {1002 btrfs_abort_transaction(trans, ret);1003 break;1004 }1005 ret = 0;1006 1007 key.offset = end_byte - 1;1008 } else {1009 truncate_one_csum(trans, path, &key, bytenr, len);1010 if (key.offset < bytenr)1011 break;1012 }1013 btrfs_release_path(path);1014 }1015 btrfs_free_path(path);1016 return ret;1017}1018 1019static int find_next_csum_offset(struct btrfs_root *root,1020 struct btrfs_path *path,1021 u64 *next_offset)1022{1023 const u32 nritems = btrfs_header_nritems(path->nodes[0]);1024 struct btrfs_key found_key;1025 int slot = path->slots[0] + 1;1026 int ret;1027 1028 if (nritems == 0 || slot >= nritems) {1029 ret = btrfs_next_leaf(root, path);1030 if (ret < 0) {1031 return ret;1032 } else if (ret > 0) {1033 *next_offset = (u64)-1;1034 return 0;1035 }1036 slot = path->slots[0];1037 }1038 1039 btrfs_item_key_to_cpu(path->nodes[0], &found_key, slot);1040 1041 if (found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||1042 found_key.type != BTRFS_EXTENT_CSUM_KEY)1043 *next_offset = (u64)-1;1044 else1045 *next_offset = found_key.offset;1046 1047 return 0;1048}1049 1050int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans,1051 struct btrfs_root *root,1052 struct btrfs_ordered_sum *sums)1053{1054 struct btrfs_fs_info *fs_info = root->fs_info;1055 struct btrfs_key file_key;1056 struct btrfs_key found_key;1057 struct btrfs_path *path;1058 struct btrfs_csum_item *item;1059 struct btrfs_csum_item *item_end;1060 struct extent_buffer *leaf = NULL;1061 u64 next_offset;1062 u64 total_bytes = 0;1063 u64 csum_offset;1064 u64 bytenr;1065 u32 ins_size;1066 int index = 0;1067 int found_next;1068 int ret;1069 const u32 csum_size = fs_info->csum_size;1070 1071 path = btrfs_alloc_path();1072 if (!path)1073 return -ENOMEM;1074again:1075 next_offset = (u64)-1;1076 found_next = 0;1077 bytenr = sums->logical + total_bytes;1078 file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;1079 file_key.offset = bytenr;1080 file_key.type = BTRFS_EXTENT_CSUM_KEY;1081 1082 item = btrfs_lookup_csum(trans, root, path, bytenr, 1);1083 if (!IS_ERR(item)) {1084 ret = 0;1085 leaf = path->nodes[0];1086 item_end = btrfs_item_ptr(leaf, path->slots[0],1087 struct btrfs_csum_item);1088 item_end = (struct btrfs_csum_item *)((char *)item_end +1089 btrfs_item_size(leaf, path->slots[0]));1090 goto found;1091 }1092 ret = PTR_ERR(item);1093 if (ret != -EFBIG && ret != -ENOENT)1094 goto out;1095 1096 if (ret == -EFBIG) {1097 u32 item_size;1098 /* we found one, but it isn't big enough yet */1099 leaf = path->nodes[0];1100 item_size = btrfs_item_size(leaf, path->slots[0]);1101 if ((item_size / csum_size) >=1102 MAX_CSUM_ITEMS(fs_info, csum_size)) {1103 /* already at max size, make a new one */1104 goto insert;1105 }1106 } else {1107 /* We didn't find a csum item, insert one. */1108 ret = find_next_csum_offset(root, path, &next_offset);1109 if (ret < 0)1110 goto out;1111 found_next = 1;1112 goto insert;1113 }1114 1115 /*1116 * At this point, we know the tree has a checksum item that ends at an1117 * offset matching the start of the checksum range we want to insert.1118 * We try to extend that item as much as possible and then add as many1119 * checksums to it as they fit.1120 *1121 * First check if the leaf has enough free space for at least one1122 * checksum. If it has go directly to the item extension code, otherwise1123 * release the path and do a search for insertion before the extension.1124 */1125 if (btrfs_leaf_free_space(leaf) >= csum_size) {1126 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);1127 csum_offset = (bytenr - found_key.offset) >>1128 fs_info->sectorsize_bits;1129 goto extend_csum;1130 }1131 1132 btrfs_release_path(path);1133 path->search_for_extension = 1;1134 ret = btrfs_search_slot(trans, root, &file_key, path,1135 csum_size, 1);1136 path->search_for_extension = 0;1137 if (ret < 0)1138 goto out;1139 1140 if (ret > 0) {1141 if (path->slots[0] == 0)1142 goto insert;1143 path->slots[0]--;1144 }1145 1146 leaf = path->nodes[0];1147 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);1148 csum_offset = (bytenr - found_key.offset) >> fs_info->sectorsize_bits;1149 1150 if (found_key.type != BTRFS_EXTENT_CSUM_KEY ||1151 found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||1152 csum_offset >= MAX_CSUM_ITEMS(fs_info, csum_size)) {1153 goto insert;1154 }1155 1156extend_csum:1157 if (csum_offset == btrfs_item_size(leaf, path->slots[0]) /1158 csum_size) {1159 int extend_nr;1160 u64 tmp;1161 u32 diff;1162 1163 tmp = sums->len - total_bytes;1164 tmp >>= fs_info->sectorsize_bits;1165 WARN_ON(tmp < 1);1166 extend_nr = max_t(int, 1, tmp);1167 1168 /*1169 * A log tree can already have checksum items with a subset of1170 * the checksums we are trying to log. This can happen after1171 * doing a sequence of partial writes into prealloc extents and1172 * fsyncs in between, with a full fsync logging a larger subrange1173 * of an extent for which a previous fast fsync logged a smaller1174 * subrange. And this happens in particular due to merging file1175 * extent items when we complete an ordered extent for a range1176 * covered by a prealloc extent - this is done at1177 * btrfs_mark_extent_written().1178 *1179 * So if we try to extend the previous checksum item, which has1180 * a range that ends at the start of the range we want to insert,1181 * make sure we don't extend beyond the start offset of the next1182 * checksum item. If we are at the last item in the leaf, then1183 * forget the optimization of extending and add a new checksum1184 * item - it is not worth the complexity of releasing the path,1185 * getting the first key for the next leaf, repeat the btree1186 * search, etc, because log trees are temporary anyway and it1187 * would only save a few bytes of leaf space.1188 */1189 if (btrfs_root_id(root) == BTRFS_TREE_LOG_OBJECTID) {1190 if (path->slots[0] + 1 >=1191 btrfs_header_nritems(path->nodes[0])) {1192 ret = find_next_csum_offset(root, path, &next_offset);1193 if (ret < 0)1194 goto out;1195 found_next = 1;1196 goto insert;1197 }1198 1199 ret = find_next_csum_offset(root, path, &next_offset);1200 if (ret < 0)1201 goto out;1202 1203 tmp = (next_offset - bytenr) >> fs_info->sectorsize_bits;1204 if (tmp <= INT_MAX)1205 extend_nr = min_t(int, extend_nr, tmp);1206 }1207 1208 diff = (csum_offset + extend_nr) * csum_size;1209 diff = min(diff,1210 MAX_CSUM_ITEMS(fs_info, csum_size) * csum_size);1211 1212 diff = diff - btrfs_item_size(leaf, path->slots[0]);1213 diff = min_t(u32, btrfs_leaf_free_space(leaf), diff);1214 diff /= csum_size;1215 diff *= csum_size;1216 1217 btrfs_extend_item(trans, path, diff);1218 ret = 0;1219 goto csum;1220 }1221 1222insert:1223 btrfs_release_path(path);1224 csum_offset = 0;1225 if (found_next) {1226 u64 tmp;1227 1228 tmp = sums->len - total_bytes;1229 tmp >>= fs_info->sectorsize_bits;1230 tmp = min(tmp, (next_offset - file_key.offset) >>1231 fs_info->sectorsize_bits);1232 1233 tmp = max_t(u64, 1, tmp);1234 tmp = min_t(u64, tmp, MAX_CSUM_ITEMS(fs_info, csum_size));1235 ins_size = csum_size * tmp;1236 } else {1237 ins_size = csum_size;1238 }1239 ret = btrfs_insert_empty_item(trans, root, path, &file_key,1240 ins_size);1241 if (ret < 0)1242 goto out;1243 leaf = path->nodes[0];1244csum:1245 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item);1246 item_end = (struct btrfs_csum_item *)((unsigned char *)item +1247 btrfs_item_size(leaf, path->slots[0]));1248 item = (struct btrfs_csum_item *)((unsigned char *)item +1249 csum_offset * csum_size);1250found:1251 ins_size = (u32)(sums->len - total_bytes) >> fs_info->sectorsize_bits;1252 ins_size *= csum_size;1253 ins_size = min_t(u32, (unsigned long)item_end - (unsigned long)item,1254 ins_size);1255 write_extent_buffer(leaf, sums->sums + index, (unsigned long)item,1256 ins_size);1257 1258 index += ins_size;1259 ins_size /= csum_size;1260 total_bytes += ins_size * fs_info->sectorsize;1261 1262 btrfs_mark_buffer_dirty(trans, path->nodes[0]);1263 if (total_bytes < sums->len) {1264 btrfs_release_path(path);1265 cond_resched();1266 goto again;1267 }1268out:1269 btrfs_free_path(path);1270 return ret;1271}1272 1273void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode,1274 const struct btrfs_path *path,1275 const struct btrfs_file_extent_item *fi,1276 struct extent_map *em)1277{1278 struct btrfs_fs_info *fs_info = inode->root->fs_info;1279 struct btrfs_root *root = inode->root;1280 struct extent_buffer *leaf = path->nodes[0];1281 const int slot = path->slots[0];1282 struct btrfs_key key;1283 u64 extent_start;1284 u8 type = btrfs_file_extent_type(leaf, fi);1285 int compress_type = btrfs_file_extent_compression(leaf, fi);1286 1287 btrfs_item_key_to_cpu(leaf, &key, slot);1288 extent_start = key.offset;1289 em->ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);1290 em->generation = btrfs_file_extent_generation(leaf, fi);1291 if (type == BTRFS_FILE_EXTENT_REG ||1292 type == BTRFS_FILE_EXTENT_PREALLOC) {1293 const u64 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);1294 1295 em->start = extent_start;1296 em->len = btrfs_file_extent_end(path) - extent_start;1297 if (disk_bytenr == 0) {1298 em->disk_bytenr = EXTENT_MAP_HOLE;1299 em->disk_num_bytes = 0;1300 em->offset = 0;1301 return;1302 }1303 em->disk_bytenr = disk_bytenr;1304 em->disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);1305 em->offset = btrfs_file_extent_offset(leaf, fi);1306 if (compress_type != BTRFS_COMPRESS_NONE) {1307 extent_map_set_compression(em, compress_type);1308 } else {1309 /*1310 * Older kernels can create regular non-hole data1311 * extents with ram_bytes smaller than disk_num_bytes.1312 * Not a big deal, just always use disk_num_bytes1313 * for ram_bytes.1314 */1315 em->ram_bytes = em->disk_num_bytes;1316 if (type == BTRFS_FILE_EXTENT_PREALLOC)1317 em->flags |= EXTENT_FLAG_PREALLOC;1318 }1319 } else if (type == BTRFS_FILE_EXTENT_INLINE) {1320 /* Tree-checker has ensured this. */1321 ASSERT(extent_start == 0);1322 1323 em->disk_bytenr = EXTENT_MAP_INLINE;1324 em->start = 0;1325 em->len = fs_info->sectorsize;1326 em->offset = 0;1327 extent_map_set_compression(em, compress_type);1328 } else {1329 btrfs_err(fs_info,1330 "unknown file extent item type %d, inode %llu, offset %llu, "1331 "root %llu", type, btrfs_ino(inode), extent_start,1332 btrfs_root_id(root));1333 }1334}1335 1336/*1337 * Returns the end offset (non inclusive) of the file extent item the given path1338 * points to. If it points to an inline extent, the returned offset is rounded1339 * up to the sector size.1340 */1341u64 btrfs_file_extent_end(const struct btrfs_path *path)1342{1343 const struct extent_buffer *leaf = path->nodes[0];1344 const int slot = path->slots[0];1345 struct btrfs_file_extent_item *fi;1346 struct btrfs_key key;1347 u64 end;1348 1349 btrfs_item_key_to_cpu(leaf, &key, slot);1350 ASSERT(key.type == BTRFS_EXTENT_DATA_KEY);1351 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);1352 1353 if (btrfs_file_extent_type(leaf, fi) == BTRFS_FILE_EXTENT_INLINE)1354 end = leaf->fs_info->sectorsize;1355 else1356 end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);1357 1358 return end;1359}1360