brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (C) 2011, 2012 STRATO.  All rights reserved.4 */5 6#include <linux/blkdev.h>7#include <linux/ratelimit.h>8#include <linux/sched/mm.h>9#include <crypto/hash.h>10#include "ctree.h"11#include "discard.h"12#include "volumes.h"13#include "disk-io.h"14#include "ordered-data.h"15#include "transaction.h"16#include "backref.h"17#include "extent_io.h"18#include "dev-replace.h"19#include "raid56.h"20#include "block-group.h"21#include "zoned.h"22#include "fs.h"23#include "accessors.h"24#include "file-item.h"25#include "scrub.h"26#include "raid-stripe-tree.h"27 28/*29 * This is only the first step towards a full-features scrub. It reads all30 * extent and super block and verifies the checksums. In case a bad checksum31 * is found or the extent cannot be read, good data will be written back if32 * any can be found.33 *34 * Future enhancements:35 *  - In case an unrepairable extent is encountered, track which files are36 *    affected and report them37 *  - track and record media errors, throw out bad devices38 *  - add a mode to also read unallocated space39 */40 41struct scrub_ctx;42 43/*44 * The following value only influences the performance.45 *46 * This determines how many stripes would be submitted in one go,47 * which is 512KiB (BTRFS_STRIPE_LEN * SCRUB_STRIPES_PER_GROUP).48 */49#define SCRUB_STRIPES_PER_GROUP		850 51/*52 * How many groups we have for each sctx.53 *54 * This would be 8M per device, the same value as the old scrub in-flight bios55 * size limit.56 */57#define SCRUB_GROUPS_PER_SCTX		1658 59#define SCRUB_TOTAL_STRIPES		(SCRUB_GROUPS_PER_SCTX * SCRUB_STRIPES_PER_GROUP)60 61/*62 * The following value times PAGE_SIZE needs to be large enough to match the63 * largest node/leaf/sector size that shall be supported.64 */65#define SCRUB_MAX_SECTORS_PER_BLOCK	(BTRFS_MAX_METADATA_BLOCKSIZE / SZ_4K)66 67/* Represent one sector and its needed info to verify the content. */68struct scrub_sector_verification {69	bool is_metadata;70 71	union {72		/*73		 * Csum pointer for data csum verification.  Should point to a74		 * sector csum inside scrub_stripe::csums.75		 *76		 * NULL if this data sector has no csum.77		 */78		u8 *csum;79 80		/*81		 * Extra info for metadata verification.  All sectors inside a82		 * tree block share the same generation.83		 */84		u64 generation;85	};86};87 88enum scrub_stripe_flags {89	/* Set when @mirror_num, @dev, @physical and @logical are set. */90	SCRUB_STRIPE_FLAG_INITIALIZED,91 92	/* Set when the read-repair is finished. */93	SCRUB_STRIPE_FLAG_REPAIR_DONE,94 95	/*96	 * Set for data stripes if it's triggered from P/Q stripe.97	 * During such scrub, we should not report errors in data stripes, nor98	 * update the accounting.99	 */100	SCRUB_STRIPE_FLAG_NO_REPORT,101};102 103#define SCRUB_STRIPE_PAGES		(BTRFS_STRIPE_LEN / PAGE_SIZE)104 105/*106 * Represent one contiguous range with a length of BTRFS_STRIPE_LEN.107 */108struct scrub_stripe {109	struct scrub_ctx *sctx;110	struct btrfs_block_group *bg;111 112	struct page *pages[SCRUB_STRIPE_PAGES];113	struct scrub_sector_verification *sectors;114 115	struct btrfs_device *dev;116	u64 logical;117	u64 physical;118 119	u16 mirror_num;120 121	/* Should be BTRFS_STRIPE_LEN / sectorsize. */122	u16 nr_sectors;123 124	/*125	 * How many data/meta extents are in this stripe.  Only for scrub status126	 * reporting purposes.127	 */128	u16 nr_data_extents;129	u16 nr_meta_extents;130 131	atomic_t pending_io;132	wait_queue_head_t io_wait;133	wait_queue_head_t repair_wait;134 135	/*136	 * Indicate the states of the stripe.  Bits are defined in137	 * scrub_stripe_flags enum.138	 */139	unsigned long state;140 141	/* Indicate which sectors are covered by extent items. */142	unsigned long extent_sector_bitmap;143 144	/*145	 * The errors hit during the initial read of the stripe.146	 *147	 * Would be utilized for error reporting and repair.148	 *149	 * The remaining init_nr_* records the number of errors hit, only used150	 * by error reporting.151	 */152	unsigned long init_error_bitmap;153	unsigned int init_nr_io_errors;154	unsigned int init_nr_csum_errors;155	unsigned int init_nr_meta_errors;156 157	/*158	 * The following error bitmaps are all for the current status.159	 * Every time we submit a new read, these bitmaps may be updated.160	 *161	 * error_bitmap = io_error_bitmap | csum_error_bitmap | meta_error_bitmap;162	 *163	 * IO and csum errors can happen for both metadata and data.164	 */165	unsigned long error_bitmap;166	unsigned long io_error_bitmap;167	unsigned long csum_error_bitmap;168	unsigned long meta_error_bitmap;169 170	/* For writeback (repair or replace) error reporting. */171	unsigned long write_error_bitmap;172 173	/* Writeback can be concurrent, thus we need to protect the bitmap. */174	spinlock_t write_error_lock;175 176	/*177	 * Checksum for the whole stripe if this stripe is inside a data block178	 * group.179	 */180	u8 *csums;181 182	struct work_struct work;183};184 185struct scrub_ctx {186	struct scrub_stripe	stripes[SCRUB_TOTAL_STRIPES];187	struct scrub_stripe	*raid56_data_stripes;188	struct btrfs_fs_info	*fs_info;189	struct btrfs_path	extent_path;190	struct btrfs_path	csum_path;191	int			first_free;192	int			cur_stripe;193	atomic_t		cancel_req;194	int			readonly;195 196	/* State of IO submission throttling affecting the associated device */197	ktime_t			throttle_deadline;198	u64			throttle_sent;199 200	int			is_dev_replace;201	u64			write_pointer;202 203	struct mutex            wr_lock;204	struct btrfs_device     *wr_tgtdev;205 206	/*207	 * statistics208	 */209	struct btrfs_scrub_progress stat;210	spinlock_t		stat_lock;211 212	/*213	 * Use a ref counter to avoid use-after-free issues. Scrub workers214	 * decrement bios_in_flight and workers_pending and then do a wakeup215	 * on the list_wait wait queue. We must ensure the main scrub task216	 * doesn't free the scrub context before or while the workers are217	 * doing the wakeup() call.218	 */219	refcount_t              refs;220};221 222struct scrub_warning {223	struct btrfs_path	*path;224	u64			extent_item_size;225	const char		*errstr;226	u64			physical;227	u64			logical;228	struct btrfs_device	*dev;229};230 231static void release_scrub_stripe(struct scrub_stripe *stripe)232{233	if (!stripe)234		return;235 236	for (int i = 0; i < SCRUB_STRIPE_PAGES; i++) {237		if (stripe->pages[i])238			__free_page(stripe->pages[i]);239		stripe->pages[i] = NULL;240	}241	kfree(stripe->sectors);242	kfree(stripe->csums);243	stripe->sectors = NULL;244	stripe->csums = NULL;245	stripe->sctx = NULL;246	stripe->state = 0;247}248 249static int init_scrub_stripe(struct btrfs_fs_info *fs_info,250			     struct scrub_stripe *stripe)251{252	int ret;253 254	memset(stripe, 0, sizeof(*stripe));255 256	stripe->nr_sectors = BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits;257	stripe->state = 0;258 259	init_waitqueue_head(&stripe->io_wait);260	init_waitqueue_head(&stripe->repair_wait);261	atomic_set(&stripe->pending_io, 0);262	spin_lock_init(&stripe->write_error_lock);263 264	ret = btrfs_alloc_page_array(SCRUB_STRIPE_PAGES, stripe->pages, false);265	if (ret < 0)266		goto error;267 268	stripe->sectors = kcalloc(stripe->nr_sectors,269				  sizeof(struct scrub_sector_verification),270				  GFP_KERNEL);271	if (!stripe->sectors)272		goto error;273 274	stripe->csums = kcalloc(BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits,275				fs_info->csum_size, GFP_KERNEL);276	if (!stripe->csums)277		goto error;278	return 0;279error:280	release_scrub_stripe(stripe);281	return -ENOMEM;282}283 284static void wait_scrub_stripe_io(struct scrub_stripe *stripe)285{286	wait_event(stripe->io_wait, atomic_read(&stripe->pending_io) == 0);287}288 289static void scrub_put_ctx(struct scrub_ctx *sctx);290 291static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)292{293	while (atomic_read(&fs_info->scrub_pause_req)) {294		mutex_unlock(&fs_info->scrub_lock);295		wait_event(fs_info->scrub_pause_wait,296		   atomic_read(&fs_info->scrub_pause_req) == 0);297		mutex_lock(&fs_info->scrub_lock);298	}299}300 301static void scrub_pause_on(struct btrfs_fs_info *fs_info)302{303	atomic_inc(&fs_info->scrubs_paused);304	wake_up(&fs_info->scrub_pause_wait);305}306 307static void scrub_pause_off(struct btrfs_fs_info *fs_info)308{309	mutex_lock(&fs_info->scrub_lock);310	__scrub_blocked_if_needed(fs_info);311	atomic_dec(&fs_info->scrubs_paused);312	mutex_unlock(&fs_info->scrub_lock);313 314	wake_up(&fs_info->scrub_pause_wait);315}316 317static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)318{319	scrub_pause_on(fs_info);320	scrub_pause_off(fs_info);321}322 323static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)324{325	int i;326 327	if (!sctx)328		return;329 330	for (i = 0; i < SCRUB_TOTAL_STRIPES; i++)331		release_scrub_stripe(&sctx->stripes[i]);332 333	kvfree(sctx);334}335 336static void scrub_put_ctx(struct scrub_ctx *sctx)337{338	if (refcount_dec_and_test(&sctx->refs))339		scrub_free_ctx(sctx);340}341 342static noinline_for_stack struct scrub_ctx *scrub_setup_ctx(343		struct btrfs_fs_info *fs_info, int is_dev_replace)344{345	struct scrub_ctx *sctx;346	int		i;347 348	/* Since sctx has inline 128 stripes, it can go beyond 64K easily.  Use349	 * kvzalloc().350	 */351	sctx = kvzalloc(sizeof(*sctx), GFP_KERNEL);352	if (!sctx)353		goto nomem;354	refcount_set(&sctx->refs, 1);355	sctx->is_dev_replace = is_dev_replace;356	sctx->fs_info = fs_info;357	sctx->extent_path.search_commit_root = 1;358	sctx->extent_path.skip_locking = 1;359	sctx->csum_path.search_commit_root = 1;360	sctx->csum_path.skip_locking = 1;361	for (i = 0; i < SCRUB_TOTAL_STRIPES; i++) {362		int ret;363 364		ret = init_scrub_stripe(fs_info, &sctx->stripes[i]);365		if (ret < 0)366			goto nomem;367		sctx->stripes[i].sctx = sctx;368	}369	sctx->first_free = 0;370	atomic_set(&sctx->cancel_req, 0);371 372	spin_lock_init(&sctx->stat_lock);373	sctx->throttle_deadline = 0;374 375	mutex_init(&sctx->wr_lock);376	if (is_dev_replace) {377		WARN_ON(!fs_info->dev_replace.tgtdev);378		sctx->wr_tgtdev = fs_info->dev_replace.tgtdev;379	}380 381	return sctx;382 383nomem:384	scrub_free_ctx(sctx);385	return ERR_PTR(-ENOMEM);386}387 388static int scrub_print_warning_inode(u64 inum, u64 offset, u64 num_bytes,389				     u64 root, void *warn_ctx)390{391	u32 nlink;392	int ret;393	int i;394	unsigned nofs_flag;395	struct extent_buffer *eb;396	struct btrfs_inode_item *inode_item;397	struct scrub_warning *swarn = warn_ctx;398	struct btrfs_fs_info *fs_info = swarn->dev->fs_info;399	struct inode_fs_paths *ipath = NULL;400	struct btrfs_root *local_root;401	struct btrfs_key key;402 403	local_root = btrfs_get_fs_root(fs_info, root, true);404	if (IS_ERR(local_root)) {405		ret = PTR_ERR(local_root);406		goto err;407	}408 409	/*410	 * this makes the path point to (inum INODE_ITEM ioff)411	 */412	key.objectid = inum;413	key.type = BTRFS_INODE_ITEM_KEY;414	key.offset = 0;415 416	ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0);417	if (ret) {418		btrfs_put_root(local_root);419		btrfs_release_path(swarn->path);420		goto err;421	}422 423	eb = swarn->path->nodes[0];424	inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],425					struct btrfs_inode_item);426	nlink = btrfs_inode_nlink(eb, inode_item);427	btrfs_release_path(swarn->path);428 429	/*430	 * init_path might indirectly call vmalloc, or use GFP_KERNEL. Scrub431	 * uses GFP_NOFS in this context, so we keep it consistent but it does432	 * not seem to be strictly necessary.433	 */434	nofs_flag = memalloc_nofs_save();435	ipath = init_ipath(4096, local_root, swarn->path);436	memalloc_nofs_restore(nofs_flag);437	if (IS_ERR(ipath)) {438		btrfs_put_root(local_root);439		ret = PTR_ERR(ipath);440		ipath = NULL;441		goto err;442	}443	ret = paths_from_inode(inum, ipath);444 445	if (ret < 0)446		goto err;447 448	/*449	 * we deliberately ignore the bit ipath might have been too small to450	 * hold all of the paths here451	 */452	for (i = 0; i < ipath->fspath->elem_cnt; ++i)453		btrfs_warn_in_rcu(fs_info,454"%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu, length %u, links %u (path: %s)",455				  swarn->errstr, swarn->logical,456				  btrfs_dev_name(swarn->dev),457				  swarn->physical,458				  root, inum, offset,459				  fs_info->sectorsize, nlink,460				  (char *)(unsigned long)ipath->fspath->val[i]);461 462	btrfs_put_root(local_root);463	free_ipath(ipath);464	return 0;465 466err:467	btrfs_warn_in_rcu(fs_info,468			  "%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu: path resolving failed with ret=%d",469			  swarn->errstr, swarn->logical,470			  btrfs_dev_name(swarn->dev),471			  swarn->physical,472			  root, inum, offset, ret);473 474	free_ipath(ipath);475	return 0;476}477 478static void scrub_print_common_warning(const char *errstr, struct btrfs_device *dev,479				       bool is_super, u64 logical, u64 physical)480{481	struct btrfs_fs_info *fs_info = dev->fs_info;482	struct btrfs_path *path;483	struct btrfs_key found_key;484	struct extent_buffer *eb;485	struct btrfs_extent_item *ei;486	struct scrub_warning swarn;487	u64 flags = 0;488	u32 item_size;489	int ret;490 491	/* Super block error, no need to search extent tree. */492	if (is_super) {493		btrfs_warn_in_rcu(fs_info, "%s on device %s, physical %llu",494				  errstr, btrfs_dev_name(dev), physical);495		return;496	}497	path = btrfs_alloc_path();498	if (!path)499		return;500 501	swarn.physical = physical;502	swarn.logical = logical;503	swarn.errstr = errstr;504	swarn.dev = NULL;505 506	ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,507				  &flags);508	if (ret < 0)509		goto out;510 511	swarn.extent_item_size = found_key.offset;512 513	eb = path->nodes[0];514	ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);515	item_size = btrfs_item_size(eb, path->slots[0]);516 517	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {518		unsigned long ptr = 0;519		u8 ref_level;520		u64 ref_root;521 522		while (true) {523			ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,524						      item_size, &ref_root,525						      &ref_level);526			if (ret < 0) {527				btrfs_warn(fs_info,528				"failed to resolve tree backref for logical %llu: %d",529						  swarn.logical, ret);530				break;531			}532			if (ret > 0)533				break;534			btrfs_warn_in_rcu(fs_info,535"%s at logical %llu on dev %s, physical %llu: metadata %s (level %d) in tree %llu",536				errstr, swarn.logical, btrfs_dev_name(dev),537				swarn.physical, (ref_level ? "node" : "leaf"),538				ref_level, ref_root);539		}540		btrfs_release_path(path);541	} else {542		struct btrfs_backref_walk_ctx ctx = { 0 };543 544		btrfs_release_path(path);545 546		ctx.bytenr = found_key.objectid;547		ctx.extent_item_pos = swarn.logical - found_key.objectid;548		ctx.fs_info = fs_info;549 550		swarn.path = path;551		swarn.dev = dev;552 553		iterate_extent_inodes(&ctx, true, scrub_print_warning_inode, &swarn);554	}555 556out:557	btrfs_free_path(path);558}559 560static int fill_writer_pointer_gap(struct scrub_ctx *sctx, u64 physical)561{562	int ret = 0;563	u64 length;564 565	if (!btrfs_is_zoned(sctx->fs_info))566		return 0;567 568	if (!btrfs_dev_is_sequential(sctx->wr_tgtdev, physical))569		return 0;570 571	if (sctx->write_pointer < physical) {572		length = physical - sctx->write_pointer;573 574		ret = btrfs_zoned_issue_zeroout(sctx->wr_tgtdev,575						sctx->write_pointer, length);576		if (!ret)577			sctx->write_pointer = physical;578	}579	return ret;580}581 582static struct page *scrub_stripe_get_page(struct scrub_stripe *stripe, int sector_nr)583{584	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;585	int page_index = (sector_nr << fs_info->sectorsize_bits) >> PAGE_SHIFT;586 587	return stripe->pages[page_index];588}589 590static unsigned int scrub_stripe_get_page_offset(struct scrub_stripe *stripe,591						 int sector_nr)592{593	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;594 595	return offset_in_page(sector_nr << fs_info->sectorsize_bits);596}597 598static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr)599{600	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;601	const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits;602	const u64 logical = stripe->logical + (sector_nr << fs_info->sectorsize_bits);603	const struct page *first_page = scrub_stripe_get_page(stripe, sector_nr);604	const unsigned int first_off = scrub_stripe_get_page_offset(stripe, sector_nr);605	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);606	u8 on_disk_csum[BTRFS_CSUM_SIZE];607	u8 calculated_csum[BTRFS_CSUM_SIZE];608	struct btrfs_header *header;609 610	/*611	 * Here we don't have a good way to attach the pages (and subpages)612	 * to a dummy extent buffer, thus we have to directly grab the members613	 * from pages.614	 */615	header = (struct btrfs_header *)(page_address(first_page) + first_off);616	memcpy(on_disk_csum, header->csum, fs_info->csum_size);617 618	if (logical != btrfs_stack_header_bytenr(header)) {619		bitmap_set(&stripe->csum_error_bitmap, sector_nr, sectors_per_tree);620		bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree);621		btrfs_warn_rl(fs_info,622		"tree block %llu mirror %u has bad bytenr, has %llu want %llu",623			      logical, stripe->mirror_num,624			      btrfs_stack_header_bytenr(header), logical);625		return;626	}627	if (memcmp(header->fsid, fs_info->fs_devices->metadata_uuid,628		   BTRFS_FSID_SIZE) != 0) {629		bitmap_set(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree);630		bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree);631		btrfs_warn_rl(fs_info,632		"tree block %llu mirror %u has bad fsid, has %pU want %pU",633			      logical, stripe->mirror_num,634			      header->fsid, fs_info->fs_devices->fsid);635		return;636	}637	if (memcmp(header->chunk_tree_uuid, fs_info->chunk_tree_uuid,638		   BTRFS_UUID_SIZE) != 0) {639		bitmap_set(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree);640		bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree);641		btrfs_warn_rl(fs_info,642		"tree block %llu mirror %u has bad chunk tree uuid, has %pU want %pU",643			      logical, stripe->mirror_num,644			      header->chunk_tree_uuid, fs_info->chunk_tree_uuid);645		return;646	}647 648	/* Now check tree block csum. */649	shash->tfm = fs_info->csum_shash;650	crypto_shash_init(shash);651	crypto_shash_update(shash, page_address(first_page) + first_off +652			    BTRFS_CSUM_SIZE, fs_info->sectorsize - BTRFS_CSUM_SIZE);653 654	for (int i = sector_nr + 1; i < sector_nr + sectors_per_tree; i++) {655		struct page *page = scrub_stripe_get_page(stripe, i);656		unsigned int page_off = scrub_stripe_get_page_offset(stripe, i);657 658		crypto_shash_update(shash, page_address(page) + page_off,659				    fs_info->sectorsize);660	}661 662	crypto_shash_final(shash, calculated_csum);663	if (memcmp(calculated_csum, on_disk_csum, fs_info->csum_size) != 0) {664		bitmap_set(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree);665		bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree);666		btrfs_warn_rl(fs_info,667		"tree block %llu mirror %u has bad csum, has " CSUM_FMT " want " CSUM_FMT,668			      logical, stripe->mirror_num,669			      CSUM_FMT_VALUE(fs_info->csum_size, on_disk_csum),670			      CSUM_FMT_VALUE(fs_info->csum_size, calculated_csum));671		return;672	}673	if (stripe->sectors[sector_nr].generation !=674	    btrfs_stack_header_generation(header)) {675		bitmap_set(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree);676		bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree);677		btrfs_warn_rl(fs_info,678		"tree block %llu mirror %u has bad generation, has %llu want %llu",679			      logical, stripe->mirror_num,680			      btrfs_stack_header_generation(header),681			      stripe->sectors[sector_nr].generation);682		return;683	}684	bitmap_clear(&stripe->error_bitmap, sector_nr, sectors_per_tree);685	bitmap_clear(&stripe->csum_error_bitmap, sector_nr, sectors_per_tree);686	bitmap_clear(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree);687}688 689static void scrub_verify_one_sector(struct scrub_stripe *stripe, int sector_nr)690{691	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;692	struct scrub_sector_verification *sector = &stripe->sectors[sector_nr];693	const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits;694	struct page *page = scrub_stripe_get_page(stripe, sector_nr);695	unsigned int pgoff = scrub_stripe_get_page_offset(stripe, sector_nr);696	u8 csum_buf[BTRFS_CSUM_SIZE];697	int ret;698 699	ASSERT(sector_nr >= 0 && sector_nr < stripe->nr_sectors);700 701	/* Sector not utilized, skip it. */702	if (!test_bit(sector_nr, &stripe->extent_sector_bitmap))703		return;704 705	/* IO error, no need to check. */706	if (test_bit(sector_nr, &stripe->io_error_bitmap))707		return;708 709	/* Metadata, verify the full tree block. */710	if (sector->is_metadata) {711		/*712		 * Check if the tree block crosses the stripe boundary.  If713		 * crossed the boundary, we cannot verify it but only give a714		 * warning.715		 *716		 * This can only happen on a very old filesystem where chunks717		 * are not ensured to be stripe aligned.718		 */719		if (unlikely(sector_nr + sectors_per_tree > stripe->nr_sectors)) {720			btrfs_warn_rl(fs_info,721			"tree block at %llu crosses stripe boundary %llu",722				      stripe->logical +723				      (sector_nr << fs_info->sectorsize_bits),724				      stripe->logical);725			return;726		}727		scrub_verify_one_metadata(stripe, sector_nr);728		return;729	}730 731	/*732	 * Data is easier, we just verify the data csum (if we have it).  For733	 * cases without csum, we have no other choice but to trust it.734	 */735	if (!sector->csum) {736		clear_bit(sector_nr, &stripe->error_bitmap);737		return;738	}739 740	ret = btrfs_check_sector_csum(fs_info, page, pgoff, csum_buf, sector->csum);741	if (ret < 0) {742		set_bit(sector_nr, &stripe->csum_error_bitmap);743		set_bit(sector_nr, &stripe->error_bitmap);744	} else {745		clear_bit(sector_nr, &stripe->csum_error_bitmap);746		clear_bit(sector_nr, &stripe->error_bitmap);747	}748}749 750/* Verify specified sectors of a stripe. */751static void scrub_verify_one_stripe(struct scrub_stripe *stripe, unsigned long bitmap)752{753	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;754	const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits;755	int sector_nr;756 757	for_each_set_bit(sector_nr, &bitmap, stripe->nr_sectors) {758		scrub_verify_one_sector(stripe, sector_nr);759		if (stripe->sectors[sector_nr].is_metadata)760			sector_nr += sectors_per_tree - 1;761	}762}763 764static int calc_sector_number(struct scrub_stripe *stripe, struct bio_vec *first_bvec)765{766	int i;767 768	for (i = 0; i < stripe->nr_sectors; i++) {769		if (scrub_stripe_get_page(stripe, i) == first_bvec->bv_page &&770		    scrub_stripe_get_page_offset(stripe, i) == first_bvec->bv_offset)771			break;772	}773	ASSERT(i < stripe->nr_sectors);774	return i;775}776 777/*778 * Repair read is different to the regular read:779 *780 * - Only reads the failed sectors781 * - May have extra blocksize limits782 */783static void scrub_repair_read_endio(struct btrfs_bio *bbio)784{785	struct scrub_stripe *stripe = bbio->private;786	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;787	struct bio_vec *bvec;788	int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio));789	u32 bio_size = 0;790	int i;791 792	ASSERT(sector_nr < stripe->nr_sectors);793 794	bio_for_each_bvec_all(bvec, &bbio->bio, i)795		bio_size += bvec->bv_len;796 797	if (bbio->bio.bi_status) {798		bitmap_set(&stripe->io_error_bitmap, sector_nr,799			   bio_size >> fs_info->sectorsize_bits);800		bitmap_set(&stripe->error_bitmap, sector_nr,801			   bio_size >> fs_info->sectorsize_bits);802	} else {803		bitmap_clear(&stripe->io_error_bitmap, sector_nr,804			     bio_size >> fs_info->sectorsize_bits);805	}806	bio_put(&bbio->bio);807	if (atomic_dec_and_test(&stripe->pending_io))808		wake_up(&stripe->io_wait);809}810 811static int calc_next_mirror(int mirror, int num_copies)812{813	ASSERT(mirror <= num_copies);814	return (mirror + 1 > num_copies) ? 1 : mirror + 1;815}816 817static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe,818					    int mirror, int blocksize, bool wait)819{820	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;821	struct btrfs_bio *bbio = NULL;822	const unsigned long old_error_bitmap = stripe->error_bitmap;823	int i;824 825	ASSERT(stripe->mirror_num >= 1);826	ASSERT(atomic_read(&stripe->pending_io) == 0);827 828	for_each_set_bit(i, &old_error_bitmap, stripe->nr_sectors) {829		struct page *page;830		int pgoff;831		int ret;832 833		page = scrub_stripe_get_page(stripe, i);834		pgoff = scrub_stripe_get_page_offset(stripe, i);835 836		/* The current sector cannot be merged, submit the bio. */837		if (bbio && ((i > 0 && !test_bit(i - 1, &stripe->error_bitmap)) ||838			     bbio->bio.bi_iter.bi_size >= blocksize)) {839			ASSERT(bbio->bio.bi_iter.bi_size);840			atomic_inc(&stripe->pending_io);841			btrfs_submit_bbio(bbio, mirror);842			if (wait)843				wait_scrub_stripe_io(stripe);844			bbio = NULL;845		}846 847		if (!bbio) {848			bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_READ,849				fs_info, scrub_repair_read_endio, stripe);850			bbio->bio.bi_iter.bi_sector = (stripe->logical +851				(i << fs_info->sectorsize_bits)) >> SECTOR_SHIFT;852		}853 854		ret = bio_add_page(&bbio->bio, page, fs_info->sectorsize, pgoff);855		ASSERT(ret == fs_info->sectorsize);856	}857	if (bbio) {858		ASSERT(bbio->bio.bi_iter.bi_size);859		atomic_inc(&stripe->pending_io);860		btrfs_submit_bbio(bbio, mirror);861		if (wait)862			wait_scrub_stripe_io(stripe);863	}864}865 866static void scrub_stripe_report_errors(struct scrub_ctx *sctx,867				       struct scrub_stripe *stripe)868{869	static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,870				      DEFAULT_RATELIMIT_BURST);871	struct btrfs_fs_info *fs_info = sctx->fs_info;872	struct btrfs_device *dev = NULL;873	u64 physical = 0;874	int nr_data_sectors = 0;875	int nr_meta_sectors = 0;876	int nr_nodatacsum_sectors = 0;877	int nr_repaired_sectors = 0;878	int sector_nr;879 880	if (test_bit(SCRUB_STRIPE_FLAG_NO_REPORT, &stripe->state))881		return;882 883	/*884	 * Init needed infos for error reporting.885	 *886	 * Although our scrub_stripe infrastructure is mostly based on btrfs_submit_bio()887	 * thus no need for dev/physical, error reporting still needs dev and physical.888	 */889	if (!bitmap_empty(&stripe->init_error_bitmap, stripe->nr_sectors)) {890		u64 mapped_len = fs_info->sectorsize;891		struct btrfs_io_context *bioc = NULL;892		int stripe_index = stripe->mirror_num - 1;893		int ret;894 895		/* For scrub, our mirror_num should always start at 1. */896		ASSERT(stripe->mirror_num >= 1);897		ret = btrfs_map_block(fs_info, BTRFS_MAP_GET_READ_MIRRORS,898				      stripe->logical, &mapped_len, &bioc,899				      NULL, NULL);900		/*901		 * If we failed, dev will be NULL, and later detailed reports902		 * will just be skipped.903		 */904		if (ret < 0)905			goto skip;906		physical = bioc->stripes[stripe_index].physical;907		dev = bioc->stripes[stripe_index].dev;908		btrfs_put_bioc(bioc);909	}910 911skip:912	for_each_set_bit(sector_nr, &stripe->extent_sector_bitmap, stripe->nr_sectors) {913		bool repaired = false;914 915		if (stripe->sectors[sector_nr].is_metadata) {916			nr_meta_sectors++;917		} else {918			nr_data_sectors++;919			if (!stripe->sectors[sector_nr].csum)920				nr_nodatacsum_sectors++;921		}922 923		if (test_bit(sector_nr, &stripe->init_error_bitmap) &&924		    !test_bit(sector_nr, &stripe->error_bitmap)) {925			nr_repaired_sectors++;926			repaired = true;927		}928 929		/* Good sector from the beginning, nothing need to be done. */930		if (!test_bit(sector_nr, &stripe->init_error_bitmap))931			continue;932 933		/*934		 * Report error for the corrupted sectors.  If repaired, just935		 * output the message of repaired message.936		 */937		if (repaired) {938			if (dev) {939				btrfs_err_rl_in_rcu(fs_info,940			"fixed up error at logical %llu on dev %s physical %llu",941					    stripe->logical, btrfs_dev_name(dev),942					    physical);943			} else {944				btrfs_err_rl_in_rcu(fs_info,945			"fixed up error at logical %llu on mirror %u",946					    stripe->logical, stripe->mirror_num);947			}948			continue;949		}950 951		/* The remaining are all for unrepaired. */952		if (dev) {953			btrfs_err_rl_in_rcu(fs_info,954	"unable to fixup (regular) error at logical %llu on dev %s physical %llu",955					    stripe->logical, btrfs_dev_name(dev),956					    physical);957		} else {958			btrfs_err_rl_in_rcu(fs_info,959	"unable to fixup (regular) error at logical %llu on mirror %u",960					    stripe->logical, stripe->mirror_num);961		}962 963		if (test_bit(sector_nr, &stripe->io_error_bitmap))964			if (__ratelimit(&rs) && dev)965				scrub_print_common_warning("i/o error", dev, false,966						     stripe->logical, physical);967		if (test_bit(sector_nr, &stripe->csum_error_bitmap))968			if (__ratelimit(&rs) && dev)969				scrub_print_common_warning("checksum error", dev, false,970						     stripe->logical, physical);971		if (test_bit(sector_nr, &stripe->meta_error_bitmap))972			if (__ratelimit(&rs) && dev)973				scrub_print_common_warning("header error", dev, false,974						     stripe->logical, physical);975	}976 977	spin_lock(&sctx->stat_lock);978	sctx->stat.data_extents_scrubbed += stripe->nr_data_extents;979	sctx->stat.tree_extents_scrubbed += stripe->nr_meta_extents;980	sctx->stat.data_bytes_scrubbed += nr_data_sectors << fs_info->sectorsize_bits;981	sctx->stat.tree_bytes_scrubbed += nr_meta_sectors << fs_info->sectorsize_bits;982	sctx->stat.no_csum += nr_nodatacsum_sectors;983	sctx->stat.read_errors += stripe->init_nr_io_errors;984	sctx->stat.csum_errors += stripe->init_nr_csum_errors;985	sctx->stat.verify_errors += stripe->init_nr_meta_errors;986	sctx->stat.uncorrectable_errors +=987		bitmap_weight(&stripe->error_bitmap, stripe->nr_sectors);988	sctx->stat.corrected_errors += nr_repaired_sectors;989	spin_unlock(&sctx->stat_lock);990}991 992static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *stripe,993				unsigned long write_bitmap, bool dev_replace);994 995/*996 * The main entrance for all read related scrub work, including:997 *998 * - Wait for the initial read to finish999 * - Verify and locate any bad sectors1000 * - Go through the remaining mirrors and try to read as large blocksize as1001 *   possible1002 * - Go through all mirrors (including the failed mirror) sector-by-sector1003 * - Submit writeback for repaired sectors1004 *1005 * Writeback for dev-replace does not happen here, it needs extra1006 * synchronization for zoned devices.1007 */1008static void scrub_stripe_read_repair_worker(struct work_struct *work)1009{1010	struct scrub_stripe *stripe = container_of(work, struct scrub_stripe, work);1011	struct scrub_ctx *sctx = stripe->sctx;1012	struct btrfs_fs_info *fs_info = sctx->fs_info;1013	int num_copies = btrfs_num_copies(fs_info, stripe->bg->start,1014					  stripe->bg->length);1015	unsigned long repaired;1016	int mirror;1017	int i;1018 1019	ASSERT(stripe->mirror_num > 0);1020 1021	wait_scrub_stripe_io(stripe);1022	scrub_verify_one_stripe(stripe, stripe->extent_sector_bitmap);1023	/* Save the initial failed bitmap for later repair and report usage. */1024	stripe->init_error_bitmap = stripe->error_bitmap;1025	stripe->init_nr_io_errors = bitmap_weight(&stripe->io_error_bitmap,1026						  stripe->nr_sectors);1027	stripe->init_nr_csum_errors = bitmap_weight(&stripe->csum_error_bitmap,1028						    stripe->nr_sectors);1029	stripe->init_nr_meta_errors = bitmap_weight(&stripe->meta_error_bitmap,1030						    stripe->nr_sectors);1031 1032	if (bitmap_empty(&stripe->init_error_bitmap, stripe->nr_sectors))1033		goto out;1034 1035	/*1036	 * Try all remaining mirrors.1037	 *1038	 * Here we still try to read as large block as possible, as this is1039	 * faster and we have extra safety nets to rely on.1040	 */1041	for (mirror = calc_next_mirror(stripe->mirror_num, num_copies);1042	     mirror != stripe->mirror_num;1043	     mirror = calc_next_mirror(mirror, num_copies)) {1044		const unsigned long old_error_bitmap = stripe->error_bitmap;1045 1046		scrub_stripe_submit_repair_read(stripe, mirror,1047						BTRFS_STRIPE_LEN, false);1048		wait_scrub_stripe_io(stripe);1049		scrub_verify_one_stripe(stripe, old_error_bitmap);1050		if (bitmap_empty(&stripe->error_bitmap, stripe->nr_sectors))1051			goto out;1052	}1053 1054	/*1055	 * Last safety net, try re-checking all mirrors, including the failed1056	 * one, sector-by-sector.1057	 *1058	 * As if one sector failed the drive's internal csum, the whole read1059	 * containing the offending sector would be marked as error.1060	 * Thus here we do sector-by-sector read.1061	 *1062	 * This can be slow, thus we only try it as the last resort.1063	 */1064 1065	for (i = 0, mirror = stripe->mirror_num;1066	     i < num_copies;1067	     i++, mirror = calc_next_mirror(mirror, num_copies)) {1068		const unsigned long old_error_bitmap = stripe->error_bitmap;1069 1070		scrub_stripe_submit_repair_read(stripe, mirror,1071						fs_info->sectorsize, true);1072		wait_scrub_stripe_io(stripe);1073		scrub_verify_one_stripe(stripe, old_error_bitmap);1074		if (bitmap_empty(&stripe->error_bitmap, stripe->nr_sectors))1075			goto out;1076	}1077out:1078	/*1079	 * Submit the repaired sectors.  For zoned case, we cannot do repair1080	 * in-place, but queue the bg to be relocated.1081	 */1082	bitmap_andnot(&repaired, &stripe->init_error_bitmap, &stripe->error_bitmap,1083		      stripe->nr_sectors);1084	if (!sctx->readonly && !bitmap_empty(&repaired, stripe->nr_sectors)) {1085		if (btrfs_is_zoned(fs_info)) {1086			btrfs_repair_one_zone(fs_info, sctx->stripes[0].bg->start);1087		} else {1088			scrub_write_sectors(sctx, stripe, repaired, false);1089			wait_scrub_stripe_io(stripe);1090		}1091	}1092 1093	scrub_stripe_report_errors(sctx, stripe);1094	set_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state);1095	wake_up(&stripe->repair_wait);1096}1097 1098static void scrub_read_endio(struct btrfs_bio *bbio)1099{1100	struct scrub_stripe *stripe = bbio->private;1101	struct bio_vec *bvec;1102	int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio));1103	int num_sectors;1104	u32 bio_size = 0;1105	int i;1106 1107	ASSERT(sector_nr < stripe->nr_sectors);1108	bio_for_each_bvec_all(bvec, &bbio->bio, i)1109		bio_size += bvec->bv_len;1110	num_sectors = bio_size >> stripe->bg->fs_info->sectorsize_bits;1111 1112	if (bbio->bio.bi_status) {1113		bitmap_set(&stripe->io_error_bitmap, sector_nr, num_sectors);1114		bitmap_set(&stripe->error_bitmap, sector_nr, num_sectors);1115	} else {1116		bitmap_clear(&stripe->io_error_bitmap, sector_nr, num_sectors);1117	}1118	bio_put(&bbio->bio);1119	if (atomic_dec_and_test(&stripe->pending_io)) {1120		wake_up(&stripe->io_wait);1121		INIT_WORK(&stripe->work, scrub_stripe_read_repair_worker);1122		queue_work(stripe->bg->fs_info->scrub_workers, &stripe->work);1123	}1124}1125 1126static void scrub_write_endio(struct btrfs_bio *bbio)1127{1128	struct scrub_stripe *stripe = bbio->private;1129	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;1130	struct bio_vec *bvec;1131	int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio));1132	u32 bio_size = 0;1133	int i;1134 1135	bio_for_each_bvec_all(bvec, &bbio->bio, i)1136		bio_size += bvec->bv_len;1137 1138	if (bbio->bio.bi_status) {1139		unsigned long flags;1140 1141		spin_lock_irqsave(&stripe->write_error_lock, flags);1142		bitmap_set(&stripe->write_error_bitmap, sector_nr,1143			   bio_size >> fs_info->sectorsize_bits);1144		spin_unlock_irqrestore(&stripe->write_error_lock, flags);1145	}1146	bio_put(&bbio->bio);1147 1148	if (atomic_dec_and_test(&stripe->pending_io))1149		wake_up(&stripe->io_wait);1150}1151 1152static void scrub_submit_write_bio(struct scrub_ctx *sctx,1153				   struct scrub_stripe *stripe,1154				   struct btrfs_bio *bbio, bool dev_replace)1155{1156	struct btrfs_fs_info *fs_info = sctx->fs_info;1157	u32 bio_len = bbio->bio.bi_iter.bi_size;1158	u32 bio_off = (bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT) -1159		      stripe->logical;1160 1161	fill_writer_pointer_gap(sctx, stripe->physical + bio_off);1162	atomic_inc(&stripe->pending_io);1163	btrfs_submit_repair_write(bbio, stripe->mirror_num, dev_replace);1164	if (!btrfs_is_zoned(fs_info))1165		return;1166	/*1167	 * For zoned writeback, queue depth must be 1, thus we must wait for1168	 * the write to finish before the next write.1169	 */1170	wait_scrub_stripe_io(stripe);1171 1172	/*1173	 * And also need to update the write pointer if write finished1174	 * successfully.1175	 */1176	if (!test_bit(bio_off >> fs_info->sectorsize_bits,1177		      &stripe->write_error_bitmap))1178		sctx->write_pointer += bio_len;1179}1180 1181/*1182 * Submit the write bio(s) for the sectors specified by @write_bitmap.1183 *1184 * Here we utilize btrfs_submit_repair_write(), which has some extra benefits:1185 *1186 * - Only needs logical bytenr and mirror_num1187 *   Just like the scrub read path1188 *1189 * - Would only result in writes to the specified mirror1190 *   Unlike the regular writeback path, which would write back to all stripes1191 *1192 * - Handle dev-replace and read-repair writeback differently1193 */1194static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *stripe,1195				unsigned long write_bitmap, bool dev_replace)1196{1197	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;1198	struct btrfs_bio *bbio = NULL;1199	int sector_nr;1200 1201	for_each_set_bit(sector_nr, &write_bitmap, stripe->nr_sectors) {1202		struct page *page = scrub_stripe_get_page(stripe, sector_nr);1203		unsigned int pgoff = scrub_stripe_get_page_offset(stripe, sector_nr);1204		int ret;1205 1206		/* We should only writeback sectors covered by an extent. */1207		ASSERT(test_bit(sector_nr, &stripe->extent_sector_bitmap));1208 1209		/* Cannot merge with previous sector, submit the current one. */1210		if (bbio && sector_nr && !test_bit(sector_nr - 1, &write_bitmap)) {1211			scrub_submit_write_bio(sctx, stripe, bbio, dev_replace);1212			bbio = NULL;1213		}1214		if (!bbio) {1215			bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_WRITE,1216					       fs_info, scrub_write_endio, stripe);1217			bbio->bio.bi_iter.bi_sector = (stripe->logical +1218				(sector_nr << fs_info->sectorsize_bits)) >>1219				SECTOR_SHIFT;1220		}1221		ret = bio_add_page(&bbio->bio, page, fs_info->sectorsize, pgoff);1222		ASSERT(ret == fs_info->sectorsize);1223	}1224	if (bbio)1225		scrub_submit_write_bio(sctx, stripe, bbio, dev_replace);1226}1227 1228/*1229 * Throttling of IO submission, bandwidth-limit based, the timeslice is 11230 * second.  Limit can be set via /sys/fs/UUID/devinfo/devid/scrub_speed_max.1231 */1232static void scrub_throttle_dev_io(struct scrub_ctx *sctx, struct btrfs_device *device,1233				  unsigned int bio_size)1234{1235	const int time_slice = 1000;1236	s64 delta;1237	ktime_t now;1238	u32 div;1239	u64 bwlimit;1240 1241	bwlimit = READ_ONCE(device->scrub_speed_max);1242	if (bwlimit == 0)1243		return;1244 1245	/*1246	 * Slice is divided into intervals when the IO is submitted, adjust by1247	 * bwlimit and maximum of 64 intervals.1248	 */1249	div = max_t(u32, 1, (u32)(bwlimit / (16 * 1024 * 1024)));1250	div = min_t(u32, 64, div);1251 1252	/* Start new epoch, set deadline */1253	now = ktime_get();1254	if (sctx->throttle_deadline == 0) {1255		sctx->throttle_deadline = ktime_add_ms(now, time_slice / div);1256		sctx->throttle_sent = 0;1257	}1258 1259	/* Still in the time to send? */1260	if (ktime_before(now, sctx->throttle_deadline)) {1261		/* If current bio is within the limit, send it */1262		sctx->throttle_sent += bio_size;1263		if (sctx->throttle_sent <= div_u64(bwlimit, div))1264			return;1265 1266		/* We're over the limit, sleep until the rest of the slice */1267		delta = ktime_ms_delta(sctx->throttle_deadline, now);1268	} else {1269		/* New request after deadline, start new epoch */1270		delta = 0;1271	}1272 1273	if (delta) {1274		long timeout;1275 1276		timeout = div_u64(delta * HZ, 1000);1277		schedule_timeout_interruptible(timeout);1278	}1279 1280	/* Next call will start the deadline period */1281	sctx->throttle_deadline = 0;1282}1283 1284/*1285 * Given a physical address, this will calculate it's1286 * logical offset. if this is a parity stripe, it will return1287 * the most left data stripe's logical offset.1288 *1289 * return 0 if it is a data stripe, 1 means parity stripe.1290 */1291static int get_raid56_logic_offset(u64 physical, int num,1292				   struct btrfs_chunk_map *map, u64 *offset,1293				   u64 *stripe_start)1294{1295	int i;1296	int j = 0;1297	u64 last_offset;1298	const int data_stripes = nr_data_stripes(map);1299 1300	last_offset = (physical - map->stripes[num].physical) * data_stripes;1301	if (stripe_start)1302		*stripe_start = last_offset;1303 1304	*offset = last_offset;1305	for (i = 0; i < data_stripes; i++) {1306		u32 stripe_nr;1307		u32 stripe_index;1308		u32 rot;1309 1310		*offset = last_offset + btrfs_stripe_nr_to_offset(i);1311 1312		stripe_nr = (u32)(*offset >> BTRFS_STRIPE_LEN_SHIFT) / data_stripes;1313 1314		/* Work out the disk rotation on this stripe-set */1315		rot = stripe_nr % map->num_stripes;1316		/* calculate which stripe this data locates */1317		rot += i;1318		stripe_index = rot % map->num_stripes;1319		if (stripe_index == num)1320			return 0;1321		if (stripe_index < num)1322			j++;1323	}1324	*offset = last_offset + btrfs_stripe_nr_to_offset(j);1325	return 1;1326}1327 1328/*1329 * Return 0 if the extent item range covers any byte of the range.1330 * Return <0 if the extent item is before @search_start.1331 * Return >0 if the extent item is after @start_start + @search_len.1332 */1333static int compare_extent_item_range(struct btrfs_path *path,1334				     u64 search_start, u64 search_len)1335{1336	struct btrfs_fs_info *fs_info = path->nodes[0]->fs_info;1337	u64 len;1338	struct btrfs_key key;1339 1340	btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);1341	ASSERT(key.type == BTRFS_EXTENT_ITEM_KEY ||1342	       key.type == BTRFS_METADATA_ITEM_KEY);1343	if (key.type == BTRFS_METADATA_ITEM_KEY)1344		len = fs_info->nodesize;1345	else1346		len = key.offset;1347 1348	if (key.objectid + len <= search_start)1349		return -1;1350	if (key.objectid >= search_start + search_len)1351		return 1;1352	return 0;1353}1354 1355/*1356 * Locate one extent item which covers any byte in range1357 * [@search_start, @search_start + @search_length)1358 *1359 * If the path is not initialized, we will initialize the search by doing1360 * a btrfs_search_slot().1361 * If the path is already initialized, we will use the path as the initial1362 * slot, to avoid duplicated btrfs_search_slot() calls.1363 *1364 * NOTE: If an extent item starts before @search_start, we will still1365 * return the extent item. This is for data extent crossing stripe boundary.1366 *1367 * Return 0 if we found such extent item, and @path will point to the extent item.1368 * Return >0 if no such extent item can be found, and @path will be released.1369 * Return <0 if hit fatal error, and @path will be released.1370 */1371static int find_first_extent_item(struct btrfs_root *extent_root,1372				  struct btrfs_path *path,1373				  u64 search_start, u64 search_len)1374{1375	struct btrfs_fs_info *fs_info = extent_root->fs_info;1376	struct btrfs_key key;1377	int ret;1378 1379	/* Continue using the existing path */1380	if (path->nodes[0])1381		goto search_forward;1382 1383	if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))1384		key.type = BTRFS_METADATA_ITEM_KEY;1385	else1386		key.type = BTRFS_EXTENT_ITEM_KEY;1387	key.objectid = search_start;1388	key.offset = (u64)-1;1389 1390	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);1391	if (ret < 0)1392		return ret;1393	if (ret == 0) {1394		/*1395		 * Key with offset -1 found, there would have to exist an extent1396		 * item with such offset, but this is out of the valid range.1397		 */1398		btrfs_release_path(path);1399		return -EUCLEAN;1400	}1401 1402	/*1403	 * Here we intentionally pass 0 as @min_objectid, as there could be1404	 * an extent item starting before @search_start.1405	 */1406	ret = btrfs_previous_extent_item(extent_root, path, 0);1407	if (ret < 0)1408		return ret;1409	/*1410	 * No matter whether we have found an extent item, the next loop will1411	 * properly do every check on the key.1412	 */1413search_forward:1414	while (true) {1415		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);1416		if (key.objectid >= search_start + search_len)1417			break;1418		if (key.type != BTRFS_METADATA_ITEM_KEY &&1419		    key.type != BTRFS_EXTENT_ITEM_KEY)1420			goto next;1421 1422		ret = compare_extent_item_range(path, search_start, search_len);1423		if (ret == 0)1424			return ret;1425		if (ret > 0)1426			break;1427next:1428		ret = btrfs_next_item(extent_root, path);1429		if (ret) {1430			/* Either no more items or a fatal error. */1431			btrfs_release_path(path);1432			return ret;1433		}1434	}1435	btrfs_release_path(path);1436	return 1;1437}1438 1439static void get_extent_info(struct btrfs_path *path, u64 *extent_start_ret,1440			    u64 *size_ret, u64 *flags_ret, u64 *generation_ret)1441{1442	struct btrfs_key key;1443	struct btrfs_extent_item *ei;1444 1445	btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);1446	ASSERT(key.type == BTRFS_METADATA_ITEM_KEY ||1447	       key.type == BTRFS_EXTENT_ITEM_KEY);1448	*extent_start_ret = key.objectid;1449	if (key.type == BTRFS_METADATA_ITEM_KEY)1450		*size_ret = path->nodes[0]->fs_info->nodesize;1451	else1452		*size_ret = key.offset;1453	ei = btrfs_item_ptr(path->nodes[0], path->slots[0], struct btrfs_extent_item);1454	*flags_ret = btrfs_extent_flags(path->nodes[0], ei);1455	*generation_ret = btrfs_extent_generation(path->nodes[0], ei);1456}1457 1458static int sync_write_pointer_for_zoned(struct scrub_ctx *sctx, u64 logical,1459					u64 physical, u64 physical_end)1460{1461	struct btrfs_fs_info *fs_info = sctx->fs_info;1462	int ret = 0;1463 1464	if (!btrfs_is_zoned(fs_info))1465		return 0;1466 1467	mutex_lock(&sctx->wr_lock);1468	if (sctx->write_pointer < physical_end) {1469		ret = btrfs_sync_zone_write_pointer(sctx->wr_tgtdev, logical,1470						    physical,1471						    sctx->write_pointer);1472		if (ret)1473			btrfs_err(fs_info,1474				  "zoned: failed to recover write pointer");1475	}1476	mutex_unlock(&sctx->wr_lock);1477	btrfs_dev_clear_zone_empty(sctx->wr_tgtdev, physical);1478 1479	return ret;1480}1481 1482static void fill_one_extent_info(struct btrfs_fs_info *fs_info,1483				 struct scrub_stripe *stripe,1484				 u64 extent_start, u64 extent_len,1485				 u64 extent_flags, u64 extent_gen)1486{1487	for (u64 cur_logical = max(stripe->logical, extent_start);1488	     cur_logical < min(stripe->logical + BTRFS_STRIPE_LEN,1489			       extent_start + extent_len);1490	     cur_logical += fs_info->sectorsize) {1491		const int nr_sector = (cur_logical - stripe->logical) >>1492				      fs_info->sectorsize_bits;1493		struct scrub_sector_verification *sector =1494						&stripe->sectors[nr_sector];1495 1496		set_bit(nr_sector, &stripe->extent_sector_bitmap);1497		if (extent_flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {1498			sector->is_metadata = true;1499			sector->generation = extent_gen;1500		}1501	}1502}1503 1504static void scrub_stripe_reset_bitmaps(struct scrub_stripe *stripe)1505{1506	stripe->extent_sector_bitmap = 0;1507	stripe->init_error_bitmap = 0;1508	stripe->init_nr_io_errors = 0;1509	stripe->init_nr_csum_errors = 0;1510	stripe->init_nr_meta_errors = 0;1511	stripe->error_bitmap = 0;1512	stripe->io_error_bitmap = 0;1513	stripe->csum_error_bitmap = 0;1514	stripe->meta_error_bitmap = 0;1515}1516 1517/*1518 * Locate one stripe which has at least one extent in its range.1519 *1520 * Return 0 if found such stripe, and store its info into @stripe.1521 * Return >0 if there is no such stripe in the specified range.1522 * Return <0 for error.1523 */1524static int scrub_find_fill_first_stripe(struct btrfs_block_group *bg,1525					struct btrfs_path *extent_path,1526					struct btrfs_path *csum_path,1527					struct btrfs_device *dev, u64 physical,1528					int mirror_num, u64 logical_start,1529					u32 logical_len,1530					struct scrub_stripe *stripe)1531{1532	struct btrfs_fs_info *fs_info = bg->fs_info;1533	struct btrfs_root *extent_root = btrfs_extent_root(fs_info, bg->start);1534	struct btrfs_root *csum_root = btrfs_csum_root(fs_info, bg->start);1535	const u64 logical_end = logical_start + logical_len;1536	u64 cur_logical = logical_start;1537	u64 stripe_end;1538	u64 extent_start;1539	u64 extent_len;1540	u64 extent_flags;1541	u64 extent_gen;1542	int ret;1543 1544	memset(stripe->sectors, 0, sizeof(struct scrub_sector_verification) *1545				   stripe->nr_sectors);1546	scrub_stripe_reset_bitmaps(stripe);1547 1548	/* The range must be inside the bg. */1549	ASSERT(logical_start >= bg->start && logical_end <= bg->start + bg->length);1550 1551	ret = find_first_extent_item(extent_root, extent_path, logical_start,1552				     logical_len);1553	/* Either error or not found. */1554	if (ret)1555		goto out;1556	get_extent_info(extent_path, &extent_start, &extent_len, &extent_flags,1557			&extent_gen);1558	if (extent_flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)1559		stripe->nr_meta_extents++;1560	if (extent_flags & BTRFS_EXTENT_FLAG_DATA)1561		stripe->nr_data_extents++;1562	cur_logical = max(extent_start, cur_logical);1563 1564	/*1565	 * Round down to stripe boundary.1566	 *1567	 * The extra calculation against bg->start is to handle block groups1568	 * whose logical bytenr is not BTRFS_STRIPE_LEN aligned.1569	 */1570	stripe->logical = round_down(cur_logical - bg->start, BTRFS_STRIPE_LEN) +1571			  bg->start;1572	stripe->physical = physical + stripe->logical - logical_start;1573	stripe->dev = dev;1574	stripe->bg = bg;1575	stripe->mirror_num = mirror_num;1576	stripe_end = stripe->logical + BTRFS_STRIPE_LEN - 1;1577 1578	/* Fill the first extent info into stripe->sectors[] array. */1579	fill_one_extent_info(fs_info, stripe, extent_start, extent_len,1580			     extent_flags, extent_gen);1581	cur_logical = extent_start + extent_len;1582 1583	/* Fill the extent info for the remaining sectors. */1584	while (cur_logical <= stripe_end) {1585		ret = find_first_extent_item(extent_root, extent_path, cur_logical,1586					     stripe_end - cur_logical + 1);1587		if (ret < 0)1588			goto out;1589		if (ret > 0) {1590			ret = 0;1591			break;1592		}1593		get_extent_info(extent_path, &extent_start, &extent_len,1594				&extent_flags, &extent_gen);1595		if (extent_flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)1596			stripe->nr_meta_extents++;1597		if (extent_flags & BTRFS_EXTENT_FLAG_DATA)1598			stripe->nr_data_extents++;1599		fill_one_extent_info(fs_info, stripe, extent_start, extent_len,1600				     extent_flags, extent_gen);1601		cur_logical = extent_start + extent_len;1602	}1603 1604	/* Now fill the data csum. */1605	if (bg->flags & BTRFS_BLOCK_GROUP_DATA) {1606		int sector_nr;1607		unsigned long csum_bitmap = 0;1608 1609		/* Csum space should have already been allocated. */1610		ASSERT(stripe->csums);1611 1612		/*1613		 * Our csum bitmap should be large enough, as BTRFS_STRIPE_LEN1614		 * should contain at most 16 sectors.1615		 */1616		ASSERT(BITS_PER_LONG >= BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits);1617 1618		ret = btrfs_lookup_csums_bitmap(csum_root, csum_path,1619						stripe->logical, stripe_end,1620						stripe->csums, &csum_bitmap);1621		if (ret < 0)1622			goto out;1623		if (ret > 0)1624			ret = 0;1625 1626		for_each_set_bit(sector_nr, &csum_bitmap, stripe->nr_sectors) {1627			stripe->sectors[sector_nr].csum = stripe->csums +1628				sector_nr * fs_info->csum_size;1629		}1630	}1631	set_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state);1632out:1633	return ret;1634}1635 1636static void scrub_reset_stripe(struct scrub_stripe *stripe)1637{1638	scrub_stripe_reset_bitmaps(stripe);1639 1640	stripe->nr_meta_extents = 0;1641	stripe->nr_data_extents = 0;1642	stripe->state = 0;1643 1644	for (int i = 0; i < stripe->nr_sectors; i++) {1645		stripe->sectors[i].is_metadata = false;1646		stripe->sectors[i].csum = NULL;1647		stripe->sectors[i].generation = 0;1648	}1649}1650 1651static u32 stripe_length(const struct scrub_stripe *stripe)1652{1653	ASSERT(stripe->bg);1654 1655	return min(BTRFS_STRIPE_LEN,1656		   stripe->bg->start + stripe->bg->length - stripe->logical);1657}1658 1659static void scrub_submit_extent_sector_read(struct scrub_ctx *sctx,1660					    struct scrub_stripe *stripe)1661{1662	struct btrfs_fs_info *fs_info = stripe->bg->fs_info;1663	struct btrfs_bio *bbio = NULL;1664	unsigned int nr_sectors = stripe_length(stripe) >> fs_info->sectorsize_bits;1665	u64 stripe_len = BTRFS_STRIPE_LEN;1666	int mirror = stripe->mirror_num;1667	int i;1668 1669	atomic_inc(&stripe->pending_io);1670 1671	for_each_set_bit(i, &stripe->extent_sector_bitmap, stripe->nr_sectors) {1672		struct page *page = scrub_stripe_get_page(stripe, i);1673		unsigned int pgoff = scrub_stripe_get_page_offset(stripe, i);1674 1675		/* We're beyond the chunk boundary, no need to read anymore. */1676		if (i >= nr_sectors)1677			break;1678 1679		/* The current sector cannot be merged, submit the bio. */1680		if (bbio &&1681		    ((i > 0 &&1682		      !test_bit(i - 1, &stripe->extent_sector_bitmap)) ||1683		     bbio->bio.bi_iter.bi_size >= stripe_len)) {1684			ASSERT(bbio->bio.bi_iter.bi_size);1685			atomic_inc(&stripe->pending_io);1686			btrfs_submit_bbio(bbio, mirror);1687			bbio = NULL;1688		}1689 1690		if (!bbio) {1691			struct btrfs_io_stripe io_stripe = {};1692			struct btrfs_io_context *bioc = NULL;1693			const u64 logical = stripe->logical +1694					    (i << fs_info->sectorsize_bits);1695			int err;1696 1697			io_stripe.rst_search_commit_root = true;1698			stripe_len = (nr_sectors - i) << fs_info->sectorsize_bits;1699			/*1700			 * For RST cases, we need to manually split the bbio to1701			 * follow the RST boundary.1702			 */1703			err = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical,1704					      &stripe_len, &bioc, &io_stripe, &mirror);1705			btrfs_put_bioc(bioc);1706			if (err < 0) {1707				set_bit(i, &stripe->io_error_bitmap);1708				set_bit(i, &stripe->error_bitmap);1709				continue;1710			}1711 1712			bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_READ,1713					       fs_info, scrub_read_endio, stripe);1714			bbio->bio.bi_iter.bi_sector = logical >> SECTOR_SHIFT;1715		}1716 1717		__bio_add_page(&bbio->bio, page, fs_info->sectorsize, pgoff);1718	}1719 1720	if (bbio) {1721		ASSERT(bbio->bio.bi_iter.bi_size);1722		atomic_inc(&stripe->pending_io);1723		btrfs_submit_bbio(bbio, mirror);1724	}1725 1726	if (atomic_dec_and_test(&stripe->pending_io)) {1727		wake_up(&stripe->io_wait);1728		INIT_WORK(&stripe->work, scrub_stripe_read_repair_worker);1729		queue_work(stripe->bg->fs_info->scrub_workers, &stripe->work);1730	}1731}1732 1733static void scrub_submit_initial_read(struct scrub_ctx *sctx,1734				      struct scrub_stripe *stripe)1735{1736	struct btrfs_fs_info *fs_info = sctx->fs_info;1737	struct btrfs_bio *bbio;1738	unsigned int nr_sectors = stripe_length(stripe) >> fs_info->sectorsize_bits;1739	int mirror = stripe->mirror_num;1740 1741	ASSERT(stripe->bg);1742	ASSERT(stripe->mirror_num > 0);1743	ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state));1744 1745	if (btrfs_need_stripe_tree_update(fs_info, stripe->bg->flags)) {1746		scrub_submit_extent_sector_read(sctx, stripe);1747		return;1748	}1749 1750	bbio = btrfs_bio_alloc(SCRUB_STRIPE_PAGES, REQ_OP_READ, fs_info,1751			       scrub_read_endio, stripe);1752 1753	bbio->bio.bi_iter.bi_sector = stripe->logical >> SECTOR_SHIFT;1754	/* Read the whole range inside the chunk boundary. */1755	for (unsigned int cur = 0; cur < nr_sectors; cur++) {1756		struct page *page = scrub_stripe_get_page(stripe, cur);1757		unsigned int pgoff = scrub_stripe_get_page_offset(stripe, cur);1758		int ret;1759 1760		ret = bio_add_page(&bbio->bio, page, fs_info->sectorsize, pgoff);1761		/* We should have allocated enough bio vectors. */1762		ASSERT(ret == fs_info->sectorsize);1763	}1764	atomic_inc(&stripe->pending_io);1765 1766	/*1767	 * For dev-replace, either user asks to avoid the source dev, or1768	 * the device is missing, we try the next mirror instead.1769	 */1770	if (sctx->is_dev_replace &&1771	    (fs_info->dev_replace.cont_reading_from_srcdev_mode ==1772	     BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_AVOID ||1773	     !stripe->dev->bdev)) {1774		int num_copies = btrfs_num_copies(fs_info, stripe->bg->start,1775						  stripe->bg->length);1776 1777		mirror = calc_next_mirror(mirror, num_copies);1778	}1779	btrfs_submit_bbio(bbio, mirror);1780}1781 1782static bool stripe_has_metadata_error(struct scrub_stripe *stripe)1783{1784	int i;1785 1786	for_each_set_bit(i, &stripe->error_bitmap, stripe->nr_sectors) {1787		if (stripe->sectors[i].is_metadata) {1788			struct btrfs_fs_info *fs_info = stripe->bg->fs_info;1789 1790			btrfs_err(fs_info,1791			"stripe %llu has unrepaired metadata sector at %llu",1792				  stripe->logical,1793				  stripe->logical + (i << fs_info->sectorsize_bits));1794			return true;1795		}1796	}1797	return false;1798}1799 1800static void submit_initial_group_read(struct scrub_ctx *sctx,1801				      unsigned int first_slot,1802				      unsigned int nr_stripes)1803{1804	struct blk_plug plug;1805 1806	ASSERT(first_slot < SCRUB_TOTAL_STRIPES);1807	ASSERT(first_slot + nr_stripes <= SCRUB_TOTAL_STRIPES);1808 1809	scrub_throttle_dev_io(sctx, sctx->stripes[0].dev,1810			      btrfs_stripe_nr_to_offset(nr_stripes));1811	blk_start_plug(&plug);1812	for (int i = 0; i < nr_stripes; i++) {1813		struct scrub_stripe *stripe = &sctx->stripes[first_slot + i];1814 1815		/* Those stripes should be initialized. */1816		ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state));1817		scrub_submit_initial_read(sctx, stripe);1818	}1819	blk_finish_plug(&plug);1820}1821 1822static int flush_scrub_stripes(struct scrub_ctx *sctx)1823{1824	struct btrfs_fs_info *fs_info = sctx->fs_info;1825	struct scrub_stripe *stripe;1826	const int nr_stripes = sctx->cur_stripe;1827	int ret = 0;1828 1829	if (!nr_stripes)1830		return 0;1831 1832	ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &sctx->stripes[0].state));1833 1834	/* Submit the stripes which are populated but not submitted. */1835	if (nr_stripes % SCRUB_STRIPES_PER_GROUP) {1836		const int first_slot = round_down(nr_stripes, SCRUB_STRIPES_PER_GROUP);1837 1838		submit_initial_group_read(sctx, first_slot, nr_stripes - first_slot);1839	}1840 1841	for (int i = 0; i < nr_stripes; i++) {1842		stripe = &sctx->stripes[i];1843 1844		wait_event(stripe->repair_wait,1845			   test_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state));1846	}1847 1848	/* Submit for dev-replace. */1849	if (sctx->is_dev_replace) {1850		/*1851		 * For dev-replace, if we know there is something wrong with1852		 * metadata, we should immediately abort.1853		 */1854		for (int i = 0; i < nr_stripes; i++) {1855			if (stripe_has_metadata_error(&sctx->stripes[i])) {1856				ret = -EIO;1857				goto out;1858			}1859		}1860		for (int i = 0; i < nr_stripes; i++) {1861			unsigned long good;1862 1863			stripe = &sctx->stripes[i];1864 1865			ASSERT(stripe->dev == fs_info->dev_replace.srcdev);1866 1867			bitmap_andnot(&good, &stripe->extent_sector_bitmap,1868				      &stripe->error_bitmap, stripe->nr_sectors);1869			scrub_write_sectors(sctx, stripe, good, true);1870		}1871	}1872 1873	/* Wait for the above writebacks to finish. */1874	for (int i = 0; i < nr_stripes; i++) {1875		stripe = &sctx->stripes[i];1876 1877		wait_scrub_stripe_io(stripe);1878		spin_lock(&sctx->stat_lock);1879		sctx->stat.last_physical = stripe->physical + stripe_length(stripe);1880		spin_unlock(&sctx->stat_lock);1881		scrub_reset_stripe(stripe);1882	}1883out:1884	sctx->cur_stripe = 0;1885	return ret;1886}1887 1888static void raid56_scrub_wait_endio(struct bio *bio)1889{1890	complete(bio->bi_private);1891}1892 1893static int queue_scrub_stripe(struct scrub_ctx *sctx, struct btrfs_block_group *bg,1894			      struct btrfs_device *dev, int mirror_num,1895			      u64 logical, u32 length, u64 physical,1896			      u64 *found_logical_ret)1897{1898	struct scrub_stripe *stripe;1899	int ret;1900 1901	/*1902	 * There should always be one slot left, as caller filling the last1903	 * slot should flush them all.1904	 */1905	ASSERT(sctx->cur_stripe < SCRUB_TOTAL_STRIPES);1906 1907	/* @found_logical_ret must be specified. */1908	ASSERT(found_logical_ret);1909 1910	stripe = &sctx->stripes[sctx->cur_stripe];1911	scrub_reset_stripe(stripe);1912	ret = scrub_find_fill_first_stripe(bg, &sctx->extent_path,1913					   &sctx->csum_path, dev, physical,1914					   mirror_num, logical, length, stripe);1915	/* Either >0 as no more extents or <0 for error. */1916	if (ret)1917		return ret;1918	*found_logical_ret = stripe->logical;1919	sctx->cur_stripe++;1920 1921	/* We filled one group, submit it. */1922	if (sctx->cur_stripe % SCRUB_STRIPES_PER_GROUP == 0) {1923		const int first_slot = sctx->cur_stripe - SCRUB_STRIPES_PER_GROUP;1924 1925		submit_initial_group_read(sctx, first_slot, SCRUB_STRIPES_PER_GROUP);1926	}1927 1928	/* Last slot used, flush them all. */1929	if (sctx->cur_stripe == SCRUB_TOTAL_STRIPES)1930		return flush_scrub_stripes(sctx);1931	return 0;1932}1933 1934static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx,1935				      struct btrfs_device *scrub_dev,1936				      struct btrfs_block_group *bg,1937				      struct btrfs_chunk_map *map,1938				      u64 full_stripe_start)1939{1940	DECLARE_COMPLETION_ONSTACK(io_done);1941	struct btrfs_fs_info *fs_info = sctx->fs_info;1942	struct btrfs_raid_bio *rbio;1943	struct btrfs_io_context *bioc = NULL;1944	struct btrfs_path extent_path = { 0 };1945	struct btrfs_path csum_path = { 0 };1946	struct bio *bio;1947	struct scrub_stripe *stripe;1948	bool all_empty = true;1949	const int data_stripes = nr_data_stripes(map);1950	unsigned long extent_bitmap = 0;1951	u64 length = btrfs_stripe_nr_to_offset(data_stripes);1952	int ret;1953 1954	ASSERT(sctx->raid56_data_stripes);1955 1956	/*1957	 * For data stripe search, we cannot re-use the same extent/csum paths,1958	 * as the data stripe bytenr may be smaller than previous extent.  Thus1959	 * we have to use our own extent/csum paths.1960	 */1961	extent_path.search_commit_root = 1;1962	extent_path.skip_locking = 1;1963	csum_path.search_commit_root = 1;1964	csum_path.skip_locking = 1;1965 1966	for (int i = 0; i < data_stripes; i++) {1967		int stripe_index;1968		int rot;1969		u64 physical;1970 1971		stripe = &sctx->raid56_data_stripes[i];1972		rot = div_u64(full_stripe_start - bg->start,1973			      data_stripes) >> BTRFS_STRIPE_LEN_SHIFT;1974		stripe_index = (i + rot) % map->num_stripes;1975		physical = map->stripes[stripe_index].physical +1976			   btrfs_stripe_nr_to_offset(rot);1977 1978		scrub_reset_stripe(stripe);1979		set_bit(SCRUB_STRIPE_FLAG_NO_REPORT, &stripe->state);1980		ret = scrub_find_fill_first_stripe(bg, &extent_path, &csum_path,1981				map->stripes[stripe_index].dev, physical, 1,1982				full_stripe_start + btrfs_stripe_nr_to_offset(i),1983				BTRFS_STRIPE_LEN, stripe);1984		if (ret < 0)1985			goto out;1986		/*1987		 * No extent in this data stripe, need to manually mark them1988		 * initialized to make later read submission happy.1989		 */1990		if (ret > 0) {1991			stripe->logical = full_stripe_start +1992					  btrfs_stripe_nr_to_offset(i);1993			stripe->dev = map->stripes[stripe_index].dev;1994			stripe->mirror_num = 1;1995			set_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state);1996		}1997	}1998 1999	/* Check if all data stripes are empty. */2000	for (int i = 0; i < data_stripes; i++) {2001		stripe = &sctx->raid56_data_stripes[i];2002		if (!bitmap_empty(&stripe->extent_sector_bitmap, stripe->nr_sectors)) {2003			all_empty = false;2004			break;2005		}2006	}2007	if (all_empty) {2008		ret = 0;2009		goto out;2010	}2011 2012	for (int i = 0; i < data_stripes; i++) {2013		stripe = &sctx->raid56_data_stripes[i];2014		scrub_submit_initial_read(sctx, stripe);2015	}2016	for (int i = 0; i < data_stripes; i++) {2017		stripe = &sctx->raid56_data_stripes[i];2018 2019		wait_event(stripe->repair_wait,2020			   test_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state));2021	}2022	/* For now, no zoned support for RAID56. */2023	ASSERT(!btrfs_is_zoned(sctx->fs_info));2024 2025	/*2026	 * Now all data stripes are properly verified. Check if we have any2027	 * unrepaired, if so abort immediately or we could further corrupt the2028	 * P/Q stripes.2029	 *2030	 * During the loop, also populate extent_bitmap.2031	 */2032	for (int i = 0; i < data_stripes; i++) {2033		unsigned long error;2034 2035		stripe = &sctx->raid56_data_stripes[i];2036 2037		/*2038		 * We should only check the errors where there is an extent.2039		 * As we may hit an empty data stripe while it's missing.2040		 */2041		bitmap_and(&error, &stripe->error_bitmap,2042			   &stripe->extent_sector_bitmap, stripe->nr_sectors);2043		if (!bitmap_empty(&error, stripe->nr_sectors)) {2044			btrfs_err(fs_info,2045"unrepaired sectors detected, full stripe %llu data stripe %u errors %*pbl",2046				  full_stripe_start, i, stripe->nr_sectors,2047				  &error);2048			ret = -EIO;2049			goto out;2050		}2051		bitmap_or(&extent_bitmap, &extent_bitmap,2052			  &stripe->extent_sector_bitmap, stripe->nr_sectors);2053	}2054 2055	/* Now we can check and regenerate the P/Q stripe. */2056	bio = bio_alloc(NULL, 1, REQ_OP_READ, GFP_NOFS);2057	bio->bi_iter.bi_sector = full_stripe_start >> SECTOR_SHIFT;2058	bio->bi_private = &io_done;2059	bio->bi_end_io = raid56_scrub_wait_endio;2060 2061	btrfs_bio_counter_inc_blocked(fs_info);2062	ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, full_stripe_start,2063			      &length, &bioc, NULL, NULL);2064	if (ret < 0) {2065		btrfs_put_bioc(bioc);2066		btrfs_bio_counter_dec(fs_info);2067		goto out;2068	}2069	rbio = raid56_parity_alloc_scrub_rbio(bio, bioc, scrub_dev, &extent_bitmap,2070				BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits);2071	btrfs_put_bioc(bioc);2072	if (!rbio) {2073		ret = -ENOMEM;2074		btrfs_bio_counter_dec(fs_info);2075		goto out;2076	}2077	/* Use the recovered stripes as cache to avoid read them from disk again. */2078	for (int i = 0; i < data_stripes; i++) {2079		stripe = &sctx->raid56_data_stripes[i];2080 2081		raid56_parity_cache_data_pages(rbio, stripe->pages,2082				full_stripe_start + (i << BTRFS_STRIPE_LEN_SHIFT));2083	}2084	raid56_parity_submit_scrub_rbio(rbio);2085	wait_for_completion_io(&io_done);2086	ret = blk_status_to_errno(bio->bi_status);2087	bio_put(bio);2088	btrfs_bio_counter_dec(fs_info);2089 2090	btrfs_release_path(&extent_path);2091	btrfs_release_path(&csum_path);2092out:2093	return ret;2094}2095 2096/*2097 * Scrub one range which can only has simple mirror based profile.2098 * (Including all range in SINGLE/DUP/RAID1/RAID1C*, and each stripe in2099 *  RAID0/RAID10).2100 *2101 * Since we may need to handle a subset of block group, we need @logical_start2102 * and @logical_length parameter.2103 */2104static int scrub_simple_mirror(struct scrub_ctx *sctx,2105			       struct btrfs_block_group *bg,2106			       struct btrfs_chunk_map *map,2107			       u64 logical_start, u64 logical_length,2108			       struct btrfs_device *device,2109			       u64 physical, int mirror_num)2110{2111	struct btrfs_fs_info *fs_info = sctx->fs_info;2112	const u64 logical_end = logical_start + logical_length;2113	u64 cur_logical = logical_start;2114	int ret = 0;2115 2116	/* The range must be inside the bg */2117	ASSERT(logical_start >= bg->start && logical_end <= bg->start + bg->length);2118 2119	/* Go through each extent items inside the logical range */2120	while (cur_logical < logical_end) {2121		u64 found_logical = U64_MAX;2122		u64 cur_physical = physical + cur_logical - logical_start;2123 2124		/* Canceled? */2125		if (atomic_read(&fs_info->scrub_cancel_req) ||2126		    atomic_read(&sctx->cancel_req)) {2127			ret = -ECANCELED;2128			break;2129		}2130		/* Paused? */2131		if (atomic_read(&fs_info->scrub_pause_req)) {2132			/* Push queued extents */2133			scrub_blocked_if_needed(fs_info);2134		}2135		/* Block group removed? */2136		spin_lock(&bg->lock);2137		if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &bg->runtime_flags)) {2138			spin_unlock(&bg->lock);2139			ret = 0;2140			break;2141		}2142		spin_unlock(&bg->lock);2143 2144		ret = queue_scrub_stripe(sctx, bg, device, mirror_num,2145					 cur_logical, logical_end - cur_logical,2146					 cur_physical, &found_logical);2147		if (ret > 0) {2148			/* No more extent, just update the accounting */2149			spin_lock(&sctx->stat_lock);2150			sctx->stat.last_physical = physical + logical_length;2151			spin_unlock(&sctx->stat_lock);2152			ret = 0;2153			break;2154		}2155		if (ret < 0)2156			break;2157 2158		/* queue_scrub_stripe() returned 0, @found_logical must be updated. */2159		ASSERT(found_logical != U64_MAX);2160		cur_logical = found_logical + BTRFS_STRIPE_LEN;2161 2162		/* Don't hold CPU for too long time */2163		cond_resched();2164	}2165	return ret;2166}2167 2168/* Calculate the full stripe length for simple stripe based profiles */2169static u64 simple_stripe_full_stripe_len(const struct btrfs_chunk_map *map)2170{2171	ASSERT(map->type & (BTRFS_BLOCK_GROUP_RAID0 |2172			    BTRFS_BLOCK_GROUP_RAID10));2173 2174	return btrfs_stripe_nr_to_offset(map->num_stripes / map->sub_stripes);2175}2176 2177/* Get the logical bytenr for the stripe */2178static u64 simple_stripe_get_logical(struct btrfs_chunk_map *map,2179				     struct btrfs_block_group *bg,2180				     int stripe_index)2181{2182	ASSERT(map->type & (BTRFS_BLOCK_GROUP_RAID0 |2183			    BTRFS_BLOCK_GROUP_RAID10));2184	ASSERT(stripe_index < map->num_stripes);2185 2186	/*2187	 * (stripe_index / sub_stripes) gives how many data stripes we need to2188	 * skip.2189	 */2190	return btrfs_stripe_nr_to_offset(stripe_index / map->sub_stripes) +2191	       bg->start;2192}2193 2194/* Get the mirror number for the stripe */2195static int simple_stripe_mirror_num(struct btrfs_chunk_map *map, int stripe_index)2196{2197	ASSERT(map->type & (BTRFS_BLOCK_GROUP_RAID0 |2198			    BTRFS_BLOCK_GROUP_RAID10));2199	ASSERT(stripe_index < map->num_stripes);2200 2201	/* For RAID0, it's fixed to 1, for RAID10 it's 0,1,0,1... */2202	return stripe_index % map->sub_stripes + 1;2203}2204 2205static int scrub_simple_stripe(struct scrub_ctx *sctx,2206			       struct btrfs_block_group *bg,2207			       struct btrfs_chunk_map *map,2208			       struct btrfs_device *device,2209			       int stripe_index)2210{2211	const u64 logical_increment = simple_stripe_full_stripe_len(map);2212	const u64 orig_logical = simple_stripe_get_logical(map, bg, stripe_index);2213	const u64 orig_physical = map->stripes[stripe_index].physical;2214	const int mirror_num = simple_stripe_mirror_num(map, stripe_index);2215	u64 cur_logical = orig_logical;2216	u64 cur_physical = orig_physical;2217	int ret = 0;2218 2219	while (cur_logical < bg->start + bg->length) {2220		/*2221		 * Inside each stripe, RAID0 is just SINGLE, and RAID10 is2222		 * just RAID1, so we can reuse scrub_simple_mirror() to scrub2223		 * this stripe.2224		 */2225		ret = scrub_simple_mirror(sctx, bg, map, cur_logical,2226					  BTRFS_STRIPE_LEN, device, cur_physical,2227					  mirror_num);2228		if (ret)2229			return ret;2230		/* Skip to next stripe which belongs to the target device */2231		cur_logical += logical_increment;2232		/* For physical offset, we just go to next stripe */2233		cur_physical += BTRFS_STRIPE_LEN;2234	}2235	return ret;2236}2237 2238static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,2239					   struct btrfs_block_group *bg,2240					   struct btrfs_chunk_map *map,2241					   struct btrfs_device *scrub_dev,2242					   int stripe_index)2243{2244	struct btrfs_fs_info *fs_info = sctx->fs_info;2245	const u64 profile = map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK;2246	const u64 chunk_logical = bg->start;2247	int ret;2248	int ret2;2249	u64 physical = map->stripes[stripe_index].physical;2250	const u64 dev_stripe_len = btrfs_calc_stripe_length(map);2251	const u64 physical_end = physical + dev_stripe_len;2252	u64 logical;2253	u64 logic_end;2254	/* The logical increment after finishing one stripe */2255	u64 increment;2256	/* Offset inside the chunk */2257	u64 offset;2258	u64 stripe_logical;2259	int stop_loop = 0;2260 2261	/* Extent_path should be released by now. */2262	ASSERT(sctx->extent_path.nodes[0] == NULL);2263 2264	scrub_blocked_if_needed(fs_info);2265 2266	if (sctx->is_dev_replace &&2267	    btrfs_dev_is_sequential(sctx->wr_tgtdev, physical)) {2268		mutex_lock(&sctx->wr_lock);2269		sctx->write_pointer = physical;2270		mutex_unlock(&sctx->wr_lock);2271	}2272 2273	/* Prepare the extra data stripes used by RAID56. */2274	if (profile & BTRFS_BLOCK_GROUP_RAID56_MASK) {2275		ASSERT(sctx->raid56_data_stripes == NULL);2276 2277		sctx->raid56_data_stripes = kcalloc(nr_data_stripes(map),2278						    sizeof(struct scrub_stripe),2279						    GFP_KERNEL);2280		if (!sctx->raid56_data_stripes) {2281			ret = -ENOMEM;2282			goto out;2283		}2284		for (int i = 0; i < nr_data_stripes(map); i++) {2285			ret = init_scrub_stripe(fs_info,2286						&sctx->raid56_data_stripes[i]);2287			if (ret < 0)2288				goto out;2289			sctx->raid56_data_stripes[i].bg = bg;2290			sctx->raid56_data_stripes[i].sctx = sctx;2291		}2292	}2293	/*2294	 * There used to be a big double loop to handle all profiles using the2295	 * same routine, which grows larger and more gross over time.2296	 *2297	 * So here we handle each profile differently, so simpler profiles2298	 * have simpler scrubbing function.2299	 */2300	if (!(profile & (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID10 |2301			 BTRFS_BLOCK_GROUP_RAID56_MASK))) {2302		/*2303		 * Above check rules out all complex profile, the remaining2304		 * profiles are SINGLE|DUP|RAID1|RAID1C*, which is simple2305		 * mirrored duplication without stripe.2306		 *2307		 * Only @physical and @mirror_num needs to calculated using2308		 * @stripe_index.2309		 */2310		ret = scrub_simple_mirror(sctx, bg, map, bg->start, bg->length,2311				scrub_dev, map->stripes[stripe_index].physical,2312				stripe_index + 1);2313		offset = 0;2314		goto out;2315	}2316	if (profile & (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID10)) {2317		ret = scrub_simple_stripe(sctx, bg, map, scrub_dev, stripe_index);2318		offset = btrfs_stripe_nr_to_offset(stripe_index / map->sub_stripes);2319		goto out;2320	}2321 2322	/* Only RAID56 goes through the old code */2323	ASSERT(map->type & BTRFS_BLOCK_GROUP_RAID56_MASK);2324	ret = 0;2325 2326	/* Calculate the logical end of the stripe */2327	get_raid56_logic_offset(physical_end, stripe_index,2328				map, &logic_end, NULL);2329	logic_end += chunk_logical;2330 2331	/* Initialize @offset in case we need to go to out: label */2332	get_raid56_logic_offset(physical, stripe_index, map, &offset, NULL);2333	increment = btrfs_stripe_nr_to_offset(nr_data_stripes(map));2334 2335	/*2336	 * Due to the rotation, for RAID56 it's better to iterate each stripe2337	 * using their physical offset.2338	 */2339	while (physical < physical_end) {2340		ret = get_raid56_logic_offset(physical, stripe_index, map,2341					      &logical, &stripe_logical);2342		logical += chunk_logical;2343		if (ret) {2344			/* it is parity strip */2345			stripe_logical += chunk_logical;2346			ret = scrub_raid56_parity_stripe(sctx, scrub_dev, bg,2347							 map, stripe_logical);2348			spin_lock(&sctx->stat_lock);2349			sctx->stat.last_physical = min(physical + BTRFS_STRIPE_LEN,2350						       physical_end);2351			spin_unlock(&sctx->stat_lock);2352			if (ret)2353				goto out;2354			goto next;2355		}2356 2357		/*2358		 * Now we're at a data stripe, scrub each extents in the range.2359		 *2360		 * At this stage, if we ignore the repair part, inside each data2361		 * stripe it is no different than SINGLE profile.2362		 * We can reuse scrub_simple_mirror() here, as the repair part2363		 * is still based on @mirror_num.2364		 */2365		ret = scrub_simple_mirror(sctx, bg, map, logical, BTRFS_STRIPE_LEN,2366					  scrub_dev, physical, 1);2367		if (ret < 0)2368			goto out;2369next:2370		logical += increment;2371		physical += BTRFS_STRIPE_LEN;2372		spin_lock(&sctx->stat_lock);2373		if (stop_loop)2374			sctx->stat.last_physical =2375				map->stripes[stripe_index].physical + dev_stripe_len;2376		else2377			sctx->stat.last_physical = physical;2378		spin_unlock(&sctx->stat_lock);2379		if (stop_loop)2380			break;2381	}2382out:2383	ret2 = flush_scrub_stripes(sctx);2384	if (!ret)2385		ret = ret2;2386	btrfs_release_path(&sctx->extent_path);2387	btrfs_release_path(&sctx->csum_path);2388 2389	if (sctx->raid56_data_stripes) {2390		for (int i = 0; i < nr_data_stripes(map); i++)2391			release_scrub_stripe(&sctx->raid56_data_stripes[i]);2392		kfree(sctx->raid56_data_stripes);2393		sctx->raid56_data_stripes = NULL;2394	}2395 2396	if (sctx->is_dev_replace && ret >= 0) {2397		int ret2;2398 2399		ret2 = sync_write_pointer_for_zoned(sctx,2400				chunk_logical + offset,2401				map->stripes[stripe_index].physical,2402				physical_end);2403		if (ret2)2404			ret = ret2;2405	}2406 2407	return ret < 0 ? ret : 0;2408}2409 2410static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,2411					  struct btrfs_block_group *bg,2412					  struct btrfs_device *scrub_dev,2413					  u64 dev_offset,2414					  u64 dev_extent_len)2415{2416	struct btrfs_fs_info *fs_info = sctx->fs_info;2417	struct btrfs_chunk_map *map;2418	int i;2419	int ret = 0;2420 2421	map = btrfs_find_chunk_map(fs_info, bg->start, bg->length);2422	if (!map) {2423		/*2424		 * Might have been an unused block group deleted by the cleaner2425		 * kthread or relocation.2426		 */2427		spin_lock(&bg->lock);2428		if (!test_bit(BLOCK_GROUP_FLAG_REMOVED, &bg->runtime_flags))2429			ret = -EINVAL;2430		spin_unlock(&bg->lock);2431 2432		return ret;2433	}2434	if (map->start != bg->start)2435		goto out;2436	if (map->chunk_len < dev_extent_len)2437		goto out;2438 2439	for (i = 0; i < map->num_stripes; ++i) {2440		if (map->stripes[i].dev->bdev == scrub_dev->bdev &&2441		    map->stripes[i].physical == dev_offset) {2442			ret = scrub_stripe(sctx, bg, map, scrub_dev, i);2443			if (ret)2444				goto out;2445		}2446	}2447out:2448	btrfs_free_chunk_map(map);2449 2450	return ret;2451}2452 2453static int finish_extent_writes_for_zoned(struct btrfs_root *root,2454					  struct btrfs_block_group *cache)2455{2456	struct btrfs_fs_info *fs_info = cache->fs_info;2457 2458	if (!btrfs_is_zoned(fs_info))2459		return 0;2460 2461	btrfs_wait_block_group_reservations(cache);2462	btrfs_wait_nocow_writers(cache);2463	btrfs_wait_ordered_roots(fs_info, U64_MAX, cache);2464 2465	return btrfs_commit_current_transaction(root);2466}2467 2468static noinline_for_stack2469int scrub_enumerate_chunks(struct scrub_ctx *sctx,2470			   struct btrfs_device *scrub_dev, u64 start, u64 end)2471{2472	struct btrfs_dev_extent *dev_extent = NULL;2473	struct btrfs_path *path;2474	struct btrfs_fs_info *fs_info = sctx->fs_info;2475	struct btrfs_root *root = fs_info->dev_root;2476	u64 chunk_offset;2477	int ret = 0;2478	int ro_set;2479	int slot;2480	struct extent_buffer *l;2481	struct btrfs_key key;2482	struct btrfs_key found_key;2483	struct btrfs_block_group *cache;2484	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;2485 2486	path = btrfs_alloc_path();2487	if (!path)2488		return -ENOMEM;2489 2490	path->reada = READA_FORWARD;2491	path->search_commit_root = 1;2492	path->skip_locking = 1;2493 2494	key.objectid = scrub_dev->devid;2495	key.offset = 0ull;2496	key.type = BTRFS_DEV_EXTENT_KEY;2497 2498	while (1) {2499		u64 dev_extent_len;2500 2501		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);2502		if (ret < 0)2503			break;2504		if (ret > 0) {2505			if (path->slots[0] >=2506			    btrfs_header_nritems(path->nodes[0])) {2507				ret = btrfs_next_leaf(root, path);2508				if (ret < 0)2509					break;2510				if (ret > 0) {2511					ret = 0;2512					break;2513				}2514			} else {2515				ret = 0;2516			}2517		}2518 2519		l = path->nodes[0];2520		slot = path->slots[0];2521 2522		btrfs_item_key_to_cpu(l, &found_key, slot);2523 2524		if (found_key.objectid != scrub_dev->devid)2525			break;2526 2527		if (found_key.type != BTRFS_DEV_EXTENT_KEY)2528			break;2529 2530		if (found_key.offset >= end)2531			break;2532 2533		if (found_key.offset < key.offset)2534			break;2535 2536		dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);2537		dev_extent_len = btrfs_dev_extent_length(l, dev_extent);2538 2539		if (found_key.offset + dev_extent_len <= start)2540			goto skip;2541 2542		chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);2543 2544		/*2545		 * get a reference on the corresponding block group to prevent2546		 * the chunk from going away while we scrub it2547		 */2548		cache = btrfs_lookup_block_group(fs_info, chunk_offset);2549 2550		/* some chunks are removed but not committed to disk yet,2551		 * continue scrubbing */2552		if (!cache)2553			goto skip;2554 2555		ASSERT(cache->start <= chunk_offset);2556		/*2557		 * We are using the commit root to search for device extents, so2558		 * that means we could have found a device extent item from a2559		 * block group that was deleted in the current transaction. The2560		 * logical start offset of the deleted block group, stored at2561		 * @chunk_offset, might be part of the logical address range of2562		 * a new block group (which uses different physical extents).2563		 * In this case btrfs_lookup_block_group() has returned the new2564		 * block group, and its start address is less than @chunk_offset.2565		 *2566		 * We skip such new block groups, because it's pointless to2567		 * process them, as we won't find their extents because we search2568		 * for them using the commit root of the extent tree. For a device2569		 * replace it's also fine to skip it, we won't miss copying them2570		 * to the target device because we have the write duplication2571		 * setup through the regular write path (by btrfs_map_block()),2572		 * and we have committed a transaction when we started the device2573		 * replace, right after setting up the device replace state.2574		 */2575		if (cache->start < chunk_offset) {2576			btrfs_put_block_group(cache);2577			goto skip;2578		}2579 2580		if (sctx->is_dev_replace && btrfs_is_zoned(fs_info)) {2581			if (!test_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags)) {2582				btrfs_put_block_group(cache);2583				goto skip;2584			}2585		}2586 2587		/*2588		 * Make sure that while we are scrubbing the corresponding block2589		 * group doesn't get its logical address and its device extents2590		 * reused for another block group, which can possibly be of a2591		 * different type and different profile. We do this to prevent2592		 * false error detections and crashes due to bogus attempts to2593		 * repair extents.2594		 */2595		spin_lock(&cache->lock);2596		if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &cache->runtime_flags)) {2597			spin_unlock(&cache->lock);2598			btrfs_put_block_group(cache);2599			goto skip;2600		}2601		btrfs_freeze_block_group(cache);2602		spin_unlock(&cache->lock);2603 2604		/*2605		 * we need call btrfs_inc_block_group_ro() with scrubs_paused,2606		 * to avoid deadlock caused by:2607		 * btrfs_inc_block_group_ro()2608		 * -> btrfs_wait_for_commit()2609		 * -> btrfs_commit_transaction()2610		 * -> btrfs_scrub_pause()2611		 */2612		scrub_pause_on(fs_info);2613 2614		/*2615		 * Don't do chunk preallocation for scrub.2616		 *2617		 * This is especially important for SYSTEM bgs, or we can hit2618		 * -EFBIG from btrfs_finish_chunk_alloc() like:2619		 * 1. The only SYSTEM bg is marked RO.2620		 *    Since SYSTEM bg is small, that's pretty common.2621		 * 2. New SYSTEM bg will be allocated2622		 *    Due to regular version will allocate new chunk.2623		 * 3. New SYSTEM bg is empty and will get cleaned up2624		 *    Before cleanup really happens, it's marked RO again.2625		 * 4. Empty SYSTEM bg get scrubbed2626		 *    We go back to 2.2627		 *2628		 * This can easily boost the amount of SYSTEM chunks if cleaner2629		 * thread can't be triggered fast enough, and use up all space2630		 * of btrfs_super_block::sys_chunk_array2631		 *2632		 * While for dev replace, we need to try our best to mark block2633		 * group RO, to prevent race between:2634		 * - Write duplication2635		 *   Contains latest data2636		 * - Scrub copy2637		 *   Contains data from commit tree2638		 *2639		 * If target block group is not marked RO, nocow writes can2640		 * be overwritten by scrub copy, causing data corruption.2641		 * So for dev-replace, it's not allowed to continue if a block2642		 * group is not RO.2643		 */2644		ret = btrfs_inc_block_group_ro(cache, sctx->is_dev_replace);2645		if (!ret && sctx->is_dev_replace) {2646			ret = finish_extent_writes_for_zoned(root, cache);2647			if (ret) {2648				btrfs_dec_block_group_ro(cache);2649				scrub_pause_off(fs_info);2650				btrfs_put_block_group(cache);2651				break;2652			}2653		}2654 2655		if (ret == 0) {2656			ro_set = 1;2657		} else if (ret == -ENOSPC && !sctx->is_dev_replace &&2658			   !(cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK)) {2659			/*2660			 * btrfs_inc_block_group_ro return -ENOSPC when it2661			 * failed in creating new chunk for metadata.2662			 * It is not a problem for scrub, because2663			 * metadata are always cowed, and our scrub paused2664			 * commit_transactions.2665			 *2666			 * For RAID56 chunks, we have to mark them read-only2667			 * for scrub, as later we would use our own cache2668			 * out of RAID56 realm.2669			 * Thus we want the RAID56 bg to be marked RO to2670			 * prevent RMW from screwing up out cache.2671			 */2672			ro_set = 0;2673		} else if (ret == -ETXTBSY) {2674			btrfs_warn(fs_info,2675		   "skipping scrub of block group %llu due to active swapfile",2676				   cache->start);2677			scrub_pause_off(fs_info);2678			ret = 0;2679			goto skip_unfreeze;2680		} else {2681			btrfs_warn(fs_info,2682				   "failed setting block group ro: %d", ret);2683			btrfs_unfreeze_block_group(cache);2684			btrfs_put_block_group(cache);2685			scrub_pause_off(fs_info);2686			break;2687		}2688 2689		/*2690		 * Now the target block is marked RO, wait for nocow writes to2691		 * finish before dev-replace.2692		 * COW is fine, as COW never overwrites extents in commit tree.2693		 */2694		if (sctx->is_dev_replace) {2695			btrfs_wait_nocow_writers(cache);2696			btrfs_wait_ordered_roots(fs_info, U64_MAX, cache);2697		}2698 2699		scrub_pause_off(fs_info);2700		down_write(&dev_replace->rwsem);2701		dev_replace->cursor_right = found_key.offset + dev_extent_len;2702		dev_replace->cursor_left = found_key.offset;2703		dev_replace->item_needs_writeback = 1;2704		up_write(&dev_replace->rwsem);2705 2706		ret = scrub_chunk(sctx, cache, scrub_dev, found_key.offset,2707				  dev_extent_len);2708		if (sctx->is_dev_replace &&2709		    !btrfs_finish_block_group_to_copy(dev_replace->srcdev,2710						      cache, found_key.offset))2711			ro_set = 0;2712 2713		down_write(&dev_replace->rwsem);2714		dev_replace->cursor_left = dev_replace->cursor_right;2715		dev_replace->item_needs_writeback = 1;2716		up_write(&dev_replace->rwsem);2717 2718		if (ro_set)2719			btrfs_dec_block_group_ro(cache);2720 2721		/*2722		 * We might have prevented the cleaner kthread from deleting2723		 * this block group if it was already unused because we raced2724		 * and set it to RO mode first. So add it back to the unused2725		 * list, otherwise it might not ever be deleted unless a manual2726		 * balance is triggered or it becomes used and unused again.2727		 */2728		spin_lock(&cache->lock);2729		if (!test_bit(BLOCK_GROUP_FLAG_REMOVED, &cache->runtime_flags) &&2730		    !cache->ro && cache->reserved == 0 && cache->used == 0) {2731			spin_unlock(&cache->lock);2732			if (btrfs_test_opt(fs_info, DISCARD_ASYNC))2733				btrfs_discard_queue_work(&fs_info->discard_ctl,2734							 cache);2735			else2736				btrfs_mark_bg_unused(cache);2737		} else {2738			spin_unlock(&cache->lock);2739		}2740skip_unfreeze:2741		btrfs_unfreeze_block_group(cache);2742		btrfs_put_block_group(cache);2743		if (ret)2744			break;2745		if (sctx->is_dev_replace &&2746		    atomic64_read(&dev_replace->num_write_errors) > 0) {2747			ret = -EIO;2748			break;2749		}2750		if (sctx->stat.malloc_errors > 0) {2751			ret = -ENOMEM;2752			break;2753		}2754skip:2755		key.offset = found_key.offset + dev_extent_len;2756		btrfs_release_path(path);2757	}2758 2759	btrfs_free_path(path);2760 2761	return ret;2762}2763 2764static int scrub_one_super(struct scrub_ctx *sctx, struct btrfs_device *dev,2765			   struct page *page, u64 physical, u64 generation)2766{2767	struct btrfs_fs_info *fs_info = sctx->fs_info;2768	struct bio_vec bvec;2769	struct bio bio;2770	struct btrfs_super_block *sb = page_address(page);2771	int ret;2772 2773	bio_init(&bio, dev->bdev, &bvec, 1, REQ_OP_READ);2774	bio.bi_iter.bi_sector = physical >> SECTOR_SHIFT;2775	__bio_add_page(&bio, page, BTRFS_SUPER_INFO_SIZE, 0);2776	ret = submit_bio_wait(&bio);2777	bio_uninit(&bio);2778 2779	if (ret < 0)2780		return ret;2781	ret = btrfs_check_super_csum(fs_info, sb);2782	if (ret != 0) {2783		btrfs_err_rl(fs_info,2784			"super block at physical %llu devid %llu has bad csum",2785			physical, dev->devid);2786		return -EIO;2787	}2788	if (btrfs_super_generation(sb) != generation) {2789		btrfs_err_rl(fs_info,2790"super block at physical %llu devid %llu has bad generation %llu expect %llu",2791			     physical, dev->devid,2792			     btrfs_super_generation(sb), generation);2793		return -EUCLEAN;2794	}2795 2796	return btrfs_validate_super(fs_info, sb, -1);2797}2798 2799static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,2800					   struct btrfs_device *scrub_dev)2801{2802	int	i;2803	u64	bytenr;2804	u64	gen;2805	int ret = 0;2806	struct page *page;2807	struct btrfs_fs_info *fs_info = sctx->fs_info;2808 2809	if (BTRFS_FS_ERROR(fs_info))2810		return -EROFS;2811 2812	page = alloc_page(GFP_KERNEL);2813	if (!page) {2814		spin_lock(&sctx->stat_lock);2815		sctx->stat.malloc_errors++;2816		spin_unlock(&sctx->stat_lock);2817		return -ENOMEM;2818	}2819 2820	/* Seed devices of a new filesystem has their own generation. */2821	if (scrub_dev->fs_devices != fs_info->fs_devices)2822		gen = scrub_dev->generation;2823	else2824		gen = btrfs_get_last_trans_committed(fs_info);2825 2826	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {2827		ret = btrfs_sb_log_location(scrub_dev, i, 0, &bytenr);2828		if (ret == -ENOENT)2829			break;2830 2831		if (ret) {2832			spin_lock(&sctx->stat_lock);2833			sctx->stat.super_errors++;2834			spin_unlock(&sctx->stat_lock);2835			continue;2836		}2837 2838		if (bytenr + BTRFS_SUPER_INFO_SIZE >2839		    scrub_dev->commit_total_bytes)2840			break;2841		if (!btrfs_check_super_location(scrub_dev, bytenr))2842			continue;2843 2844		ret = scrub_one_super(sctx, scrub_dev, page, bytenr, gen);2845		if (ret) {2846			spin_lock(&sctx->stat_lock);2847			sctx->stat.super_errors++;2848			spin_unlock(&sctx->stat_lock);2849		}2850	}2851	__free_page(page);2852	return 0;2853}2854 2855static void scrub_workers_put(struct btrfs_fs_info *fs_info)2856{2857	if (refcount_dec_and_mutex_lock(&fs_info->scrub_workers_refcnt,2858					&fs_info->scrub_lock)) {2859		struct workqueue_struct *scrub_workers = fs_info->scrub_workers;2860 2861		fs_info->scrub_workers = NULL;2862		mutex_unlock(&fs_info->scrub_lock);2863 2864		if (scrub_workers)2865			destroy_workqueue(scrub_workers);2866	}2867}2868 2869/*2870 * get a reference count on fs_info->scrub_workers. start worker if necessary2871 */2872static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info)2873{2874	struct workqueue_struct *scrub_workers = NULL;2875	unsigned int flags = WQ_FREEZABLE | WQ_UNBOUND;2876	int max_active = fs_info->thread_pool_size;2877	int ret = -ENOMEM;2878 2879	if (refcount_inc_not_zero(&fs_info->scrub_workers_refcnt))2880		return 0;2881 2882	scrub_workers = alloc_workqueue("btrfs-scrub", flags, max_active);2883	if (!scrub_workers)2884		return -ENOMEM;2885 2886	mutex_lock(&fs_info->scrub_lock);2887	if (refcount_read(&fs_info->scrub_workers_refcnt) == 0) {2888		ASSERT(fs_info->scrub_workers == NULL);2889		fs_info->scrub_workers = scrub_workers;2890		refcount_set(&fs_info->scrub_workers_refcnt, 1);2891		mutex_unlock(&fs_info->scrub_lock);2892		return 0;2893	}2894	/* Other thread raced in and created the workers for us */2895	refcount_inc(&fs_info->scrub_workers_refcnt);2896	mutex_unlock(&fs_info->scrub_lock);2897 2898	ret = 0;2899 2900	destroy_workqueue(scrub_workers);2901	return ret;2902}2903 2904int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,2905		    u64 end, struct btrfs_scrub_progress *progress,2906		    int readonly, int is_dev_replace)2907{2908	struct btrfs_dev_lookup_args args = { .devid = devid };2909	struct scrub_ctx *sctx;2910	int ret;2911	struct btrfs_device *dev;2912	unsigned int nofs_flag;2913	bool need_commit = false;2914 2915	if (btrfs_fs_closing(fs_info))2916		return -EAGAIN;2917 2918	/* At mount time we have ensured nodesize is in the range of [4K, 64K]. */2919	ASSERT(fs_info->nodesize <= BTRFS_STRIPE_LEN);2920 2921	/*2922	 * SCRUB_MAX_SECTORS_PER_BLOCK is calculated using the largest possible2923	 * value (max nodesize / min sectorsize), thus nodesize should always2924	 * be fine.2925	 */2926	ASSERT(fs_info->nodesize <=2927	       SCRUB_MAX_SECTORS_PER_BLOCK << fs_info->sectorsize_bits);2928 2929	/* Allocate outside of device_list_mutex */2930	sctx = scrub_setup_ctx(fs_info, is_dev_replace);2931	if (IS_ERR(sctx))2932		return PTR_ERR(sctx);2933 2934	ret = scrub_workers_get(fs_info);2935	if (ret)2936		goto out_free_ctx;2937 2938	mutex_lock(&fs_info->fs_devices->device_list_mutex);2939	dev = btrfs_find_device(fs_info->fs_devices, &args);2940	if (!dev || (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) &&2941		     !is_dev_replace)) {2942		mutex_unlock(&fs_info->fs_devices->device_list_mutex);2943		ret = -ENODEV;2944		goto out;2945	}2946 2947	if (!is_dev_replace && !readonly &&2948	    !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {2949		mutex_unlock(&fs_info->fs_devices->device_list_mutex);2950		btrfs_err_in_rcu(fs_info,2951			"scrub on devid %llu: filesystem on %s is not writable",2952				 devid, btrfs_dev_name(dev));2953		ret = -EROFS;2954		goto out;2955	}2956 2957	mutex_lock(&fs_info->scrub_lock);2958	if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||2959	    test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &dev->dev_state)) {2960		mutex_unlock(&fs_info->scrub_lock);2961		mutex_unlock(&fs_info->fs_devices->device_list_mutex);2962		ret = -EIO;2963		goto out;2964	}2965 2966	down_read(&fs_info->dev_replace.rwsem);2967	if (dev->scrub_ctx ||2968	    (!is_dev_replace &&2969	     btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {2970		up_read(&fs_info->dev_replace.rwsem);2971		mutex_unlock(&fs_info->scrub_lock);2972		mutex_unlock(&fs_info->fs_devices->device_list_mutex);2973		ret = -EINPROGRESS;2974		goto out;2975	}2976	up_read(&fs_info->dev_replace.rwsem);2977 2978	sctx->readonly = readonly;2979	dev->scrub_ctx = sctx;2980	mutex_unlock(&fs_info->fs_devices->device_list_mutex);2981 2982	/*2983	 * checking @scrub_pause_req here, we can avoid2984	 * race between committing transaction and scrubbing.2985	 */2986	__scrub_blocked_if_needed(fs_info);2987	atomic_inc(&fs_info->scrubs_running);2988	mutex_unlock(&fs_info->scrub_lock);2989 2990	/*2991	 * In order to avoid deadlock with reclaim when there is a transaction2992	 * trying to pause scrub, make sure we use GFP_NOFS for all the2993	 * allocations done at btrfs_scrub_sectors() and scrub_sectors_for_parity()2994	 * invoked by our callees. The pausing request is done when the2995	 * transaction commit starts, and it blocks the transaction until scrub2996	 * is paused (done at specific points at scrub_stripe() or right above2997	 * before incrementing fs_info->scrubs_running).2998	 */2999	nofs_flag = memalloc_nofs_save();3000	if (!is_dev_replace) {3001		u64 old_super_errors;3002 3003		spin_lock(&sctx->stat_lock);3004		old_super_errors = sctx->stat.super_errors;3005		spin_unlock(&sctx->stat_lock);3006 3007		btrfs_info(fs_info, "scrub: started on devid %llu", devid);3008		/*3009		 * by holding device list mutex, we can3010		 * kick off writing super in log tree sync.3011		 */3012		mutex_lock(&fs_info->fs_devices->device_list_mutex);3013		ret = scrub_supers(sctx, dev);3014		mutex_unlock(&fs_info->fs_devices->device_list_mutex);3015 3016		spin_lock(&sctx->stat_lock);3017		/*3018		 * Super block errors found, but we can not commit transaction3019		 * at current context, since btrfs_commit_transaction() needs3020		 * to pause the current running scrub (hold by ourselves).3021		 */3022		if (sctx->stat.super_errors > old_super_errors && !sctx->readonly)3023			need_commit = true;3024		spin_unlock(&sctx->stat_lock);3025	}3026 3027	if (!ret)3028		ret = scrub_enumerate_chunks(sctx, dev, start, end);3029	memalloc_nofs_restore(nofs_flag);3030 3031	atomic_dec(&fs_info->scrubs_running);3032	wake_up(&fs_info->scrub_pause_wait);3033 3034	if (progress)3035		memcpy(progress, &sctx->stat, sizeof(*progress));3036 3037	if (!is_dev_replace)3038		btrfs_info(fs_info, "scrub: %s on devid %llu with status: %d",3039			ret ? "not finished" : "finished", devid, ret);3040 3041	mutex_lock(&fs_info->scrub_lock);3042	dev->scrub_ctx = NULL;3043	mutex_unlock(&fs_info->scrub_lock);3044 3045	scrub_workers_put(fs_info);3046	scrub_put_ctx(sctx);3047 3048	/*3049	 * We found some super block errors before, now try to force a3050	 * transaction commit, as scrub has finished.3051	 */3052	if (need_commit) {3053		struct btrfs_trans_handle *trans;3054 3055		trans = btrfs_start_transaction(fs_info->tree_root, 0);3056		if (IS_ERR(trans)) {3057			ret = PTR_ERR(trans);3058			btrfs_err(fs_info,3059	"scrub: failed to start transaction to fix super block errors: %d", ret);3060			return ret;3061		}3062		ret = btrfs_commit_transaction(trans);3063		if (ret < 0)3064			btrfs_err(fs_info,3065	"scrub: failed to commit transaction to fix super block errors: %d", ret);3066	}3067	return ret;3068out:3069	scrub_workers_put(fs_info);3070out_free_ctx:3071	scrub_free_ctx(sctx);3072 3073	return ret;3074}3075 3076void btrfs_scrub_pause(struct btrfs_fs_info *fs_info)3077{3078	mutex_lock(&fs_info->scrub_lock);3079	atomic_inc(&fs_info->scrub_pause_req);3080	while (atomic_read(&fs_info->scrubs_paused) !=3081	       atomic_read(&fs_info->scrubs_running)) {3082		mutex_unlock(&fs_info->scrub_lock);3083		wait_event(fs_info->scrub_pause_wait,3084			   atomic_read(&fs_info->scrubs_paused) ==3085			   atomic_read(&fs_info->scrubs_running));3086		mutex_lock(&fs_info->scrub_lock);3087	}3088	mutex_unlock(&fs_info->scrub_lock);3089}3090 3091void btrfs_scrub_continue(struct btrfs_fs_info *fs_info)3092{3093	atomic_dec(&fs_info->scrub_pause_req);3094	wake_up(&fs_info->scrub_pause_wait);3095}3096 3097int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)3098{3099	mutex_lock(&fs_info->scrub_lock);3100	if (!atomic_read(&fs_info->scrubs_running)) {3101		mutex_unlock(&fs_info->scrub_lock);3102		return -ENOTCONN;3103	}3104 3105	atomic_inc(&fs_info->scrub_cancel_req);3106	while (atomic_read(&fs_info->scrubs_running)) {3107		mutex_unlock(&fs_info->scrub_lock);3108		wait_event(fs_info->scrub_pause_wait,3109			   atomic_read(&fs_info->scrubs_running) == 0);3110		mutex_lock(&fs_info->scrub_lock);3111	}3112	atomic_dec(&fs_info->scrub_cancel_req);3113	mutex_unlock(&fs_info->scrub_lock);3114 3115	return 0;3116}3117 3118int btrfs_scrub_cancel_dev(struct btrfs_device *dev)3119{3120	struct btrfs_fs_info *fs_info = dev->fs_info;3121	struct scrub_ctx *sctx;3122 3123	mutex_lock(&fs_info->scrub_lock);3124	sctx = dev->scrub_ctx;3125	if (!sctx) {3126		mutex_unlock(&fs_info->scrub_lock);3127		return -ENOTCONN;3128	}3129	atomic_inc(&sctx->cancel_req);3130	while (dev->scrub_ctx) {3131		mutex_unlock(&fs_info->scrub_lock);3132		wait_event(fs_info->scrub_pause_wait,3133			   dev->scrub_ctx == NULL);3134		mutex_lock(&fs_info->scrub_lock);3135	}3136	mutex_unlock(&fs_info->scrub_lock);3137 3138	return 0;3139}3140 3141int btrfs_scrub_progress(struct btrfs_fs_info *fs_info, u64 devid,3142			 struct btrfs_scrub_progress *progress)3143{3144	struct btrfs_dev_lookup_args args = { .devid = devid };3145	struct btrfs_device *dev;3146	struct scrub_ctx *sctx = NULL;3147 3148	mutex_lock(&fs_info->fs_devices->device_list_mutex);3149	dev = btrfs_find_device(fs_info->fs_devices, &args);3150	if (dev)3151		sctx = dev->scrub_ctx;3152	if (sctx)3153		memcpy(progress, &sctx->stat, sizeof(*progress));3154	mutex_unlock(&fs_info->fs_devices->device_list_mutex);3155 3156	return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;3157}3158