912 lines · c
1// SPDX-License-Identifier: GPL-2.02 3#include <linux/slab.h>4#include "messages.h"5#include "ctree.h"6#include "subpage.h"7#include "btrfs_inode.h"8 9/*10 * Subpage (sectorsize < PAGE_SIZE) support overview:11 *12 * Limitations:13 *14 * - Only support 64K page size for now15 * This is to make metadata handling easier, as 64K page would ensure16 * all nodesize would fit inside one page, thus we don't need to handle17 * cases where a tree block crosses several pages.18 *19 * - Only metadata read-write for now20 * The data read-write part is in development.21 *22 * - Metadata can't cross 64K page boundary23 * btrfs-progs and kernel have done that for a while, thus only ancient24 * filesystems could have such problem. For such case, do a graceful25 * rejection.26 *27 * Special behavior:28 *29 * - Metadata30 * Metadata read is fully supported.31 * Meaning when reading one tree block will only trigger the read for the32 * needed range, other unrelated range in the same page will not be touched.33 *34 * Metadata write support is partial.35 * The writeback is still for the full page, but we will only submit36 * the dirty extent buffers in the page.37 *38 * This means, if we have a metadata page like this:39 *40 * Page offset41 * 0 16K 32K 48K 64K42 * |/////////| |///////////|43 * \- Tree block A \- Tree block B44 *45 * Even if we just want to writeback tree block A, we will also writeback46 * tree block B if it's also dirty.47 *48 * This may cause extra metadata writeback which results more COW.49 *50 * Implementation:51 *52 * - Common53 * Both metadata and data will use a new structure, btrfs_subpage, to54 * record the status of each sector inside a page. This provides the extra55 * granularity needed.56 *57 * - Metadata58 * Since we have multiple tree blocks inside one page, we can't rely on page59 * locking anymore, or we will have greatly reduced concurrency or even60 * deadlocks (hold one tree lock while trying to lock another tree lock in61 * the same page).62 *63 * Thus for metadata locking, subpage support relies on io_tree locking only.64 * This means a slightly higher tree locking latency.65 */66 67#if PAGE_SIZE > SZ_4K68bool btrfs_is_subpage(const struct btrfs_fs_info *fs_info, struct address_space *mapping)69{70 if (fs_info->sectorsize >= PAGE_SIZE)71 return false;72 73 /*74 * Only data pages (either through DIO or compression) can have no75 * mapping. And if page->mapping->host is data inode, it's subpage.76 * As we have ruled our sectorsize >= PAGE_SIZE case already.77 */78 if (!mapping || !mapping->host || is_data_inode(BTRFS_I(mapping->host)))79 return true;80 81 /*82 * Now the only remaining case is metadata, which we only go subpage83 * routine if nodesize < PAGE_SIZE.84 */85 if (fs_info->nodesize < PAGE_SIZE)86 return true;87 return false;88}89#endif90 91int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info,92 struct folio *folio, enum btrfs_subpage_type type)93{94 struct btrfs_subpage *subpage;95 96 /*97 * We have cases like a dummy extent buffer page, which is not mapped98 * and doesn't need to be locked.99 */100 if (folio->mapping)101 ASSERT(folio_test_locked(folio));102 103 /* Either not subpage, or the folio already has private attached. */104 if (!btrfs_is_subpage(fs_info, folio->mapping) || folio_test_private(folio))105 return 0;106 107 subpage = btrfs_alloc_subpage(fs_info, type);108 if (IS_ERR(subpage))109 return PTR_ERR(subpage);110 111 folio_attach_private(folio, subpage);112 return 0;113}114 115void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info, struct folio *folio)116{117 struct btrfs_subpage *subpage;118 119 /* Either not subpage, or the folio already has private attached. */120 if (!btrfs_is_subpage(fs_info, folio->mapping) || !folio_test_private(folio))121 return;122 123 subpage = folio_detach_private(folio);124 ASSERT(subpage);125 btrfs_free_subpage(subpage);126}127 128struct btrfs_subpage *btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info,129 enum btrfs_subpage_type type)130{131 struct btrfs_subpage *ret;132 unsigned int real_size;133 134 ASSERT(fs_info->sectorsize < PAGE_SIZE);135 136 real_size = struct_size(ret, bitmaps,137 BITS_TO_LONGS(btrfs_bitmap_nr_max * fs_info->sectors_per_page));138 ret = kzalloc(real_size, GFP_NOFS);139 if (!ret)140 return ERR_PTR(-ENOMEM);141 142 spin_lock_init(&ret->lock);143 if (type == BTRFS_SUBPAGE_METADATA) {144 atomic_set(&ret->eb_refs, 0);145 } else {146 atomic_set(&ret->readers, 0);147 atomic_set(&ret->writers, 0);148 }149 return ret;150}151 152void btrfs_free_subpage(struct btrfs_subpage *subpage)153{154 kfree(subpage);155}156 157/*158 * Increase the eb_refs of current subpage.159 *160 * This is important for eb allocation, to prevent race with last eb freeing161 * of the same page.162 * With the eb_refs increased before the eb inserted into radix tree,163 * detach_extent_buffer_page() won't detach the folio private while we're still164 * allocating the extent buffer.165 */166void btrfs_folio_inc_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio)167{168 struct btrfs_subpage *subpage;169 170 if (!btrfs_is_subpage(fs_info, folio->mapping))171 return;172 173 ASSERT(folio_test_private(folio) && folio->mapping);174 lockdep_assert_held(&folio->mapping->i_private_lock);175 176 subpage = folio_get_private(folio);177 atomic_inc(&subpage->eb_refs);178}179 180void btrfs_folio_dec_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio)181{182 struct btrfs_subpage *subpage;183 184 if (!btrfs_is_subpage(fs_info, folio->mapping))185 return;186 187 ASSERT(folio_test_private(folio) && folio->mapping);188 lockdep_assert_held(&folio->mapping->i_private_lock);189 190 subpage = folio_get_private(folio);191 ASSERT(atomic_read(&subpage->eb_refs));192 atomic_dec(&subpage->eb_refs);193}194 195static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info,196 struct folio *folio, u64 start, u32 len)197{198 /* For subpage support, the folio must be single page. */199 ASSERT(folio_order(folio) == 0);200 201 /* Basic checks */202 ASSERT(folio_test_private(folio) && folio_get_private(folio));203 ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&204 IS_ALIGNED(len, fs_info->sectorsize));205 /*206 * The range check only works for mapped page, we can still have207 * unmapped page like dummy extent buffer pages.208 */209 if (folio->mapping)210 ASSERT(folio_pos(folio) <= start &&211 start + len <= folio_pos(folio) + PAGE_SIZE);212}213 214#define subpage_calc_start_bit(fs_info, folio, name, start, len) \215({ \216 unsigned int __start_bit; \217 \218 btrfs_subpage_assert(fs_info, folio, start, len); \219 __start_bit = offset_in_page(start) >> fs_info->sectorsize_bits; \220 __start_bit += fs_info->sectors_per_page * btrfs_bitmap_nr_##name; \221 __start_bit; \222})223 224void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info,225 struct folio *folio, u64 start, u32 len)226{227 struct btrfs_subpage *subpage = folio_get_private(folio);228 const int start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len);229 const int nbits = len >> fs_info->sectorsize_bits;230 unsigned long flags;231 232 233 btrfs_subpage_assert(fs_info, folio, start, len);234 235 spin_lock_irqsave(&subpage->lock, flags);236 /*237 * Even though it's just for reading the page, no one should have238 * locked the subpage range.239 */240 ASSERT(bitmap_test_range_all_zero(subpage->bitmaps, start_bit, nbits));241 bitmap_set(subpage->bitmaps, start_bit, nbits);242 atomic_add(nbits, &subpage->readers);243 spin_unlock_irqrestore(&subpage->lock, flags);244}245 246void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info,247 struct folio *folio, u64 start, u32 len)248{249 struct btrfs_subpage *subpage = folio_get_private(folio);250 const int start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len);251 const int nbits = len >> fs_info->sectorsize_bits;252 unsigned long flags;253 bool is_data;254 bool last;255 256 btrfs_subpage_assert(fs_info, folio, start, len);257 is_data = is_data_inode(BTRFS_I(folio->mapping->host));258 259 spin_lock_irqsave(&subpage->lock, flags);260 261 /* The range should have already been locked. */262 ASSERT(bitmap_test_range_all_set(subpage->bitmaps, start_bit, nbits));263 ASSERT(atomic_read(&subpage->readers) >= nbits);264 265 bitmap_clear(subpage->bitmaps, start_bit, nbits);266 last = atomic_sub_and_test(nbits, &subpage->readers);267 268 /*269 * For data we need to unlock the page if the last read has finished.270 *271 * And please don't replace @last with atomic_sub_and_test() call272 * inside if () condition.273 * As we want the atomic_sub_and_test() to be always executed.274 */275 if (is_data && last)276 folio_unlock(folio);277 spin_unlock_irqrestore(&subpage->lock, flags);278}279 280static void btrfs_subpage_clamp_range(struct folio *folio, u64 *start, u32 *len)281{282 u64 orig_start = *start;283 u32 orig_len = *len;284 285 *start = max_t(u64, folio_pos(folio), orig_start);286 /*287 * For certain call sites like btrfs_drop_pages(), we may have pages288 * beyond the target range. In that case, just set @len to 0, subpage289 * helpers can handle @len == 0 without any problem.290 */291 if (folio_pos(folio) >= orig_start + orig_len)292 *len = 0;293 else294 *len = min_t(u64, folio_pos(folio) + PAGE_SIZE,295 orig_start + orig_len) - *start;296}297 298static void btrfs_subpage_start_writer(const struct btrfs_fs_info *fs_info,299 struct folio *folio, u64 start, u32 len)300{301 struct btrfs_subpage *subpage = folio_get_private(folio);302 const int start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len);303 const int nbits = (len >> fs_info->sectorsize_bits);304 unsigned long flags;305 int ret;306 307 btrfs_subpage_assert(fs_info, folio, start, len);308 309 spin_lock_irqsave(&subpage->lock, flags);310 ASSERT(atomic_read(&subpage->readers) == 0);311 ASSERT(bitmap_test_range_all_zero(subpage->bitmaps, start_bit, nbits));312 bitmap_set(subpage->bitmaps, start_bit, nbits);313 ret = atomic_add_return(nbits, &subpage->writers);314 ASSERT(ret == nbits);315 spin_unlock_irqrestore(&subpage->lock, flags);316}317 318static bool btrfs_subpage_end_and_test_writer(const struct btrfs_fs_info *fs_info,319 struct folio *folio, u64 start, u32 len)320{321 struct btrfs_subpage *subpage = folio_get_private(folio);322 const int start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len);323 const int nbits = (len >> fs_info->sectorsize_bits);324 unsigned long flags;325 unsigned int cleared = 0;326 int bit = start_bit;327 bool last;328 329 btrfs_subpage_assert(fs_info, folio, start, len);330 331 spin_lock_irqsave(&subpage->lock, flags);332 /*333 * We have call sites passing @lock_page into334 * extent_clear_unlock_delalloc() for compression path.335 *336 * This @locked_page is locked by plain lock_page(), thus its337 * subpage::writers is 0. Handle them in a special way.338 */339 if (atomic_read(&subpage->writers) == 0) {340 spin_unlock_irqrestore(&subpage->lock, flags);341 return true;342 }343 344 for_each_set_bit_from(bit, subpage->bitmaps, start_bit + nbits) {345 clear_bit(bit, subpage->bitmaps);346 cleared++;347 }348 ASSERT(atomic_read(&subpage->writers) >= cleared);349 last = atomic_sub_and_test(cleared, &subpage->writers);350 spin_unlock_irqrestore(&subpage->lock, flags);351 return last;352}353 354/*355 * Lock a folio for delalloc page writeback.356 *357 * Return -EAGAIN if the page is not properly initialized.358 * Return 0 with the page locked, and writer counter updated.359 *360 * Even with 0 returned, the page still need extra check to make sure361 * it's really the correct page, as the caller is using362 * filemap_get_folios_contig(), which can race with page invalidating.363 */364int btrfs_folio_start_writer_lock(const struct btrfs_fs_info *fs_info,365 struct folio *folio, u64 start, u32 len)366{367 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio->mapping)) {368 folio_lock(folio);369 return 0;370 }371 folio_lock(folio);372 if (!folio_test_private(folio) || !folio_get_private(folio)) {373 folio_unlock(folio);374 return -EAGAIN;375 }376 btrfs_subpage_clamp_range(folio, &start, &len);377 btrfs_subpage_start_writer(fs_info, folio, start, len);378 return 0;379}380 381/*382 * Handle different locked folios:383 *384 * - Non-subpage folio385 * Just unlock it.386 *387 * - folio locked but without any subpage locked388 * This happens either before writepage_delalloc() or the delalloc range is389 * already handled by previous folio.390 * We can simple unlock it.391 *392 * - folio locked with subpage range locked.393 * We go through the locked sectors inside the range and clear their locked394 * bitmap, reduce the writer lock number, and unlock the page if that's395 * the last locked range.396 */397void btrfs_folio_end_writer_lock(const struct btrfs_fs_info *fs_info,398 struct folio *folio, u64 start, u32 len)399{400 struct btrfs_subpage *subpage = folio_get_private(folio);401 402 ASSERT(folio_test_locked(folio));403 404 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio->mapping)) {405 folio_unlock(folio);406 return;407 }408 409 /*410 * For subpage case, there are two types of locked page. With or411 * without writers number.412 *413 * Since we own the page lock, no one else could touch subpage::writers414 * and we are safe to do several atomic operations without spinlock.415 */416 if (atomic_read(&subpage->writers) == 0) {417 /* No writers, locked by plain lock_page(). */418 folio_unlock(folio);419 return;420 }421 422 btrfs_subpage_clamp_range(folio, &start, &len);423 if (btrfs_subpage_end_and_test_writer(fs_info, folio, start, len))424 folio_unlock(folio);425}426 427void btrfs_folio_end_writer_lock_bitmap(const struct btrfs_fs_info *fs_info,428 struct folio *folio, unsigned long bitmap)429{430 struct btrfs_subpage *subpage = folio_get_private(folio);431 const int start_bit = fs_info->sectors_per_page * btrfs_bitmap_nr_locked;432 unsigned long flags;433 bool last = false;434 int cleared = 0;435 int bit;436 437 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio->mapping)) {438 folio_unlock(folio);439 return;440 }441 442 if (atomic_read(&subpage->writers) == 0) {443 /* No writers, locked by plain lock_page(). */444 folio_unlock(folio);445 return;446 }447 448 spin_lock_irqsave(&subpage->lock, flags);449 for_each_set_bit(bit, &bitmap, fs_info->sectors_per_page) {450 if (test_and_clear_bit(bit + start_bit, subpage->bitmaps))451 cleared++;452 }453 ASSERT(atomic_read(&subpage->writers) >= cleared);454 last = atomic_sub_and_test(cleared, &subpage->writers);455 spin_unlock_irqrestore(&subpage->lock, flags);456 if (last)457 folio_unlock(folio);458}459 460#define subpage_test_bitmap_all_set(fs_info, subpage, name) \461 bitmap_test_range_all_set(subpage->bitmaps, \462 fs_info->sectors_per_page * btrfs_bitmap_nr_##name, \463 fs_info->sectors_per_page)464 465#define subpage_test_bitmap_all_zero(fs_info, subpage, name) \466 bitmap_test_range_all_zero(subpage->bitmaps, \467 fs_info->sectors_per_page * btrfs_bitmap_nr_##name, \468 fs_info->sectors_per_page)469 470void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info,471 struct folio *folio, u64 start, u32 len)472{473 struct btrfs_subpage *subpage = folio_get_private(folio);474 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,475 uptodate, start, len);476 unsigned long flags;477 478 spin_lock_irqsave(&subpage->lock, flags);479 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);480 if (subpage_test_bitmap_all_set(fs_info, subpage, uptodate))481 folio_mark_uptodate(folio);482 spin_unlock_irqrestore(&subpage->lock, flags);483}484 485void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info,486 struct folio *folio, u64 start, u32 len)487{488 struct btrfs_subpage *subpage = folio_get_private(folio);489 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,490 uptodate, start, len);491 unsigned long flags;492 493 spin_lock_irqsave(&subpage->lock, flags);494 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);495 folio_clear_uptodate(folio);496 spin_unlock_irqrestore(&subpage->lock, flags);497}498 499void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info,500 struct folio *folio, u64 start, u32 len)501{502 struct btrfs_subpage *subpage = folio_get_private(folio);503 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,504 dirty, start, len);505 unsigned long flags;506 507 spin_lock_irqsave(&subpage->lock, flags);508 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);509 spin_unlock_irqrestore(&subpage->lock, flags);510 folio_mark_dirty(folio);511}512 513/*514 * Extra clear_and_test function for subpage dirty bitmap.515 *516 * Return true if we're the last bits in the dirty_bitmap and clear the517 * dirty_bitmap.518 * Return false otherwise.519 *520 * NOTE: Callers should manually clear page dirty for true case, as we have521 * extra handling for tree blocks.522 */523bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info,524 struct folio *folio, u64 start, u32 len)525{526 struct btrfs_subpage *subpage = folio_get_private(folio);527 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,528 dirty, start, len);529 unsigned long flags;530 bool last = false;531 532 spin_lock_irqsave(&subpage->lock, flags);533 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);534 if (subpage_test_bitmap_all_zero(fs_info, subpage, dirty))535 last = true;536 spin_unlock_irqrestore(&subpage->lock, flags);537 return last;538}539 540void btrfs_subpage_clear_dirty(const struct btrfs_fs_info *fs_info,541 struct folio *folio, u64 start, u32 len)542{543 bool last;544 545 last = btrfs_subpage_clear_and_test_dirty(fs_info, folio, start, len);546 if (last)547 folio_clear_dirty_for_io(folio);548}549 550void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info,551 struct folio *folio, u64 start, u32 len)552{553 struct btrfs_subpage *subpage = folio_get_private(folio);554 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,555 writeback, start, len);556 unsigned long flags;557 558 spin_lock_irqsave(&subpage->lock, flags);559 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);560 if (!folio_test_writeback(folio))561 folio_start_writeback(folio);562 spin_unlock_irqrestore(&subpage->lock, flags);563}564 565void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info,566 struct folio *folio, u64 start, u32 len)567{568 struct btrfs_subpage *subpage = folio_get_private(folio);569 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,570 writeback, start, len);571 unsigned long flags;572 573 spin_lock_irqsave(&subpage->lock, flags);574 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);575 if (subpage_test_bitmap_all_zero(fs_info, subpage, writeback)) {576 ASSERT(folio_test_writeback(folio));577 folio_end_writeback(folio);578 }579 spin_unlock_irqrestore(&subpage->lock, flags);580}581 582void btrfs_subpage_set_ordered(const struct btrfs_fs_info *fs_info,583 struct folio *folio, u64 start, u32 len)584{585 struct btrfs_subpage *subpage = folio_get_private(folio);586 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,587 ordered, start, len);588 unsigned long flags;589 590 spin_lock_irqsave(&subpage->lock, flags);591 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);592 folio_set_ordered(folio);593 spin_unlock_irqrestore(&subpage->lock, flags);594}595 596void btrfs_subpage_clear_ordered(const struct btrfs_fs_info *fs_info,597 struct folio *folio, u64 start, u32 len)598{599 struct btrfs_subpage *subpage = folio_get_private(folio);600 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,601 ordered, start, len);602 unsigned long flags;603 604 spin_lock_irqsave(&subpage->lock, flags);605 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);606 if (subpage_test_bitmap_all_zero(fs_info, subpage, ordered))607 folio_clear_ordered(folio);608 spin_unlock_irqrestore(&subpage->lock, flags);609}610 611void btrfs_subpage_set_checked(const struct btrfs_fs_info *fs_info,612 struct folio *folio, u64 start, u32 len)613{614 struct btrfs_subpage *subpage = folio_get_private(folio);615 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,616 checked, start, len);617 unsigned long flags;618 619 spin_lock_irqsave(&subpage->lock, flags);620 bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);621 if (subpage_test_bitmap_all_set(fs_info, subpage, checked))622 folio_set_checked(folio);623 spin_unlock_irqrestore(&subpage->lock, flags);624}625 626void btrfs_subpage_clear_checked(const struct btrfs_fs_info *fs_info,627 struct folio *folio, u64 start, u32 len)628{629 struct btrfs_subpage *subpage = folio_get_private(folio);630 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,631 checked, start, len);632 unsigned long flags;633 634 spin_lock_irqsave(&subpage->lock, flags);635 bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);636 folio_clear_checked(folio);637 spin_unlock_irqrestore(&subpage->lock, flags);638}639 640/*641 * Unlike set/clear which is dependent on each page status, for test all bits642 * are tested in the same way.643 */644#define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name) \645bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \646 struct folio *folio, u64 start, u32 len) \647{ \648 struct btrfs_subpage *subpage = folio_get_private(folio); \649 unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, \650 name, start, len); \651 unsigned long flags; \652 bool ret; \653 \654 spin_lock_irqsave(&subpage->lock, flags); \655 ret = bitmap_test_range_all_set(subpage->bitmaps, start_bit, \656 len >> fs_info->sectorsize_bits); \657 spin_unlock_irqrestore(&subpage->lock, flags); \658 return ret; \659}660IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate);661IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty);662IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback);663IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(ordered);664IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(checked);665 666/*667 * Note that, in selftests (extent-io-tests), we can have empty fs_info passed668 * in. We only test sectorsize == PAGE_SIZE cases so far, thus we can fall669 * back to regular sectorsize branch.670 */671#define IMPLEMENT_BTRFS_PAGE_OPS(name, folio_set_func, \672 folio_clear_func, folio_test_func) \673void btrfs_folio_set_##name(const struct btrfs_fs_info *fs_info, \674 struct folio *folio, u64 start, u32 len) \675{ \676 if (unlikely(!fs_info) || \677 !btrfs_is_subpage(fs_info, folio->mapping)) { \678 folio_set_func(folio); \679 return; \680 } \681 btrfs_subpage_set_##name(fs_info, folio, start, len); \682} \683void btrfs_folio_clear_##name(const struct btrfs_fs_info *fs_info, \684 struct folio *folio, u64 start, u32 len) \685{ \686 if (unlikely(!fs_info) || \687 !btrfs_is_subpage(fs_info, folio->mapping)) { \688 folio_clear_func(folio); \689 return; \690 } \691 btrfs_subpage_clear_##name(fs_info, folio, start, len); \692} \693bool btrfs_folio_test_##name(const struct btrfs_fs_info *fs_info, \694 struct folio *folio, u64 start, u32 len) \695{ \696 if (unlikely(!fs_info) || \697 !btrfs_is_subpage(fs_info, folio->mapping)) \698 return folio_test_func(folio); \699 return btrfs_subpage_test_##name(fs_info, folio, start, len); \700} \701void btrfs_folio_clamp_set_##name(const struct btrfs_fs_info *fs_info, \702 struct folio *folio, u64 start, u32 len) \703{ \704 if (unlikely(!fs_info) || \705 !btrfs_is_subpage(fs_info, folio->mapping)) { \706 folio_set_func(folio); \707 return; \708 } \709 btrfs_subpage_clamp_range(folio, &start, &len); \710 btrfs_subpage_set_##name(fs_info, folio, start, len); \711} \712void btrfs_folio_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \713 struct folio *folio, u64 start, u32 len) \714{ \715 if (unlikely(!fs_info) || \716 !btrfs_is_subpage(fs_info, folio->mapping)) { \717 folio_clear_func(folio); \718 return; \719 } \720 btrfs_subpage_clamp_range(folio, &start, &len); \721 btrfs_subpage_clear_##name(fs_info, folio, start, len); \722} \723bool btrfs_folio_clamp_test_##name(const struct btrfs_fs_info *fs_info, \724 struct folio *folio, u64 start, u32 len) \725{ \726 if (unlikely(!fs_info) || \727 !btrfs_is_subpage(fs_info, folio->mapping)) \728 return folio_test_func(folio); \729 btrfs_subpage_clamp_range(folio, &start, &len); \730 return btrfs_subpage_test_##name(fs_info, folio, start, len); \731}732IMPLEMENT_BTRFS_PAGE_OPS(uptodate, folio_mark_uptodate, folio_clear_uptodate,733 folio_test_uptodate);734IMPLEMENT_BTRFS_PAGE_OPS(dirty, folio_mark_dirty, folio_clear_dirty_for_io,735 folio_test_dirty);736IMPLEMENT_BTRFS_PAGE_OPS(writeback, folio_start_writeback, folio_end_writeback,737 folio_test_writeback);738IMPLEMENT_BTRFS_PAGE_OPS(ordered, folio_set_ordered, folio_clear_ordered,739 folio_test_ordered);740IMPLEMENT_BTRFS_PAGE_OPS(checked, folio_set_checked, folio_clear_checked,741 folio_test_checked);742 743/*744 * Make sure not only the page dirty bit is cleared, but also subpage dirty bit745 * is cleared.746 */747void btrfs_folio_assert_not_dirty(const struct btrfs_fs_info *fs_info,748 struct folio *folio, u64 start, u32 len)749{750 struct btrfs_subpage *subpage;751 unsigned int start_bit;752 unsigned int nbits;753 unsigned long flags;754 755 if (!IS_ENABLED(CONFIG_BTRFS_ASSERT))756 return;757 758 if (!btrfs_is_subpage(fs_info, folio->mapping)) {759 ASSERT(!folio_test_dirty(folio));760 return;761 }762 763 start_bit = subpage_calc_start_bit(fs_info, folio, dirty, start, len);764 nbits = len >> fs_info->sectorsize_bits;765 subpage = folio_get_private(folio);766 ASSERT(subpage);767 spin_lock_irqsave(&subpage->lock, flags);768 ASSERT(bitmap_test_range_all_zero(subpage->bitmaps, start_bit, nbits));769 spin_unlock_irqrestore(&subpage->lock, flags);770}771 772/*773 * This is for folio already locked by plain lock_page()/folio_lock(), which774 * doesn't have any subpage awareness.775 *776 * This populates the involved subpage ranges so that subpage helpers can777 * properly unlock them.778 */779void btrfs_folio_set_writer_lock(const struct btrfs_fs_info *fs_info,780 struct folio *folio, u64 start, u32 len)781{782 struct btrfs_subpage *subpage;783 unsigned long flags;784 unsigned int start_bit;785 unsigned int nbits;786 int ret;787 788 ASSERT(folio_test_locked(folio));789 if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio->mapping))790 return;791 792 subpage = folio_get_private(folio);793 start_bit = subpage_calc_start_bit(fs_info, folio, locked, start, len);794 nbits = len >> fs_info->sectorsize_bits;795 spin_lock_irqsave(&subpage->lock, flags);796 /* Target range should not yet be locked. */797 ASSERT(bitmap_test_range_all_zero(subpage->bitmaps, start_bit, nbits));798 bitmap_set(subpage->bitmaps, start_bit, nbits);799 ret = atomic_add_return(nbits, &subpage->writers);800 ASSERT(ret <= fs_info->sectors_per_page);801 spin_unlock_irqrestore(&subpage->lock, flags);802}803 804/*805 * Find any subpage writer locked range inside @folio, starting at file offset806 * @search_start. The caller should ensure the folio is locked.807 *808 * Return true and update @found_start_ret and @found_len_ret to the first809 * writer locked range.810 * Return false if there is no writer locked range.811 */812bool btrfs_subpage_find_writer_locked(const struct btrfs_fs_info *fs_info,813 struct folio *folio, u64 search_start,814 u64 *found_start_ret, u32 *found_len_ret)815{816 struct btrfs_subpage *subpage = folio_get_private(folio);817 const u32 sectors_per_page = fs_info->sectors_per_page;818 const unsigned int len = PAGE_SIZE - offset_in_page(search_start);819 const unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,820 locked, search_start, len);821 const unsigned int locked_bitmap_start = sectors_per_page * btrfs_bitmap_nr_locked;822 const unsigned int locked_bitmap_end = locked_bitmap_start + sectors_per_page;823 unsigned long flags;824 int first_zero;825 int first_set;826 bool found = false;827 828 ASSERT(folio_test_locked(folio));829 spin_lock_irqsave(&subpage->lock, flags);830 first_set = find_next_bit(subpage->bitmaps, locked_bitmap_end, start_bit);831 if (first_set >= locked_bitmap_end)832 goto out;833 834 found = true;835 836 *found_start_ret = folio_pos(folio) +837 ((first_set - locked_bitmap_start) << fs_info->sectorsize_bits);838 /*839 * Since @first_set is ensured to be smaller than locked_bitmap_end840 * here, @found_start_ret should be inside the folio.841 */842 ASSERT(*found_start_ret < folio_pos(folio) + PAGE_SIZE);843 844 first_zero = find_next_zero_bit(subpage->bitmaps, locked_bitmap_end, first_set);845 *found_len_ret = (first_zero - first_set) << fs_info->sectorsize_bits;846out:847 spin_unlock_irqrestore(&subpage->lock, flags);848 return found;849}850 851#define GET_SUBPAGE_BITMAP(subpage, fs_info, name, dst) \852{ \853 const int sectors_per_page = fs_info->sectors_per_page; \854 \855 ASSERT(sectors_per_page < BITS_PER_LONG); \856 *dst = bitmap_read(subpage->bitmaps, \857 sectors_per_page * btrfs_bitmap_nr_##name, \858 sectors_per_page); \859}860 861void __cold btrfs_subpage_dump_bitmap(const struct btrfs_fs_info *fs_info,862 struct folio *folio, u64 start, u32 len)863{864 struct btrfs_subpage *subpage;865 const u32 sectors_per_page = fs_info->sectors_per_page;866 unsigned long uptodate_bitmap;867 unsigned long dirty_bitmap;868 unsigned long writeback_bitmap;869 unsigned long ordered_bitmap;870 unsigned long checked_bitmap;871 unsigned long flags;872 873 ASSERT(folio_test_private(folio) && folio_get_private(folio));874 ASSERT(sectors_per_page > 1);875 subpage = folio_get_private(folio);876 877 spin_lock_irqsave(&subpage->lock, flags);878 GET_SUBPAGE_BITMAP(subpage, fs_info, uptodate, &uptodate_bitmap);879 GET_SUBPAGE_BITMAP(subpage, fs_info, dirty, &dirty_bitmap);880 GET_SUBPAGE_BITMAP(subpage, fs_info, writeback, &writeback_bitmap);881 GET_SUBPAGE_BITMAP(subpage, fs_info, ordered, &ordered_bitmap);882 GET_SUBPAGE_BITMAP(subpage, fs_info, checked, &checked_bitmap);883 GET_SUBPAGE_BITMAP(subpage, fs_info, locked, &checked_bitmap);884 spin_unlock_irqrestore(&subpage->lock, flags);885 886 dump_page(folio_page(folio, 0), "btrfs subpage dump");887 btrfs_warn(fs_info,888"start=%llu len=%u page=%llu, bitmaps uptodate=%*pbl dirty=%*pbl writeback=%*pbl ordered=%*pbl checked=%*pbl",889 start, len, folio_pos(folio),890 sectors_per_page, &uptodate_bitmap,891 sectors_per_page, &dirty_bitmap,892 sectors_per_page, &writeback_bitmap,893 sectors_per_page, &ordered_bitmap,894 sectors_per_page, &checked_bitmap);895}896 897void btrfs_get_subpage_dirty_bitmap(struct btrfs_fs_info *fs_info,898 struct folio *folio,899 unsigned long *ret_bitmap)900{901 struct btrfs_subpage *subpage;902 unsigned long flags;903 904 ASSERT(folio_test_private(folio) && folio_get_private(folio));905 ASSERT(fs_info->sectors_per_page > 1);906 subpage = folio_get_private(folio);907 908 spin_lock_irqsave(&subpage->lock, flags);909 GET_SUBPAGE_BITMAP(subpage, fs_info, dirty, ret_bitmap);910 spin_unlock_irqrestore(&subpage->lock, flags);911}912