2087 lines · c
1// SPDX-License-Identifier: GPL-2.02 3#include "linux/spinlock.h"4#include <linux/minmax.h>5#include "misc.h"6#include "ctree.h"7#include "space-info.h"8#include "sysfs.h"9#include "volumes.h"10#include "free-space-cache.h"11#include "ordered-data.h"12#include "transaction.h"13#include "block-group.h"14#include "fs.h"15#include "accessors.h"16#include "extent-tree.h"17 18/*19 * HOW DOES SPACE RESERVATION WORK20 *21 * If you want to know about delalloc specifically, there is a separate comment22 * for that with the delalloc code. This comment is about how the whole system23 * works generally.24 *25 * BASIC CONCEPTS26 *27 * 1) space_info. This is the ultimate arbiter of how much space we can use.28 * There's a description of the bytes_ fields with the struct declaration,29 * refer to that for specifics on each field. Suffice it to say that for30 * reservations we care about total_bytes - SUM(space_info->bytes_) when31 * determining if there is space to make an allocation. There is a space_info32 * for METADATA, SYSTEM, and DATA areas.33 *34 * 2) block_rsv's. These are basically buckets for every different type of35 * metadata reservation we have. You can see the comment in the block_rsv36 * code on the rules for each type, but generally block_rsv->reserved is how37 * much space is accounted for in space_info->bytes_may_use.38 *39 * 3) btrfs_calc*_size. These are the worst case calculations we used based40 * on the number of items we will want to modify. We have one for changing41 * items, and one for inserting new items. Generally we use these helpers to42 * determine the size of the block reserves, and then use the actual bytes43 * values to adjust the space_info counters.44 *45 * MAKING RESERVATIONS, THE NORMAL CASE46 *47 * We call into either btrfs_reserve_data_bytes() or48 * btrfs_reserve_metadata_bytes(), depending on which we're looking for, with49 * num_bytes we want to reserve.50 *51 * ->reserve52 * space_info->bytes_may_reserve += num_bytes53 *54 * ->extent allocation55 * Call btrfs_add_reserved_bytes() which does56 * space_info->bytes_may_reserve -= num_bytes57 * space_info->bytes_reserved += extent_bytes58 *59 * ->insert reference60 * Call btrfs_update_block_group() which does61 * space_info->bytes_reserved -= extent_bytes62 * space_info->bytes_used += extent_bytes63 *64 * MAKING RESERVATIONS, FLUSHING NORMALLY (non-priority)65 *66 * Assume we are unable to simply make the reservation because we do not have67 * enough space68 *69 * -> __reserve_bytes70 * create a reserve_ticket with ->bytes set to our reservation, add it to71 * the tail of space_info->tickets, kick async flush thread72 *73 * ->handle_reserve_ticket74 * wait on ticket->wait for ->bytes to be reduced to 0, or ->error to be set75 * on the ticket.76 *77 * -> btrfs_async_reclaim_metadata_space/btrfs_async_reclaim_data_space78 * Flushes various things attempting to free up space.79 *80 * -> btrfs_try_granting_tickets()81 * This is called by anything that either subtracts space from82 * space_info->bytes_may_use, ->bytes_pinned, etc, or adds to the83 * space_info->total_bytes. This loops through the ->priority_tickets and84 * then the ->tickets list checking to see if the reservation can be85 * completed. If it can the space is added to space_info->bytes_may_use and86 * the ticket is woken up.87 *88 * -> ticket wakeup89 * Check if ->bytes == 0, if it does we got our reservation and we can carry90 * on, if not return the appropriate error (ENOSPC, but can be EINTR if we91 * were interrupted.)92 *93 * MAKING RESERVATIONS, FLUSHING HIGH PRIORITY94 *95 * Same as the above, except we add ourselves to the96 * space_info->priority_tickets, and we do not use ticket->wait, we simply97 * call flush_space() ourselves for the states that are safe for us to call98 * without deadlocking and hope for the best.99 *100 * THE FLUSHING STATES101 *102 * Generally speaking we will have two cases for each state, a "nice" state103 * and a "ALL THE THINGS" state. In btrfs we delay a lot of work in order to104 * reduce the locking over head on the various trees, and even to keep from105 * doing any work at all in the case of delayed refs. Each of these delayed106 * things however hold reservations, and so letting them run allows us to107 * reclaim space so we can make new reservations.108 *109 * FLUSH_DELAYED_ITEMS110 * Every inode has a delayed item to update the inode. Take a simple write111 * for example, we would update the inode item at write time to update the112 * mtime, and then again at finish_ordered_io() time in order to update the113 * isize or bytes. We keep these delayed items to coalesce these operations114 * into a single operation done on demand. These are an easy way to reclaim115 * metadata space.116 *117 * FLUSH_DELALLOC118 * Look at the delalloc comment to get an idea of how much space is reserved119 * for delayed allocation. We can reclaim some of this space simply by120 * running delalloc, but usually we need to wait for ordered extents to121 * reclaim the bulk of this space.122 *123 * FLUSH_DELAYED_REFS124 * We have a block reserve for the outstanding delayed refs space, and every125 * delayed ref operation holds a reservation. Running these is a quick way126 * to reclaim space, but we want to hold this until the end because COW can127 * churn a lot and we can avoid making some extent tree modifications if we128 * are able to delay for as long as possible.129 *130 * ALLOC_CHUNK131 * We will skip this the first time through space reservation, because of132 * overcommit and we don't want to have a lot of useless metadata space when133 * our worst case reservations will likely never come true.134 *135 * RUN_DELAYED_IPUTS136 * If we're freeing inodes we're likely freeing checksums, file extent137 * items, and extent tree items. Loads of space could be freed up by these138 * operations, however they won't be usable until the transaction commits.139 *140 * COMMIT_TRANS141 * This will commit the transaction. Historically we had a lot of logic142 * surrounding whether or not we'd commit the transaction, but this waits born143 * out of a pre-tickets era where we could end up committing the transaction144 * thousands of times in a row without making progress. Now thanks to our145 * ticketing system we know if we're not making progress and can error146 * everybody out after a few commits rather than burning the disk hoping for147 * a different answer.148 *149 * OVERCOMMIT150 *151 * Because we hold so many reservations for metadata we will allow you to152 * reserve more space than is currently free in the currently allocate153 * metadata space. This only happens with metadata, data does not allow154 * overcommitting.155 *156 * You can see the current logic for when we allow overcommit in157 * btrfs_can_overcommit(), but it only applies to unallocated space. If there158 * is no unallocated space to be had, all reservations are kept within the159 * free space in the allocated metadata chunks.160 *161 * Because of overcommitting, you generally want to use the162 * btrfs_can_overcommit() logic for metadata allocations, as it does the right163 * thing with or without extra unallocated space.164 */165 166u64 __pure btrfs_space_info_used(const struct btrfs_space_info *s_info,167 bool may_use_included)168{169 ASSERT(s_info);170 return s_info->bytes_used + s_info->bytes_reserved +171 s_info->bytes_pinned + s_info->bytes_readonly +172 s_info->bytes_zone_unusable +173 (may_use_included ? s_info->bytes_may_use : 0);174}175 176/*177 * after adding space to the filesystem, we need to clear the full flags178 * on all the space infos.179 */180void btrfs_clear_space_info_full(struct btrfs_fs_info *info)181{182 struct list_head *head = &info->space_info;183 struct btrfs_space_info *found;184 185 list_for_each_entry(found, head, list)186 found->full = 0;187}188 189/*190 * Block groups with more than this value (percents) of unusable space will be191 * scheduled for background reclaim.192 */193#define BTRFS_DEFAULT_ZONED_RECLAIM_THRESH (75)194 195#define BTRFS_UNALLOC_BLOCK_GROUP_TARGET (10ULL)196 197/*198 * Calculate chunk size depending on volume type (regular or zoned).199 */200static u64 calc_chunk_size(const struct btrfs_fs_info *fs_info, u64 flags)201{202 if (btrfs_is_zoned(fs_info))203 return fs_info->zone_size;204 205 ASSERT(flags & BTRFS_BLOCK_GROUP_TYPE_MASK);206 207 if (flags & BTRFS_BLOCK_GROUP_DATA)208 return BTRFS_MAX_DATA_CHUNK_SIZE;209 else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)210 return SZ_32M;211 212 /* Handle BTRFS_BLOCK_GROUP_METADATA */213 if (fs_info->fs_devices->total_rw_bytes > 50ULL * SZ_1G)214 return SZ_1G;215 216 return SZ_256M;217}218 219/*220 * Update default chunk size.221 */222void btrfs_update_space_info_chunk_size(struct btrfs_space_info *space_info,223 u64 chunk_size)224{225 WRITE_ONCE(space_info->chunk_size, chunk_size);226}227 228static int create_space_info(struct btrfs_fs_info *info, u64 flags)229{230 231 struct btrfs_space_info *space_info;232 int i;233 int ret;234 235 space_info = kzalloc(sizeof(*space_info), GFP_NOFS);236 if (!space_info)237 return -ENOMEM;238 239 space_info->fs_info = info;240 for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)241 INIT_LIST_HEAD(&space_info->block_groups[i]);242 init_rwsem(&space_info->groups_sem);243 spin_lock_init(&space_info->lock);244 space_info->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK;245 space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;246 INIT_LIST_HEAD(&space_info->ro_bgs);247 INIT_LIST_HEAD(&space_info->tickets);248 INIT_LIST_HEAD(&space_info->priority_tickets);249 space_info->clamp = 1;250 btrfs_update_space_info_chunk_size(space_info, calc_chunk_size(info, flags));251 252 if (btrfs_is_zoned(info))253 space_info->bg_reclaim_threshold = BTRFS_DEFAULT_ZONED_RECLAIM_THRESH;254 255 ret = btrfs_sysfs_add_space_info_type(info, space_info);256 if (ret)257 return ret;258 259 list_add(&space_info->list, &info->space_info);260 if (flags & BTRFS_BLOCK_GROUP_DATA)261 info->data_sinfo = space_info;262 263 return ret;264}265 266int btrfs_init_space_info(struct btrfs_fs_info *fs_info)267{268 struct btrfs_super_block *disk_super;269 u64 features;270 u64 flags;271 int mixed = 0;272 int ret;273 274 disk_super = fs_info->super_copy;275 if (!btrfs_super_root(disk_super))276 return -EINVAL;277 278 features = btrfs_super_incompat_flags(disk_super);279 if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)280 mixed = 1;281 282 flags = BTRFS_BLOCK_GROUP_SYSTEM;283 ret = create_space_info(fs_info, flags);284 if (ret)285 goto out;286 287 if (mixed) {288 flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA;289 ret = create_space_info(fs_info, flags);290 } else {291 flags = BTRFS_BLOCK_GROUP_METADATA;292 ret = create_space_info(fs_info, flags);293 if (ret)294 goto out;295 296 flags = BTRFS_BLOCK_GROUP_DATA;297 ret = create_space_info(fs_info, flags);298 }299out:300 return ret;301}302 303void btrfs_add_bg_to_space_info(struct btrfs_fs_info *info,304 struct btrfs_block_group *block_group)305{306 struct btrfs_space_info *found;307 int factor, index;308 309 factor = btrfs_bg_type_to_factor(block_group->flags);310 311 found = btrfs_find_space_info(info, block_group->flags);312 ASSERT(found);313 spin_lock(&found->lock);314 found->total_bytes += block_group->length;315 found->disk_total += block_group->length * factor;316 found->bytes_used += block_group->used;317 found->disk_used += block_group->used * factor;318 found->bytes_readonly += block_group->bytes_super;319 btrfs_space_info_update_bytes_zone_unusable(info, found, block_group->zone_unusable);320 if (block_group->length > 0)321 found->full = 0;322 btrfs_try_granting_tickets(info, found);323 spin_unlock(&found->lock);324 325 block_group->space_info = found;326 327 index = btrfs_bg_flags_to_raid_index(block_group->flags);328 down_write(&found->groups_sem);329 list_add_tail(&block_group->list, &found->block_groups[index]);330 up_write(&found->groups_sem);331}332 333struct btrfs_space_info *btrfs_find_space_info(struct btrfs_fs_info *info,334 u64 flags)335{336 struct list_head *head = &info->space_info;337 struct btrfs_space_info *found;338 339 flags &= BTRFS_BLOCK_GROUP_TYPE_MASK;340 341 list_for_each_entry(found, head, list) {342 if (found->flags & flags)343 return found;344 }345 return NULL;346}347 348static u64 calc_effective_data_chunk_size(struct btrfs_fs_info *fs_info)349{350 struct btrfs_space_info *data_sinfo;351 u64 data_chunk_size;352 353 /*354 * Calculate the data_chunk_size, space_info->chunk_size is the355 * "optimal" chunk size based on the fs size. However when we actually356 * allocate the chunk we will strip this down further, making it no357 * more than 10% of the disk or 1G, whichever is smaller.358 *359 * On the zoned mode, we need to use zone_size (= data_sinfo->chunk_size)360 * as it is.361 */362 data_sinfo = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);363 if (btrfs_is_zoned(fs_info))364 return data_sinfo->chunk_size;365 data_chunk_size = min(data_sinfo->chunk_size,366 mult_perc(fs_info->fs_devices->total_rw_bytes, 10));367 return min_t(u64, data_chunk_size, SZ_1G);368}369 370static u64 calc_available_free_space(struct btrfs_fs_info *fs_info,371 const struct btrfs_space_info *space_info,372 enum btrfs_reserve_flush_enum flush)373{374 u64 profile;375 u64 avail;376 u64 data_chunk_size;377 int factor;378 379 if (space_info->flags & BTRFS_BLOCK_GROUP_SYSTEM)380 profile = btrfs_system_alloc_profile(fs_info);381 else382 profile = btrfs_metadata_alloc_profile(fs_info);383 384 avail = atomic64_read(&fs_info->free_chunk_space);385 386 /*387 * If we have dup, raid1 or raid10 then only half of the free388 * space is actually usable. For raid56, the space info used389 * doesn't include the parity drive, so we don't have to390 * change the math391 */392 factor = btrfs_bg_type_to_factor(profile);393 avail = div_u64(avail, factor);394 if (avail == 0)395 return 0;396 397 data_chunk_size = calc_effective_data_chunk_size(fs_info);398 399 /*400 * Since data allocations immediately use block groups as part of the401 * reservation, because we assume that data reservations will == actual402 * usage, we could potentially overcommit and then immediately have that403 * available space used by a data allocation, which could put us in a404 * bind when we get close to filling the file system.405 *406 * To handle this simply remove the data_chunk_size from the available407 * space. If we are relatively empty this won't affect our ability to408 * overcommit much, and if we're very close to full it'll keep us from409 * getting into a position where we've given ourselves very little410 * metadata wiggle room.411 */412 if (avail <= data_chunk_size)413 return 0;414 avail -= data_chunk_size;415 416 /*417 * If we aren't flushing all things, let us overcommit up to418 * 1/2th of the space. If we can flush, don't let us overcommit419 * too much, let it overcommit up to 1/8 of the space.420 */421 if (flush == BTRFS_RESERVE_FLUSH_ALL)422 avail >>= 3;423 else424 avail >>= 1;425 426 /*427 * On the zoned mode, we always allocate one zone as one chunk.428 * Returning non-zone size alingned bytes here will result in429 * less pressure for the async metadata reclaim process, and it430 * will over-commit too much leading to ENOSPC. Align down to the431 * zone size to avoid that.432 */433 if (btrfs_is_zoned(fs_info))434 avail = ALIGN_DOWN(avail, fs_info->zone_size);435 436 return avail;437}438 439int btrfs_can_overcommit(struct btrfs_fs_info *fs_info,440 const struct btrfs_space_info *space_info, u64 bytes,441 enum btrfs_reserve_flush_enum flush)442{443 u64 avail;444 u64 used;445 446 /* Don't overcommit when in mixed mode */447 if (space_info->flags & BTRFS_BLOCK_GROUP_DATA)448 return 0;449 450 used = btrfs_space_info_used(space_info, true);451 avail = calc_available_free_space(fs_info, space_info, flush);452 453 if (used + bytes < space_info->total_bytes + avail)454 return 1;455 return 0;456}457 458static void remove_ticket(struct btrfs_space_info *space_info,459 struct reserve_ticket *ticket)460{461 if (!list_empty(&ticket->list)) {462 list_del_init(&ticket->list);463 ASSERT(space_info->reclaim_size >= ticket->bytes);464 space_info->reclaim_size -= ticket->bytes;465 }466}467 468/*469 * This is for space we already have accounted in space_info->bytes_may_use, so470 * basically when we're returning space from block_rsv's.471 */472void btrfs_try_granting_tickets(struct btrfs_fs_info *fs_info,473 struct btrfs_space_info *space_info)474{475 struct list_head *head;476 enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_NO_FLUSH;477 478 lockdep_assert_held(&space_info->lock);479 480 head = &space_info->priority_tickets;481again:482 while (!list_empty(head)) {483 struct reserve_ticket *ticket;484 u64 used = btrfs_space_info_used(space_info, true);485 486 ticket = list_first_entry(head, struct reserve_ticket, list);487 488 /* Check and see if our ticket can be satisfied now. */489 if ((used + ticket->bytes <= space_info->total_bytes) ||490 btrfs_can_overcommit(fs_info, space_info, ticket->bytes,491 flush)) {492 btrfs_space_info_update_bytes_may_use(fs_info,493 space_info,494 ticket->bytes);495 remove_ticket(space_info, ticket);496 ticket->bytes = 0;497 space_info->tickets_id++;498 wake_up(&ticket->wait);499 } else {500 break;501 }502 }503 504 if (head == &space_info->priority_tickets) {505 head = &space_info->tickets;506 flush = BTRFS_RESERVE_FLUSH_ALL;507 goto again;508 }509}510 511#define DUMP_BLOCK_RSV(fs_info, rsv_name) \512do { \513 struct btrfs_block_rsv *__rsv = &(fs_info)->rsv_name; \514 spin_lock(&__rsv->lock); \515 btrfs_info(fs_info, #rsv_name ": size %llu reserved %llu", \516 __rsv->size, __rsv->reserved); \517 spin_unlock(&__rsv->lock); \518} while (0)519 520static const char *space_info_flag_to_str(const struct btrfs_space_info *space_info)521{522 switch (space_info->flags) {523 case BTRFS_BLOCK_GROUP_SYSTEM:524 return "SYSTEM";525 case BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA:526 return "DATA+METADATA";527 case BTRFS_BLOCK_GROUP_DATA:528 return "DATA";529 case BTRFS_BLOCK_GROUP_METADATA:530 return "METADATA";531 default:532 return "UNKNOWN";533 }534}535 536static void dump_global_block_rsv(struct btrfs_fs_info *fs_info)537{538 DUMP_BLOCK_RSV(fs_info, global_block_rsv);539 DUMP_BLOCK_RSV(fs_info, trans_block_rsv);540 DUMP_BLOCK_RSV(fs_info, chunk_block_rsv);541 DUMP_BLOCK_RSV(fs_info, delayed_block_rsv);542 DUMP_BLOCK_RSV(fs_info, delayed_refs_rsv);543}544 545static void __btrfs_dump_space_info(const struct btrfs_fs_info *fs_info,546 const struct btrfs_space_info *info)547{548 const char *flag_str = space_info_flag_to_str(info);549 lockdep_assert_held(&info->lock);550 551 /* The free space could be negative in case of overcommit */552 btrfs_info(fs_info, "space_info %s has %lld free, is %sfull",553 flag_str,554 (s64)(info->total_bytes - btrfs_space_info_used(info, true)),555 info->full ? "" : "not ");556 btrfs_info(fs_info,557"space_info total=%llu, used=%llu, pinned=%llu, reserved=%llu, may_use=%llu, readonly=%llu zone_unusable=%llu",558 info->total_bytes, info->bytes_used, info->bytes_pinned,559 info->bytes_reserved, info->bytes_may_use,560 info->bytes_readonly, info->bytes_zone_unusable);561}562 563void btrfs_dump_space_info(struct btrfs_fs_info *fs_info,564 struct btrfs_space_info *info, u64 bytes,565 int dump_block_groups)566{567 struct btrfs_block_group *cache;568 u64 total_avail = 0;569 int index = 0;570 571 spin_lock(&info->lock);572 __btrfs_dump_space_info(fs_info, info);573 dump_global_block_rsv(fs_info);574 spin_unlock(&info->lock);575 576 if (!dump_block_groups)577 return;578 579 down_read(&info->groups_sem);580again:581 list_for_each_entry(cache, &info->block_groups[index], list) {582 u64 avail;583 584 spin_lock(&cache->lock);585 avail = cache->length - cache->used - cache->pinned -586 cache->reserved - cache->bytes_super - cache->zone_unusable;587 btrfs_info(fs_info,588"block group %llu has %llu bytes, %llu used %llu pinned %llu reserved %llu delalloc %llu super %llu zone_unusable (%llu bytes available) %s",589 cache->start, cache->length, cache->used, cache->pinned,590 cache->reserved, cache->delalloc_bytes,591 cache->bytes_super, cache->zone_unusable,592 avail, cache->ro ? "[readonly]" : "");593 spin_unlock(&cache->lock);594 btrfs_dump_free_space(cache, bytes);595 total_avail += avail;596 }597 if (++index < BTRFS_NR_RAID_TYPES)598 goto again;599 up_read(&info->groups_sem);600 601 btrfs_info(fs_info, "%llu bytes available across all block groups", total_avail);602}603 604static inline u64 calc_reclaim_items_nr(const struct btrfs_fs_info *fs_info,605 u64 to_reclaim)606{607 u64 bytes;608 u64 nr;609 610 bytes = btrfs_calc_insert_metadata_size(fs_info, 1);611 nr = div64_u64(to_reclaim, bytes);612 if (!nr)613 nr = 1;614 return nr;615}616 617/*618 * shrink metadata reservation for delalloc619 */620static void shrink_delalloc(struct btrfs_fs_info *fs_info,621 struct btrfs_space_info *space_info,622 u64 to_reclaim, bool wait_ordered,623 bool for_preempt)624{625 struct btrfs_trans_handle *trans;626 u64 delalloc_bytes;627 u64 ordered_bytes;628 u64 items;629 long time_left;630 int loops;631 632 delalloc_bytes = percpu_counter_sum_positive(&fs_info->delalloc_bytes);633 ordered_bytes = percpu_counter_sum_positive(&fs_info->ordered_bytes);634 if (delalloc_bytes == 0 && ordered_bytes == 0)635 return;636 637 /* Calc the number of the pages we need flush for space reservation */638 if (to_reclaim == U64_MAX) {639 items = U64_MAX;640 } else {641 /*642 * to_reclaim is set to however much metadata we need to643 * reclaim, but reclaiming that much data doesn't really track644 * exactly. What we really want to do is reclaim full inode's645 * worth of reservations, however that's not available to us646 * here. We will take a fraction of the delalloc bytes for our647 * flushing loops and hope for the best. Delalloc will expand648 * the amount we write to cover an entire dirty extent, which649 * will reclaim the metadata reservation for that range. If650 * it's not enough subsequent flush stages will be more651 * aggressive.652 */653 to_reclaim = max(to_reclaim, delalloc_bytes >> 3);654 items = calc_reclaim_items_nr(fs_info, to_reclaim) * 2;655 }656 657 trans = current->journal_info;658 659 /*660 * If we are doing more ordered than delalloc we need to just wait on661 * ordered extents, otherwise we'll waste time trying to flush delalloc662 * that likely won't give us the space back we need.663 */664 if (ordered_bytes > delalloc_bytes && !for_preempt)665 wait_ordered = true;666 667 loops = 0;668 while ((delalloc_bytes || ordered_bytes) && loops < 3) {669 u64 temp = min(delalloc_bytes, to_reclaim) >> PAGE_SHIFT;670 long nr_pages = min_t(u64, temp, LONG_MAX);671 int async_pages;672 673 btrfs_start_delalloc_roots(fs_info, nr_pages, true);674 675 /*676 * We need to make sure any outstanding async pages are now677 * processed before we continue. This is because things like678 * sync_inode() try to be smart and skip writing if the inode is679 * marked clean. We don't use filemap_fwrite for flushing680 * because we want to control how many pages we write out at a681 * time, thus this is the only safe way to make sure we've682 * waited for outstanding compressed workers to have started683 * their jobs and thus have ordered extents set up properly.684 *685 * This exists because we do not want to wait for each686 * individual inode to finish its async work, we simply want to687 * start the IO on everybody, and then come back here and wait688 * for all of the async work to catch up. Once we're done with689 * that we know we'll have ordered extents for everything and we690 * can decide if we wait for that or not.691 *692 * If we choose to replace this in the future, make absolutely693 * sure that the proper waiting is being done in the async case,694 * as there have been bugs in that area before.695 */696 async_pages = atomic_read(&fs_info->async_delalloc_pages);697 if (!async_pages)698 goto skip_async;699 700 /*701 * We don't want to wait forever, if we wrote less pages in this702 * loop than we have outstanding, only wait for that number of703 * pages, otherwise we can wait for all async pages to finish704 * before continuing.705 */706 if (async_pages > nr_pages)707 async_pages -= nr_pages;708 else709 async_pages = 0;710 wait_event(fs_info->async_submit_wait,711 atomic_read(&fs_info->async_delalloc_pages) <=712 async_pages);713skip_async:714 loops++;715 if (wait_ordered && !trans) {716 btrfs_wait_ordered_roots(fs_info, items, NULL);717 } else {718 time_left = schedule_timeout_killable(1);719 if (time_left)720 break;721 }722 723 /*724 * If we are for preemption we just want a one-shot of delalloc725 * flushing so we can stop flushing if we decide we don't need726 * to anymore.727 */728 if (for_preempt)729 break;730 731 spin_lock(&space_info->lock);732 if (list_empty(&space_info->tickets) &&733 list_empty(&space_info->priority_tickets)) {734 spin_unlock(&space_info->lock);735 break;736 }737 spin_unlock(&space_info->lock);738 739 delalloc_bytes = percpu_counter_sum_positive(740 &fs_info->delalloc_bytes);741 ordered_bytes = percpu_counter_sum_positive(742 &fs_info->ordered_bytes);743 }744}745 746/*747 * Try to flush some data based on policy set by @state. This is only advisory748 * and may fail for various reasons. The caller is supposed to examine the749 * state of @space_info to detect the outcome.750 */751static void flush_space(struct btrfs_fs_info *fs_info,752 struct btrfs_space_info *space_info, u64 num_bytes,753 enum btrfs_flush_state state, bool for_preempt)754{755 struct btrfs_root *root = fs_info->tree_root;756 struct btrfs_trans_handle *trans;757 int nr;758 int ret = 0;759 760 switch (state) {761 case FLUSH_DELAYED_ITEMS_NR:762 case FLUSH_DELAYED_ITEMS:763 if (state == FLUSH_DELAYED_ITEMS_NR)764 nr = calc_reclaim_items_nr(fs_info, num_bytes) * 2;765 else766 nr = -1;767 768 trans = btrfs_join_transaction_nostart(root);769 if (IS_ERR(trans)) {770 ret = PTR_ERR(trans);771 if (ret == -ENOENT)772 ret = 0;773 break;774 }775 ret = btrfs_run_delayed_items_nr(trans, nr);776 btrfs_end_transaction(trans);777 break;778 case FLUSH_DELALLOC:779 case FLUSH_DELALLOC_WAIT:780 case FLUSH_DELALLOC_FULL:781 if (state == FLUSH_DELALLOC_FULL)782 num_bytes = U64_MAX;783 shrink_delalloc(fs_info, space_info, num_bytes,784 state != FLUSH_DELALLOC, for_preempt);785 break;786 case FLUSH_DELAYED_REFS_NR:787 case FLUSH_DELAYED_REFS:788 trans = btrfs_join_transaction_nostart(root);789 if (IS_ERR(trans)) {790 ret = PTR_ERR(trans);791 if (ret == -ENOENT)792 ret = 0;793 break;794 }795 if (state == FLUSH_DELAYED_REFS_NR)796 btrfs_run_delayed_refs(trans, num_bytes);797 else798 btrfs_run_delayed_refs(trans, 0);799 btrfs_end_transaction(trans);800 break;801 case ALLOC_CHUNK:802 case ALLOC_CHUNK_FORCE:803 trans = btrfs_join_transaction(root);804 if (IS_ERR(trans)) {805 ret = PTR_ERR(trans);806 break;807 }808 ret = btrfs_chunk_alloc(trans,809 btrfs_get_alloc_profile(fs_info, space_info->flags),810 (state == ALLOC_CHUNK) ? CHUNK_ALLOC_NO_FORCE :811 CHUNK_ALLOC_FORCE);812 btrfs_end_transaction(trans);813 814 if (ret > 0 || ret == -ENOSPC)815 ret = 0;816 break;817 case RUN_DELAYED_IPUTS:818 /*819 * If we have pending delayed iputs then we could free up a820 * bunch of pinned space, so make sure we run the iputs before821 * we do our pinned bytes check below.822 */823 btrfs_run_delayed_iputs(fs_info);824 btrfs_wait_on_delayed_iputs(fs_info);825 break;826 case COMMIT_TRANS:827 ASSERT(current->journal_info == NULL);828 /*829 * We don't want to start a new transaction, just attach to the830 * current one or wait it fully commits in case its commit is831 * happening at the moment. Note: we don't use a nostart join832 * because that does not wait for a transaction to fully commit833 * (only for it to be unblocked, state TRANS_STATE_UNBLOCKED).834 */835 ret = btrfs_commit_current_transaction(root);836 break;837 default:838 ret = -ENOSPC;839 break;840 }841 842 trace_btrfs_flush_space(fs_info, space_info->flags, num_bytes, state,843 ret, for_preempt);844 return;845}846 847static u64 btrfs_calc_reclaim_metadata_size(struct btrfs_fs_info *fs_info,848 const struct btrfs_space_info *space_info)849{850 u64 used;851 u64 avail;852 u64 to_reclaim = space_info->reclaim_size;853 854 lockdep_assert_held(&space_info->lock);855 856 avail = calc_available_free_space(fs_info, space_info,857 BTRFS_RESERVE_FLUSH_ALL);858 used = btrfs_space_info_used(space_info, true);859 860 /*861 * We may be flushing because suddenly we have less space than we had862 * before, and now we're well over-committed based on our current free863 * space. If that's the case add in our overage so we make sure to put864 * appropriate pressure on the flushing state machine.865 */866 if (space_info->total_bytes + avail < used)867 to_reclaim += used - (space_info->total_bytes + avail);868 869 return to_reclaim;870}871 872static bool need_preemptive_reclaim(struct btrfs_fs_info *fs_info,873 const struct btrfs_space_info *space_info)874{875 const u64 global_rsv_size = btrfs_block_rsv_reserved(&fs_info->global_block_rsv);876 u64 ordered, delalloc;877 u64 thresh;878 u64 used;879 880 thresh = mult_perc(space_info->total_bytes, 90);881 882 lockdep_assert_held(&space_info->lock);883 884 /* If we're just plain full then async reclaim just slows us down. */885 if ((space_info->bytes_used + space_info->bytes_reserved +886 global_rsv_size) >= thresh)887 return false;888 889 used = space_info->bytes_may_use + space_info->bytes_pinned;890 891 /* The total flushable belongs to the global rsv, don't flush. */892 if (global_rsv_size >= used)893 return false;894 895 /*896 * 128MiB is 1/4 of the maximum global rsv size. If we have less than897 * that devoted to other reservations then there's no sense in flushing,898 * we don't have a lot of things that need flushing.899 */900 if (used - global_rsv_size <= SZ_128M)901 return false;902 903 /*904 * We have tickets queued, bail so we don't compete with the async905 * flushers.906 */907 if (space_info->reclaim_size)908 return false;909 910 /*911 * If we have over half of the free space occupied by reservations or912 * pinned then we want to start flushing.913 *914 * We do not do the traditional thing here, which is to say915 *916 * if (used >= ((total_bytes + avail) / 2))917 * return 1;918 *919 * because this doesn't quite work how we want. If we had more than 50%920 * of the space_info used by bytes_used and we had 0 available we'd just921 * constantly run the background flusher. Instead we want it to kick in922 * if our reclaimable space exceeds our clamped free space.923 *924 * Our clamping range is 2^1 -> 2^8. Practically speaking that means925 * the following:926 *927 * Amount of RAM Minimum threshold Maximum threshold928 *929 * 256GiB 1GiB 128GiB930 * 128GiB 512MiB 64GiB931 * 64GiB 256MiB 32GiB932 * 32GiB 128MiB 16GiB933 * 16GiB 64MiB 8GiB934 *935 * These are the range our thresholds will fall in, corresponding to how936 * much delalloc we need for the background flusher to kick in.937 */938 939 thresh = calc_available_free_space(fs_info, space_info,940 BTRFS_RESERVE_FLUSH_ALL);941 used = space_info->bytes_used + space_info->bytes_reserved +942 space_info->bytes_readonly + global_rsv_size;943 if (used < space_info->total_bytes)944 thresh += space_info->total_bytes - used;945 thresh >>= space_info->clamp;946 947 used = space_info->bytes_pinned;948 949 /*950 * If we have more ordered bytes than delalloc bytes then we're either951 * doing a lot of DIO, or we simply don't have a lot of delalloc waiting952 * around. Preemptive flushing is only useful in that it can free up953 * space before tickets need to wait for things to finish. In the case954 * of ordered extents, preemptively waiting on ordered extents gets us955 * nothing, if our reservations are tied up in ordered extents we'll956 * simply have to slow down writers by forcing them to wait on ordered957 * extents.958 *959 * In the case that ordered is larger than delalloc, only include the960 * block reserves that we would actually be able to directly reclaim961 * from. In this case if we're heavy on metadata operations this will962 * clearly be heavy enough to warrant preemptive flushing. In the case963 * of heavy DIO or ordered reservations, preemptive flushing will just964 * waste time and cause us to slow down.965 *966 * We want to make sure we truly are maxed out on ordered however, so967 * cut ordered in half, and if it's still higher than delalloc then we968 * can keep flushing. This is to avoid the case where we start969 * flushing, and now delalloc == ordered and we stop preemptively970 * flushing when we could still have several gigs of delalloc to flush.971 */972 ordered = percpu_counter_read_positive(&fs_info->ordered_bytes) >> 1;973 delalloc = percpu_counter_read_positive(&fs_info->delalloc_bytes);974 if (ordered >= delalloc)975 used += btrfs_block_rsv_reserved(&fs_info->delayed_refs_rsv) +976 btrfs_block_rsv_reserved(&fs_info->delayed_block_rsv);977 else978 used += space_info->bytes_may_use - global_rsv_size;979 980 return (used >= thresh && !btrfs_fs_closing(fs_info) &&981 !test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state));982}983 984static bool steal_from_global_rsv(struct btrfs_fs_info *fs_info,985 struct btrfs_space_info *space_info,986 struct reserve_ticket *ticket)987{988 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;989 u64 min_bytes;990 991 if (!ticket->steal)992 return false;993 994 if (global_rsv->space_info != space_info)995 return false;996 997 spin_lock(&global_rsv->lock);998 min_bytes = mult_perc(global_rsv->size, 10);999 if (global_rsv->reserved < min_bytes + ticket->bytes) {1000 spin_unlock(&global_rsv->lock);1001 return false;1002 }1003 global_rsv->reserved -= ticket->bytes;1004 remove_ticket(space_info, ticket);1005 ticket->bytes = 0;1006 wake_up(&ticket->wait);1007 space_info->tickets_id++;1008 if (global_rsv->reserved < global_rsv->size)1009 global_rsv->full = 0;1010 spin_unlock(&global_rsv->lock);1011 1012 return true;1013}1014 1015/*1016 * We've exhausted our flushing, start failing tickets.1017 *1018 * @fs_info - fs_info for this fs1019 * @space_info - the space info we were flushing1020 *1021 * We call this when we've exhausted our flushing ability and haven't made1022 * progress in satisfying tickets. The reservation code handles tickets in1023 * order, so if there is a large ticket first and then smaller ones we could1024 * very well satisfy the smaller tickets. This will attempt to wake up any1025 * tickets in the list to catch this case.1026 *1027 * This function returns true if it was able to make progress by clearing out1028 * other tickets, or if it stumbles across a ticket that was smaller than the1029 * first ticket.1030 */1031static bool maybe_fail_all_tickets(struct btrfs_fs_info *fs_info,1032 struct btrfs_space_info *space_info)1033{1034 struct reserve_ticket *ticket;1035 u64 tickets_id = space_info->tickets_id;1036 const bool aborted = BTRFS_FS_ERROR(fs_info);1037 1038 trace_btrfs_fail_all_tickets(fs_info, space_info);1039 1040 if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {1041 btrfs_info(fs_info, "cannot satisfy tickets, dumping space info");1042 __btrfs_dump_space_info(fs_info, space_info);1043 }1044 1045 while (!list_empty(&space_info->tickets) &&1046 tickets_id == space_info->tickets_id) {1047 ticket = list_first_entry(&space_info->tickets,1048 struct reserve_ticket, list);1049 1050 if (!aborted && steal_from_global_rsv(fs_info, space_info, ticket))1051 return true;1052 1053 if (!aborted && btrfs_test_opt(fs_info, ENOSPC_DEBUG))1054 btrfs_info(fs_info, "failing ticket with %llu bytes",1055 ticket->bytes);1056 1057 remove_ticket(space_info, ticket);1058 if (aborted)1059 ticket->error = -EIO;1060 else1061 ticket->error = -ENOSPC;1062 wake_up(&ticket->wait);1063 1064 /*1065 * We're just throwing tickets away, so more flushing may not1066 * trip over btrfs_try_granting_tickets, so we need to call it1067 * here to see if we can make progress with the next ticket in1068 * the list.1069 */1070 if (!aborted)1071 btrfs_try_granting_tickets(fs_info, space_info);1072 }1073 return (tickets_id != space_info->tickets_id);1074}1075 1076/*1077 * This is for normal flushers, we can wait all goddamned day if we want to. We1078 * will loop and continuously try to flush as long as we are making progress.1079 * We count progress as clearing off tickets each time we have to loop.1080 */1081static void btrfs_async_reclaim_metadata_space(struct work_struct *work)1082{1083 struct btrfs_fs_info *fs_info;1084 struct btrfs_space_info *space_info;1085 u64 to_reclaim;1086 enum btrfs_flush_state flush_state;1087 int commit_cycles = 0;1088 u64 last_tickets_id;1089 1090 fs_info = container_of(work, struct btrfs_fs_info, async_reclaim_work);1091 space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);1092 1093 spin_lock(&space_info->lock);1094 to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info, space_info);1095 if (!to_reclaim) {1096 space_info->flush = 0;1097 spin_unlock(&space_info->lock);1098 return;1099 }1100 last_tickets_id = space_info->tickets_id;1101 spin_unlock(&space_info->lock);1102 1103 flush_state = FLUSH_DELAYED_ITEMS_NR;1104 do {1105 flush_space(fs_info, space_info, to_reclaim, flush_state, false);1106 spin_lock(&space_info->lock);1107 if (list_empty(&space_info->tickets)) {1108 space_info->flush = 0;1109 spin_unlock(&space_info->lock);1110 return;1111 }1112 to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info,1113 space_info);1114 if (last_tickets_id == space_info->tickets_id) {1115 flush_state++;1116 } else {1117 last_tickets_id = space_info->tickets_id;1118 flush_state = FLUSH_DELAYED_ITEMS_NR;1119 if (commit_cycles)1120 commit_cycles--;1121 }1122 1123 /*1124 * We do not want to empty the system of delalloc unless we're1125 * under heavy pressure, so allow one trip through the flushing1126 * logic before we start doing a FLUSH_DELALLOC_FULL.1127 */1128 if (flush_state == FLUSH_DELALLOC_FULL && !commit_cycles)1129 flush_state++;1130 1131 /*1132 * We don't want to force a chunk allocation until we've tried1133 * pretty hard to reclaim space. Think of the case where we1134 * freed up a bunch of space and so have a lot of pinned space1135 * to reclaim. We would rather use that than possibly create a1136 * underutilized metadata chunk. So if this is our first run1137 * through the flushing state machine skip ALLOC_CHUNK_FORCE and1138 * commit the transaction. If nothing has changed the next go1139 * around then we can force a chunk allocation.1140 */1141 if (flush_state == ALLOC_CHUNK_FORCE && !commit_cycles)1142 flush_state++;1143 1144 if (flush_state > COMMIT_TRANS) {1145 commit_cycles++;1146 if (commit_cycles > 2) {1147 if (maybe_fail_all_tickets(fs_info, space_info)) {1148 flush_state = FLUSH_DELAYED_ITEMS_NR;1149 commit_cycles--;1150 } else {1151 space_info->flush = 0;1152 }1153 } else {1154 flush_state = FLUSH_DELAYED_ITEMS_NR;1155 }1156 }1157 spin_unlock(&space_info->lock);1158 } while (flush_state <= COMMIT_TRANS);1159}1160 1161/*1162 * This handles pre-flushing of metadata space before we get to the point that1163 * we need to start blocking threads on tickets. The logic here is different1164 * from the other flush paths because it doesn't rely on tickets to tell us how1165 * much we need to flush, instead it attempts to keep us below the 80% full1166 * watermark of space by flushing whichever reservation pool is currently the1167 * largest.1168 */1169static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work)1170{1171 struct btrfs_fs_info *fs_info;1172 struct btrfs_space_info *space_info;1173 struct btrfs_block_rsv *delayed_block_rsv;1174 struct btrfs_block_rsv *delayed_refs_rsv;1175 struct btrfs_block_rsv *global_rsv;1176 struct btrfs_block_rsv *trans_rsv;1177 int loops = 0;1178 1179 fs_info = container_of(work, struct btrfs_fs_info,1180 preempt_reclaim_work);1181 space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);1182 delayed_block_rsv = &fs_info->delayed_block_rsv;1183 delayed_refs_rsv = &fs_info->delayed_refs_rsv;1184 global_rsv = &fs_info->global_block_rsv;1185 trans_rsv = &fs_info->trans_block_rsv;1186 1187 spin_lock(&space_info->lock);1188 while (need_preemptive_reclaim(fs_info, space_info)) {1189 enum btrfs_flush_state flush;1190 u64 delalloc_size = 0;1191 u64 to_reclaim, block_rsv_size;1192 const u64 global_rsv_size = btrfs_block_rsv_reserved(global_rsv);1193 1194 loops++;1195 1196 /*1197 * We don't have a precise counter for the metadata being1198 * reserved for delalloc, so we'll approximate it by subtracting1199 * out the block rsv's space from the bytes_may_use. If that1200 * amount is higher than the individual reserves, then we can1201 * assume it's tied up in delalloc reservations.1202 */1203 block_rsv_size = global_rsv_size +1204 btrfs_block_rsv_reserved(delayed_block_rsv) +1205 btrfs_block_rsv_reserved(delayed_refs_rsv) +1206 btrfs_block_rsv_reserved(trans_rsv);1207 if (block_rsv_size < space_info->bytes_may_use)1208 delalloc_size = space_info->bytes_may_use - block_rsv_size;1209 1210 /*1211 * We don't want to include the global_rsv in our calculation,1212 * because that's space we can't touch. Subtract it from the1213 * block_rsv_size for the next checks.1214 */1215 block_rsv_size -= global_rsv_size;1216 1217 /*1218 * We really want to avoid flushing delalloc too much, as it1219 * could result in poor allocation patterns, so only flush it if1220 * it's larger than the rest of the pools combined.1221 */1222 if (delalloc_size > block_rsv_size) {1223 to_reclaim = delalloc_size;1224 flush = FLUSH_DELALLOC;1225 } else if (space_info->bytes_pinned >1226 (btrfs_block_rsv_reserved(delayed_block_rsv) +1227 btrfs_block_rsv_reserved(delayed_refs_rsv))) {1228 to_reclaim = space_info->bytes_pinned;1229 flush = COMMIT_TRANS;1230 } else if (btrfs_block_rsv_reserved(delayed_block_rsv) >1231 btrfs_block_rsv_reserved(delayed_refs_rsv)) {1232 to_reclaim = btrfs_block_rsv_reserved(delayed_block_rsv);1233 flush = FLUSH_DELAYED_ITEMS_NR;1234 } else {1235 to_reclaim = btrfs_block_rsv_reserved(delayed_refs_rsv);1236 flush = FLUSH_DELAYED_REFS_NR;1237 }1238 1239 spin_unlock(&space_info->lock);1240 1241 /*1242 * We don't want to reclaim everything, just a portion, so scale1243 * down the to_reclaim by 1/4. If it takes us down to 0,1244 * reclaim 1 items worth.1245 */1246 to_reclaim >>= 2;1247 if (!to_reclaim)1248 to_reclaim = btrfs_calc_insert_metadata_size(fs_info, 1);1249 flush_space(fs_info, space_info, to_reclaim, flush, true);1250 cond_resched();1251 spin_lock(&space_info->lock);1252 }1253 1254 /* We only went through once, back off our clamping. */1255 if (loops == 1 && !space_info->reclaim_size)1256 space_info->clamp = max(1, space_info->clamp - 1);1257 trace_btrfs_done_preemptive_reclaim(fs_info, space_info);1258 spin_unlock(&space_info->lock);1259}1260 1261/*1262 * FLUSH_DELALLOC_WAIT:1263 * Space is freed from flushing delalloc in one of two ways.1264 *1265 * 1) compression is on and we allocate less space than we reserved1266 * 2) we are overwriting existing space1267 *1268 * For #1 that extra space is reclaimed as soon as the delalloc pages are1269 * COWed, by way of btrfs_add_reserved_bytes() which adds the actual extent1270 * length to ->bytes_reserved, and subtracts the reserved space from1271 * ->bytes_may_use.1272 *1273 * For #2 this is trickier. Once the ordered extent runs we will drop the1274 * extent in the range we are overwriting, which creates a delayed ref for1275 * that freed extent. This however is not reclaimed until the transaction1276 * commits, thus the next stages.1277 *1278 * RUN_DELAYED_IPUTS1279 * If we are freeing inodes, we want to make sure all delayed iputs have1280 * completed, because they could have been on an inode with i_nlink == 0, and1281 * thus have been truncated and freed up space. But again this space is not1282 * immediately re-usable, it comes in the form of a delayed ref, which must be1283 * run and then the transaction must be committed.1284 *1285 * COMMIT_TRANS1286 * This is where we reclaim all of the pinned space generated by running the1287 * iputs1288 *1289 * ALLOC_CHUNK_FORCE1290 * For data we start with alloc chunk force, however we could have been full1291 * before, and then the transaction commit could have freed new block groups,1292 * so if we now have space to allocate do the force chunk allocation.1293 */1294static const enum btrfs_flush_state data_flush_states[] = {1295 FLUSH_DELALLOC_FULL,1296 RUN_DELAYED_IPUTS,1297 COMMIT_TRANS,1298 ALLOC_CHUNK_FORCE,1299};1300 1301static void btrfs_async_reclaim_data_space(struct work_struct *work)1302{1303 struct btrfs_fs_info *fs_info;1304 struct btrfs_space_info *space_info;1305 u64 last_tickets_id;1306 enum btrfs_flush_state flush_state = 0;1307 1308 fs_info = container_of(work, struct btrfs_fs_info, async_data_reclaim_work);1309 space_info = fs_info->data_sinfo;1310 1311 spin_lock(&space_info->lock);1312 if (list_empty(&space_info->tickets)) {1313 space_info->flush = 0;1314 spin_unlock(&space_info->lock);1315 return;1316 }1317 last_tickets_id = space_info->tickets_id;1318 spin_unlock(&space_info->lock);1319 1320 while (!space_info->full) {1321 flush_space(fs_info, space_info, U64_MAX, ALLOC_CHUNK_FORCE, false);1322 spin_lock(&space_info->lock);1323 if (list_empty(&space_info->tickets)) {1324 space_info->flush = 0;1325 spin_unlock(&space_info->lock);1326 return;1327 }1328 1329 /* Something happened, fail everything and bail. */1330 if (BTRFS_FS_ERROR(fs_info))1331 goto aborted_fs;1332 last_tickets_id = space_info->tickets_id;1333 spin_unlock(&space_info->lock);1334 }1335 1336 while (flush_state < ARRAY_SIZE(data_flush_states)) {1337 flush_space(fs_info, space_info, U64_MAX,1338 data_flush_states[flush_state], false);1339 spin_lock(&space_info->lock);1340 if (list_empty(&space_info->tickets)) {1341 space_info->flush = 0;1342 spin_unlock(&space_info->lock);1343 return;1344 }1345 1346 if (last_tickets_id == space_info->tickets_id) {1347 flush_state++;1348 } else {1349 last_tickets_id = space_info->tickets_id;1350 flush_state = 0;1351 }1352 1353 if (flush_state >= ARRAY_SIZE(data_flush_states)) {1354 if (space_info->full) {1355 if (maybe_fail_all_tickets(fs_info, space_info))1356 flush_state = 0;1357 else1358 space_info->flush = 0;1359 } else {1360 flush_state = 0;1361 }1362 1363 /* Something happened, fail everything and bail. */1364 if (BTRFS_FS_ERROR(fs_info))1365 goto aborted_fs;1366 1367 }1368 spin_unlock(&space_info->lock);1369 }1370 return;1371 1372aborted_fs:1373 maybe_fail_all_tickets(fs_info, space_info);1374 space_info->flush = 0;1375 spin_unlock(&space_info->lock);1376}1377 1378void btrfs_init_async_reclaim_work(struct btrfs_fs_info *fs_info)1379{1380 INIT_WORK(&fs_info->async_reclaim_work, btrfs_async_reclaim_metadata_space);1381 INIT_WORK(&fs_info->async_data_reclaim_work, btrfs_async_reclaim_data_space);1382 INIT_WORK(&fs_info->preempt_reclaim_work,1383 btrfs_preempt_reclaim_metadata_space);1384}1385 1386static const enum btrfs_flush_state priority_flush_states[] = {1387 FLUSH_DELAYED_ITEMS_NR,1388 FLUSH_DELAYED_ITEMS,1389 ALLOC_CHUNK,1390};1391 1392static const enum btrfs_flush_state evict_flush_states[] = {1393 FLUSH_DELAYED_ITEMS_NR,1394 FLUSH_DELAYED_ITEMS,1395 FLUSH_DELAYED_REFS_NR,1396 FLUSH_DELAYED_REFS,1397 FLUSH_DELALLOC,1398 FLUSH_DELALLOC_WAIT,1399 FLUSH_DELALLOC_FULL,1400 ALLOC_CHUNK,1401 COMMIT_TRANS,1402};1403 1404static void priority_reclaim_metadata_space(struct btrfs_fs_info *fs_info,1405 struct btrfs_space_info *space_info,1406 struct reserve_ticket *ticket,1407 const enum btrfs_flush_state *states,1408 int states_nr)1409{1410 u64 to_reclaim;1411 int flush_state = 0;1412 1413 spin_lock(&space_info->lock);1414 to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info, space_info);1415 /*1416 * This is the priority reclaim path, so to_reclaim could be >0 still1417 * because we may have only satisfied the priority tickets and still1418 * left non priority tickets on the list. We would then have1419 * to_reclaim but ->bytes == 0.1420 */1421 if (ticket->bytes == 0) {1422 spin_unlock(&space_info->lock);1423 return;1424 }1425 1426 while (flush_state < states_nr) {1427 spin_unlock(&space_info->lock);1428 flush_space(fs_info, space_info, to_reclaim, states[flush_state],1429 false);1430 flush_state++;1431 spin_lock(&space_info->lock);1432 if (ticket->bytes == 0) {1433 spin_unlock(&space_info->lock);1434 return;1435 }1436 }1437 1438 /*1439 * Attempt to steal from the global rsv if we can, except if the fs was1440 * turned into error mode due to a transaction abort when flushing space1441 * above, in that case fail with the abort error instead of returning1442 * success to the caller if we can steal from the global rsv - this is1443 * just to have caller fail immeditelly instead of later when trying to1444 * modify the fs, making it easier to debug -ENOSPC problems.1445 */1446 if (BTRFS_FS_ERROR(fs_info)) {1447 ticket->error = BTRFS_FS_ERROR(fs_info);1448 remove_ticket(space_info, ticket);1449 } else if (!steal_from_global_rsv(fs_info, space_info, ticket)) {1450 ticket->error = -ENOSPC;1451 remove_ticket(space_info, ticket);1452 }1453 1454 /*1455 * We must run try_granting_tickets here because we could be a large1456 * ticket in front of a smaller ticket that can now be satisfied with1457 * the available space.1458 */1459 btrfs_try_granting_tickets(fs_info, space_info);1460 spin_unlock(&space_info->lock);1461}1462 1463static void priority_reclaim_data_space(struct btrfs_fs_info *fs_info,1464 struct btrfs_space_info *space_info,1465 struct reserve_ticket *ticket)1466{1467 spin_lock(&space_info->lock);1468 1469 /* We could have been granted before we got here. */1470 if (ticket->bytes == 0) {1471 spin_unlock(&space_info->lock);1472 return;1473 }1474 1475 while (!space_info->full) {1476 spin_unlock(&space_info->lock);1477 flush_space(fs_info, space_info, U64_MAX, ALLOC_CHUNK_FORCE, false);1478 spin_lock(&space_info->lock);1479 if (ticket->bytes == 0) {1480 spin_unlock(&space_info->lock);1481 return;1482 }1483 }1484 1485 ticket->error = -ENOSPC;1486 remove_ticket(space_info, ticket);1487 btrfs_try_granting_tickets(fs_info, space_info);1488 spin_unlock(&space_info->lock);1489}1490 1491static void wait_reserve_ticket(struct btrfs_fs_info *fs_info,1492 struct btrfs_space_info *space_info,1493 struct reserve_ticket *ticket)1494 1495{1496 DEFINE_WAIT(wait);1497 int ret = 0;1498 1499 spin_lock(&space_info->lock);1500 while (ticket->bytes > 0 && ticket->error == 0) {1501 ret = prepare_to_wait_event(&ticket->wait, &wait, TASK_KILLABLE);1502 if (ret) {1503 /*1504 * Delete us from the list. After we unlock the space1505 * info, we don't want the async reclaim job to reserve1506 * space for this ticket. If that would happen, then the1507 * ticket's task would not known that space was reserved1508 * despite getting an error, resulting in a space leak1509 * (bytes_may_use counter of our space_info).1510 */1511 remove_ticket(space_info, ticket);1512 ticket->error = -EINTR;1513 break;1514 }1515 spin_unlock(&space_info->lock);1516 1517 schedule();1518 1519 finish_wait(&ticket->wait, &wait);1520 spin_lock(&space_info->lock);1521 }1522 spin_unlock(&space_info->lock);1523}1524 1525/*1526 * Do the appropriate flushing and waiting for a ticket.1527 *1528 * @fs_info: the filesystem1529 * @space_info: space info for the reservation1530 * @ticket: ticket for the reservation1531 * @start_ns: timestamp when the reservation started1532 * @orig_bytes: amount of bytes originally reserved1533 * @flush: how much we can flush1534 *1535 * This does the work of figuring out how to flush for the ticket, waiting for1536 * the reservation, and returning the appropriate error if there is one.1537 */1538static int handle_reserve_ticket(struct btrfs_fs_info *fs_info,1539 struct btrfs_space_info *space_info,1540 struct reserve_ticket *ticket,1541 u64 start_ns, u64 orig_bytes,1542 enum btrfs_reserve_flush_enum flush)1543{1544 int ret;1545 1546 switch (flush) {1547 case BTRFS_RESERVE_FLUSH_DATA:1548 case BTRFS_RESERVE_FLUSH_ALL:1549 case BTRFS_RESERVE_FLUSH_ALL_STEAL:1550 wait_reserve_ticket(fs_info, space_info, ticket);1551 break;1552 case BTRFS_RESERVE_FLUSH_LIMIT:1553 priority_reclaim_metadata_space(fs_info, space_info, ticket,1554 priority_flush_states,1555 ARRAY_SIZE(priority_flush_states));1556 break;1557 case BTRFS_RESERVE_FLUSH_EVICT:1558 priority_reclaim_metadata_space(fs_info, space_info, ticket,1559 evict_flush_states,1560 ARRAY_SIZE(evict_flush_states));1561 break;1562 case BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE:1563 priority_reclaim_data_space(fs_info, space_info, ticket);1564 break;1565 default:1566 ASSERT(0);1567 break;1568 }1569 1570 ret = ticket->error;1571 ASSERT(list_empty(&ticket->list));1572 /*1573 * Check that we can't have an error set if the reservation succeeded,1574 * as that would confuse tasks and lead them to error out without1575 * releasing reserved space (if an error happens the expectation is that1576 * space wasn't reserved at all).1577 */1578 ASSERT(!(ticket->bytes == 0 && ticket->error));1579 trace_btrfs_reserve_ticket(fs_info, space_info->flags, orig_bytes,1580 start_ns, flush, ticket->error);1581 return ret;1582}1583 1584/*1585 * This returns true if this flush state will go through the ordinary flushing1586 * code.1587 */1588static inline bool is_normal_flushing(enum btrfs_reserve_flush_enum flush)1589{1590 return (flush == BTRFS_RESERVE_FLUSH_ALL) ||1591 (flush == BTRFS_RESERVE_FLUSH_ALL_STEAL);1592}1593 1594static inline void maybe_clamp_preempt(struct btrfs_fs_info *fs_info,1595 struct btrfs_space_info *space_info)1596{1597 u64 ordered = percpu_counter_sum_positive(&fs_info->ordered_bytes);1598 u64 delalloc = percpu_counter_sum_positive(&fs_info->delalloc_bytes);1599 1600 /*1601 * If we're heavy on ordered operations then clamping won't help us. We1602 * need to clamp specifically to keep up with dirty'ing buffered1603 * writers, because there's not a 1:1 correlation of writing delalloc1604 * and freeing space, like there is with flushing delayed refs or1605 * delayed nodes. If we're already more ordered than delalloc then1606 * we're keeping up, otherwise we aren't and should probably clamp.1607 */1608 if (ordered < delalloc)1609 space_info->clamp = min(space_info->clamp + 1, 8);1610}1611 1612static inline bool can_steal(enum btrfs_reserve_flush_enum flush)1613{1614 return (flush == BTRFS_RESERVE_FLUSH_ALL_STEAL ||1615 flush == BTRFS_RESERVE_FLUSH_EVICT);1616}1617 1618/*1619 * NO_FLUSH and FLUSH_EMERGENCY don't want to create a ticket, they just want to1620 * fail as quickly as possible.1621 */1622static inline bool can_ticket(enum btrfs_reserve_flush_enum flush)1623{1624 return (flush != BTRFS_RESERVE_NO_FLUSH &&1625 flush != BTRFS_RESERVE_FLUSH_EMERGENCY);1626}1627 1628/*1629 * Try to reserve bytes from the block_rsv's space.1630 *1631 * @fs_info: the filesystem1632 * @space_info: space info we want to allocate from1633 * @orig_bytes: number of bytes we want1634 * @flush: whether or not we can flush to make our reservation1635 *1636 * This will reserve orig_bytes number of bytes from the space info associated1637 * with the block_rsv. If there is not enough space it will make an attempt to1638 * flush out space to make room. It will do this by flushing delalloc if1639 * possible or committing the transaction. If flush is 0 then no attempts to1640 * regain reservations will be made and this will fail if there is not enough1641 * space already.1642 */1643static int __reserve_bytes(struct btrfs_fs_info *fs_info,1644 struct btrfs_space_info *space_info, u64 orig_bytes,1645 enum btrfs_reserve_flush_enum flush)1646{1647 struct work_struct *async_work;1648 struct reserve_ticket ticket;1649 u64 start_ns = 0;1650 u64 used;1651 int ret = -ENOSPC;1652 bool pending_tickets;1653 1654 ASSERT(orig_bytes);1655 /*1656 * If have a transaction handle (current->journal_info != NULL), then1657 * the flush method can not be neither BTRFS_RESERVE_FLUSH_ALL* nor1658 * BTRFS_RESERVE_FLUSH_EVICT, as we could deadlock because those1659 * flushing methods can trigger transaction commits.1660 */1661 if (current->journal_info) {1662 /* One assert per line for easier debugging. */1663 ASSERT(flush != BTRFS_RESERVE_FLUSH_ALL);1664 ASSERT(flush != BTRFS_RESERVE_FLUSH_ALL_STEAL);1665 ASSERT(flush != BTRFS_RESERVE_FLUSH_EVICT);1666 }1667 1668 if (flush == BTRFS_RESERVE_FLUSH_DATA)1669 async_work = &fs_info->async_data_reclaim_work;1670 else1671 async_work = &fs_info->async_reclaim_work;1672 1673 spin_lock(&space_info->lock);1674 used = btrfs_space_info_used(space_info, true);1675 1676 /*1677 * We don't want NO_FLUSH allocations to jump everybody, they can1678 * generally handle ENOSPC in a different way, so treat them the same as1679 * normal flushers when it comes to skipping pending tickets.1680 */1681 if (is_normal_flushing(flush) || (flush == BTRFS_RESERVE_NO_FLUSH))1682 pending_tickets = !list_empty(&space_info->tickets) ||1683 !list_empty(&space_info->priority_tickets);1684 else1685 pending_tickets = !list_empty(&space_info->priority_tickets);1686 1687 /*1688 * Carry on if we have enough space (short-circuit) OR call1689 * can_overcommit() to ensure we can overcommit to continue.1690 */1691 if (!pending_tickets &&1692 ((used + orig_bytes <= space_info->total_bytes) ||1693 btrfs_can_overcommit(fs_info, space_info, orig_bytes, flush))) {1694 btrfs_space_info_update_bytes_may_use(fs_info, space_info,1695 orig_bytes);1696 ret = 0;1697 }1698 1699 /*1700 * Things are dire, we need to make a reservation so we don't abort. We1701 * will let this reservation go through as long as we have actual space1702 * left to allocate for the block.1703 */1704 if (ret && unlikely(flush == BTRFS_RESERVE_FLUSH_EMERGENCY)) {1705 used = btrfs_space_info_used(space_info, false);1706 if (used + orig_bytes <= space_info->total_bytes) {1707 btrfs_space_info_update_bytes_may_use(fs_info, space_info,1708 orig_bytes);1709 ret = 0;1710 }1711 }1712 1713 /*1714 * If we couldn't make a reservation then setup our reservation ticket1715 * and kick the async worker if it's not already running.1716 *1717 * If we are a priority flusher then we just need to add our ticket to1718 * the list and we will do our own flushing further down.1719 */1720 if (ret && can_ticket(flush)) {1721 ticket.bytes = orig_bytes;1722 ticket.error = 0;1723 space_info->reclaim_size += ticket.bytes;1724 init_waitqueue_head(&ticket.wait);1725 ticket.steal = can_steal(flush);1726 if (trace_btrfs_reserve_ticket_enabled())1727 start_ns = ktime_get_ns();1728 1729 if (flush == BTRFS_RESERVE_FLUSH_ALL ||1730 flush == BTRFS_RESERVE_FLUSH_ALL_STEAL ||1731 flush == BTRFS_RESERVE_FLUSH_DATA) {1732 list_add_tail(&ticket.list, &space_info->tickets);1733 if (!space_info->flush) {1734 /*1735 * We were forced to add a reserve ticket, so1736 * our preemptive flushing is unable to keep1737 * up. Clamp down on the threshold for the1738 * preemptive flushing in order to keep up with1739 * the workload.1740 */1741 maybe_clamp_preempt(fs_info, space_info);1742 1743 space_info->flush = 1;1744 trace_btrfs_trigger_flush(fs_info,1745 space_info->flags,1746 orig_bytes, flush,1747 "enospc");1748 queue_work(system_unbound_wq, async_work);1749 }1750 } else {1751 list_add_tail(&ticket.list,1752 &space_info->priority_tickets);1753 }1754 } else if (!ret && space_info->flags & BTRFS_BLOCK_GROUP_METADATA) {1755 /*1756 * We will do the space reservation dance during log replay,1757 * which means we won't have fs_info->fs_root set, so don't do1758 * the async reclaim as we will panic.1759 */1760 if (!test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags) &&1761 !work_busy(&fs_info->preempt_reclaim_work) &&1762 need_preemptive_reclaim(fs_info, space_info)) {1763 trace_btrfs_trigger_flush(fs_info, space_info->flags,1764 orig_bytes, flush, "preempt");1765 queue_work(system_unbound_wq,1766 &fs_info->preempt_reclaim_work);1767 }1768 }1769 spin_unlock(&space_info->lock);1770 if (!ret || !can_ticket(flush))1771 return ret;1772 1773 return handle_reserve_ticket(fs_info, space_info, &ticket, start_ns,1774 orig_bytes, flush);1775}1776 1777/*1778 * Try to reserve metadata bytes from the block_rsv's space.1779 *1780 * @fs_info: the filesystem1781 * @space_info: the space_info we're allocating for1782 * @orig_bytes: number of bytes we want1783 * @flush: whether or not we can flush to make our reservation1784 *1785 * This will reserve orig_bytes number of bytes from the space info associated1786 * with the block_rsv. If there is not enough space it will make an attempt to1787 * flush out space to make room. It will do this by flushing delalloc if1788 * possible or committing the transaction. If flush is 0 then no attempts to1789 * regain reservations will be made and this will fail if there is not enough1790 * space already.1791 */1792int btrfs_reserve_metadata_bytes(struct btrfs_fs_info *fs_info,1793 struct btrfs_space_info *space_info,1794 u64 orig_bytes,1795 enum btrfs_reserve_flush_enum flush)1796{1797 int ret;1798 1799 ret = __reserve_bytes(fs_info, space_info, orig_bytes, flush);1800 if (ret == -ENOSPC) {1801 trace_btrfs_space_reservation(fs_info, "space_info:enospc",1802 space_info->flags, orig_bytes, 1);1803 1804 if (btrfs_test_opt(fs_info, ENOSPC_DEBUG))1805 btrfs_dump_space_info(fs_info, space_info, orig_bytes, 0);1806 }1807 return ret;1808}1809 1810/*1811 * Try to reserve data bytes for an allocation.1812 *1813 * @fs_info: the filesystem1814 * @bytes: number of bytes we need1815 * @flush: how we are allowed to flush1816 *1817 * This will reserve bytes from the data space info. If there is not enough1818 * space then we will attempt to flush space as specified by flush.1819 */1820int btrfs_reserve_data_bytes(struct btrfs_fs_info *fs_info, u64 bytes,1821 enum btrfs_reserve_flush_enum flush)1822{1823 struct btrfs_space_info *data_sinfo = fs_info->data_sinfo;1824 int ret;1825 1826 ASSERT(flush == BTRFS_RESERVE_FLUSH_DATA ||1827 flush == BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE ||1828 flush == BTRFS_RESERVE_NO_FLUSH);1829 ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_DATA);1830 1831 ret = __reserve_bytes(fs_info, data_sinfo, bytes, flush);1832 if (ret == -ENOSPC) {1833 trace_btrfs_space_reservation(fs_info, "space_info:enospc",1834 data_sinfo->flags, bytes, 1);1835 if (btrfs_test_opt(fs_info, ENOSPC_DEBUG))1836 btrfs_dump_space_info(fs_info, data_sinfo, bytes, 0);1837 }1838 return ret;1839}1840 1841/* Dump all the space infos when we abort a transaction due to ENOSPC. */1842__cold void btrfs_dump_space_info_for_trans_abort(struct btrfs_fs_info *fs_info)1843{1844 struct btrfs_space_info *space_info;1845 1846 btrfs_info(fs_info, "dumping space info:");1847 list_for_each_entry(space_info, &fs_info->space_info, list) {1848 spin_lock(&space_info->lock);1849 __btrfs_dump_space_info(fs_info, space_info);1850 spin_unlock(&space_info->lock);1851 }1852 dump_global_block_rsv(fs_info);1853}1854 1855/*1856 * Account the unused space of all the readonly block group in the space_info.1857 * takes mirrors into account.1858 */1859u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)1860{1861 struct btrfs_block_group *block_group;1862 u64 free_bytes = 0;1863 int factor;1864 1865 /* It's df, we don't care if it's racy */1866 if (list_empty(&sinfo->ro_bgs))1867 return 0;1868 1869 spin_lock(&sinfo->lock);1870 list_for_each_entry(block_group, &sinfo->ro_bgs, ro_list) {1871 spin_lock(&block_group->lock);1872 1873 if (!block_group->ro) {1874 spin_unlock(&block_group->lock);1875 continue;1876 }1877 1878 factor = btrfs_bg_type_to_factor(block_group->flags);1879 free_bytes += (block_group->length -1880 block_group->used) * factor;1881 1882 spin_unlock(&block_group->lock);1883 }1884 spin_unlock(&sinfo->lock);1885 1886 return free_bytes;1887}1888 1889static u64 calc_pct_ratio(u64 x, u64 y)1890{1891 int err;1892 1893 if (!y)1894 return 0;1895again:1896 err = check_mul_overflow(100, x, &x);1897 if (err)1898 goto lose_precision;1899 return div64_u64(x, y);1900lose_precision:1901 x >>= 10;1902 y >>= 10;1903 if (!y)1904 y = 1;1905 goto again;1906}1907 1908/*1909 * A reasonable buffer for unallocated space is 10 data block_groups.1910 * If we claw this back repeatedly, we can still achieve efficient1911 * utilization when near full, and not do too much reclaim while1912 * always maintaining a solid buffer for workloads that quickly1913 * allocate and pressure the unallocated space.1914 */1915static u64 calc_unalloc_target(struct btrfs_fs_info *fs_info)1916{1917 u64 chunk_sz = calc_effective_data_chunk_size(fs_info);1918 1919 return BTRFS_UNALLOC_BLOCK_GROUP_TARGET * chunk_sz;1920}1921 1922/*1923 * The fundamental goal of automatic reclaim is to protect the filesystem's1924 * unallocated space and thus minimize the probability of the filesystem going1925 * read only when a metadata allocation failure causes a transaction abort.1926 *1927 * However, relocations happen into the space_info's unused space, therefore1928 * automatic reclaim must also back off as that space runs low. There is no1929 * value in doing trivial "relocations" of re-writing the same block group1930 * into a fresh one.1931 *1932 * Furthermore, we want to avoid doing too much reclaim even if there are good1933 * candidates. This is because the allocator is pretty good at filling up the1934 * holes with writes. So we want to do just enough reclaim to try and stay1935 * safe from running out of unallocated space but not be wasteful about it.1936 *1937 * Therefore, the dynamic reclaim threshold is calculated as follows:1938 * - calculate a target unallocated amount of 5 block group sized chunks1939 * - ratchet up the intensity of reclaim depending on how far we are from1940 * that target by using a formula of unalloc / target to set the threshold.1941 *1942 * Typically with 10 block groups as the target, the discrete values this comes1943 * out to are 0, 10, 20, ... , 80, 90, and 99.1944 */1945static int calc_dynamic_reclaim_threshold(const struct btrfs_space_info *space_info)1946{1947 struct btrfs_fs_info *fs_info = space_info->fs_info;1948 u64 unalloc = atomic64_read(&fs_info->free_chunk_space);1949 u64 target = calc_unalloc_target(fs_info);1950 u64 alloc = space_info->total_bytes;1951 u64 used = btrfs_space_info_used(space_info, false);1952 u64 unused = alloc - used;1953 u64 want = target > unalloc ? target - unalloc : 0;1954 u64 data_chunk_size = calc_effective_data_chunk_size(fs_info);1955 1956 /* If we have no unused space, don't bother, it won't work anyway. */1957 if (unused < data_chunk_size)1958 return 0;1959 1960 /* Cast to int is OK because want <= target. */1961 return calc_pct_ratio(want, target);1962}1963 1964int btrfs_calc_reclaim_threshold(const struct btrfs_space_info *space_info)1965{1966 lockdep_assert_held(&space_info->lock);1967 1968 if (READ_ONCE(space_info->dynamic_reclaim))1969 return calc_dynamic_reclaim_threshold(space_info);1970 return READ_ONCE(space_info->bg_reclaim_threshold);1971}1972 1973/*1974 * Under "urgent" reclaim, we will reclaim even fresh block groups that have1975 * recently seen successful allocations, as we are desperate to reclaim1976 * whatever we can to avoid ENOSPC in a transaction leading to a readonly fs.1977 */1978static bool is_reclaim_urgent(struct btrfs_space_info *space_info)1979{1980 struct btrfs_fs_info *fs_info = space_info->fs_info;1981 u64 unalloc = atomic64_read(&fs_info->free_chunk_space);1982 u64 data_chunk_size = calc_effective_data_chunk_size(fs_info);1983 1984 return unalloc < data_chunk_size;1985}1986 1987static void do_reclaim_sweep(const struct btrfs_fs_info *fs_info,1988 struct btrfs_space_info *space_info, int raid)1989{1990 struct btrfs_block_group *bg;1991 int thresh_pct;1992 bool try_again = true;1993 bool urgent;1994 1995 spin_lock(&space_info->lock);1996 urgent = is_reclaim_urgent(space_info);1997 thresh_pct = btrfs_calc_reclaim_threshold(space_info);1998 spin_unlock(&space_info->lock);1999 2000 down_read(&space_info->groups_sem);2001again:2002 list_for_each_entry(bg, &space_info->block_groups[raid], list) {2003 u64 thresh;2004 bool reclaim = false;2005 2006 btrfs_get_block_group(bg);2007 spin_lock(&bg->lock);2008 thresh = mult_perc(bg->length, thresh_pct);2009 if (bg->used < thresh && bg->reclaim_mark) {2010 try_again = false;2011 reclaim = true;2012 }2013 bg->reclaim_mark++;2014 spin_unlock(&bg->lock);2015 if (reclaim)2016 btrfs_mark_bg_to_reclaim(bg);2017 btrfs_put_block_group(bg);2018 }2019 2020 /*2021 * In situations where we are very motivated to reclaim (low unalloc)2022 * use two passes to make the reclaim mark check best effort.2023 *2024 * If we have any staler groups, we don't touch the fresher ones, but if we2025 * really need a block group, do take a fresh one.2026 */2027 if (try_again && urgent) {2028 try_again = false;2029 goto again;2030 }2031 2032 up_read(&space_info->groups_sem);2033}2034 2035void btrfs_space_info_update_reclaimable(struct btrfs_space_info *space_info, s64 bytes)2036{2037 u64 chunk_sz = calc_effective_data_chunk_size(space_info->fs_info);2038 2039 lockdep_assert_held(&space_info->lock);2040 space_info->reclaimable_bytes += bytes;2041 2042 if (space_info->reclaimable_bytes >= chunk_sz)2043 btrfs_set_periodic_reclaim_ready(space_info, true);2044}2045 2046void btrfs_set_periodic_reclaim_ready(struct btrfs_space_info *space_info, bool ready)2047{2048 lockdep_assert_held(&space_info->lock);2049 if (!READ_ONCE(space_info->periodic_reclaim))2050 return;2051 if (ready != space_info->periodic_reclaim_ready) {2052 space_info->periodic_reclaim_ready = ready;2053 if (!ready)2054 space_info->reclaimable_bytes = 0;2055 }2056}2057 2058bool btrfs_should_periodic_reclaim(struct btrfs_space_info *space_info)2059{2060 bool ret;2061 2062 if (space_info->flags & BTRFS_BLOCK_GROUP_SYSTEM)2063 return false;2064 if (!READ_ONCE(space_info->periodic_reclaim))2065 return false;2066 2067 spin_lock(&space_info->lock);2068 ret = space_info->periodic_reclaim_ready;2069 btrfs_set_periodic_reclaim_ready(space_info, false);2070 spin_unlock(&space_info->lock);2071 2072 return ret;2073}2074 2075void btrfs_reclaim_sweep(const struct btrfs_fs_info *fs_info)2076{2077 int raid;2078 struct btrfs_space_info *space_info;2079 2080 list_for_each_entry(space_info, &fs_info->space_info, list) {2081 if (!btrfs_should_periodic_reclaim(space_info))2082 continue;2083 for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++)2084 do_reclaim_sweep(fs_info, space_info, raid);2085 }2086}2087