1328 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Code for manipulating bucket marks for garbage collection.4 *5 * Copyright 2014 Datera, Inc.6 */7 8#include "bcachefs.h"9#include "alloc_background.h"10#include "backpointers.h"11#include "bset.h"12#include "btree_gc.h"13#include "btree_update.h"14#include "buckets.h"15#include "buckets_waiting_for_journal.h"16#include "disk_accounting.h"17#include "ec.h"18#include "error.h"19#include "inode.h"20#include "movinggc.h"21#include "recovery.h"22#include "reflink.h"23#include "replicas.h"24#include "subvolume.h"25#include "trace.h"26 27#include <linux/preempt.h>28 29void bch2_dev_usage_read_fast(struct bch_dev *ca, struct bch_dev_usage *usage)30{31 memset(usage, 0, sizeof(*usage));32 acc_u64s_percpu((u64 *) usage, (u64 __percpu *) ca->usage, dev_usage_u64s());33}34 35static u64 reserve_factor(u64 r)36{37 return r + (round_up(r, (1 << RESERVE_FACTOR)) >> RESERVE_FACTOR);38}39 40static struct bch_fs_usage_short41__bch2_fs_usage_read_short(struct bch_fs *c)42{43 struct bch_fs_usage_short ret;44 u64 data, reserved;45 46 ret.capacity = c->capacity -47 percpu_u64_get(&c->usage->hidden);48 49 data = percpu_u64_get(&c->usage->data) +50 percpu_u64_get(&c->usage->btree);51 reserved = percpu_u64_get(&c->usage->reserved) +52 percpu_u64_get(c->online_reserved);53 54 ret.used = min(ret.capacity, data + reserve_factor(reserved));55 ret.free = ret.capacity - ret.used;56 57 ret.nr_inodes = percpu_u64_get(&c->usage->nr_inodes);58 59 return ret;60}61 62struct bch_fs_usage_short63bch2_fs_usage_read_short(struct bch_fs *c)64{65 struct bch_fs_usage_short ret;66 67 percpu_down_read(&c->mark_lock);68 ret = __bch2_fs_usage_read_short(c);69 percpu_up_read(&c->mark_lock);70 71 return ret;72}73 74void bch2_dev_usage_to_text(struct printbuf *out,75 struct bch_dev *ca,76 struct bch_dev_usage *usage)77{78 if (out->nr_tabstops < 5) {79 printbuf_tabstops_reset(out);80 printbuf_tabstop_push(out, 12);81 printbuf_tabstop_push(out, 16);82 printbuf_tabstop_push(out, 16);83 printbuf_tabstop_push(out, 16);84 printbuf_tabstop_push(out, 16);85 }86 87 prt_printf(out, "\tbuckets\rsectors\rfragmented\r\n");88 89 for (unsigned i = 0; i < BCH_DATA_NR; i++) {90 bch2_prt_data_type(out, i);91 prt_printf(out, "\t%llu\r%llu\r%llu\r\n",92 usage->d[i].buckets,93 usage->d[i].sectors,94 usage->d[i].fragmented);95 }96 97 prt_printf(out, "capacity\t%llu\r\n", ca->mi.nbuckets);98}99 100static int bch2_check_fix_ptr(struct btree_trans *trans,101 struct bkey_s_c k,102 struct extent_ptr_decoded p,103 const union bch_extent_entry *entry,104 bool *do_update)105{106 struct bch_fs *c = trans->c;107 struct printbuf buf = PRINTBUF;108 int ret = 0;109 110 struct bch_dev *ca = bch2_dev_tryget(c, p.ptr.dev);111 if (!ca) {112 if (fsck_err_on(p.ptr.dev != BCH_SB_MEMBER_INVALID,113 trans, ptr_to_invalid_device,114 "pointer to missing device %u\n"115 "while marking %s",116 p.ptr.dev,117 (printbuf_reset(&buf),118 bch2_bkey_val_to_text(&buf, c, k), buf.buf)))119 *do_update = true;120 return 0;121 }122 123 struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr);124 if (!g) {125 if (fsck_err(trans, ptr_to_invalid_device,126 "pointer to invalid bucket on device %u\n"127 "while marking %s",128 p.ptr.dev,129 (printbuf_reset(&buf),130 bch2_bkey_val_to_text(&buf, c, k), buf.buf)))131 *do_update = true;132 goto out;133 }134 135 enum bch_data_type data_type = bch2_bkey_ptr_data_type(k, p, entry);136 137 if (fsck_err_on(!g->gen_valid,138 trans, ptr_to_missing_alloc_key,139 "bucket %u:%zu data type %s ptr gen %u missing in alloc btree\n"140 "while marking %s",141 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),142 bch2_data_type_str(ptr_data_type(k.k, &p.ptr)),143 p.ptr.gen,144 (printbuf_reset(&buf),145 bch2_bkey_val_to_text(&buf, c, k), buf.buf))) {146 if (!p.ptr.cached) {147 g->gen_valid = true;148 g->gen = p.ptr.gen;149 } else {150 *do_update = true;151 }152 }153 154 if (fsck_err_on(gen_cmp(p.ptr.gen, g->gen) > 0,155 trans, ptr_gen_newer_than_bucket_gen,156 "bucket %u:%zu data type %s ptr gen in the future: %u > %u\n"157 "while marking %s",158 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),159 bch2_data_type_str(ptr_data_type(k.k, &p.ptr)),160 p.ptr.gen, g->gen,161 (printbuf_reset(&buf),162 bch2_bkey_val_to_text(&buf, c, k), buf.buf))) {163 if (!p.ptr.cached &&164 (g->data_type != BCH_DATA_btree ||165 data_type == BCH_DATA_btree)) {166 g->gen_valid = true;167 g->gen = p.ptr.gen;168 g->data_type = 0;169 g->stripe_sectors = 0;170 g->dirty_sectors = 0;171 g->cached_sectors = 0;172 } else {173 *do_update = true;174 }175 }176 177 if (fsck_err_on(gen_cmp(g->gen, p.ptr.gen) > BUCKET_GC_GEN_MAX,178 trans, ptr_gen_newer_than_bucket_gen,179 "bucket %u:%zu gen %u data type %s: ptr gen %u too stale\n"180 "while marking %s",181 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), g->gen,182 bch2_data_type_str(ptr_data_type(k.k, &p.ptr)),183 p.ptr.gen,184 (printbuf_reset(&buf),185 bch2_bkey_val_to_text(&buf, c, k), buf.buf)))186 *do_update = true;187 188 if (fsck_err_on(!p.ptr.cached && gen_cmp(p.ptr.gen, g->gen) < 0,189 trans, stale_dirty_ptr,190 "bucket %u:%zu data type %s stale dirty ptr: %u < %u\n"191 "while marking %s",192 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),193 bch2_data_type_str(ptr_data_type(k.k, &p.ptr)),194 p.ptr.gen, g->gen,195 (printbuf_reset(&buf),196 bch2_bkey_val_to_text(&buf, c, k), buf.buf)))197 *do_update = true;198 199 if (data_type != BCH_DATA_btree && p.ptr.gen != g->gen)200 goto out;201 202 if (fsck_err_on(bucket_data_type_mismatch(g->data_type, data_type),203 trans, ptr_bucket_data_type_mismatch,204 "bucket %u:%zu gen %u different types of data in same bucket: %s, %s\n"205 "while marking %s",206 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr), g->gen,207 bch2_data_type_str(g->data_type),208 bch2_data_type_str(data_type),209 (printbuf_reset(&buf),210 bch2_bkey_val_to_text(&buf, c, k), buf.buf))) {211 if (data_type == BCH_DATA_btree) {212 g->gen_valid = true;213 g->gen = p.ptr.gen;214 g->data_type = data_type;215 g->stripe_sectors = 0;216 g->dirty_sectors = 0;217 g->cached_sectors = 0;218 } else {219 *do_update = true;220 }221 }222 223 if (p.has_ec) {224 struct gc_stripe *m = genradix_ptr(&c->gc_stripes, p.ec.idx);225 226 if (fsck_err_on(!m || !m->alive,227 trans, ptr_to_missing_stripe,228 "pointer to nonexistent stripe %llu\n"229 "while marking %s",230 (u64) p.ec.idx,231 (printbuf_reset(&buf),232 bch2_bkey_val_to_text(&buf, c, k), buf.buf)))233 *do_update = true;234 235 if (fsck_err_on(m && m->alive && !bch2_ptr_matches_stripe_m(m, p),236 trans, ptr_to_incorrect_stripe,237 "pointer does not match stripe %llu\n"238 "while marking %s",239 (u64) p.ec.idx,240 (printbuf_reset(&buf),241 bch2_bkey_val_to_text(&buf, c, k), buf.buf)))242 *do_update = true;243 }244out:245fsck_err:246 bch2_dev_put(ca);247 printbuf_exit(&buf);248 return ret;249}250 251int bch2_check_fix_ptrs(struct btree_trans *trans,252 enum btree_id btree, unsigned level, struct bkey_s_c k,253 enum btree_iter_update_trigger_flags flags)254{255 struct bch_fs *c = trans->c;256 struct bkey_ptrs_c ptrs_c = bch2_bkey_ptrs_c(k);257 const union bch_extent_entry *entry_c;258 struct extent_ptr_decoded p = { 0 };259 bool do_update = false;260 struct printbuf buf = PRINTBUF;261 int ret = 0;262 263 percpu_down_read(&c->mark_lock);264 265 bkey_for_each_ptr_decode(k.k, ptrs_c, p, entry_c) {266 ret = bch2_check_fix_ptr(trans, k, p, entry_c, &do_update);267 if (ret)268 goto err;269 }270 271 if (do_update) {272 if (flags & BTREE_TRIGGER_is_root) {273 bch_err(c, "cannot update btree roots yet");274 ret = -EINVAL;275 goto err;276 }277 278 struct bkey_i *new = bch2_bkey_make_mut_noupdate(trans, k);279 ret = PTR_ERR_OR_ZERO(new);280 if (ret)281 goto err;282 283 rcu_read_lock();284 bch2_bkey_drop_ptrs(bkey_i_to_s(new), ptr, !bch2_dev_exists(c, ptr->dev));285 rcu_read_unlock();286 287 if (level) {288 /*289 * We don't want to drop btree node pointers - if the290 * btree node isn't there anymore, the read path will291 * sort it out:292 */293 struct bkey_ptrs ptrs = bch2_bkey_ptrs(bkey_i_to_s(new));294 rcu_read_lock();295 bkey_for_each_ptr(ptrs, ptr) {296 struct bch_dev *ca = bch2_dev_rcu(c, ptr->dev);297 struct bucket *g = PTR_GC_BUCKET(ca, ptr);298 299 ptr->gen = g->gen;300 }301 rcu_read_unlock();302 } else {303 struct bkey_ptrs ptrs;304 union bch_extent_entry *entry;305 306 rcu_read_lock();307restart_drop_ptrs:308 ptrs = bch2_bkey_ptrs(bkey_i_to_s(new));309 bkey_for_each_ptr_decode(bkey_i_to_s(new).k, ptrs, p, entry) {310 struct bch_dev *ca = bch2_dev_rcu(c, p.ptr.dev);311 struct bucket *g = PTR_GC_BUCKET(ca, &p.ptr);312 enum bch_data_type data_type = bch2_bkey_ptr_data_type(bkey_i_to_s_c(new), p, entry);313 314 if ((p.ptr.cached &&315 (!g->gen_valid || gen_cmp(p.ptr.gen, g->gen) > 0)) ||316 (!p.ptr.cached &&317 gen_cmp(p.ptr.gen, g->gen) < 0) ||318 gen_cmp(g->gen, p.ptr.gen) > BUCKET_GC_GEN_MAX ||319 (g->data_type &&320 g->data_type != data_type)) {321 bch2_bkey_drop_ptr(bkey_i_to_s(new), &entry->ptr);322 goto restart_drop_ptrs;323 }324 }325 rcu_read_unlock();326again:327 ptrs = bch2_bkey_ptrs(bkey_i_to_s(new));328 bkey_extent_entry_for_each(ptrs, entry) {329 if (extent_entry_type(entry) == BCH_EXTENT_ENTRY_stripe_ptr) {330 struct gc_stripe *m = genradix_ptr(&c->gc_stripes,331 entry->stripe_ptr.idx);332 union bch_extent_entry *next_ptr;333 334 bkey_extent_entry_for_each_from(ptrs, next_ptr, entry)335 if (extent_entry_type(next_ptr) == BCH_EXTENT_ENTRY_ptr)336 goto found;337 next_ptr = NULL;338found:339 if (!next_ptr) {340 bch_err(c, "aieee, found stripe ptr with no data ptr");341 continue;342 }343 344 if (!m || !m->alive ||345 !__bch2_ptr_matches_stripe(&m->ptrs[entry->stripe_ptr.block],346 &next_ptr->ptr,347 m->sectors)) {348 bch2_bkey_extent_entry_drop(new, entry);349 goto again;350 }351 }352 }353 }354 355 if (0) {356 printbuf_reset(&buf);357 bch2_bkey_val_to_text(&buf, c, k);358 bch_info(c, "updated %s", buf.buf);359 360 printbuf_reset(&buf);361 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(new));362 bch_info(c, "new key %s", buf.buf);363 }364 365 percpu_up_read(&c->mark_lock);366 struct btree_iter iter;367 bch2_trans_node_iter_init(trans, &iter, btree, new->k.p, 0, level,368 BTREE_ITER_intent|BTREE_ITER_all_snapshots);369 ret = bch2_btree_iter_traverse(&iter) ?:370 bch2_trans_update(trans, &iter, new,371 BTREE_UPDATE_internal_snapshot_node|372 BTREE_TRIGGER_norun);373 bch2_trans_iter_exit(trans, &iter);374 percpu_down_read(&c->mark_lock);375 376 if (ret)377 goto err;378 379 if (level)380 bch2_btree_node_update_key_early(trans, btree, level - 1, k, new);381 }382err:383 percpu_up_read(&c->mark_lock);384 printbuf_exit(&buf);385 return ret;386}387 388int bch2_bucket_ref_update(struct btree_trans *trans, struct bch_dev *ca,389 struct bkey_s_c k,390 const struct bch_extent_ptr *ptr,391 s64 sectors, enum bch_data_type ptr_data_type,392 u8 b_gen, u8 bucket_data_type,393 u32 *bucket_sectors)394{395 struct bch_fs *c = trans->c;396 size_t bucket_nr = PTR_BUCKET_NR(ca, ptr);397 struct printbuf buf = PRINTBUF;398 bool inserting = sectors > 0;399 int ret = 0;400 401 BUG_ON(!sectors);402 403 if (gen_after(ptr->gen, b_gen)) {404 bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,405 ptr_gen_newer_than_bucket_gen,406 "bucket %u:%zu gen %u data type %s: ptr gen %u newer than bucket gen\n"407 "while marking %s",408 ptr->dev, bucket_nr, b_gen,409 bch2_data_type_str(bucket_data_type ?: ptr_data_type),410 ptr->gen,411 (bch2_bkey_val_to_text(&buf, c, k), buf.buf));412 if (inserting)413 goto err;414 goto out;415 }416 417 if (gen_cmp(b_gen, ptr->gen) > BUCKET_GC_GEN_MAX) {418 bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,419 ptr_too_stale,420 "bucket %u:%zu gen %u data type %s: ptr gen %u too stale\n"421 "while marking %s",422 ptr->dev, bucket_nr, b_gen,423 bch2_data_type_str(bucket_data_type ?: ptr_data_type),424 ptr->gen,425 (printbuf_reset(&buf),426 bch2_bkey_val_to_text(&buf, c, k), buf.buf));427 if (inserting)428 goto err;429 goto out;430 }431 432 if (b_gen != ptr->gen && ptr->cached) {433 ret = 1;434 goto out;435 }436 437 if (b_gen != ptr->gen) {438 bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,439 stale_dirty_ptr,440 "bucket %u:%zu gen %u (mem gen %u) data type %s: stale dirty ptr (gen %u)\n"441 "while marking %s",442 ptr->dev, bucket_nr, b_gen,443 bucket_gen_get(ca, bucket_nr),444 bch2_data_type_str(bucket_data_type ?: ptr_data_type),445 ptr->gen,446 (printbuf_reset(&buf),447 bch2_bkey_val_to_text(&buf, c, k), buf.buf));448 if (inserting)449 goto err;450 goto out;451 }452 453 if (bucket_data_type_mismatch(bucket_data_type, ptr_data_type)) {454 bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,455 ptr_bucket_data_type_mismatch,456 "bucket %u:%zu gen %u different types of data in same bucket: %s, %s\n"457 "while marking %s",458 ptr->dev, bucket_nr, b_gen,459 bch2_data_type_str(bucket_data_type),460 bch2_data_type_str(ptr_data_type),461 (printbuf_reset(&buf),462 bch2_bkey_val_to_text(&buf, c, k), buf.buf));463 if (inserting)464 goto err;465 goto out;466 }467 468 if ((u64) *bucket_sectors + sectors > U32_MAX) {469 bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,470 bucket_sector_count_overflow,471 "bucket %u:%zu gen %u data type %s sector count overflow: %u + %lli > U32_MAX\n"472 "while marking %s",473 ptr->dev, bucket_nr, b_gen,474 bch2_data_type_str(bucket_data_type ?: ptr_data_type),475 *bucket_sectors, sectors,476 (printbuf_reset(&buf),477 bch2_bkey_val_to_text(&buf, c, k), buf.buf));478 if (inserting)479 goto err;480 sectors = -*bucket_sectors;481 }482 483 *bucket_sectors += sectors;484out:485 printbuf_exit(&buf);486 return ret;487err:488 bch2_dump_trans_updates(trans);489 ret = -BCH_ERR_bucket_ref_update;490 goto out;491}492 493void bch2_trans_account_disk_usage_change(struct btree_trans *trans)494{495 struct bch_fs *c = trans->c;496 u64 disk_res_sectors = trans->disk_res ? trans->disk_res->sectors : 0;497 static int warned_disk_usage = 0;498 bool warn = false;499 500 percpu_down_read(&c->mark_lock);501 struct bch_fs_usage_base *src = &trans->fs_usage_delta;502 503 s64 added = src->btree + src->data + src->reserved;504 505 /*506 * Not allowed to reduce sectors_available except by getting a507 * reservation:508 */509 s64 should_not_have_added = added - (s64) disk_res_sectors;510 if (unlikely(should_not_have_added > 0)) {511 u64 old, new;512 513 old = atomic64_read(&c->sectors_available);514 do {515 new = max_t(s64, 0, old - should_not_have_added);516 } while (!atomic64_try_cmpxchg(&c->sectors_available,517 &old, new));518 519 added -= should_not_have_added;520 warn = true;521 }522 523 if (added > 0) {524 trans->disk_res->sectors -= added;525 this_cpu_sub(*c->online_reserved, added);526 }527 528 preempt_disable();529 struct bch_fs_usage_base *dst = this_cpu_ptr(c->usage);530 acc_u64s((u64 *) dst, (u64 *) src, sizeof(*src) / sizeof(u64));531 preempt_enable();532 percpu_up_read(&c->mark_lock);533 534 if (unlikely(warn) && !xchg(&warned_disk_usage, 1))535 bch2_trans_inconsistent(trans,536 "disk usage increased %lli more than %llu sectors reserved)",537 should_not_have_added, disk_res_sectors);538}539 540/* KEY_TYPE_extent: */541 542static int __mark_pointer(struct btree_trans *trans, struct bch_dev *ca,543 struct bkey_s_c k,544 const struct extent_ptr_decoded *p,545 s64 sectors, enum bch_data_type ptr_data_type,546 struct bch_alloc_v4 *a)547{548 u32 *dst_sectors = p->has_ec ? &a->stripe_sectors :549 !p->ptr.cached ? &a->dirty_sectors :550 &a->cached_sectors;551 int ret = bch2_bucket_ref_update(trans, ca, k, &p->ptr, sectors, ptr_data_type,552 a->gen, a->data_type, dst_sectors);553 554 if (ret)555 return ret;556 557 alloc_data_type_set(a, ptr_data_type);558 return 0;559}560 561static int bch2_trigger_pointer(struct btree_trans *trans,562 enum btree_id btree_id, unsigned level,563 struct bkey_s_c k, struct extent_ptr_decoded p,564 const union bch_extent_entry *entry,565 s64 *sectors,566 enum btree_iter_update_trigger_flags flags)567{568 struct bch_fs *c = trans->c;569 bool insert = !(flags & BTREE_TRIGGER_overwrite);570 struct printbuf buf = PRINTBUF;571 int ret = 0;572 573 u64 abs_sectors = ptr_disk_sectors(level ? btree_sectors(c) : k.k->size, p);574 *sectors = insert ? abs_sectors : -abs_sectors;575 576 struct bch_dev *ca = bch2_dev_tryget(c, p.ptr.dev);577 if (unlikely(!ca)) {578 if (insert && p.ptr.dev != BCH_SB_MEMBER_INVALID)579 ret = -BCH_ERR_trigger_pointer;580 goto err;581 }582 583 struct bpos bucket;584 struct bch_backpointer bp;585 __bch2_extent_ptr_to_bp(trans->c, ca, btree_id, level, k, p, entry, &bucket, &bp, abs_sectors);586 587 if (flags & BTREE_TRIGGER_transactional) {588 struct bkey_i_alloc_v4 *a = bch2_trans_start_alloc_update(trans, bucket, 0);589 ret = PTR_ERR_OR_ZERO(a) ?:590 __mark_pointer(trans, ca, k, &p, *sectors, bp.data_type, &a->v);591 if (ret)592 goto err;593 594 if (!p.ptr.cached) {595 ret = bch2_bucket_backpointer_mod(trans, ca, bucket, bp, k, insert);596 if (ret)597 goto err;598 }599 }600 601 if (flags & BTREE_TRIGGER_gc) {602 percpu_down_read(&c->mark_lock);603 struct bucket *g = gc_bucket(ca, bucket.offset);604 if (bch2_fs_inconsistent_on(!g, c, "reference to invalid bucket on device %u\n %s",605 p.ptr.dev,606 (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) {607 ret = -BCH_ERR_trigger_pointer;608 goto err_unlock;609 }610 611 bucket_lock(g);612 struct bch_alloc_v4 old = bucket_m_to_alloc(*g), new = old;613 ret = __mark_pointer(trans, ca, k, &p, *sectors, bp.data_type, &new);614 alloc_to_bucket(g, new);615 bucket_unlock(g);616err_unlock:617 percpu_up_read(&c->mark_lock);618 619 if (!ret)620 ret = bch2_alloc_key_to_dev_counters(trans, ca, &old, &new, flags);621 }622err:623 bch2_dev_put(ca);624 printbuf_exit(&buf);625 return ret;626}627 628static int bch2_trigger_stripe_ptr(struct btree_trans *trans,629 struct bkey_s_c k,630 struct extent_ptr_decoded p,631 enum bch_data_type data_type,632 s64 sectors,633 enum btree_iter_update_trigger_flags flags)634{635 if (flags & BTREE_TRIGGER_transactional) {636 struct btree_iter iter;637 struct bkey_i_stripe *s = bch2_bkey_get_mut_typed(trans, &iter,638 BTREE_ID_stripes, POS(0, p.ec.idx),639 BTREE_ITER_with_updates, stripe);640 int ret = PTR_ERR_OR_ZERO(s);641 if (unlikely(ret)) {642 bch2_trans_inconsistent_on(bch2_err_matches(ret, ENOENT), trans,643 "pointer to nonexistent stripe %llu",644 (u64) p.ec.idx);645 goto err;646 }647 648 if (!bch2_ptr_matches_stripe(&s->v, p)) {649 bch2_trans_inconsistent(trans,650 "stripe pointer doesn't match stripe %llu",651 (u64) p.ec.idx);652 ret = -BCH_ERR_trigger_stripe_pointer;653 goto err;654 }655 656 stripe_blockcount_set(&s->v, p.ec.block,657 stripe_blockcount_get(&s->v, p.ec.block) +658 sectors);659 660 struct disk_accounting_pos acc = {661 .type = BCH_DISK_ACCOUNTING_replicas,662 };663 bch2_bkey_to_replicas(&acc.replicas, bkey_i_to_s_c(&s->k_i));664 acc.replicas.data_type = data_type;665 ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, false);666err:667 bch2_trans_iter_exit(trans, &iter);668 return ret;669 }670 671 if (flags & BTREE_TRIGGER_gc) {672 struct bch_fs *c = trans->c;673 674 struct gc_stripe *m = genradix_ptr_alloc(&c->gc_stripes, p.ec.idx, GFP_KERNEL);675 if (!m) {676 bch_err(c, "error allocating memory for gc_stripes, idx %llu",677 (u64) p.ec.idx);678 return -BCH_ERR_ENOMEM_mark_stripe_ptr;679 }680 681 mutex_lock(&c->ec_stripes_heap_lock);682 683 if (!m || !m->alive) {684 mutex_unlock(&c->ec_stripes_heap_lock);685 struct printbuf buf = PRINTBUF;686 bch2_bkey_val_to_text(&buf, c, k);687 bch_err_ratelimited(c, "pointer to nonexistent stripe %llu\n while marking %s",688 (u64) p.ec.idx, buf.buf);689 printbuf_exit(&buf);690 bch2_inconsistent_error(c);691 return -BCH_ERR_trigger_stripe_pointer;692 }693 694 m->block_sectors[p.ec.block] += sectors;695 696 struct disk_accounting_pos acc = {697 .type = BCH_DISK_ACCOUNTING_replicas,698 };699 memcpy(&acc.replicas, &m->r.e, replicas_entry_bytes(&m->r.e));700 mutex_unlock(&c->ec_stripes_heap_lock);701 702 acc.replicas.data_type = data_type;703 int ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, true);704 if (ret)705 return ret;706 }707 708 return 0;709}710 711static int __trigger_extent(struct btree_trans *trans,712 enum btree_id btree_id, unsigned level,713 struct bkey_s_c k,714 enum btree_iter_update_trigger_flags flags,715 s64 *replicas_sectors)716{717 bool gc = flags & BTREE_TRIGGER_gc;718 struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);719 const union bch_extent_entry *entry;720 struct extent_ptr_decoded p;721 enum bch_data_type data_type = bkey_is_btree_ptr(k.k)722 ? BCH_DATA_btree723 : BCH_DATA_user;724 int ret = 0;725 726 struct disk_accounting_pos acc_replicas_key = {727 .type = BCH_DISK_ACCOUNTING_replicas,728 .replicas.data_type = data_type,729 .replicas.nr_devs = 0,730 .replicas.nr_required = 1,731 };732 733 struct disk_accounting_pos acct_compression_key = {734 .type = BCH_DISK_ACCOUNTING_compression,735 };736 u64 compression_acct[3] = { 1, 0, 0 };737 738 bkey_for_each_ptr_decode(k.k, ptrs, p, entry) {739 s64 disk_sectors = 0;740 ret = bch2_trigger_pointer(trans, btree_id, level, k, p, entry, &disk_sectors, flags);741 if (ret < 0)742 return ret;743 744 bool stale = ret > 0;745 746 if (p.ptr.cached && stale)747 continue;748 749 if (p.ptr.cached) {750 ret = bch2_mod_dev_cached_sectors(trans, p.ptr.dev, disk_sectors, gc);751 if (ret)752 return ret;753 } else if (!p.has_ec) {754 *replicas_sectors += disk_sectors;755 replicas_entry_add_dev(&acc_replicas_key.replicas, p.ptr.dev);756 } else {757 ret = bch2_trigger_stripe_ptr(trans, k, p, data_type, disk_sectors, flags);758 if (ret)759 return ret;760 761 /*762 * There may be other dirty pointers in this extent, but763 * if so they're not required for mounting if we have an764 * erasure coded pointer in this extent:765 */766 acc_replicas_key.replicas.nr_required = 0;767 }768 769 if (acct_compression_key.compression.type &&770 acct_compression_key.compression.type != p.crc.compression_type) {771 if (flags & BTREE_TRIGGER_overwrite)772 bch2_u64s_neg(compression_acct, ARRAY_SIZE(compression_acct));773 774 ret = bch2_disk_accounting_mod(trans, &acct_compression_key, compression_acct,775 ARRAY_SIZE(compression_acct), gc);776 if (ret)777 return ret;778 779 compression_acct[0] = 1;780 compression_acct[1] = 0;781 compression_acct[2] = 0;782 }783 784 acct_compression_key.compression.type = p.crc.compression_type;785 if (p.crc.compression_type) {786 compression_acct[1] += p.crc.uncompressed_size;787 compression_acct[2] += p.crc.compressed_size;788 }789 }790 791 if (acc_replicas_key.replicas.nr_devs) {792 ret = bch2_disk_accounting_mod(trans, &acc_replicas_key, replicas_sectors, 1, gc);793 if (ret)794 return ret;795 }796 797 if (acc_replicas_key.replicas.nr_devs && !level && k.k->p.snapshot) {798 struct disk_accounting_pos acc_snapshot_key = {799 .type = BCH_DISK_ACCOUNTING_snapshot,800 .snapshot.id = k.k->p.snapshot,801 };802 ret = bch2_disk_accounting_mod(trans, &acc_snapshot_key, replicas_sectors, 1, gc);803 if (ret)804 return ret;805 }806 807 if (acct_compression_key.compression.type) {808 if (flags & BTREE_TRIGGER_overwrite)809 bch2_u64s_neg(compression_acct, ARRAY_SIZE(compression_acct));810 811 ret = bch2_disk_accounting_mod(trans, &acct_compression_key, compression_acct,812 ARRAY_SIZE(compression_acct), gc);813 if (ret)814 return ret;815 }816 817 if (level) {818 struct disk_accounting_pos acc_btree_key = {819 .type = BCH_DISK_ACCOUNTING_btree,820 .btree.id = btree_id,821 };822 ret = bch2_disk_accounting_mod(trans, &acc_btree_key, replicas_sectors, 1, gc);823 if (ret)824 return ret;825 } else {826 bool insert = !(flags & BTREE_TRIGGER_overwrite);827 struct disk_accounting_pos acc_inum_key = {828 .type = BCH_DISK_ACCOUNTING_inum,829 .inum.inum = k.k->p.inode,830 };831 s64 v[3] = {832 insert ? 1 : -1,833 insert ? k.k->size : -((s64) k.k->size),834 *replicas_sectors,835 };836 ret = bch2_disk_accounting_mod(trans, &acc_inum_key, v, ARRAY_SIZE(v), gc);837 if (ret)838 return ret;839 }840 841 return 0;842}843 844int bch2_trigger_extent(struct btree_trans *trans,845 enum btree_id btree, unsigned level,846 struct bkey_s_c old, struct bkey_s new,847 enum btree_iter_update_trigger_flags flags)848{849 struct bch_fs *c = trans->c;850 struct bkey_ptrs_c new_ptrs = bch2_bkey_ptrs_c(new.s_c);851 struct bkey_ptrs_c old_ptrs = bch2_bkey_ptrs_c(old);852 unsigned new_ptrs_bytes = (void *) new_ptrs.end - (void *) new_ptrs.start;853 unsigned old_ptrs_bytes = (void *) old_ptrs.end - (void *) old_ptrs.start;854 855 if (unlikely(flags & BTREE_TRIGGER_check_repair))856 return bch2_check_fix_ptrs(trans, btree, level, new.s_c, flags);857 858 /* if pointers aren't changing - nothing to do: */859 if (new_ptrs_bytes == old_ptrs_bytes &&860 !memcmp(new_ptrs.start,861 old_ptrs.start,862 new_ptrs_bytes))863 return 0;864 865 if (flags & (BTREE_TRIGGER_transactional|BTREE_TRIGGER_gc)) {866 s64 old_replicas_sectors = 0, new_replicas_sectors = 0;867 868 if (old.k->type) {869 int ret = __trigger_extent(trans, btree, level, old,870 flags & ~BTREE_TRIGGER_insert,871 &old_replicas_sectors);872 if (ret)873 return ret;874 }875 876 if (new.k->type) {877 int ret = __trigger_extent(trans, btree, level, new.s_c,878 flags & ~BTREE_TRIGGER_overwrite,879 &new_replicas_sectors);880 if (ret)881 return ret;882 }883 884 int need_rebalance_delta = 0;885 s64 need_rebalance_sectors_delta = 0;886 887 s64 s = bch2_bkey_sectors_need_rebalance(c, old);888 need_rebalance_delta -= s != 0;889 need_rebalance_sectors_delta -= s;890 891 s = bch2_bkey_sectors_need_rebalance(c, new.s_c);892 need_rebalance_delta += s != 0;893 need_rebalance_sectors_delta += s;894 895 if ((flags & BTREE_TRIGGER_transactional) && need_rebalance_delta) {896 int ret = bch2_btree_bit_mod_buffered(trans, BTREE_ID_rebalance_work,897 new.k->p, need_rebalance_delta > 0);898 if (ret)899 return ret;900 }901 902 if (need_rebalance_sectors_delta) {903 struct disk_accounting_pos acc = {904 .type = BCH_DISK_ACCOUNTING_rebalance_work,905 };906 int ret = bch2_disk_accounting_mod(trans, &acc, &need_rebalance_sectors_delta, 1,907 flags & BTREE_TRIGGER_gc);908 if (ret)909 return ret;910 }911 }912 913 return 0;914}915 916/* KEY_TYPE_reservation */917 918static int __trigger_reservation(struct btree_trans *trans,919 enum btree_id btree_id, unsigned level, struct bkey_s_c k,920 enum btree_iter_update_trigger_flags flags)921{922 if (flags & (BTREE_TRIGGER_transactional|BTREE_TRIGGER_gc)) {923 s64 sectors = k.k->size;924 925 if (flags & BTREE_TRIGGER_overwrite)926 sectors = -sectors;927 928 struct disk_accounting_pos acc = {929 .type = BCH_DISK_ACCOUNTING_persistent_reserved,930 .persistent_reserved.nr_replicas = bkey_s_c_to_reservation(k).v->nr_replicas,931 };932 933 return bch2_disk_accounting_mod(trans, &acc, §ors, 1, flags & BTREE_TRIGGER_gc);934 }935 936 return 0;937}938 939int bch2_trigger_reservation(struct btree_trans *trans,940 enum btree_id btree_id, unsigned level,941 struct bkey_s_c old, struct bkey_s new,942 enum btree_iter_update_trigger_flags flags)943{944 return trigger_run_overwrite_then_insert(__trigger_reservation, trans, btree_id, level, old, new, flags);945}946 947/* Mark superblocks: */948 949static int __bch2_trans_mark_metadata_bucket(struct btree_trans *trans,950 struct bch_dev *ca, u64 b,951 enum bch_data_type type,952 unsigned sectors)953{954 struct btree_iter iter;955 int ret = 0;956 957 struct bkey_i_alloc_v4 *a =958 bch2_trans_start_alloc_update_noupdate(trans, &iter, POS(ca->dev_idx, b));959 if (IS_ERR(a))960 return PTR_ERR(a);961 962 if (a->v.data_type && type && a->v.data_type != type) {963 bch2_fsck_err(trans, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,964 bucket_metadata_type_mismatch,965 "bucket %llu:%llu gen %u different types of data in same bucket: %s, %s\n"966 "while marking %s",967 iter.pos.inode, iter.pos.offset, a->v.gen,968 bch2_data_type_str(a->v.data_type),969 bch2_data_type_str(type),970 bch2_data_type_str(type));971 ret = -BCH_ERR_metadata_bucket_inconsistency;972 goto err;973 }974 975 if (a->v.data_type != type ||976 a->v.dirty_sectors != sectors) {977 a->v.data_type = type;978 a->v.dirty_sectors = sectors;979 ret = bch2_trans_update(trans, &iter, &a->k_i, 0);980 }981err:982 bch2_trans_iter_exit(trans, &iter);983 return ret;984}985 986static int bch2_mark_metadata_bucket(struct btree_trans *trans, struct bch_dev *ca,987 u64 b, enum bch_data_type data_type, unsigned sectors,988 enum btree_iter_update_trigger_flags flags)989{990 struct bch_fs *c = trans->c;991 int ret = 0;992 993 percpu_down_read(&c->mark_lock);994 struct bucket *g = gc_bucket(ca, b);995 if (bch2_fs_inconsistent_on(!g, c, "reference to invalid bucket on device %u when marking metadata type %s",996 ca->dev_idx, bch2_data_type_str(data_type)))997 goto err_unlock;998 999 bucket_lock(g);1000 struct bch_alloc_v4 old = bucket_m_to_alloc(*g);1001 1002 if (bch2_fs_inconsistent_on(g->data_type &&1003 g->data_type != data_type, c,1004 "different types of data in same bucket: %s, %s",1005 bch2_data_type_str(g->data_type),1006 bch2_data_type_str(data_type)))1007 goto err;1008 1009 if (bch2_fs_inconsistent_on((u64) g->dirty_sectors + sectors > ca->mi.bucket_size, c,1010 "bucket %u:%llu gen %u data type %s sector count overflow: %u + %u > bucket size",1011 ca->dev_idx, b, g->gen,1012 bch2_data_type_str(g->data_type ?: data_type),1013 g->dirty_sectors, sectors))1014 goto err;1015 1016 g->data_type = data_type;1017 g->dirty_sectors += sectors;1018 struct bch_alloc_v4 new = bucket_m_to_alloc(*g);1019 bucket_unlock(g);1020 percpu_up_read(&c->mark_lock);1021 ret = bch2_alloc_key_to_dev_counters(trans, ca, &old, &new, flags);1022 return ret;1023err:1024 bucket_unlock(g);1025err_unlock:1026 percpu_up_read(&c->mark_lock);1027 return -BCH_ERR_metadata_bucket_inconsistency;1028}1029 1030int bch2_trans_mark_metadata_bucket(struct btree_trans *trans,1031 struct bch_dev *ca, u64 b,1032 enum bch_data_type type, unsigned sectors,1033 enum btree_iter_update_trigger_flags flags)1034{1035 BUG_ON(type != BCH_DATA_free &&1036 type != BCH_DATA_sb &&1037 type != BCH_DATA_journal);1038 1039 /*1040 * Backup superblock might be past the end of our normal usable space:1041 */1042 if (b >= ca->mi.nbuckets)1043 return 0;1044 1045 if (flags & BTREE_TRIGGER_gc)1046 return bch2_mark_metadata_bucket(trans, ca, b, type, sectors, flags);1047 else if (flags & BTREE_TRIGGER_transactional)1048 return commit_do(trans, NULL, NULL, 0,1049 __bch2_trans_mark_metadata_bucket(trans, ca, b, type, sectors));1050 else1051 BUG();1052}1053 1054static int bch2_trans_mark_metadata_sectors(struct btree_trans *trans,1055 struct bch_dev *ca, u64 start, u64 end,1056 enum bch_data_type type, u64 *bucket, unsigned *bucket_sectors,1057 enum btree_iter_update_trigger_flags flags)1058{1059 do {1060 u64 b = sector_to_bucket(ca, start);1061 unsigned sectors =1062 min_t(u64, bucket_to_sector(ca, b + 1), end) - start;1063 1064 if (b != *bucket && *bucket_sectors) {1065 int ret = bch2_trans_mark_metadata_bucket(trans, ca, *bucket,1066 type, *bucket_sectors, flags);1067 if (ret)1068 return ret;1069 1070 *bucket_sectors = 0;1071 }1072 1073 *bucket = b;1074 *bucket_sectors += sectors;1075 start += sectors;1076 } while (start < end);1077 1078 return 0;1079}1080 1081static int __bch2_trans_mark_dev_sb(struct btree_trans *trans, struct bch_dev *ca,1082 enum btree_iter_update_trigger_flags flags)1083{1084 struct bch_fs *c = trans->c;1085 1086 mutex_lock(&c->sb_lock);1087 struct bch_sb_layout layout = ca->disk_sb.sb->layout;1088 mutex_unlock(&c->sb_lock);1089 1090 u64 bucket = 0;1091 unsigned i, bucket_sectors = 0;1092 int ret;1093 1094 for (i = 0; i < layout.nr_superblocks; i++) {1095 u64 offset = le64_to_cpu(layout.sb_offset[i]);1096 1097 if (offset == BCH_SB_SECTOR) {1098 ret = bch2_trans_mark_metadata_sectors(trans, ca,1099 0, BCH_SB_SECTOR,1100 BCH_DATA_sb, &bucket, &bucket_sectors, flags);1101 if (ret)1102 return ret;1103 }1104 1105 ret = bch2_trans_mark_metadata_sectors(trans, ca, offset,1106 offset + (1 << layout.sb_max_size_bits),1107 BCH_DATA_sb, &bucket, &bucket_sectors, flags);1108 if (ret)1109 return ret;1110 }1111 1112 if (bucket_sectors) {1113 ret = bch2_trans_mark_metadata_bucket(trans, ca,1114 bucket, BCH_DATA_sb, bucket_sectors, flags);1115 if (ret)1116 return ret;1117 }1118 1119 for (i = 0; i < ca->journal.nr; i++) {1120 ret = bch2_trans_mark_metadata_bucket(trans, ca,1121 ca->journal.buckets[i],1122 BCH_DATA_journal, ca->mi.bucket_size, flags);1123 if (ret)1124 return ret;1125 }1126 1127 return 0;1128}1129 1130int bch2_trans_mark_dev_sb(struct bch_fs *c, struct bch_dev *ca,1131 enum btree_iter_update_trigger_flags flags)1132{1133 int ret = bch2_trans_run(c,1134 __bch2_trans_mark_dev_sb(trans, ca, flags));1135 bch_err_fn(c, ret);1136 return ret;1137}1138 1139int bch2_trans_mark_dev_sbs_flags(struct bch_fs *c,1140 enum btree_iter_update_trigger_flags flags)1141{1142 for_each_online_member(c, ca) {1143 int ret = bch2_trans_mark_dev_sb(c, ca, flags);1144 if (ret) {1145 percpu_ref_put(&ca->io_ref);1146 return ret;1147 }1148 }1149 1150 return 0;1151}1152 1153int bch2_trans_mark_dev_sbs(struct bch_fs *c)1154{1155 return bch2_trans_mark_dev_sbs_flags(c, BTREE_TRIGGER_transactional);1156}1157 1158/* Disk reservations: */1159 1160#define SECTORS_CACHE 10241161 1162int __bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res,1163 u64 sectors, enum bch_reservation_flags flags)1164{1165 struct bch_fs_pcpu *pcpu;1166 u64 old, get;1167 u64 sectors_available;1168 int ret;1169 1170 percpu_down_read(&c->mark_lock);1171 preempt_disable();1172 pcpu = this_cpu_ptr(c->pcpu);1173 1174 if (sectors <= pcpu->sectors_available)1175 goto out;1176 1177 old = atomic64_read(&c->sectors_available);1178 do {1179 get = min((u64) sectors + SECTORS_CACHE, old);1180 1181 if (get < sectors) {1182 preempt_enable();1183 goto recalculate;1184 }1185 } while (!atomic64_try_cmpxchg(&c->sectors_available,1186 &old, old - get));1187 1188 pcpu->sectors_available += get;1189 1190out:1191 pcpu->sectors_available -= sectors;1192 this_cpu_add(*c->online_reserved, sectors);1193 res->sectors += sectors;1194 1195 preempt_enable();1196 percpu_up_read(&c->mark_lock);1197 return 0;1198 1199recalculate:1200 mutex_lock(&c->sectors_available_lock);1201 1202 percpu_u64_set(&c->pcpu->sectors_available, 0);1203 sectors_available = avail_factor(__bch2_fs_usage_read_short(c).free);1204 1205 if (sectors_available && (flags & BCH_DISK_RESERVATION_PARTIAL))1206 sectors = min(sectors, sectors_available);1207 1208 if (sectors <= sectors_available ||1209 (flags & BCH_DISK_RESERVATION_NOFAIL)) {1210 atomic64_set(&c->sectors_available,1211 max_t(s64, 0, sectors_available - sectors));1212 this_cpu_add(*c->online_reserved, sectors);1213 res->sectors += sectors;1214 ret = 0;1215 } else {1216 atomic64_set(&c->sectors_available, sectors_available);1217 ret = -BCH_ERR_ENOSPC_disk_reservation;1218 }1219 1220 mutex_unlock(&c->sectors_available_lock);1221 percpu_up_read(&c->mark_lock);1222 1223 return ret;1224}1225 1226/* Startup/shutdown: */1227 1228void bch2_buckets_nouse_free(struct bch_fs *c)1229{1230 for_each_member_device(c, ca) {1231 kvfree_rcu_mightsleep(ca->buckets_nouse);1232 ca->buckets_nouse = NULL;1233 }1234}1235 1236int bch2_buckets_nouse_alloc(struct bch_fs *c)1237{1238 for_each_member_device(c, ca) {1239 BUG_ON(ca->buckets_nouse);1240 1241 ca->buckets_nouse = kvmalloc(BITS_TO_LONGS(ca->mi.nbuckets) *1242 sizeof(unsigned long),1243 GFP_KERNEL|__GFP_ZERO);1244 if (!ca->buckets_nouse) {1245 bch2_dev_put(ca);1246 return -BCH_ERR_ENOMEM_buckets_nouse;1247 }1248 }1249 1250 return 0;1251}1252 1253static void bucket_gens_free_rcu(struct rcu_head *rcu)1254{1255 struct bucket_gens *buckets =1256 container_of(rcu, struct bucket_gens, rcu);1257 1258 kvfree(buckets);1259}1260 1261int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)1262{1263 struct bucket_gens *bucket_gens = NULL, *old_bucket_gens = NULL;1264 bool resize = ca->bucket_gens != NULL;1265 int ret;1266 1267 BUG_ON(resize && ca->buckets_nouse);1268 1269 if (!(bucket_gens = kvmalloc(sizeof(struct bucket_gens) + nbuckets,1270 GFP_KERNEL|__GFP_ZERO))) {1271 ret = -BCH_ERR_ENOMEM_bucket_gens;1272 goto err;1273 }1274 1275 bucket_gens->first_bucket = ca->mi.first_bucket;1276 bucket_gens->nbuckets = nbuckets;1277 bucket_gens->nbuckets_minus_first =1278 bucket_gens->nbuckets - bucket_gens->first_bucket;1279 1280 if (resize) {1281 down_write(&ca->bucket_lock);1282 percpu_down_write(&c->mark_lock);1283 }1284 1285 old_bucket_gens = rcu_dereference_protected(ca->bucket_gens, 1);1286 1287 if (resize) {1288 size_t n = min(bucket_gens->nbuckets, old_bucket_gens->nbuckets);1289 1290 memcpy(bucket_gens->b,1291 old_bucket_gens->b,1292 n);1293 }1294 1295 rcu_assign_pointer(ca->bucket_gens, bucket_gens);1296 bucket_gens = old_bucket_gens;1297 1298 nbuckets = ca->mi.nbuckets;1299 1300 if (resize) {1301 percpu_up_write(&c->mark_lock);1302 up_write(&ca->bucket_lock);1303 }1304 1305 ret = 0;1306err:1307 if (bucket_gens)1308 call_rcu(&bucket_gens->rcu, bucket_gens_free_rcu);1309 1310 return ret;1311}1312 1313void bch2_dev_buckets_free(struct bch_dev *ca)1314{1315 kvfree(ca->buckets_nouse);1316 kvfree(rcu_dereference_protected(ca->bucket_gens, 1));1317 free_percpu(ca->usage);1318}1319 1320int bch2_dev_buckets_alloc(struct bch_fs *c, struct bch_dev *ca)1321{1322 ca->usage = alloc_percpu(struct bch_dev_usage);1323 if (!ca->usage)1324 return -BCH_ERR_ENOMEM_usage_init;1325 1326 return bch2_dev_buckets_resize(c, ca, ca->mi.nbuckets);1327}1328