977 lines · c
1// SPDX-License-Identifier: GPL-2.02 3#include "bcachefs.h"4#include "bcachefs_ioctl.h"5#include "btree_cache.h"6#include "btree_journal_iter.h"7#include "btree_update.h"8#include "btree_write_buffer.h"9#include "buckets.h"10#include "compress.h"11#include "disk_accounting.h"12#include "error.h"13#include "journal_io.h"14#include "replicas.h"15 16/*17 * Notes on disk accounting:18 *19 * We have two parallel sets of counters to be concerned with, and both must be20 * kept in sync.21 *22 * - Persistent/on disk accounting, stored in the accounting btree and updated23 * via btree write buffer updates that treat new accounting keys as deltas to24 * apply to existing values. But reading from a write buffer btree is25 * expensive, so we also have26 *27 * - In memory accounting, where accounting is stored as an array of percpu28 * counters, indexed by an eytzinger array of disk acounting keys/bpos (which29 * are the same thing, excepting byte swabbing on big endian).30 *31 * Cheap to read, but non persistent.32 *33 * Disk accounting updates are generated by transactional triggers; these run as34 * keys enter and leave the btree, and can compare old and new versions of keys;35 * the output of these triggers are deltas to the various counters.36 *37 * Disk accounting updates are done as btree write buffer updates, where the38 * counters in the disk accounting key are deltas that will be applied to the39 * counter in the btree when the key is flushed by the write buffer (or journal40 * replay).41 *42 * To do a disk accounting update:43 * - initialize a disk_accounting_pos, to specify which counter is being update44 * - initialize counter deltas, as an array of 1-3 s64s45 * - call bch2_disk_accounting_mod()46 *47 * This queues up the accounting update to be done at transaction commit time.48 * Underneath, it's a normal btree write buffer update.49 *50 * The transaction commit path is responsible for propagating updates to the in51 * memory counters, with bch2_accounting_mem_mod().52 *53 * The commit path also assigns every disk accounting update a unique version54 * number, based on the journal sequence number and offset within that journal55 * buffer; this is used by journal replay to determine which updates have been56 * done.57 *58 * The transaction commit path also ensures that replicas entry accounting59 * updates are properly marked in the superblock (so that we know whether we can60 * mount without data being unavailable); it will update the superblock if61 * bch2_accounting_mem_mod() tells it to.62 */63 64static const char * const disk_accounting_type_strs[] = {65#define x(t, n, ...) [n] = #t,66 BCH_DISK_ACCOUNTING_TYPES()67#undef x68 NULL69};70 71static inline void accounting_key_init(struct bkey_i *k, struct disk_accounting_pos *pos,72 s64 *d, unsigned nr)73{74 struct bkey_i_accounting *acc = bkey_accounting_init(k);75 76 acc->k.p = disk_accounting_pos_to_bpos(pos);77 set_bkey_val_u64s(&acc->k, sizeof(struct bch_accounting) / sizeof(u64) + nr);78 79 memcpy_u64s_small(acc->v.d, d, nr);80}81 82int bch2_disk_accounting_mod(struct btree_trans *trans,83 struct disk_accounting_pos *k,84 s64 *d, unsigned nr, bool gc)85{86 /* Normalize: */87 switch (k->type) {88 case BCH_DISK_ACCOUNTING_replicas:89 bubble_sort(k->replicas.devs, k->replicas.nr_devs, u8_cmp);90 break;91 }92 93 BUG_ON(nr > BCH_ACCOUNTING_MAX_COUNTERS);94 95 struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i;96 97 accounting_key_init(&k_i.k, k, d, nr);98 99 return likely(!gc)100 ? bch2_trans_update_buffered(trans, BTREE_ID_accounting, &k_i.k)101 : bch2_accounting_mem_add(trans, bkey_i_to_s_c_accounting(&k_i.k), true);102}103 104int bch2_mod_dev_cached_sectors(struct btree_trans *trans,105 unsigned dev, s64 sectors,106 bool gc)107{108 struct disk_accounting_pos acc = {109 .type = BCH_DISK_ACCOUNTING_replicas,110 };111 112 bch2_replicas_entry_cached(&acc.replicas, dev);113 114 return bch2_disk_accounting_mod(trans, &acc, §ors, 1, gc);115}116 117static inline bool is_zero(char *start, char *end)118{119 BUG_ON(start > end);120 121 for (; start < end; start++)122 if (*start)123 return false;124 return true;125}126 127#define field_end(p, member) (((void *) (&p.member)) + sizeof(p.member))128 129int bch2_accounting_validate(struct bch_fs *c, struct bkey_s_c k,130 enum bch_validate_flags flags)131{132 struct disk_accounting_pos acc_k;133 bpos_to_disk_accounting_pos(&acc_k, k.k->p);134 void *end = &acc_k + 1;135 int ret = 0;136 137 bkey_fsck_err_on(bversion_zero(k.k->bversion),138 c, accounting_key_version_0,139 "accounting key with version=0");140 141 switch (acc_k.type) {142 case BCH_DISK_ACCOUNTING_nr_inodes:143 end = field_end(acc_k, nr_inodes);144 break;145 case BCH_DISK_ACCOUNTING_persistent_reserved:146 end = field_end(acc_k, persistent_reserved);147 break;148 case BCH_DISK_ACCOUNTING_replicas:149 bkey_fsck_err_on(!acc_k.replicas.nr_devs,150 c, accounting_key_replicas_nr_devs_0,151 "accounting key replicas entry with nr_devs=0");152 153 bkey_fsck_err_on(acc_k.replicas.nr_required > acc_k.replicas.nr_devs ||154 (acc_k.replicas.nr_required > 1 &&155 acc_k.replicas.nr_required == acc_k.replicas.nr_devs),156 c, accounting_key_replicas_nr_required_bad,157 "accounting key replicas entry with bad nr_required");158 159 for (unsigned i = 0; i + 1 < acc_k.replicas.nr_devs; i++)160 bkey_fsck_err_on(acc_k.replicas.devs[i] >= acc_k.replicas.devs[i + 1],161 c, accounting_key_replicas_devs_unsorted,162 "accounting key replicas entry with unsorted devs");163 164 end = (void *) &acc_k.replicas + replicas_entry_bytes(&acc_k.replicas);165 break;166 case BCH_DISK_ACCOUNTING_dev_data_type:167 end = field_end(acc_k, dev_data_type);168 break;169 case BCH_DISK_ACCOUNTING_compression:170 end = field_end(acc_k, compression);171 break;172 case BCH_DISK_ACCOUNTING_snapshot:173 end = field_end(acc_k, snapshot);174 break;175 case BCH_DISK_ACCOUNTING_btree:176 end = field_end(acc_k, btree);177 break;178 case BCH_DISK_ACCOUNTING_rebalance_work:179 end = field_end(acc_k, rebalance_work);180 break;181 }182 183 bkey_fsck_err_on(!is_zero(end, (void *) (&acc_k + 1)),184 c, accounting_key_junk_at_end,185 "junk at end of accounting key");186fsck_err:187 return ret;188}189 190void bch2_accounting_key_to_text(struct printbuf *out, struct disk_accounting_pos *k)191{192 if (k->type >= BCH_DISK_ACCOUNTING_TYPE_NR) {193 prt_printf(out, "unknown type %u", k->type);194 return;195 }196 197 prt_str(out, disk_accounting_type_strs[k->type]);198 prt_str(out, " ");199 200 switch (k->type) {201 case BCH_DISK_ACCOUNTING_nr_inodes:202 break;203 case BCH_DISK_ACCOUNTING_persistent_reserved:204 prt_printf(out, "replicas=%u", k->persistent_reserved.nr_replicas);205 break;206 case BCH_DISK_ACCOUNTING_replicas:207 bch2_replicas_entry_to_text(out, &k->replicas);208 break;209 case BCH_DISK_ACCOUNTING_dev_data_type:210 prt_printf(out, "dev=%u data_type=", k->dev_data_type.dev);211 bch2_prt_data_type(out, k->dev_data_type.data_type);212 break;213 case BCH_DISK_ACCOUNTING_compression:214 bch2_prt_compression_type(out, k->compression.type);215 break;216 case BCH_DISK_ACCOUNTING_snapshot:217 prt_printf(out, "id=%u", k->snapshot.id);218 break;219 case BCH_DISK_ACCOUNTING_btree:220 prt_printf(out, "btree=%s", bch2_btree_id_str(k->btree.id));221 break;222 }223}224 225void bch2_accounting_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k)226{227 struct bkey_s_c_accounting acc = bkey_s_c_to_accounting(k);228 struct disk_accounting_pos acc_k;229 bpos_to_disk_accounting_pos(&acc_k, k.k->p);230 231 bch2_accounting_key_to_text(out, &acc_k);232 233 for (unsigned i = 0; i < bch2_accounting_counters(k.k); i++)234 prt_printf(out, " %lli", acc.v->d[i]);235}236 237void bch2_accounting_swab(struct bkey_s k)238{239 for (u64 *p = (u64 *) k.v;240 p < (u64 *) bkey_val_end(k);241 p++)242 *p = swab64(*p);243}244 245static inline void __accounting_to_replicas(struct bch_replicas_entry_v1 *r,246 struct disk_accounting_pos acc)247{248 unsafe_memcpy(r, &acc.replicas,249 replicas_entry_bytes(&acc.replicas),250 "variable length struct");251}252 253static inline bool accounting_to_replicas(struct bch_replicas_entry_v1 *r, struct bpos p)254{255 struct disk_accounting_pos acc_k;256 bpos_to_disk_accounting_pos(&acc_k, p);257 258 switch (acc_k.type) {259 case BCH_DISK_ACCOUNTING_replicas:260 __accounting_to_replicas(r, acc_k);261 return true;262 default:263 return false;264 }265}266 267static int bch2_accounting_update_sb_one(struct bch_fs *c, struct bpos p)268{269 struct bch_replicas_padded r;270 return accounting_to_replicas(&r.e, p)271 ? bch2_mark_replicas(c, &r.e)272 : 0;273}274 275/*276 * Ensure accounting keys being updated are present in the superblock, when277 * applicable (i.e. replicas updates)278 */279int bch2_accounting_update_sb(struct btree_trans *trans)280{281 for (struct jset_entry *i = trans->journal_entries;282 i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s);283 i = vstruct_next(i))284 if (jset_entry_is_key(i) && i->start->k.type == KEY_TYPE_accounting) {285 int ret = bch2_accounting_update_sb_one(trans->c, i->start->k.p);286 if (ret)287 return ret;288 }289 290 return 0;291}292 293static int __bch2_accounting_mem_insert(struct bch_fs *c, struct bkey_s_c_accounting a)294{295 struct bch_accounting_mem *acc = &c->accounting;296 297 /* raced with another insert, already present: */298 if (eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),299 accounting_pos_cmp, &a.k->p) < acc->k.nr)300 return 0;301 302 struct accounting_mem_entry n = {303 .pos = a.k->p,304 .bversion = a.k->bversion,305 .nr_counters = bch2_accounting_counters(a.k),306 .v[0] = __alloc_percpu_gfp(n.nr_counters * sizeof(u64),307 sizeof(u64), GFP_KERNEL),308 };309 310 if (!n.v[0])311 goto err;312 313 if (acc->gc_running) {314 n.v[1] = __alloc_percpu_gfp(n.nr_counters * sizeof(u64),315 sizeof(u64), GFP_KERNEL);316 if (!n.v[1])317 goto err;318 }319 320 if (darray_push(&acc->k, n))321 goto err;322 323 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),324 accounting_pos_cmp, NULL);325 return 0;326err:327 free_percpu(n.v[1]);328 free_percpu(n.v[0]);329 return -BCH_ERR_ENOMEM_disk_accounting;330}331 332int bch2_accounting_mem_insert(struct bch_fs *c, struct bkey_s_c_accounting a,333 enum bch_accounting_mode mode)334{335 struct bch_replicas_padded r;336 337 if (mode != BCH_ACCOUNTING_read &&338 accounting_to_replicas(&r.e, a.k->p) &&339 !bch2_replicas_marked_locked(c, &r.e))340 return -BCH_ERR_btree_insert_need_mark_replicas;341 342 percpu_up_read(&c->mark_lock);343 percpu_down_write(&c->mark_lock);344 int ret = __bch2_accounting_mem_insert(c, a);345 percpu_up_write(&c->mark_lock);346 percpu_down_read(&c->mark_lock);347 return ret;348}349 350static bool accounting_mem_entry_is_zero(struct accounting_mem_entry *e)351{352 for (unsigned i = 0; i < e->nr_counters; i++)353 if (percpu_u64_get(e->v[0] + i) ||354 (e->v[1] &&355 percpu_u64_get(e->v[1] + i)))356 return false;357 return true;358}359 360void bch2_accounting_mem_gc(struct bch_fs *c)361{362 struct bch_accounting_mem *acc = &c->accounting;363 364 percpu_down_write(&c->mark_lock);365 struct accounting_mem_entry *dst = acc->k.data;366 367 darray_for_each(acc->k, src) {368 if (accounting_mem_entry_is_zero(src)) {369 free_percpu(src->v[0]);370 free_percpu(src->v[1]);371 } else {372 *dst++ = *src;373 }374 }375 376 acc->k.nr = dst - acc->k.data;377 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),378 accounting_pos_cmp, NULL);379 percpu_up_write(&c->mark_lock);380}381 382/*383 * Read out accounting keys for replicas entries, as an array of384 * bch_replicas_usage entries.385 *386 * Note: this may be deprecated/removed at smoe point in the future and replaced387 * with something more general, it exists to support the ioctl used by the388 * 'bcachefs fs usage' command.389 */390int bch2_fs_replicas_usage_read(struct bch_fs *c, darray_char *usage)391{392 struct bch_accounting_mem *acc = &c->accounting;393 int ret = 0;394 395 darray_init(usage);396 397 percpu_down_read(&c->mark_lock);398 darray_for_each(acc->k, i) {399 struct {400 struct bch_replicas_usage r;401 u8 pad[BCH_BKEY_PTRS_MAX];402 } u;403 404 if (!accounting_to_replicas(&u.r.r, i->pos))405 continue;406 407 u64 sectors;408 bch2_accounting_mem_read_counters(acc, i - acc->k.data, §ors, 1, false);409 u.r.sectors = sectors;410 411 ret = darray_make_room(usage, replicas_usage_bytes(&u.r));412 if (ret)413 break;414 415 memcpy(&darray_top(*usage), &u.r, replicas_usage_bytes(&u.r));416 usage->nr += replicas_usage_bytes(&u.r);417 }418 percpu_up_read(&c->mark_lock);419 420 if (ret)421 darray_exit(usage);422 return ret;423}424 425int bch2_fs_accounting_read(struct bch_fs *c, darray_char *out_buf, unsigned accounting_types_mask)426{427 428 struct bch_accounting_mem *acc = &c->accounting;429 int ret = 0;430 431 darray_init(out_buf);432 433 percpu_down_read(&c->mark_lock);434 darray_for_each(acc->k, i) {435 struct disk_accounting_pos a_p;436 bpos_to_disk_accounting_pos(&a_p, i->pos);437 438 if (!(accounting_types_mask & BIT(a_p.type)))439 continue;440 441 ret = darray_make_room(out_buf, sizeof(struct bkey_i_accounting) +442 sizeof(u64) * i->nr_counters);443 if (ret)444 break;445 446 struct bkey_i_accounting *a_out =447 bkey_accounting_init((void *) &darray_top(*out_buf));448 set_bkey_val_u64s(&a_out->k, i->nr_counters);449 a_out->k.p = i->pos;450 bch2_accounting_mem_read_counters(acc, i - acc->k.data,451 a_out->v.d, i->nr_counters, false);452 453 if (!bch2_accounting_key_is_zero(accounting_i_to_s_c(a_out)))454 out_buf->nr += bkey_bytes(&a_out->k);455 }456 457 percpu_up_read(&c->mark_lock);458 459 if (ret)460 darray_exit(out_buf);461 return ret;462}463 464void bch2_fs_accounting_to_text(struct printbuf *out, struct bch_fs *c)465{466 struct bch_accounting_mem *acc = &c->accounting;467 468 percpu_down_read(&c->mark_lock);469 out->atomic++;470 471 eytzinger0_for_each(i, acc->k.nr) {472 struct disk_accounting_pos acc_k;473 bpos_to_disk_accounting_pos(&acc_k, acc->k.data[i].pos);474 475 bch2_accounting_key_to_text(out, &acc_k);476 477 u64 v[BCH_ACCOUNTING_MAX_COUNTERS];478 bch2_accounting_mem_read_counters(acc, i, v, ARRAY_SIZE(v), false);479 480 prt_str(out, ":");481 for (unsigned j = 0; j < acc->k.data[i].nr_counters; j++)482 prt_printf(out, " %llu", v[j]);483 prt_newline(out);484 }485 486 --out->atomic;487 percpu_up_read(&c->mark_lock);488}489 490static void bch2_accounting_free_counters(struct bch_accounting_mem *acc, bool gc)491{492 darray_for_each(acc->k, e) {493 free_percpu(e->v[gc]);494 e->v[gc] = NULL;495 }496}497 498int bch2_gc_accounting_start(struct bch_fs *c)499{500 struct bch_accounting_mem *acc = &c->accounting;501 int ret = 0;502 503 percpu_down_write(&c->mark_lock);504 darray_for_each(acc->k, e) {505 e->v[1] = __alloc_percpu_gfp(e->nr_counters * sizeof(u64),506 sizeof(u64), GFP_KERNEL);507 if (!e->v[1]) {508 bch2_accounting_free_counters(acc, true);509 ret = -BCH_ERR_ENOMEM_disk_accounting;510 break;511 }512 }513 514 acc->gc_running = !ret;515 percpu_up_write(&c->mark_lock);516 517 return ret;518}519 520int bch2_gc_accounting_done(struct bch_fs *c)521{522 struct bch_accounting_mem *acc = &c->accounting;523 struct btree_trans *trans = bch2_trans_get(c);524 struct printbuf buf = PRINTBUF;525 struct bpos pos = POS_MIN;526 int ret = 0;527 528 percpu_down_write(&c->mark_lock);529 while (1) {530 unsigned idx = eytzinger0_find_ge(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),531 accounting_pos_cmp, &pos);532 533 if (idx >= acc->k.nr)534 break;535 536 struct accounting_mem_entry *e = acc->k.data + idx;537 pos = bpos_successor(e->pos);538 539 struct disk_accounting_pos acc_k;540 bpos_to_disk_accounting_pos(&acc_k, e->pos);541 542 if (acc_k.type >= BCH_DISK_ACCOUNTING_TYPE_NR)543 continue;544 545 u64 src_v[BCH_ACCOUNTING_MAX_COUNTERS];546 u64 dst_v[BCH_ACCOUNTING_MAX_COUNTERS];547 548 unsigned nr = e->nr_counters;549 bch2_accounting_mem_read_counters(acc, idx, dst_v, nr, false);550 bch2_accounting_mem_read_counters(acc, idx, src_v, nr, true);551 552 if (memcmp(dst_v, src_v, nr * sizeof(u64))) {553 printbuf_reset(&buf);554 prt_str(&buf, "accounting mismatch for ");555 bch2_accounting_key_to_text(&buf, &acc_k);556 557 prt_str(&buf, ": got");558 for (unsigned j = 0; j < nr; j++)559 prt_printf(&buf, " %llu", dst_v[j]);560 561 prt_str(&buf, " should be");562 for (unsigned j = 0; j < nr; j++)563 prt_printf(&buf, " %llu", src_v[j]);564 565 for (unsigned j = 0; j < nr; j++)566 src_v[j] -= dst_v[j];567 568 if (fsck_err(trans, accounting_mismatch, "%s", buf.buf)) {569 percpu_up_write(&c->mark_lock);570 ret = commit_do(trans, NULL, NULL, 0,571 bch2_disk_accounting_mod(trans, &acc_k, src_v, nr, false));572 percpu_down_write(&c->mark_lock);573 if (ret)574 goto err;575 576 if (!test_bit(BCH_FS_may_go_rw, &c->flags)) {577 memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta));578 struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i;579 580 accounting_key_init(&k_i.k, &acc_k, src_v, nr);581 bch2_accounting_mem_mod_locked(trans,582 bkey_i_to_s_c_accounting(&k_i.k),583 BCH_ACCOUNTING_normal);584 585 preempt_disable();586 struct bch_fs_usage_base *dst = this_cpu_ptr(c->usage);587 struct bch_fs_usage_base *src = &trans->fs_usage_delta;588 acc_u64s((u64 *) dst, (u64 *) src, sizeof(*src) / sizeof(u64));589 preempt_enable();590 }591 }592 }593 }594err:595fsck_err:596 percpu_up_write(&c->mark_lock);597 printbuf_exit(&buf);598 bch2_trans_put(trans);599 bch_err_fn(c, ret);600 return ret;601}602 603static int accounting_read_key(struct btree_trans *trans, struct bkey_s_c k)604{605 struct bch_fs *c = trans->c;606 607 if (k.k->type != KEY_TYPE_accounting)608 return 0;609 610 percpu_down_read(&c->mark_lock);611 int ret = bch2_accounting_mem_mod_locked(trans, bkey_s_c_to_accounting(k),612 BCH_ACCOUNTING_read);613 percpu_up_read(&c->mark_lock);614 return ret;615}616 617static int bch2_disk_accounting_validate_late(struct btree_trans *trans,618 struct disk_accounting_pos acc,619 u64 *v, unsigned nr)620{621 struct bch_fs *c = trans->c;622 struct printbuf buf = PRINTBUF;623 int ret = 0, invalid_dev = -1;624 625 switch (acc.type) {626 case BCH_DISK_ACCOUNTING_replicas: {627 struct bch_replicas_padded r;628 __accounting_to_replicas(&r.e, acc);629 630 for (unsigned i = 0; i < r.e.nr_devs; i++)631 if (r.e.devs[i] != BCH_SB_MEMBER_INVALID &&632 !bch2_dev_exists(c, r.e.devs[i])) {633 invalid_dev = r.e.devs[i];634 goto invalid_device;635 }636 637 /*638 * All replicas entry checks except for invalid device are done639 * in bch2_accounting_validate640 */641 BUG_ON(bch2_replicas_entry_validate(&r.e, c, &buf));642 643 if (fsck_err_on(!bch2_replicas_marked_locked(c, &r.e),644 trans, accounting_replicas_not_marked,645 "accounting not marked in superblock replicas\n %s",646 (printbuf_reset(&buf),647 bch2_accounting_key_to_text(&buf, &acc),648 buf.buf))) {649 /*650 * We're not RW yet and still single threaded, dropping651 * and retaking lock is ok:652 */653 percpu_up_write(&c->mark_lock);654 ret = bch2_mark_replicas(c, &r.e);655 if (ret)656 goto fsck_err;657 percpu_down_write(&c->mark_lock);658 }659 break;660 }661 662 case BCH_DISK_ACCOUNTING_dev_data_type:663 if (!bch2_dev_exists(c, acc.dev_data_type.dev)) {664 invalid_dev = acc.dev_data_type.dev;665 goto invalid_device;666 }667 break;668 }669 670fsck_err:671 printbuf_exit(&buf);672 return ret;673invalid_device:674 if (fsck_err(trans, accounting_to_invalid_device,675 "accounting entry points to invalid device %i\n %s",676 invalid_dev,677 (printbuf_reset(&buf),678 bch2_accounting_key_to_text(&buf, &acc),679 buf.buf))) {680 for (unsigned i = 0; i < nr; i++)681 v[i] = -v[i];682 683 ret = commit_do(trans, NULL, NULL, 0,684 bch2_disk_accounting_mod(trans, &acc, v, nr, false)) ?:685 -BCH_ERR_remove_disk_accounting_entry;686 } else {687 ret = -BCH_ERR_remove_disk_accounting_entry;688 }689 goto fsck_err;690}691 692/*693 * At startup time, initialize the in memory accounting from the btree (and694 * journal)695 */696int bch2_accounting_read(struct bch_fs *c)697{698 struct bch_accounting_mem *acc = &c->accounting;699 struct btree_trans *trans = bch2_trans_get(c);700 struct printbuf buf = PRINTBUF;701 702 int ret = for_each_btree_key(trans, iter,703 BTREE_ID_accounting, POS_MIN,704 BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, ({705 struct bkey u;706 struct bkey_s_c k = bch2_btree_path_peek_slot_exact(btree_iter_path(trans, &iter), &u);707 accounting_read_key(trans, k);708 }));709 if (ret)710 goto err;711 712 struct journal_keys *keys = &c->journal_keys;713 struct journal_key *dst = keys->data;714 move_gap(keys, keys->nr);715 716 darray_for_each(*keys, i) {717 if (i->k->k.type == KEY_TYPE_accounting) {718 struct bkey_s_c k = bkey_i_to_s_c(i->k);719 unsigned idx = eytzinger0_find(acc->k.data, acc->k.nr,720 sizeof(acc->k.data[0]),721 accounting_pos_cmp, &k.k->p);722 723 bool applied = idx < acc->k.nr &&724 bversion_cmp(acc->k.data[idx].bversion, k.k->bversion) >= 0;725 726 if (applied)727 continue;728 729 if (i + 1 < &darray_top(*keys) &&730 i[1].k->k.type == KEY_TYPE_accounting &&731 !journal_key_cmp(i, i + 1)) {732 WARN_ON(bversion_cmp(i[0].k->k.bversion, i[1].k->k.bversion) >= 0);733 734 i[1].journal_seq = i[0].journal_seq;735 736 bch2_accounting_accumulate(bkey_i_to_accounting(i[1].k),737 bkey_s_c_to_accounting(k));738 continue;739 }740 741 ret = accounting_read_key(trans, k);742 if (ret)743 goto err;744 }745 746 *dst++ = *i;747 }748 keys->gap = keys->nr = dst - keys->data;749 750 percpu_down_write(&c->mark_lock);751 unsigned i = 0;752 while (i < acc->k.nr) {753 unsigned idx = inorder_to_eytzinger0(i, acc->k.nr);754 755 struct disk_accounting_pos acc_k;756 bpos_to_disk_accounting_pos(&acc_k, acc->k.data[idx].pos);757 758 u64 v[BCH_ACCOUNTING_MAX_COUNTERS];759 bch2_accounting_mem_read_counters(acc, idx, v, ARRAY_SIZE(v), false);760 761 /*762 * If the entry counters are zeroed, it should be treated as763 * nonexistent - it might point to an invalid device.764 *765 * Remove it, so that if it's re-added it gets re-marked in the766 * superblock:767 */768 ret = bch2_is_zero(v, sizeof(v[0]) * acc->k.data[idx].nr_counters)769 ? -BCH_ERR_remove_disk_accounting_entry770 : bch2_disk_accounting_validate_late(trans, acc_k,771 v, acc->k.data[idx].nr_counters);772 773 if (ret == -BCH_ERR_remove_disk_accounting_entry) {774 free_percpu(acc->k.data[idx].v[0]);775 free_percpu(acc->k.data[idx].v[1]);776 darray_remove_item(&acc->k, &acc->k.data[idx]);777 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]),778 accounting_pos_cmp, NULL);779 ret = 0;780 continue;781 }782 783 if (ret)784 goto fsck_err;785 i++;786 }787 788 preempt_disable();789 struct bch_fs_usage_base *usage = this_cpu_ptr(c->usage);790 791 for (unsigned i = 0; i < acc->k.nr; i++) {792 struct disk_accounting_pos k;793 bpos_to_disk_accounting_pos(&k, acc->k.data[i].pos);794 795 u64 v[BCH_ACCOUNTING_MAX_COUNTERS];796 bch2_accounting_mem_read_counters(acc, i, v, ARRAY_SIZE(v), false);797 798 switch (k.type) {799 case BCH_DISK_ACCOUNTING_persistent_reserved:800 usage->reserved += v[0] * k.persistent_reserved.nr_replicas;801 break;802 case BCH_DISK_ACCOUNTING_replicas:803 fs_usage_data_type_to_base(usage, k.replicas.data_type, v[0]);804 break;805 case BCH_DISK_ACCOUNTING_dev_data_type:806 rcu_read_lock();807 struct bch_dev *ca = bch2_dev_rcu(c, k.dev_data_type.dev);808 if (ca) {809 struct bch_dev_usage_type __percpu *d = &ca->usage->d[k.dev_data_type.data_type];810 percpu_u64_set(&d->buckets, v[0]);811 percpu_u64_set(&d->sectors, v[1]);812 percpu_u64_set(&d->fragmented, v[2]);813 814 if (k.dev_data_type.data_type == BCH_DATA_sb ||815 k.dev_data_type.data_type == BCH_DATA_journal)816 usage->hidden += v[0] * ca->mi.bucket_size;817 }818 rcu_read_unlock();819 break;820 }821 }822 preempt_enable();823fsck_err:824 percpu_up_write(&c->mark_lock);825err:826 printbuf_exit(&buf);827 bch2_trans_put(trans);828 bch_err_fn(c, ret);829 return ret;830}831 832int bch2_dev_usage_remove(struct bch_fs *c, unsigned dev)833{834 return bch2_trans_run(c,835 bch2_btree_write_buffer_flush_sync(trans) ?:836 for_each_btree_key_commit(trans, iter, BTREE_ID_accounting, POS_MIN,837 BTREE_ITER_all_snapshots, k, NULL, NULL, 0, ({838 struct disk_accounting_pos acc;839 bpos_to_disk_accounting_pos(&acc, k.k->p);840 841 acc.type == BCH_DISK_ACCOUNTING_dev_data_type &&842 acc.dev_data_type.dev == dev843 ? bch2_btree_bit_mod_buffered(trans, BTREE_ID_accounting, k.k->p, 0)844 : 0;845 })) ?:846 bch2_btree_write_buffer_flush_sync(trans));847}848 849int bch2_dev_usage_init(struct bch_dev *ca, bool gc)850{851 struct bch_fs *c = ca->fs;852 struct disk_accounting_pos acc = {853 .type = BCH_DISK_ACCOUNTING_dev_data_type,854 .dev_data_type.dev = ca->dev_idx,855 .dev_data_type.data_type = BCH_DATA_free,856 };857 u64 v[3] = { ca->mi.nbuckets - ca->mi.first_bucket, 0, 0 };858 859 int ret = bch2_trans_do(c, ({860 bch2_disk_accounting_mod(trans, &acc, v, ARRAY_SIZE(v), gc) ?:861 (!gc ? bch2_trans_commit(trans, NULL, NULL, 0) : 0);862 }));863 bch_err_fn(c, ret);864 return ret;865}866 867void bch2_verify_accounting_clean(struct bch_fs *c)868{869 bool mismatch = false;870 struct bch_fs_usage_base base = {}, base_inmem = {};871 872 bch2_trans_run(c,873 for_each_btree_key(trans, iter,874 BTREE_ID_accounting, POS_MIN,875 BTREE_ITER_all_snapshots, k, ({876 u64 v[BCH_ACCOUNTING_MAX_COUNTERS];877 struct bkey_s_c_accounting a = bkey_s_c_to_accounting(k);878 unsigned nr = bch2_accounting_counters(k.k);879 880 struct disk_accounting_pos acc_k;881 bpos_to_disk_accounting_pos(&acc_k, k.k->p);882 883 if (acc_k.type >= BCH_DISK_ACCOUNTING_TYPE_NR)884 continue;885 886 if (acc_k.type == BCH_DISK_ACCOUNTING_inum)887 continue;888 889 bch2_accounting_mem_read(c, k.k->p, v, nr);890 891 if (memcmp(a.v->d, v, nr * sizeof(u64))) {892 struct printbuf buf = PRINTBUF;893 894 bch2_bkey_val_to_text(&buf, c, k);895 prt_str(&buf, " !=");896 for (unsigned j = 0; j < nr; j++)897 prt_printf(&buf, " %llu", v[j]);898 899 pr_err("%s", buf.buf);900 printbuf_exit(&buf);901 mismatch = true;902 }903 904 switch (acc_k.type) {905 case BCH_DISK_ACCOUNTING_persistent_reserved:906 base.reserved += acc_k.persistent_reserved.nr_replicas * a.v->d[0];907 break;908 case BCH_DISK_ACCOUNTING_replicas:909 fs_usage_data_type_to_base(&base, acc_k.replicas.data_type, a.v->d[0]);910 break;911 case BCH_DISK_ACCOUNTING_dev_data_type: {912 rcu_read_lock();913 struct bch_dev *ca = bch2_dev_rcu(c, acc_k.dev_data_type.dev);914 if (!ca) {915 rcu_read_unlock();916 continue;917 }918 919 v[0] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].buckets);920 v[1] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].sectors);921 v[2] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].fragmented);922 rcu_read_unlock();923 924 if (memcmp(a.v->d, v, 3 * sizeof(u64))) {925 struct printbuf buf = PRINTBUF;926 927 bch2_bkey_val_to_text(&buf, c, k);928 prt_str(&buf, " in mem");929 for (unsigned j = 0; j < nr; j++)930 prt_printf(&buf, " %llu", v[j]);931 932 pr_err("dev accounting mismatch: %s", buf.buf);933 printbuf_exit(&buf);934 mismatch = true;935 }936 }937 }938 939 0;940 })));941 942 acc_u64s_percpu(&base_inmem.hidden, &c->usage->hidden, sizeof(base_inmem) / sizeof(u64));943 944#define check(x) \945 if (base.x != base_inmem.x) { \946 pr_err("fs_usage_base.%s mismatch: %llu != %llu", #x, base.x, base_inmem.x); \947 mismatch = true; \948 }949 950 //check(hidden);951 check(btree);952 check(data);953 check(cached);954 check(reserved);955 check(nr_inodes);956 957 WARN_ON(mismatch);958}959 960void bch2_accounting_gc_free(struct bch_fs *c)961{962 lockdep_assert_held(&c->mark_lock);963 964 struct bch_accounting_mem *acc = &c->accounting;965 966 bch2_accounting_free_counters(acc, true);967 acc->gc_running = false;968}969 970void bch2_fs_accounting_exit(struct bch_fs *c)971{972 struct bch_accounting_mem *acc = &c->accounting;973 974 bch2_accounting_free_counters(acc, false);975 darray_exit(&acc->k);976}977