2506 lines · c
1// SPDX-License-Identifier: GPL-2.02 3/* erasure coding */4 5#include "bcachefs.h"6#include "alloc_background.h"7#include "alloc_foreground.h"8#include "backpointers.h"9#include "bkey_buf.h"10#include "bset.h"11#include "btree_gc.h"12#include "btree_update.h"13#include "btree_write_buffer.h"14#include "buckets.h"15#include "checksum.h"16#include "disk_accounting.h"17#include "disk_groups.h"18#include "ec.h"19#include "error.h"20#include "io_read.h"21#include "io_write.h"22#include "keylist.h"23#include "recovery.h"24#include "replicas.h"25#include "super-io.h"26#include "util.h"27 28#include <linux/sort.h>29 30#ifdef __KERNEL__31 32#include <linux/raid/pq.h>33#include <linux/raid/xor.h>34 35static void raid5_recov(unsigned disks, unsigned failed_idx,36 size_t size, void **data)37{38 unsigned i = 2, nr;39 40 BUG_ON(failed_idx >= disks);41 42 swap(data[0], data[failed_idx]);43 memcpy(data[0], data[1], size);44 45 while (i < disks) {46 nr = min_t(unsigned, disks - i, MAX_XOR_BLOCKS);47 xor_blocks(nr, size, data[0], data + i);48 i += nr;49 }50 51 swap(data[0], data[failed_idx]);52}53 54static void raid_gen(int nd, int np, size_t size, void **v)55{56 if (np >= 1)57 raid5_recov(nd + np, nd, size, v);58 if (np >= 2)59 raid6_call.gen_syndrome(nd + np, size, v);60 BUG_ON(np > 2);61}62 63static void raid_rec(int nr, int *ir, int nd, int np, size_t size, void **v)64{65 switch (nr) {66 case 0:67 break;68 case 1:69 if (ir[0] < nd + 1)70 raid5_recov(nd + 1, ir[0], size, v);71 else72 raid6_call.gen_syndrome(nd + np, size, v);73 break;74 case 2:75 if (ir[1] < nd) {76 /* data+data failure. */77 raid6_2data_recov(nd + np, size, ir[0], ir[1], v);78 } else if (ir[0] < nd) {79 /* data + p/q failure */80 81 if (ir[1] == nd) /* data + p failure */82 raid6_datap_recov(nd + np, size, ir[0], v);83 else { /* data + q failure */84 raid5_recov(nd + 1, ir[0], size, v);85 raid6_call.gen_syndrome(nd + np, size, v);86 }87 } else {88 raid_gen(nd, np, size, v);89 }90 break;91 default:92 BUG();93 }94}95 96#else97 98#include <raid/raid.h>99 100#endif101 102struct ec_bio {103 struct bch_dev *ca;104 struct ec_stripe_buf *buf;105 size_t idx;106 struct bio bio;107};108 109/* Stripes btree keys: */110 111int bch2_stripe_validate(struct bch_fs *c, struct bkey_s_c k,112 enum bch_validate_flags flags)113{114 const struct bch_stripe *s = bkey_s_c_to_stripe(k).v;115 int ret = 0;116 117 bkey_fsck_err_on(bkey_eq(k.k->p, POS_MIN) ||118 bpos_gt(k.k->p, POS(0, U32_MAX)),119 c, stripe_pos_bad,120 "stripe at bad pos");121 122 bkey_fsck_err_on(bkey_val_u64s(k.k) < stripe_val_u64s(s),123 c, stripe_val_size_bad,124 "incorrect value size (%zu < %u)",125 bkey_val_u64s(k.k), stripe_val_u64s(s));126 127 bkey_fsck_err_on(s->csum_granularity_bits >= 64,128 c, stripe_csum_granularity_bad,129 "invalid csum granularity (%u >= 64)",130 s->csum_granularity_bits);131 132 ret = bch2_bkey_ptrs_validate(c, k, flags);133fsck_err:134 return ret;135}136 137void bch2_stripe_to_text(struct printbuf *out, struct bch_fs *c,138 struct bkey_s_c k)139{140 const struct bch_stripe *sp = bkey_s_c_to_stripe(k).v;141 struct bch_stripe s = {};142 143 memcpy(&s, sp, min(sizeof(s), bkey_val_bytes(k.k)));144 145 unsigned nr_data = s.nr_blocks - s.nr_redundant;146 147 prt_printf(out, "algo %u sectors %u blocks %u:%u csum ",148 s.algorithm,149 le16_to_cpu(s.sectors),150 nr_data,151 s.nr_redundant);152 bch2_prt_csum_type(out, s.csum_type);153 prt_str(out, " gran ");154 if (s.csum_granularity_bits < 64)155 prt_printf(out, "%llu", 1ULL << s.csum_granularity_bits);156 else157 prt_printf(out, "(invalid shift %u)", s.csum_granularity_bits);158 159 if (s.disk_label) {160 prt_str(out, " label");161 bch2_disk_path_to_text(out, c, s.disk_label - 1);162 }163 164 for (unsigned i = 0; i < s.nr_blocks; i++) {165 const struct bch_extent_ptr *ptr = sp->ptrs + i;166 167 if ((void *) ptr >= bkey_val_end(k))168 break;169 170 prt_char(out, ' ');171 bch2_extent_ptr_to_text(out, c, ptr);172 173 if (s.csum_type < BCH_CSUM_NR &&174 i < nr_data &&175 stripe_blockcount_offset(&s, i) < bkey_val_bytes(k.k))176 prt_printf(out, "#%u", stripe_blockcount_get(sp, i));177 }178}179 180/* Triggers: */181 182static int __mark_stripe_bucket(struct btree_trans *trans,183 struct bch_dev *ca,184 struct bkey_s_c_stripe s,185 unsigned ptr_idx, bool deleting,186 struct bpos bucket,187 struct bch_alloc_v4 *a,188 enum btree_iter_update_trigger_flags flags)189{190 const struct bch_extent_ptr *ptr = s.v->ptrs + ptr_idx;191 unsigned nr_data = s.v->nr_blocks - s.v->nr_redundant;192 bool parity = ptr_idx >= nr_data;193 enum bch_data_type data_type = parity ? BCH_DATA_parity : BCH_DATA_stripe;194 s64 sectors = parity ? le16_to_cpu(s.v->sectors) : 0;195 struct printbuf buf = PRINTBUF;196 int ret = 0;197 198 struct bch_fs *c = trans->c;199 if (deleting)200 sectors = -sectors;201 202 if (!deleting) {203 if (bch2_trans_inconsistent_on(a->stripe ||204 a->stripe_redundancy, trans,205 "bucket %llu:%llu gen %u data type %s dirty_sectors %u: multiple stripes using same bucket (%u, %llu)\n%s",206 bucket.inode, bucket.offset, a->gen,207 bch2_data_type_str(a->data_type),208 a->dirty_sectors,209 a->stripe, s.k->p.offset,210 (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) {211 ret = -BCH_ERR_mark_stripe;212 goto err;213 }214 215 if (bch2_trans_inconsistent_on(parity && bch2_bucket_sectors_total(*a), trans,216 "bucket %llu:%llu gen %u data type %s dirty_sectors %u cached_sectors %u: data already in parity bucket\n%s",217 bucket.inode, bucket.offset, a->gen,218 bch2_data_type_str(a->data_type),219 a->dirty_sectors,220 a->cached_sectors,221 (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) {222 ret = -BCH_ERR_mark_stripe;223 goto err;224 }225 } else {226 if (bch2_trans_inconsistent_on(a->stripe != s.k->p.offset ||227 a->stripe_redundancy != s.v->nr_redundant, trans,228 "bucket %llu:%llu gen %u: not marked as stripe when deleting stripe (got %u)\n%s",229 bucket.inode, bucket.offset, a->gen,230 a->stripe,231 (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) {232 ret = -BCH_ERR_mark_stripe;233 goto err;234 }235 236 if (bch2_trans_inconsistent_on(a->data_type != data_type, trans,237 "bucket %llu:%llu gen %u data type %s: wrong data type when stripe, should be %s\n%s",238 bucket.inode, bucket.offset, a->gen,239 bch2_data_type_str(a->data_type),240 bch2_data_type_str(data_type),241 (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) {242 ret = -BCH_ERR_mark_stripe;243 goto err;244 }245 246 if (bch2_trans_inconsistent_on(parity &&247 (a->dirty_sectors != -sectors ||248 a->cached_sectors), trans,249 "bucket %llu:%llu gen %u dirty_sectors %u cached_sectors %u: wrong sectors when deleting parity block of stripe\n%s",250 bucket.inode, bucket.offset, a->gen,251 a->dirty_sectors,252 a->cached_sectors,253 (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) {254 ret = -BCH_ERR_mark_stripe;255 goto err;256 }257 }258 259 if (sectors) {260 ret = bch2_bucket_ref_update(trans, ca, s.s_c, ptr, sectors, data_type,261 a->gen, a->data_type, &a->dirty_sectors);262 if (ret)263 goto err;264 }265 266 if (!deleting) {267 a->stripe = s.k->p.offset;268 a->stripe_redundancy = s.v->nr_redundant;269 alloc_data_type_set(a, data_type);270 } else {271 a->stripe = 0;272 a->stripe_redundancy = 0;273 alloc_data_type_set(a, BCH_DATA_user);274 }275err:276 printbuf_exit(&buf);277 return ret;278}279 280static int mark_stripe_bucket(struct btree_trans *trans,281 struct bkey_s_c_stripe s,282 unsigned ptr_idx, bool deleting,283 enum btree_iter_update_trigger_flags flags)284{285 struct bch_fs *c = trans->c;286 const struct bch_extent_ptr *ptr = s.v->ptrs + ptr_idx;287 struct printbuf buf = PRINTBUF;288 int ret = 0;289 290 struct bch_dev *ca = bch2_dev_tryget(c, ptr->dev);291 if (unlikely(!ca)) {292 if (ptr->dev != BCH_SB_MEMBER_INVALID && !(flags & BTREE_TRIGGER_overwrite))293 ret = -BCH_ERR_mark_stripe;294 goto err;295 }296 297 struct bpos bucket = PTR_BUCKET_POS(ca, ptr);298 299 if (flags & BTREE_TRIGGER_transactional) {300 struct bkey_i_alloc_v4 *a =301 bch2_trans_start_alloc_update(trans, bucket, 0);302 ret = PTR_ERR_OR_ZERO(a) ?:303 __mark_stripe_bucket(trans, ca, s, ptr_idx, deleting, bucket, &a->v, flags);304 }305 306 if (flags & BTREE_TRIGGER_gc) {307 percpu_down_read(&c->mark_lock);308 struct bucket *g = gc_bucket(ca, bucket.offset);309 if (bch2_fs_inconsistent_on(!g, c, "reference to invalid bucket on device %u\n %s",310 ptr->dev,311 (bch2_bkey_val_to_text(&buf, c, s.s_c), buf.buf))) {312 ret = -BCH_ERR_mark_stripe;313 goto err_unlock;314 }315 316 bucket_lock(g);317 struct bch_alloc_v4 old = bucket_m_to_alloc(*g), new = old;318 ret = __mark_stripe_bucket(trans, ca, s, ptr_idx, deleting, bucket, &new, flags);319 alloc_to_bucket(g, new);320 bucket_unlock(g);321err_unlock:322 percpu_up_read(&c->mark_lock);323 if (!ret)324 ret = bch2_alloc_key_to_dev_counters(trans, ca, &old, &new, flags);325 }326err:327 bch2_dev_put(ca);328 printbuf_exit(&buf);329 return ret;330}331 332static int mark_stripe_buckets(struct btree_trans *trans,333 struct bkey_s_c old, struct bkey_s_c new,334 enum btree_iter_update_trigger_flags flags)335{336 const struct bch_stripe *old_s = old.k->type == KEY_TYPE_stripe337 ? bkey_s_c_to_stripe(old).v : NULL;338 const struct bch_stripe *new_s = new.k->type == KEY_TYPE_stripe339 ? bkey_s_c_to_stripe(new).v : NULL;340 341 BUG_ON(old_s && new_s && old_s->nr_blocks != new_s->nr_blocks);342 343 unsigned nr_blocks = new_s ? new_s->nr_blocks : old_s->nr_blocks;344 345 for (unsigned i = 0; i < nr_blocks; i++) {346 if (new_s && old_s &&347 !memcmp(&new_s->ptrs[i],348 &old_s->ptrs[i],349 sizeof(new_s->ptrs[i])))350 continue;351 352 if (new_s) {353 int ret = mark_stripe_bucket(trans,354 bkey_s_c_to_stripe(new), i, false, flags);355 if (ret)356 return ret;357 }358 359 if (old_s) {360 int ret = mark_stripe_bucket(trans,361 bkey_s_c_to_stripe(old), i, true, flags);362 if (ret)363 return ret;364 }365 }366 367 return 0;368}369 370static inline void stripe_to_mem(struct stripe *m, const struct bch_stripe *s)371{372 m->sectors = le16_to_cpu(s->sectors);373 m->algorithm = s->algorithm;374 m->nr_blocks = s->nr_blocks;375 m->nr_redundant = s->nr_redundant;376 m->disk_label = s->disk_label;377 m->blocks_nonempty = 0;378 379 for (unsigned i = 0; i < s->nr_blocks; i++)380 m->blocks_nonempty += !!stripe_blockcount_get(s, i);381}382 383int bch2_trigger_stripe(struct btree_trans *trans,384 enum btree_id btree, unsigned level,385 struct bkey_s_c old, struct bkey_s _new,386 enum btree_iter_update_trigger_flags flags)387{388 struct bkey_s_c new = _new.s_c;389 struct bch_fs *c = trans->c;390 u64 idx = new.k->p.offset;391 const struct bch_stripe *old_s = old.k->type == KEY_TYPE_stripe392 ? bkey_s_c_to_stripe(old).v : NULL;393 const struct bch_stripe *new_s = new.k->type == KEY_TYPE_stripe394 ? bkey_s_c_to_stripe(new).v : NULL;395 396 if (unlikely(flags & BTREE_TRIGGER_check_repair))397 return bch2_check_fix_ptrs(trans, btree, level, _new.s_c, flags);398 399 BUG_ON(new_s && old_s &&400 (new_s->nr_blocks != old_s->nr_blocks ||401 new_s->nr_redundant != old_s->nr_redundant));402 403 404 if (flags & (BTREE_TRIGGER_transactional|BTREE_TRIGGER_gc)) {405 /*406 * If the pointers aren't changing, we don't need to do anything:407 */408 if (new_s && old_s &&409 new_s->nr_blocks == old_s->nr_blocks &&410 new_s->nr_redundant == old_s->nr_redundant &&411 !memcmp(old_s->ptrs, new_s->ptrs,412 new_s->nr_blocks * sizeof(struct bch_extent_ptr)))413 return 0;414 415 struct gc_stripe *gc = NULL;416 if (flags & BTREE_TRIGGER_gc) {417 gc = genradix_ptr_alloc(&c->gc_stripes, idx, GFP_KERNEL);418 if (!gc) {419 bch_err(c, "error allocating memory for gc_stripes, idx %llu", idx);420 return -BCH_ERR_ENOMEM_mark_stripe;421 }422 423 /*424 * This will be wrong when we bring back runtime gc: we should425 * be unmarking the old key and then marking the new key426 *427 * Also: when we bring back runtime gc, locking428 */429 gc->alive = true;430 gc->sectors = le16_to_cpu(new_s->sectors);431 gc->nr_blocks = new_s->nr_blocks;432 gc->nr_redundant = new_s->nr_redundant;433 434 for (unsigned i = 0; i < new_s->nr_blocks; i++)435 gc->ptrs[i] = new_s->ptrs[i];436 437 /*438 * gc recalculates this field from stripe ptr439 * references:440 */441 memset(gc->block_sectors, 0, sizeof(gc->block_sectors));442 }443 444 if (new_s) {445 s64 sectors = (u64) le16_to_cpu(new_s->sectors) * new_s->nr_redundant;446 447 struct disk_accounting_pos acc = {448 .type = BCH_DISK_ACCOUNTING_replicas,449 };450 bch2_bkey_to_replicas(&acc.replicas, new);451 int ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, gc);452 if (ret)453 return ret;454 455 if (gc)456 memcpy(&gc->r.e, &acc.replicas, replicas_entry_bytes(&acc.replicas));457 }458 459 if (old_s) {460 s64 sectors = -((s64) le16_to_cpu(old_s->sectors)) * old_s->nr_redundant;461 462 struct disk_accounting_pos acc = {463 .type = BCH_DISK_ACCOUNTING_replicas,464 };465 bch2_bkey_to_replicas(&acc.replicas, old);466 int ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, gc);467 if (ret)468 return ret;469 }470 471 int ret = mark_stripe_buckets(trans, old, new, flags);472 if (ret)473 return ret;474 }475 476 if (flags & BTREE_TRIGGER_atomic) {477 struct stripe *m = genradix_ptr(&c->stripes, idx);478 479 if (!m) {480 struct printbuf buf1 = PRINTBUF;481 struct printbuf buf2 = PRINTBUF;482 483 bch2_bkey_val_to_text(&buf1, c, old);484 bch2_bkey_val_to_text(&buf2, c, new);485 bch_err_ratelimited(c, "error marking nonexistent stripe %llu while marking\n"486 "old %s\n"487 "new %s", idx, buf1.buf, buf2.buf);488 printbuf_exit(&buf2);489 printbuf_exit(&buf1);490 bch2_inconsistent_error(c);491 return -1;492 }493 494 if (!new_s) {495 bch2_stripes_heap_del(c, m, idx);496 497 memset(m, 0, sizeof(*m));498 } else {499 stripe_to_mem(m, new_s);500 501 if (!old_s)502 bch2_stripes_heap_insert(c, m, idx);503 else504 bch2_stripes_heap_update(c, m, idx);505 }506 }507 508 return 0;509}510 511/* returns blocknr in stripe that we matched: */512static const struct bch_extent_ptr *bkey_matches_stripe(struct bch_stripe *s,513 struct bkey_s_c k, unsigned *block)514{515 struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);516 unsigned i, nr_data = s->nr_blocks - s->nr_redundant;517 518 bkey_for_each_ptr(ptrs, ptr)519 for (i = 0; i < nr_data; i++)520 if (__bch2_ptr_matches_stripe(&s->ptrs[i], ptr,521 le16_to_cpu(s->sectors))) {522 *block = i;523 return ptr;524 }525 526 return NULL;527}528 529static bool extent_has_stripe_ptr(struct bkey_s_c k, u64 idx)530{531 switch (k.k->type) {532 case KEY_TYPE_extent: {533 struct bkey_s_c_extent e = bkey_s_c_to_extent(k);534 const union bch_extent_entry *entry;535 536 extent_for_each_entry(e, entry)537 if (extent_entry_type(entry) ==538 BCH_EXTENT_ENTRY_stripe_ptr &&539 entry->stripe_ptr.idx == idx)540 return true;541 542 break;543 }544 }545 546 return false;547}548 549/* Stripe bufs: */550 551static void ec_stripe_buf_exit(struct ec_stripe_buf *buf)552{553 if (buf->key.k.type == KEY_TYPE_stripe) {554 struct bkey_i_stripe *s = bkey_i_to_stripe(&buf->key);555 unsigned i;556 557 for (i = 0; i < s->v.nr_blocks; i++) {558 kvfree(buf->data[i]);559 buf->data[i] = NULL;560 }561 }562}563 564/* XXX: this is a non-mempoolified memory allocation: */565static int ec_stripe_buf_init(struct ec_stripe_buf *buf,566 unsigned offset, unsigned size)567{568 struct bch_stripe *v = &bkey_i_to_stripe(&buf->key)->v;569 unsigned csum_granularity = 1U << v->csum_granularity_bits;570 unsigned end = offset + size;571 unsigned i;572 573 BUG_ON(end > le16_to_cpu(v->sectors));574 575 offset = round_down(offset, csum_granularity);576 end = min_t(unsigned, le16_to_cpu(v->sectors),577 round_up(end, csum_granularity));578 579 buf->offset = offset;580 buf->size = end - offset;581 582 memset(buf->valid, 0xFF, sizeof(buf->valid));583 584 for (i = 0; i < v->nr_blocks; i++) {585 buf->data[i] = kvmalloc(buf->size << 9, GFP_KERNEL);586 if (!buf->data[i])587 goto err;588 }589 590 return 0;591err:592 ec_stripe_buf_exit(buf);593 return -BCH_ERR_ENOMEM_stripe_buf;594}595 596/* Checksumming: */597 598static struct bch_csum ec_block_checksum(struct ec_stripe_buf *buf,599 unsigned block, unsigned offset)600{601 struct bch_stripe *v = &bkey_i_to_stripe(&buf->key)->v;602 unsigned csum_granularity = 1 << v->csum_granularity_bits;603 unsigned end = buf->offset + buf->size;604 unsigned len = min(csum_granularity, end - offset);605 606 BUG_ON(offset >= end);607 BUG_ON(offset < buf->offset);608 BUG_ON(offset & (csum_granularity - 1));609 BUG_ON(offset + len != le16_to_cpu(v->sectors) &&610 (len & (csum_granularity - 1)));611 612 return bch2_checksum(NULL, v->csum_type,613 null_nonce(),614 buf->data[block] + ((offset - buf->offset) << 9),615 len << 9);616}617 618static void ec_generate_checksums(struct ec_stripe_buf *buf)619{620 struct bch_stripe *v = &bkey_i_to_stripe(&buf->key)->v;621 unsigned i, j, csums_per_device = stripe_csums_per_device(v);622 623 if (!v->csum_type)624 return;625 626 BUG_ON(buf->offset);627 BUG_ON(buf->size != le16_to_cpu(v->sectors));628 629 for (i = 0; i < v->nr_blocks; i++)630 for (j = 0; j < csums_per_device; j++)631 stripe_csum_set(v, i, j,632 ec_block_checksum(buf, i, j << v->csum_granularity_bits));633}634 635static void ec_validate_checksums(struct bch_fs *c, struct ec_stripe_buf *buf)636{637 struct bch_stripe *v = &bkey_i_to_stripe(&buf->key)->v;638 unsigned csum_granularity = 1 << v->csum_granularity_bits;639 unsigned i;640 641 if (!v->csum_type)642 return;643 644 for (i = 0; i < v->nr_blocks; i++) {645 unsigned offset = buf->offset;646 unsigned end = buf->offset + buf->size;647 648 if (!test_bit(i, buf->valid))649 continue;650 651 while (offset < end) {652 unsigned j = offset >> v->csum_granularity_bits;653 unsigned len = min(csum_granularity, end - offset);654 struct bch_csum want = stripe_csum_get(v, i, j);655 struct bch_csum got = ec_block_checksum(buf, i, offset);656 657 if (bch2_crc_cmp(want, got)) {658 struct bch_dev *ca = bch2_dev_tryget(c, v->ptrs[i].dev);659 if (ca) {660 struct printbuf err = PRINTBUF;661 662 prt_str(&err, "stripe ");663 bch2_csum_err_msg(&err, v->csum_type, want, got);664 prt_printf(&err, " for %ps at %u of\n ", (void *) _RET_IP_, i);665 bch2_bkey_val_to_text(&err, c, bkey_i_to_s_c(&buf->key));666 bch_err_ratelimited(ca, "%s", err.buf);667 printbuf_exit(&err);668 669 bch2_io_error(ca, BCH_MEMBER_ERROR_checksum);670 }671 672 clear_bit(i, buf->valid);673 break;674 }675 676 offset += len;677 }678 }679}680 681/* Erasure coding: */682 683static void ec_generate_ec(struct ec_stripe_buf *buf)684{685 struct bch_stripe *v = &bkey_i_to_stripe(&buf->key)->v;686 unsigned nr_data = v->nr_blocks - v->nr_redundant;687 unsigned bytes = le16_to_cpu(v->sectors) << 9;688 689 raid_gen(nr_data, v->nr_redundant, bytes, buf->data);690}691 692static unsigned ec_nr_failed(struct ec_stripe_buf *buf)693{694 struct bch_stripe *v = &bkey_i_to_stripe(&buf->key)->v;695 696 return v->nr_blocks - bitmap_weight(buf->valid, v->nr_blocks);697}698 699static int ec_do_recov(struct bch_fs *c, struct ec_stripe_buf *buf)700{701 struct bch_stripe *v = &bkey_i_to_stripe(&buf->key)->v;702 unsigned i, failed[BCH_BKEY_PTRS_MAX], nr_failed = 0;703 unsigned nr_data = v->nr_blocks - v->nr_redundant;704 unsigned bytes = buf->size << 9;705 706 if (ec_nr_failed(buf) > v->nr_redundant) {707 bch_err_ratelimited(c,708 "error doing reconstruct read: unable to read enough blocks");709 return -1;710 }711 712 for (i = 0; i < nr_data; i++)713 if (!test_bit(i, buf->valid))714 failed[nr_failed++] = i;715 716 raid_rec(nr_failed, failed, nr_data, v->nr_redundant, bytes, buf->data);717 return 0;718}719 720/* IO: */721 722static void ec_block_endio(struct bio *bio)723{724 struct ec_bio *ec_bio = container_of(bio, struct ec_bio, bio);725 struct bch_stripe *v = &bkey_i_to_stripe(&ec_bio->buf->key)->v;726 struct bch_extent_ptr *ptr = &v->ptrs[ec_bio->idx];727 struct bch_dev *ca = ec_bio->ca;728 struct closure *cl = bio->bi_private;729 730 if (bch2_dev_io_err_on(bio->bi_status, ca,731 bio_data_dir(bio)732 ? BCH_MEMBER_ERROR_write733 : BCH_MEMBER_ERROR_read,734 "erasure coding %s error: %s",735 bio_data_dir(bio) ? "write" : "read",736 bch2_blk_status_to_str(bio->bi_status)))737 clear_bit(ec_bio->idx, ec_bio->buf->valid);738 739 int stale = dev_ptr_stale(ca, ptr);740 if (stale) {741 bch_err_ratelimited(ca->fs,742 "error %s stripe: stale/invalid pointer (%i) after io",743 bio_data_dir(bio) == READ ? "reading from" : "writing to",744 stale);745 clear_bit(ec_bio->idx, ec_bio->buf->valid);746 }747 748 bio_put(&ec_bio->bio);749 percpu_ref_put(&ca->io_ref);750 closure_put(cl);751}752 753static void ec_block_io(struct bch_fs *c, struct ec_stripe_buf *buf,754 blk_opf_t opf, unsigned idx, struct closure *cl)755{756 struct bch_stripe *v = &bkey_i_to_stripe(&buf->key)->v;757 unsigned offset = 0, bytes = buf->size << 9;758 struct bch_extent_ptr *ptr = &v->ptrs[idx];759 enum bch_data_type data_type = idx < v->nr_blocks - v->nr_redundant760 ? BCH_DATA_user761 : BCH_DATA_parity;762 int rw = op_is_write(opf);763 764 struct bch_dev *ca = bch2_dev_get_ioref(c, ptr->dev, rw);765 if (!ca) {766 clear_bit(idx, buf->valid);767 return;768 }769 770 int stale = dev_ptr_stale(ca, ptr);771 if (stale) {772 bch_err_ratelimited(c,773 "error %s stripe: stale pointer (%i)",774 rw == READ ? "reading from" : "writing to",775 stale);776 clear_bit(idx, buf->valid);777 return;778 }779 780 781 this_cpu_add(ca->io_done->sectors[rw][data_type], buf->size);782 783 while (offset < bytes) {784 unsigned nr_iovecs = min_t(size_t, BIO_MAX_VECS,785 DIV_ROUND_UP(bytes, PAGE_SIZE));786 unsigned b = min_t(size_t, bytes - offset,787 nr_iovecs << PAGE_SHIFT);788 struct ec_bio *ec_bio;789 790 ec_bio = container_of(bio_alloc_bioset(ca->disk_sb.bdev,791 nr_iovecs,792 opf,793 GFP_KERNEL,794 &c->ec_bioset),795 struct ec_bio, bio);796 797 ec_bio->ca = ca;798 ec_bio->buf = buf;799 ec_bio->idx = idx;800 801 ec_bio->bio.bi_iter.bi_sector = ptr->offset + buf->offset + (offset >> 9);802 ec_bio->bio.bi_end_io = ec_block_endio;803 ec_bio->bio.bi_private = cl;804 805 bch2_bio_map(&ec_bio->bio, buf->data[idx] + offset, b);806 807 closure_get(cl);808 percpu_ref_get(&ca->io_ref);809 810 submit_bio(&ec_bio->bio);811 812 offset += b;813 }814 815 percpu_ref_put(&ca->io_ref);816}817 818static int get_stripe_key_trans(struct btree_trans *trans, u64 idx,819 struct ec_stripe_buf *stripe)820{821 struct btree_iter iter;822 struct bkey_s_c k;823 int ret;824 825 k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_stripes,826 POS(0, idx), BTREE_ITER_slots);827 ret = bkey_err(k);828 if (ret)829 goto err;830 if (k.k->type != KEY_TYPE_stripe) {831 ret = -ENOENT;832 goto err;833 }834 bkey_reassemble(&stripe->key, k);835err:836 bch2_trans_iter_exit(trans, &iter);837 return ret;838}839 840/* recovery read path: */841int bch2_ec_read_extent(struct btree_trans *trans, struct bch_read_bio *rbio,842 struct bkey_s_c orig_k)843{844 struct bch_fs *c = trans->c;845 struct ec_stripe_buf *buf = NULL;846 struct closure cl;847 struct bch_stripe *v;848 unsigned i, offset;849 const char *msg = NULL;850 struct printbuf msgbuf = PRINTBUF;851 int ret = 0;852 853 closure_init_stack(&cl);854 855 BUG_ON(!rbio->pick.has_ec);856 857 buf = kzalloc(sizeof(*buf), GFP_NOFS);858 if (!buf)859 return -BCH_ERR_ENOMEM_ec_read_extent;860 861 ret = lockrestart_do(trans, get_stripe_key_trans(trans, rbio->pick.ec.idx, buf));862 if (ret) {863 msg = "stripe not found";864 goto err;865 }866 867 v = &bkey_i_to_stripe(&buf->key)->v;868 869 if (!bch2_ptr_matches_stripe(v, rbio->pick)) {870 msg = "pointer doesn't match stripe";871 goto err;872 }873 874 offset = rbio->bio.bi_iter.bi_sector - v->ptrs[rbio->pick.ec.block].offset;875 if (offset + bio_sectors(&rbio->bio) > le16_to_cpu(v->sectors)) {876 msg = "read is bigger than stripe";877 goto err;878 }879 880 ret = ec_stripe_buf_init(buf, offset, bio_sectors(&rbio->bio));881 if (ret) {882 msg = "-ENOMEM";883 goto err;884 }885 886 for (i = 0; i < v->nr_blocks; i++)887 ec_block_io(c, buf, REQ_OP_READ, i, &cl);888 889 closure_sync(&cl);890 891 if (ec_nr_failed(buf) > v->nr_redundant) {892 msg = "unable to read enough blocks";893 goto err;894 }895 896 ec_validate_checksums(c, buf);897 898 ret = ec_do_recov(c, buf);899 if (ret)900 goto err;901 902 memcpy_to_bio(&rbio->bio, rbio->bio.bi_iter,903 buf->data[rbio->pick.ec.block] + ((offset - buf->offset) << 9));904out:905 ec_stripe_buf_exit(buf);906 kfree(buf);907 return ret;908err:909 bch2_bkey_val_to_text(&msgbuf, c, orig_k);910 bch_err_ratelimited(c,911 "error doing reconstruct read: %s\n %s", msg, msgbuf.buf);912 printbuf_exit(&msgbuf);;913 ret = -BCH_ERR_stripe_reconstruct;914 goto out;915}916 917/* stripe bucket accounting: */918 919static int __ec_stripe_mem_alloc(struct bch_fs *c, size_t idx, gfp_t gfp)920{921 ec_stripes_heap n, *h = &c->ec_stripes_heap;922 923 if (idx >= h->size) {924 if (!init_heap(&n, max(1024UL, roundup_pow_of_two(idx + 1)), gfp))925 return -BCH_ERR_ENOMEM_ec_stripe_mem_alloc;926 927 mutex_lock(&c->ec_stripes_heap_lock);928 if (n.size > h->size) {929 memcpy(n.data, h->data, h->nr * sizeof(h->data[0]));930 n.nr = h->nr;931 swap(*h, n);932 }933 mutex_unlock(&c->ec_stripes_heap_lock);934 935 free_heap(&n);936 }937 938 if (!genradix_ptr_alloc(&c->stripes, idx, gfp))939 return -BCH_ERR_ENOMEM_ec_stripe_mem_alloc;940 941 if (c->gc_pos.phase != GC_PHASE_not_running &&942 !genradix_ptr_alloc(&c->gc_stripes, idx, gfp))943 return -BCH_ERR_ENOMEM_ec_stripe_mem_alloc;944 945 return 0;946}947 948static int ec_stripe_mem_alloc(struct btree_trans *trans,949 struct btree_iter *iter)950{951 return allocate_dropping_locks_errcode(trans,952 __ec_stripe_mem_alloc(trans->c, iter->pos.offset, _gfp));953}954 955/*956 * Hash table of open stripes:957 * Stripes that are being created or modified are kept in a hash table, so that958 * stripe deletion can skip them.959 */960 961static bool __bch2_stripe_is_open(struct bch_fs *c, u64 idx)962{963 unsigned hash = hash_64(idx, ilog2(ARRAY_SIZE(c->ec_stripes_new)));964 struct ec_stripe_new *s;965 966 hlist_for_each_entry(s, &c->ec_stripes_new[hash], hash)967 if (s->idx == idx)968 return true;969 return false;970}971 972static bool bch2_stripe_is_open(struct bch_fs *c, u64 idx)973{974 bool ret = false;975 976 spin_lock(&c->ec_stripes_new_lock);977 ret = __bch2_stripe_is_open(c, idx);978 spin_unlock(&c->ec_stripes_new_lock);979 980 return ret;981}982 983static bool bch2_try_open_stripe(struct bch_fs *c,984 struct ec_stripe_new *s,985 u64 idx)986{987 bool ret;988 989 spin_lock(&c->ec_stripes_new_lock);990 ret = !__bch2_stripe_is_open(c, idx);991 if (ret) {992 unsigned hash = hash_64(idx, ilog2(ARRAY_SIZE(c->ec_stripes_new)));993 994 s->idx = idx;995 hlist_add_head(&s->hash, &c->ec_stripes_new[hash]);996 }997 spin_unlock(&c->ec_stripes_new_lock);998 999 return ret;1000}1001 1002static void bch2_stripe_close(struct bch_fs *c, struct ec_stripe_new *s)1003{1004 BUG_ON(!s->idx);1005 1006 spin_lock(&c->ec_stripes_new_lock);1007 hlist_del_init(&s->hash);1008 spin_unlock(&c->ec_stripes_new_lock);1009 1010 s->idx = 0;1011}1012 1013/* Heap of all existing stripes, ordered by blocks_nonempty */1014 1015static u64 stripe_idx_to_delete(struct bch_fs *c)1016{1017 ec_stripes_heap *h = &c->ec_stripes_heap;1018 1019 lockdep_assert_held(&c->ec_stripes_heap_lock);1020 1021 if (h->nr &&1022 h->data[0].blocks_nonempty == 0 &&1023 !bch2_stripe_is_open(c, h->data[0].idx))1024 return h->data[0].idx;1025 1026 return 0;1027}1028 1029static inline void ec_stripes_heap_set_backpointer(ec_stripes_heap *h,1030 size_t i)1031{1032 struct bch_fs *c = container_of(h, struct bch_fs, ec_stripes_heap);1033 1034 genradix_ptr(&c->stripes, h->data[i].idx)->heap_idx = i;1035}1036 1037static inline bool ec_stripes_heap_cmp(const void *l, const void *r, void __always_unused *args)1038{1039 struct ec_stripe_heap_entry *_l = (struct ec_stripe_heap_entry *)l;1040 struct ec_stripe_heap_entry *_r = (struct ec_stripe_heap_entry *)r;1041 1042 return ((_l->blocks_nonempty > _r->blocks_nonempty) <1043 (_l->blocks_nonempty < _r->blocks_nonempty));1044}1045 1046static inline void ec_stripes_heap_swap(void *l, void *r, void *h)1047{1048 struct ec_stripe_heap_entry *_l = (struct ec_stripe_heap_entry *)l;1049 struct ec_stripe_heap_entry *_r = (struct ec_stripe_heap_entry *)r;1050 ec_stripes_heap *_h = (ec_stripes_heap *)h;1051 size_t i = _l - _h->data;1052 size_t j = _r - _h->data;1053 1054 swap(*_l, *_r);1055 1056 ec_stripes_heap_set_backpointer(_h, i);1057 ec_stripes_heap_set_backpointer(_h, j);1058}1059 1060static void heap_verify_backpointer(struct bch_fs *c, size_t idx)1061{1062 ec_stripes_heap *h = &c->ec_stripes_heap;1063 struct stripe *m = genradix_ptr(&c->stripes, idx);1064 1065 BUG_ON(m->heap_idx >= h->nr);1066 BUG_ON(h->data[m->heap_idx].idx != idx);1067}1068 1069void bch2_stripes_heap_del(struct bch_fs *c,1070 struct stripe *m, size_t idx)1071{1072 const struct min_heap_callbacks callbacks = {1073 .less = ec_stripes_heap_cmp,1074 .swp = ec_stripes_heap_swap,1075 };1076 1077 mutex_lock(&c->ec_stripes_heap_lock);1078 heap_verify_backpointer(c, idx);1079 1080 min_heap_del(&c->ec_stripes_heap, m->heap_idx, &callbacks, &c->ec_stripes_heap);1081 mutex_unlock(&c->ec_stripes_heap_lock);1082}1083 1084void bch2_stripes_heap_insert(struct bch_fs *c,1085 struct stripe *m, size_t idx)1086{1087 const struct min_heap_callbacks callbacks = {1088 .less = ec_stripes_heap_cmp,1089 .swp = ec_stripes_heap_swap,1090 };1091 1092 mutex_lock(&c->ec_stripes_heap_lock);1093 BUG_ON(min_heap_full(&c->ec_stripes_heap));1094 1095 genradix_ptr(&c->stripes, idx)->heap_idx = c->ec_stripes_heap.nr;1096 min_heap_push(&c->ec_stripes_heap, &((struct ec_stripe_heap_entry) {1097 .idx = idx,1098 .blocks_nonempty = m->blocks_nonempty,1099 }),1100 &callbacks,1101 &c->ec_stripes_heap);1102 1103 heap_verify_backpointer(c, idx);1104 mutex_unlock(&c->ec_stripes_heap_lock);1105}1106 1107void bch2_stripes_heap_update(struct bch_fs *c,1108 struct stripe *m, size_t idx)1109{1110 const struct min_heap_callbacks callbacks = {1111 .less = ec_stripes_heap_cmp,1112 .swp = ec_stripes_heap_swap,1113 };1114 ec_stripes_heap *h = &c->ec_stripes_heap;1115 bool do_deletes;1116 size_t i;1117 1118 mutex_lock(&c->ec_stripes_heap_lock);1119 heap_verify_backpointer(c, idx);1120 1121 h->data[m->heap_idx].blocks_nonempty = m->blocks_nonempty;1122 1123 i = m->heap_idx;1124 min_heap_sift_up(h, i, &callbacks, &c->ec_stripes_heap);1125 min_heap_sift_down(h, i, &callbacks, &c->ec_stripes_heap);1126 1127 heap_verify_backpointer(c, idx);1128 1129 do_deletes = stripe_idx_to_delete(c) != 0;1130 mutex_unlock(&c->ec_stripes_heap_lock);1131 1132 if (do_deletes)1133 bch2_do_stripe_deletes(c);1134}1135 1136/* stripe deletion */1137 1138static int ec_stripe_delete(struct btree_trans *trans, u64 idx)1139{1140 struct bch_fs *c = trans->c;1141 struct btree_iter iter;1142 struct bkey_s_c k;1143 struct bkey_s_c_stripe s;1144 int ret;1145 1146 k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_stripes, POS(0, idx),1147 BTREE_ITER_intent);1148 ret = bkey_err(k);1149 if (ret)1150 goto err;1151 1152 if (k.k->type != KEY_TYPE_stripe) {1153 bch2_fs_inconsistent(c, "attempting to delete nonexistent stripe %llu", idx);1154 ret = -EINVAL;1155 goto err;1156 }1157 1158 s = bkey_s_c_to_stripe(k);1159 for (unsigned i = 0; i < s.v->nr_blocks; i++)1160 if (stripe_blockcount_get(s.v, i)) {1161 struct printbuf buf = PRINTBUF;1162 1163 bch2_bkey_val_to_text(&buf, c, k);1164 bch2_fs_inconsistent(c, "attempting to delete nonempty stripe %s", buf.buf);1165 printbuf_exit(&buf);1166 ret = -EINVAL;1167 goto err;1168 }1169 1170 ret = bch2_btree_delete_at(trans, &iter, 0);1171err:1172 bch2_trans_iter_exit(trans, &iter);1173 return ret;1174}1175 1176static void ec_stripe_delete_work(struct work_struct *work)1177{1178 struct bch_fs *c =1179 container_of(work, struct bch_fs, ec_stripe_delete_work);1180 1181 while (1) {1182 mutex_lock(&c->ec_stripes_heap_lock);1183 u64 idx = stripe_idx_to_delete(c);1184 mutex_unlock(&c->ec_stripes_heap_lock);1185 1186 if (!idx)1187 break;1188 1189 int ret = bch2_trans_commit_do(c, NULL, NULL, BCH_TRANS_COMMIT_no_enospc,1190 ec_stripe_delete(trans, idx));1191 bch_err_fn(c, ret);1192 if (ret)1193 break;1194 }1195 1196 bch2_write_ref_put(c, BCH_WRITE_REF_stripe_delete);1197}1198 1199void bch2_do_stripe_deletes(struct bch_fs *c)1200{1201 if (bch2_write_ref_tryget(c, BCH_WRITE_REF_stripe_delete) &&1202 !queue_work(c->write_ref_wq, &c->ec_stripe_delete_work))1203 bch2_write_ref_put(c, BCH_WRITE_REF_stripe_delete);1204}1205 1206/* stripe creation: */1207 1208static int ec_stripe_key_update(struct btree_trans *trans,1209 struct bkey_i_stripe *old,1210 struct bkey_i_stripe *new)1211{1212 struct bch_fs *c = trans->c;1213 bool create = !old;1214 1215 struct btree_iter iter;1216 struct bkey_s_c k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_stripes,1217 new->k.p, BTREE_ITER_intent);1218 int ret = bkey_err(k);1219 if (ret)1220 goto err;1221 1222 if (bch2_fs_inconsistent_on(k.k->type != (create ? KEY_TYPE_deleted : KEY_TYPE_stripe),1223 c, "error %s stripe: got existing key type %s",1224 create ? "creating" : "updating",1225 bch2_bkey_types[k.k->type])) {1226 ret = -EINVAL;1227 goto err;1228 }1229 1230 if (k.k->type == KEY_TYPE_stripe) {1231 const struct bch_stripe *v = bkey_s_c_to_stripe(k).v;1232 1233 BUG_ON(old->v.nr_blocks != new->v.nr_blocks);1234 BUG_ON(old->v.nr_blocks != v->nr_blocks);1235 1236 for (unsigned i = 0; i < new->v.nr_blocks; i++) {1237 unsigned sectors = stripe_blockcount_get(v, i);1238 1239 if (!bch2_extent_ptr_eq(old->v.ptrs[i], new->v.ptrs[i]) && sectors) {1240 struct printbuf buf = PRINTBUF;1241 1242 prt_printf(&buf, "stripe changed nonempty block %u", i);1243 prt_str(&buf, "\nold: ");1244 bch2_bkey_val_to_text(&buf, c, k);1245 prt_str(&buf, "\nnew: ");1246 bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(&new->k_i));1247 bch2_fs_inconsistent(c, "%s", buf.buf);1248 printbuf_exit(&buf);1249 ret = -EINVAL;1250 goto err;1251 }1252 1253 /*1254 * If the stripe ptr changed underneath us, it must have1255 * been dev_remove_stripes() -> * invalidate_stripe_to_dev()1256 */1257 if (!bch2_extent_ptr_eq(old->v.ptrs[i], v->ptrs[i])) {1258 BUG_ON(v->ptrs[i].dev != BCH_SB_MEMBER_INVALID);1259 1260 if (bch2_extent_ptr_eq(old->v.ptrs[i], new->v.ptrs[i]))1261 new->v.ptrs[i].dev = BCH_SB_MEMBER_INVALID;1262 }1263 1264 stripe_blockcount_set(&new->v, i, sectors);1265 }1266 }1267 1268 ret = bch2_trans_update(trans, &iter, &new->k_i, 0);1269err:1270 bch2_trans_iter_exit(trans, &iter);1271 return ret;1272}1273 1274static int ec_stripe_update_extent(struct btree_trans *trans,1275 struct bch_dev *ca,1276 struct bpos bucket, u8 gen,1277 struct ec_stripe_buf *s,1278 struct bpos *bp_pos)1279{1280 struct bch_stripe *v = &bkey_i_to_stripe(&s->key)->v;1281 struct bch_fs *c = trans->c;1282 struct bch_backpointer bp;1283 struct btree_iter iter;1284 struct bkey_s_c k;1285 const struct bch_extent_ptr *ptr_c;1286 struct bch_extent_ptr *ec_ptr = NULL;1287 struct bch_extent_stripe_ptr stripe_ptr;1288 struct bkey_i *n;1289 int ret, dev, block;1290 1291 ret = bch2_get_next_backpointer(trans, ca, bucket, gen,1292 bp_pos, &bp, BTREE_ITER_cached);1293 if (ret)1294 return ret;1295 if (bpos_eq(*bp_pos, SPOS_MAX))1296 return 0;1297 1298 if (bp.level) {1299 struct printbuf buf = PRINTBUF;1300 struct btree_iter node_iter;1301 struct btree *b;1302 1303 b = bch2_backpointer_get_node(trans, &node_iter, *bp_pos, bp);1304 bch2_trans_iter_exit(trans, &node_iter);1305 1306 if (!b)1307 return 0;1308 1309 prt_printf(&buf, "found btree node in erasure coded bucket: b=%px\n", b);1310 bch2_backpointer_to_text(&buf, &bp);1311 1312 bch2_fs_inconsistent(c, "%s", buf.buf);1313 printbuf_exit(&buf);1314 return -EIO;1315 }1316 1317 k = bch2_backpointer_get_key(trans, &iter, *bp_pos, bp, BTREE_ITER_intent);1318 ret = bkey_err(k);1319 if (ret)1320 return ret;1321 if (!k.k) {1322 /*1323 * extent no longer exists - we could flush the btree1324 * write buffer and retry to verify, but no need:1325 */1326 return 0;1327 }1328 1329 if (extent_has_stripe_ptr(k, s->key.k.p.offset))1330 goto out;1331 1332 ptr_c = bkey_matches_stripe(v, k, &block);1333 /*1334 * It doesn't generally make sense to erasure code cached ptrs:1335 * XXX: should we be incrementing a counter?1336 */1337 if (!ptr_c || ptr_c->cached)1338 goto out;1339 1340 dev = v->ptrs[block].dev;1341 1342 n = bch2_trans_kmalloc(trans, bkey_bytes(k.k) + sizeof(stripe_ptr));1343 ret = PTR_ERR_OR_ZERO(n);1344 if (ret)1345 goto out;1346 1347 bkey_reassemble(n, k);1348 1349 bch2_bkey_drop_ptrs_noerror(bkey_i_to_s(n), ptr, ptr->dev != dev);1350 ec_ptr = bch2_bkey_has_device(bkey_i_to_s(n), dev);1351 BUG_ON(!ec_ptr);1352 1353 stripe_ptr = (struct bch_extent_stripe_ptr) {1354 .type = 1 << BCH_EXTENT_ENTRY_stripe_ptr,1355 .block = block,1356 .redundancy = v->nr_redundant,1357 .idx = s->key.k.p.offset,1358 };1359 1360 __extent_entry_insert(n,1361 (union bch_extent_entry *) ec_ptr,1362 (union bch_extent_entry *) &stripe_ptr);1363 1364 ret = bch2_trans_update(trans, &iter, n, 0);1365out:1366 bch2_trans_iter_exit(trans, &iter);1367 return ret;1368}1369 1370static int ec_stripe_update_bucket(struct btree_trans *trans, struct ec_stripe_buf *s,1371 unsigned block)1372{1373 struct bch_fs *c = trans->c;1374 struct bch_stripe *v = &bkey_i_to_stripe(&s->key)->v;1375 struct bch_extent_ptr ptr = v->ptrs[block];1376 struct bpos bp_pos = POS_MIN;1377 int ret = 0;1378 1379 struct bch_dev *ca = bch2_dev_tryget(c, ptr.dev);1380 if (!ca)1381 return -EIO;1382 1383 struct bpos bucket_pos = PTR_BUCKET_POS(ca, &ptr);1384 1385 while (1) {1386 ret = commit_do(trans, NULL, NULL,1387 BCH_TRANS_COMMIT_no_check_rw|1388 BCH_TRANS_COMMIT_no_enospc,1389 ec_stripe_update_extent(trans, ca, bucket_pos, ptr.gen, s, &bp_pos));1390 if (ret)1391 break;1392 if (bkey_eq(bp_pos, POS_MAX))1393 break;1394 1395 bp_pos = bpos_nosnap_successor(bp_pos);1396 }1397 1398 bch2_dev_put(ca);1399 return ret;1400}1401 1402static int ec_stripe_update_extents(struct bch_fs *c, struct ec_stripe_buf *s)1403{1404 struct btree_trans *trans = bch2_trans_get(c);1405 struct bch_stripe *v = &bkey_i_to_stripe(&s->key)->v;1406 unsigned i, nr_data = v->nr_blocks - v->nr_redundant;1407 int ret = 0;1408 1409 ret = bch2_btree_write_buffer_flush_sync(trans);1410 if (ret)1411 goto err;1412 1413 for (i = 0; i < nr_data; i++) {1414 ret = ec_stripe_update_bucket(trans, s, i);1415 if (ret)1416 break;1417 }1418err:1419 bch2_trans_put(trans);1420 1421 return ret;1422}1423 1424static void zero_out_rest_of_ec_bucket(struct bch_fs *c,1425 struct ec_stripe_new *s,1426 unsigned block,1427 struct open_bucket *ob)1428{1429 struct bch_dev *ca = bch2_dev_get_ioref(c, ob->dev, WRITE);1430 if (!ca) {1431 s->err = -BCH_ERR_erofs_no_writes;1432 return;1433 }1434 1435 unsigned offset = ca->mi.bucket_size - ob->sectors_free;1436 memset(s->new_stripe.data[block] + (offset << 9),1437 0,1438 ob->sectors_free << 9);1439 1440 int ret = blkdev_issue_zeroout(ca->disk_sb.bdev,1441 ob->bucket * ca->mi.bucket_size + offset,1442 ob->sectors_free,1443 GFP_KERNEL, 0);1444 1445 percpu_ref_put(&ca->io_ref);1446 1447 if (ret)1448 s->err = ret;1449}1450 1451void bch2_ec_stripe_new_free(struct bch_fs *c, struct ec_stripe_new *s)1452{1453 if (s->idx)1454 bch2_stripe_close(c, s);1455 kfree(s);1456}1457 1458/*1459 * data buckets of new stripe all written: create the stripe1460 */1461static void ec_stripe_create(struct ec_stripe_new *s)1462{1463 struct bch_fs *c = s->c;1464 struct open_bucket *ob;1465 struct bch_stripe *v = &bkey_i_to_stripe(&s->new_stripe.key)->v;1466 unsigned i, nr_data = v->nr_blocks - v->nr_redundant;1467 int ret;1468 1469 BUG_ON(s->h->s == s);1470 1471 closure_sync(&s->iodone);1472 1473 if (!s->err) {1474 for (i = 0; i < nr_data; i++)1475 if (s->blocks[i]) {1476 ob = c->open_buckets + s->blocks[i];1477 1478 if (ob->sectors_free)1479 zero_out_rest_of_ec_bucket(c, s, i, ob);1480 }1481 }1482 1483 if (s->err) {1484 if (!bch2_err_matches(s->err, EROFS))1485 bch_err(c, "error creating stripe: error writing data buckets");1486 goto err;1487 }1488 1489 if (s->have_existing_stripe) {1490 ec_validate_checksums(c, &s->existing_stripe);1491 1492 if (ec_do_recov(c, &s->existing_stripe)) {1493 bch_err(c, "error creating stripe: error reading existing stripe");1494 goto err;1495 }1496 1497 for (i = 0; i < nr_data; i++)1498 if (stripe_blockcount_get(&bkey_i_to_stripe(&s->existing_stripe.key)->v, i))1499 swap(s->new_stripe.data[i],1500 s->existing_stripe.data[i]);1501 1502 ec_stripe_buf_exit(&s->existing_stripe);1503 }1504 1505 BUG_ON(!s->allocated);1506 BUG_ON(!s->idx);1507 1508 ec_generate_ec(&s->new_stripe);1509 1510 ec_generate_checksums(&s->new_stripe);1511 1512 /* write p/q: */1513 for (i = nr_data; i < v->nr_blocks; i++)1514 ec_block_io(c, &s->new_stripe, REQ_OP_WRITE, i, &s->iodone);1515 closure_sync(&s->iodone);1516 1517 if (ec_nr_failed(&s->new_stripe)) {1518 bch_err(c, "error creating stripe: error writing redundancy buckets");1519 goto err;1520 }1521 1522 ret = bch2_trans_commit_do(c, &s->res, NULL,1523 BCH_TRANS_COMMIT_no_check_rw|1524 BCH_TRANS_COMMIT_no_enospc,1525 ec_stripe_key_update(trans,1526 s->have_existing_stripe1527 ? bkey_i_to_stripe(&s->existing_stripe.key)1528 : NULL,1529 bkey_i_to_stripe(&s->new_stripe.key)));1530 bch_err_msg(c, ret, "creating stripe key");1531 if (ret) {1532 goto err;1533 }1534 1535 ret = ec_stripe_update_extents(c, &s->new_stripe);1536 bch_err_msg(c, ret, "error updating extents");1537 if (ret)1538 goto err;1539err:1540 bch2_disk_reservation_put(c, &s->res);1541 1542 for (i = 0; i < v->nr_blocks; i++)1543 if (s->blocks[i]) {1544 ob = c->open_buckets + s->blocks[i];1545 1546 if (i < nr_data) {1547 ob->ec = NULL;1548 __bch2_open_bucket_put(c, ob);1549 } else {1550 bch2_open_bucket_put(c, ob);1551 }1552 }1553 1554 mutex_lock(&c->ec_stripe_new_lock);1555 list_del(&s->list);1556 mutex_unlock(&c->ec_stripe_new_lock);1557 wake_up(&c->ec_stripe_new_wait);1558 1559 ec_stripe_buf_exit(&s->existing_stripe);1560 ec_stripe_buf_exit(&s->new_stripe);1561 closure_debug_destroy(&s->iodone);1562 1563 ec_stripe_new_put(c, s, STRIPE_REF_stripe);1564}1565 1566static struct ec_stripe_new *get_pending_stripe(struct bch_fs *c)1567{1568 struct ec_stripe_new *s;1569 1570 mutex_lock(&c->ec_stripe_new_lock);1571 list_for_each_entry(s, &c->ec_stripe_new_list, list)1572 if (!atomic_read(&s->ref[STRIPE_REF_io]))1573 goto out;1574 s = NULL;1575out:1576 mutex_unlock(&c->ec_stripe_new_lock);1577 1578 return s;1579}1580 1581static void ec_stripe_create_work(struct work_struct *work)1582{1583 struct bch_fs *c = container_of(work,1584 struct bch_fs, ec_stripe_create_work);1585 struct ec_stripe_new *s;1586 1587 while ((s = get_pending_stripe(c)))1588 ec_stripe_create(s);1589 1590 bch2_write_ref_put(c, BCH_WRITE_REF_stripe_create);1591}1592 1593void bch2_ec_do_stripe_creates(struct bch_fs *c)1594{1595 bch2_write_ref_get(c, BCH_WRITE_REF_stripe_create);1596 1597 if (!queue_work(system_long_wq, &c->ec_stripe_create_work))1598 bch2_write_ref_put(c, BCH_WRITE_REF_stripe_create);1599}1600 1601static void ec_stripe_new_set_pending(struct bch_fs *c, struct ec_stripe_head *h)1602{1603 struct ec_stripe_new *s = h->s;1604 1605 lockdep_assert_held(&h->lock);1606 1607 BUG_ON(!s->allocated && !s->err);1608 1609 h->s = NULL;1610 s->pending = true;1611 1612 mutex_lock(&c->ec_stripe_new_lock);1613 list_add(&s->list, &c->ec_stripe_new_list);1614 mutex_unlock(&c->ec_stripe_new_lock);1615 1616 ec_stripe_new_put(c, s, STRIPE_REF_io);1617}1618 1619static void ec_stripe_new_cancel(struct bch_fs *c, struct ec_stripe_head *h, int err)1620{1621 h->s->err = err;1622 ec_stripe_new_set_pending(c, h);1623}1624 1625void bch2_ec_bucket_cancel(struct bch_fs *c, struct open_bucket *ob)1626{1627 struct ec_stripe_new *s = ob->ec;1628 1629 s->err = -EIO;1630}1631 1632void *bch2_writepoint_ec_buf(struct bch_fs *c, struct write_point *wp)1633{1634 struct open_bucket *ob = ec_open_bucket(c, &wp->ptrs);1635 if (!ob)1636 return NULL;1637 1638 BUG_ON(!ob->ec->new_stripe.data[ob->ec_idx]);1639 1640 struct bch_dev *ca = ob_dev(c, ob);1641 unsigned offset = ca->mi.bucket_size - ob->sectors_free;1642 1643 return ob->ec->new_stripe.data[ob->ec_idx] + (offset << 9);1644}1645 1646static int unsigned_cmp(const void *_l, const void *_r)1647{1648 unsigned l = *((const unsigned *) _l);1649 unsigned r = *((const unsigned *) _r);1650 1651 return cmp_int(l, r);1652}1653 1654/* pick most common bucket size: */1655static unsigned pick_blocksize(struct bch_fs *c,1656 struct bch_devs_mask *devs)1657{1658 unsigned nr = 0, sizes[BCH_SB_MEMBERS_MAX];1659 struct {1660 unsigned nr, size;1661 } cur = { 0, 0 }, best = { 0, 0 };1662 1663 for_each_member_device_rcu(c, ca, devs)1664 sizes[nr++] = ca->mi.bucket_size;1665 1666 sort(sizes, nr, sizeof(unsigned), unsigned_cmp, NULL);1667 1668 for (unsigned i = 0; i < nr; i++) {1669 if (sizes[i] != cur.size) {1670 if (cur.nr > best.nr)1671 best = cur;1672 1673 cur.nr = 0;1674 cur.size = sizes[i];1675 }1676 1677 cur.nr++;1678 }1679 1680 if (cur.nr > best.nr)1681 best = cur;1682 1683 return best.size;1684}1685 1686static bool may_create_new_stripe(struct bch_fs *c)1687{1688 return false;1689}1690 1691static void ec_stripe_key_init(struct bch_fs *c,1692 struct bkey_i *k,1693 unsigned nr_data,1694 unsigned nr_parity,1695 unsigned stripe_size,1696 unsigned disk_label)1697{1698 struct bkey_i_stripe *s = bkey_stripe_init(k);1699 unsigned u64s;1700 1701 s->v.sectors = cpu_to_le16(stripe_size);1702 s->v.algorithm = 0;1703 s->v.nr_blocks = nr_data + nr_parity;1704 s->v.nr_redundant = nr_parity;1705 s->v.csum_granularity_bits = ilog2(c->opts.encoded_extent_max >> 9);1706 s->v.csum_type = BCH_CSUM_crc32c;1707 s->v.disk_label = disk_label;1708 1709 while ((u64s = stripe_val_u64s(&s->v)) > BKEY_VAL_U64s_MAX) {1710 BUG_ON(1 << s->v.csum_granularity_bits >=1711 le16_to_cpu(s->v.sectors) ||1712 s->v.csum_granularity_bits == U8_MAX);1713 s->v.csum_granularity_bits++;1714 }1715 1716 set_bkey_val_u64s(&s->k, u64s);1717}1718 1719static int ec_new_stripe_alloc(struct bch_fs *c, struct ec_stripe_head *h)1720{1721 struct ec_stripe_new *s;1722 1723 lockdep_assert_held(&h->lock);1724 1725 s = kzalloc(sizeof(*s), GFP_KERNEL);1726 if (!s)1727 return -BCH_ERR_ENOMEM_ec_new_stripe_alloc;1728 1729 mutex_init(&s->lock);1730 closure_init(&s->iodone, NULL);1731 atomic_set(&s->ref[STRIPE_REF_stripe], 1);1732 atomic_set(&s->ref[STRIPE_REF_io], 1);1733 s->c = c;1734 s->h = h;1735 s->nr_data = min_t(unsigned, h->nr_active_devs,1736 BCH_BKEY_PTRS_MAX) - h->redundancy;1737 s->nr_parity = h->redundancy;1738 1739 ec_stripe_key_init(c, &s->new_stripe.key,1740 s->nr_data, s->nr_parity,1741 h->blocksize, h->disk_label);1742 1743 h->s = s;1744 h->nr_created++;1745 return 0;1746}1747 1748static void ec_stripe_head_devs_update(struct bch_fs *c, struct ec_stripe_head *h)1749{1750 struct bch_devs_mask devs = h->devs;1751 1752 rcu_read_lock();1753 h->devs = target_rw_devs(c, BCH_DATA_user, h->disk_label1754 ? group_to_target(h->disk_label - 1)1755 : 0);1756 unsigned nr_devs = dev_mask_nr(&h->devs);1757 1758 for_each_member_device_rcu(c, ca, &h->devs)1759 if (!ca->mi.durability)1760 __clear_bit(ca->dev_idx, h->devs.d);1761 unsigned nr_devs_with_durability = dev_mask_nr(&h->devs);1762 1763 h->blocksize = pick_blocksize(c, &h->devs);1764 1765 h->nr_active_devs = 0;1766 for_each_member_device_rcu(c, ca, &h->devs)1767 if (ca->mi.bucket_size == h->blocksize)1768 h->nr_active_devs++;1769 1770 rcu_read_unlock();1771 1772 /*1773 * If we only have redundancy + 1 devices, we're better off with just1774 * replication:1775 */1776 h->insufficient_devs = h->nr_active_devs < h->redundancy + 2;1777 1778 if (h->insufficient_devs) {1779 const char *err;1780 1781 if (nr_devs < h->redundancy + 2)1782 err = NULL;1783 else if (nr_devs_with_durability < h->redundancy + 2)1784 err = "cannot use durability=0 devices";1785 else1786 err = "mismatched bucket sizes";1787 1788 if (err)1789 bch_err(c, "insufficient devices available to create stripe (have %u, need %u): %s",1790 h->nr_active_devs, h->redundancy + 2, err);1791 }1792 1793 struct bch_devs_mask devs_leaving;1794 bitmap_andnot(devs_leaving.d, devs.d, h->devs.d, BCH_SB_MEMBERS_MAX);1795 1796 if (h->s && !h->s->allocated && dev_mask_nr(&devs_leaving))1797 ec_stripe_new_cancel(c, h, -EINTR);1798 1799 h->rw_devs_change_count = c->rw_devs_change_count;1800}1801 1802static struct ec_stripe_head *1803ec_new_stripe_head_alloc(struct bch_fs *c, unsigned disk_label,1804 unsigned algo, unsigned redundancy,1805 enum bch_watermark watermark)1806{1807 struct ec_stripe_head *h;1808 1809 h = kzalloc(sizeof(*h), GFP_KERNEL);1810 if (!h)1811 return NULL;1812 1813 mutex_init(&h->lock);1814 BUG_ON(!mutex_trylock(&h->lock));1815 1816 h->disk_label = disk_label;1817 h->algo = algo;1818 h->redundancy = redundancy;1819 h->watermark = watermark;1820 1821 list_add(&h->list, &c->ec_stripe_head_list);1822 return h;1823}1824 1825void bch2_ec_stripe_head_put(struct bch_fs *c, struct ec_stripe_head *h)1826{1827 if (h->s &&1828 h->s->allocated &&1829 bitmap_weight(h->s->blocks_allocated,1830 h->s->nr_data) == h->s->nr_data)1831 ec_stripe_new_set_pending(c, h);1832 1833 mutex_unlock(&h->lock);1834}1835 1836static struct ec_stripe_head *1837__bch2_ec_stripe_head_get(struct btree_trans *trans,1838 unsigned disk_label,1839 unsigned algo,1840 unsigned redundancy,1841 enum bch_watermark watermark)1842{1843 struct bch_fs *c = trans->c;1844 struct ec_stripe_head *h;1845 int ret;1846 1847 if (!redundancy)1848 return NULL;1849 1850 ret = bch2_trans_mutex_lock(trans, &c->ec_stripe_head_lock);1851 if (ret)1852 return ERR_PTR(ret);1853 1854 if (test_bit(BCH_FS_going_ro, &c->flags)) {1855 h = ERR_PTR(-BCH_ERR_erofs_no_writes);1856 goto err;1857 }1858 1859 list_for_each_entry(h, &c->ec_stripe_head_list, list)1860 if (h->disk_label == disk_label &&1861 h->algo == algo &&1862 h->redundancy == redundancy &&1863 h->watermark == watermark) {1864 ret = bch2_trans_mutex_lock(trans, &h->lock);1865 if (ret) {1866 h = ERR_PTR(ret);1867 goto err;1868 }1869 goto found;1870 }1871 1872 h = ec_new_stripe_head_alloc(c, disk_label, algo, redundancy, watermark);1873 if (!h) {1874 h = ERR_PTR(-BCH_ERR_ENOMEM_stripe_head_alloc);1875 goto err;1876 }1877found:1878 if (h->rw_devs_change_count != c->rw_devs_change_count)1879 ec_stripe_head_devs_update(c, h);1880 1881 if (h->insufficient_devs) {1882 mutex_unlock(&h->lock);1883 h = NULL;1884 }1885err:1886 mutex_unlock(&c->ec_stripe_head_lock);1887 return h;1888}1889 1890static int new_stripe_alloc_buckets(struct btree_trans *trans, struct ec_stripe_head *h,1891 enum bch_watermark watermark, struct closure *cl)1892{1893 struct bch_fs *c = trans->c;1894 struct bch_devs_mask devs = h->devs;1895 struct open_bucket *ob;1896 struct open_buckets buckets;1897 struct bch_stripe *v = &bkey_i_to_stripe(&h->s->new_stripe.key)->v;1898 unsigned i, j, nr_have_parity = 0, nr_have_data = 0;1899 bool have_cache = true;1900 int ret = 0;1901 1902 BUG_ON(v->nr_blocks != h->s->nr_data + h->s->nr_parity);1903 BUG_ON(v->nr_redundant != h->s->nr_parity);1904 1905 /* * We bypass the sector allocator which normally does this: */1906 bitmap_and(devs.d, devs.d, c->rw_devs[BCH_DATA_user].d, BCH_SB_MEMBERS_MAX);1907 1908 for_each_set_bit(i, h->s->blocks_gotten, v->nr_blocks) {1909 /*1910 * Note: we don't yet repair invalid blocks (failed/removed1911 * devices) when reusing stripes - we still need a codepath to1912 * walk backpointers and update all extents that point to that1913 * block when updating the stripe1914 */1915 if (v->ptrs[i].dev != BCH_SB_MEMBER_INVALID)1916 __clear_bit(v->ptrs[i].dev, devs.d);1917 1918 if (i < h->s->nr_data)1919 nr_have_data++;1920 else1921 nr_have_parity++;1922 }1923 1924 BUG_ON(nr_have_data > h->s->nr_data);1925 BUG_ON(nr_have_parity > h->s->nr_parity);1926 1927 buckets.nr = 0;1928 if (nr_have_parity < h->s->nr_parity) {1929 ret = bch2_bucket_alloc_set_trans(trans, &buckets,1930 &h->parity_stripe,1931 &devs,1932 h->s->nr_parity,1933 &nr_have_parity,1934 &have_cache, 0,1935 BCH_DATA_parity,1936 watermark,1937 cl);1938 1939 open_bucket_for_each(c, &buckets, ob, i) {1940 j = find_next_zero_bit(h->s->blocks_gotten,1941 h->s->nr_data + h->s->nr_parity,1942 h->s->nr_data);1943 BUG_ON(j >= h->s->nr_data + h->s->nr_parity);1944 1945 h->s->blocks[j] = buckets.v[i];1946 v->ptrs[j] = bch2_ob_ptr(c, ob);1947 __set_bit(j, h->s->blocks_gotten);1948 }1949 1950 if (ret)1951 return ret;1952 }1953 1954 buckets.nr = 0;1955 if (nr_have_data < h->s->nr_data) {1956 ret = bch2_bucket_alloc_set_trans(trans, &buckets,1957 &h->block_stripe,1958 &devs,1959 h->s->nr_data,1960 &nr_have_data,1961 &have_cache, 0,1962 BCH_DATA_user,1963 watermark,1964 cl);1965 1966 open_bucket_for_each(c, &buckets, ob, i) {1967 j = find_next_zero_bit(h->s->blocks_gotten,1968 h->s->nr_data, 0);1969 BUG_ON(j >= h->s->nr_data);1970 1971 h->s->blocks[j] = buckets.v[i];1972 v->ptrs[j] = bch2_ob_ptr(c, ob);1973 __set_bit(j, h->s->blocks_gotten);1974 }1975 1976 if (ret)1977 return ret;1978 }1979 1980 return 0;1981}1982 1983static s64 get_existing_stripe(struct bch_fs *c,1984 struct ec_stripe_head *head)1985{1986 ec_stripes_heap *h = &c->ec_stripes_heap;1987 struct stripe *m;1988 size_t heap_idx;1989 u64 stripe_idx;1990 s64 ret = -1;1991 1992 if (may_create_new_stripe(c))1993 return -1;1994 1995 mutex_lock(&c->ec_stripes_heap_lock);1996 for (heap_idx = 0; heap_idx < h->nr; heap_idx++) {1997 /* No blocks worth reusing, stripe will just be deleted: */1998 if (!h->data[heap_idx].blocks_nonempty)1999 continue;2000 2001 stripe_idx = h->data[heap_idx].idx;2002 2003 m = genradix_ptr(&c->stripes, stripe_idx);2004 2005 if (m->disk_label == head->disk_label &&2006 m->algorithm == head->algo &&2007 m->nr_redundant == head->redundancy &&2008 m->sectors == head->blocksize &&2009 m->blocks_nonempty < m->nr_blocks - m->nr_redundant &&2010 bch2_try_open_stripe(c, head->s, stripe_idx)) {2011 ret = stripe_idx;2012 break;2013 }2014 }2015 mutex_unlock(&c->ec_stripes_heap_lock);2016 return ret;2017}2018 2019static int __bch2_ec_stripe_head_reuse(struct btree_trans *trans, struct ec_stripe_head *h)2020{2021 struct bch_fs *c = trans->c;2022 struct bch_stripe *new_v = &bkey_i_to_stripe(&h->s->new_stripe.key)->v;2023 struct bch_stripe *existing_v;2024 unsigned i;2025 s64 idx;2026 int ret;2027 2028 /*2029 * If we can't allocate a new stripe, and there's no stripes with empty2030 * blocks for us to reuse, that means we have to wait on copygc:2031 */2032 idx = get_existing_stripe(c, h);2033 if (idx < 0)2034 return -BCH_ERR_stripe_alloc_blocked;2035 2036 ret = get_stripe_key_trans(trans, idx, &h->s->existing_stripe);2037 bch2_fs_fatal_err_on(ret && !bch2_err_matches(ret, BCH_ERR_transaction_restart), c,2038 "reading stripe key: %s", bch2_err_str(ret));2039 if (ret) {2040 bch2_stripe_close(c, h->s);2041 return ret;2042 }2043 2044 existing_v = &bkey_i_to_stripe(&h->s->existing_stripe.key)->v;2045 2046 BUG_ON(existing_v->nr_redundant != h->s->nr_parity);2047 h->s->nr_data = existing_v->nr_blocks -2048 existing_v->nr_redundant;2049 2050 ret = ec_stripe_buf_init(&h->s->existing_stripe, 0, h->blocksize);2051 if (ret) {2052 bch2_stripe_close(c, h->s);2053 return ret;2054 }2055 2056 BUG_ON(h->s->existing_stripe.size != h->blocksize);2057 BUG_ON(h->s->existing_stripe.size != le16_to_cpu(existing_v->sectors));2058 2059 /*2060 * Free buckets we initially allocated - they might conflict with2061 * blocks from the stripe we're reusing:2062 */2063 for_each_set_bit(i, h->s->blocks_gotten, new_v->nr_blocks) {2064 bch2_open_bucket_put(c, c->open_buckets + h->s->blocks[i]);2065 h->s->blocks[i] = 0;2066 }2067 memset(h->s->blocks_gotten, 0, sizeof(h->s->blocks_gotten));2068 memset(h->s->blocks_allocated, 0, sizeof(h->s->blocks_allocated));2069 2070 for (i = 0; i < existing_v->nr_blocks; i++) {2071 if (stripe_blockcount_get(existing_v, i)) {2072 __set_bit(i, h->s->blocks_gotten);2073 __set_bit(i, h->s->blocks_allocated);2074 }2075 2076 ec_block_io(c, &h->s->existing_stripe, READ, i, &h->s->iodone);2077 }2078 2079 bkey_copy(&h->s->new_stripe.key, &h->s->existing_stripe.key);2080 h->s->have_existing_stripe = true;2081 2082 return 0;2083}2084 2085static int __bch2_ec_stripe_head_reserve(struct btree_trans *trans, struct ec_stripe_head *h)2086{2087 struct bch_fs *c = trans->c;2088 struct btree_iter iter;2089 struct bkey_s_c k;2090 struct bpos min_pos = POS(0, 1);2091 struct bpos start_pos = bpos_max(min_pos, POS(0, c->ec_stripe_hint));2092 int ret;2093 2094 if (!h->s->res.sectors) {2095 ret = bch2_disk_reservation_get(c, &h->s->res,2096 h->blocksize,2097 h->s->nr_parity,2098 BCH_DISK_RESERVATION_NOFAIL);2099 if (ret)2100 return ret;2101 }2102 2103 for_each_btree_key_norestart(trans, iter, BTREE_ID_stripes, start_pos,2104 BTREE_ITER_slots|BTREE_ITER_intent, k, ret) {2105 if (bkey_gt(k.k->p, POS(0, U32_MAX))) {2106 if (start_pos.offset) {2107 start_pos = min_pos;2108 bch2_btree_iter_set_pos(&iter, start_pos);2109 continue;2110 }2111 2112 ret = -BCH_ERR_ENOSPC_stripe_create;2113 break;2114 }2115 2116 if (bkey_deleted(k.k) &&2117 bch2_try_open_stripe(c, h->s, k.k->p.offset))2118 break;2119 }2120 2121 c->ec_stripe_hint = iter.pos.offset;2122 2123 if (ret)2124 goto err;2125 2126 ret = ec_stripe_mem_alloc(trans, &iter);2127 if (ret) {2128 bch2_stripe_close(c, h->s);2129 goto err;2130 }2131 2132 h->s->new_stripe.key.k.p = iter.pos;2133out:2134 bch2_trans_iter_exit(trans, &iter);2135 return ret;2136err:2137 bch2_disk_reservation_put(c, &h->s->res);2138 goto out;2139}2140 2141struct ec_stripe_head *bch2_ec_stripe_head_get(struct btree_trans *trans,2142 unsigned target,2143 unsigned algo,2144 unsigned redundancy,2145 enum bch_watermark watermark,2146 struct closure *cl)2147{2148 struct bch_fs *c = trans->c;2149 struct ec_stripe_head *h;2150 bool waiting = false;2151 unsigned disk_label = 0;2152 struct target t = target_decode(target);2153 int ret;2154 2155 if (t.type == TARGET_GROUP) {2156 if (t.group > U8_MAX) {2157 bch_err(c, "cannot create a stripe when disk_label > U8_MAX");2158 return NULL;2159 }2160 disk_label = t.group + 1; /* 0 == no label */2161 }2162 2163 h = __bch2_ec_stripe_head_get(trans, disk_label, algo, redundancy, watermark);2164 if (IS_ERR_OR_NULL(h))2165 return h;2166 2167 if (!h->s) {2168 ret = ec_new_stripe_alloc(c, h);2169 if (ret) {2170 bch_err(c, "failed to allocate new stripe");2171 goto err;2172 }2173 }2174 2175 if (h->s->allocated)2176 goto allocated;2177 2178 if (h->s->have_existing_stripe)2179 goto alloc_existing;2180 2181 /* First, try to allocate a full stripe: */2182 ret = new_stripe_alloc_buckets(trans, h, BCH_WATERMARK_stripe, NULL) ?:2183 __bch2_ec_stripe_head_reserve(trans, h);2184 if (!ret)2185 goto allocate_buf;2186 if (bch2_err_matches(ret, BCH_ERR_transaction_restart) ||2187 bch2_err_matches(ret, ENOMEM))2188 goto err;2189 2190 /*2191 * Not enough buckets available for a full stripe: we must reuse an2192 * existing stripe:2193 */2194 while (1) {2195 ret = __bch2_ec_stripe_head_reuse(trans, h);2196 if (!ret)2197 break;2198 if (waiting || !cl || ret != -BCH_ERR_stripe_alloc_blocked)2199 goto err;2200 2201 if (watermark == BCH_WATERMARK_copygc) {2202 ret = new_stripe_alloc_buckets(trans, h, watermark, NULL) ?:2203 __bch2_ec_stripe_head_reserve(trans, h);2204 if (ret)2205 goto err;2206 goto allocate_buf;2207 }2208 2209 /* XXX freelist_wait? */2210 closure_wait(&c->freelist_wait, cl);2211 waiting = true;2212 }2213 2214 if (waiting)2215 closure_wake_up(&c->freelist_wait);2216alloc_existing:2217 /*2218 * Retry allocating buckets, with the watermark for this2219 * particular write:2220 */2221 ret = new_stripe_alloc_buckets(trans, h, watermark, cl);2222 if (ret)2223 goto err;2224 2225allocate_buf:2226 ret = ec_stripe_buf_init(&h->s->new_stripe, 0, h->blocksize);2227 if (ret)2228 goto err;2229 2230 h->s->allocated = true;2231allocated:2232 BUG_ON(!h->s->idx);2233 BUG_ON(!h->s->new_stripe.data[0]);2234 BUG_ON(trans->restarted);2235 return h;2236err:2237 bch2_ec_stripe_head_put(c, h);2238 return ERR_PTR(ret);2239}2240 2241/* device removal */2242 2243static int bch2_invalidate_stripe_to_dev(struct btree_trans *trans, struct bkey_s_c k_a)2244{2245 struct bch_alloc_v4 a_convert;2246 const struct bch_alloc_v4 *a = bch2_alloc_to_v4(k_a, &a_convert);2247 2248 if (!a->stripe)2249 return 0;2250 2251 if (a->stripe_sectors) {2252 bch_err(trans->c, "trying to invalidate device in stripe when bucket has stripe data");2253 return -BCH_ERR_invalidate_stripe_to_dev;2254 }2255 2256 struct btree_iter iter;2257 struct bkey_i_stripe *s =2258 bch2_bkey_get_mut_typed(trans, &iter, BTREE_ID_stripes, POS(0, a->stripe),2259 BTREE_ITER_slots, stripe);2260 int ret = PTR_ERR_OR_ZERO(s);2261 if (ret)2262 return ret;2263 2264 struct disk_accounting_pos acc = {2265 .type = BCH_DISK_ACCOUNTING_replicas,2266 };2267 2268 s64 sectors = 0;2269 for (unsigned i = 0; i < s->v.nr_blocks; i++)2270 sectors -= stripe_blockcount_get(&s->v, i);2271 2272 bch2_bkey_to_replicas(&acc.replicas, bkey_i_to_s_c(&s->k_i));2273 acc.replicas.data_type = BCH_DATA_user;2274 ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, false);2275 if (ret)2276 goto err;2277 2278 struct bkey_ptrs ptrs = bch2_bkey_ptrs(bkey_i_to_s(&s->k_i));2279 bkey_for_each_ptr(ptrs, ptr)2280 if (ptr->dev == k_a.k->p.inode)2281 ptr->dev = BCH_SB_MEMBER_INVALID;2282 2283 sectors = -sectors;2284 2285 bch2_bkey_to_replicas(&acc.replicas, bkey_i_to_s_c(&s->k_i));2286 acc.replicas.data_type = BCH_DATA_user;2287 ret = bch2_disk_accounting_mod(trans, &acc, §ors, 1, false);2288 if (ret)2289 goto err;2290err:2291 bch2_trans_iter_exit(trans, &iter);2292 return ret;2293}2294 2295int bch2_dev_remove_stripes(struct bch_fs *c, unsigned dev_idx)2296{2297 return bch2_trans_run(c,2298 for_each_btree_key_upto_commit(trans, iter,2299 BTREE_ID_alloc, POS(dev_idx, 0), POS(dev_idx, U64_MAX),2300 BTREE_ITER_intent, k,2301 NULL, NULL, 0, ({2302 bch2_invalidate_stripe_to_dev(trans, k);2303 })));2304}2305 2306/* startup/shutdown */2307 2308static void __bch2_ec_stop(struct bch_fs *c, struct bch_dev *ca)2309{2310 struct ec_stripe_head *h;2311 struct open_bucket *ob;2312 unsigned i;2313 2314 mutex_lock(&c->ec_stripe_head_lock);2315 list_for_each_entry(h, &c->ec_stripe_head_list, list) {2316 mutex_lock(&h->lock);2317 if (!h->s)2318 goto unlock;2319 2320 if (!ca)2321 goto found;2322 2323 for (i = 0; i < bkey_i_to_stripe(&h->s->new_stripe.key)->v.nr_blocks; i++) {2324 if (!h->s->blocks[i])2325 continue;2326 2327 ob = c->open_buckets + h->s->blocks[i];2328 if (ob->dev == ca->dev_idx)2329 goto found;2330 }2331 goto unlock;2332found:2333 ec_stripe_new_cancel(c, h, -BCH_ERR_erofs_no_writes);2334unlock:2335 mutex_unlock(&h->lock);2336 }2337 mutex_unlock(&c->ec_stripe_head_lock);2338}2339 2340void bch2_ec_stop_dev(struct bch_fs *c, struct bch_dev *ca)2341{2342 __bch2_ec_stop(c, ca);2343}2344 2345void bch2_fs_ec_stop(struct bch_fs *c)2346{2347 __bch2_ec_stop(c, NULL);2348}2349 2350static bool bch2_fs_ec_flush_done(struct bch_fs *c)2351{2352 bool ret;2353 2354 mutex_lock(&c->ec_stripe_new_lock);2355 ret = list_empty(&c->ec_stripe_new_list);2356 mutex_unlock(&c->ec_stripe_new_lock);2357 2358 return ret;2359}2360 2361void bch2_fs_ec_flush(struct bch_fs *c)2362{2363 wait_event(c->ec_stripe_new_wait, bch2_fs_ec_flush_done(c));2364}2365 2366int bch2_stripes_read(struct bch_fs *c)2367{2368 int ret = bch2_trans_run(c,2369 for_each_btree_key(trans, iter, BTREE_ID_stripes, POS_MIN,2370 BTREE_ITER_prefetch, k, ({2371 if (k.k->type != KEY_TYPE_stripe)2372 continue;2373 2374 ret = __ec_stripe_mem_alloc(c, k.k->p.offset, GFP_KERNEL);2375 if (ret)2376 break;2377 2378 struct stripe *m = genradix_ptr(&c->stripes, k.k->p.offset);2379 2380 stripe_to_mem(m, bkey_s_c_to_stripe(k).v);2381 2382 bch2_stripes_heap_insert(c, m, k.k->p.offset);2383 0;2384 })));2385 bch_err_fn(c, ret);2386 return ret;2387}2388 2389void bch2_stripes_heap_to_text(struct printbuf *out, struct bch_fs *c)2390{2391 ec_stripes_heap *h = &c->ec_stripes_heap;2392 struct stripe *m;2393 size_t i;2394 2395 mutex_lock(&c->ec_stripes_heap_lock);2396 for (i = 0; i < min_t(size_t, h->nr, 50); i++) {2397 m = genradix_ptr(&c->stripes, h->data[i].idx);2398 2399 prt_printf(out, "%zu %u/%u+%u", h->data[i].idx,2400 h->data[i].blocks_nonempty,2401 m->nr_blocks - m->nr_redundant,2402 m->nr_redundant);2403 if (bch2_stripe_is_open(c, h->data[i].idx))2404 prt_str(out, " open");2405 prt_newline(out);2406 }2407 mutex_unlock(&c->ec_stripes_heap_lock);2408}2409 2410static void bch2_new_stripe_to_text(struct printbuf *out, struct bch_fs *c,2411 struct ec_stripe_new *s)2412{2413 prt_printf(out, "\tidx %llu blocks %u+%u allocated %u ref %u %u %s obs",2414 s->idx, s->nr_data, s->nr_parity,2415 bitmap_weight(s->blocks_allocated, s->nr_data),2416 atomic_read(&s->ref[STRIPE_REF_io]),2417 atomic_read(&s->ref[STRIPE_REF_stripe]),2418 bch2_watermarks[s->h->watermark]);2419 2420 struct bch_stripe *v = &bkey_i_to_stripe(&s->new_stripe.key)->v;2421 unsigned i;2422 for_each_set_bit(i, s->blocks_gotten, v->nr_blocks)2423 prt_printf(out, " %u", s->blocks[i]);2424 prt_newline(out);2425 bch2_bkey_val_to_text(out, c, bkey_i_to_s_c(&s->new_stripe.key));2426 prt_newline(out);2427}2428 2429void bch2_new_stripes_to_text(struct printbuf *out, struct bch_fs *c)2430{2431 struct ec_stripe_head *h;2432 struct ec_stripe_new *s;2433 2434 mutex_lock(&c->ec_stripe_head_lock);2435 list_for_each_entry(h, &c->ec_stripe_head_list, list) {2436 prt_printf(out, "disk label %u algo %u redundancy %u %s nr created %llu:\n",2437 h->disk_label, h->algo, h->redundancy,2438 bch2_watermarks[h->watermark],2439 h->nr_created);2440 2441 if (h->s)2442 bch2_new_stripe_to_text(out, c, h->s);2443 }2444 mutex_unlock(&c->ec_stripe_head_lock);2445 2446 prt_printf(out, "in flight:\n");2447 2448 mutex_lock(&c->ec_stripe_new_lock);2449 list_for_each_entry(s, &c->ec_stripe_new_list, list)2450 bch2_new_stripe_to_text(out, c, s);2451 mutex_unlock(&c->ec_stripe_new_lock);2452}2453 2454void bch2_fs_ec_exit(struct bch_fs *c)2455{2456 struct ec_stripe_head *h;2457 unsigned i;2458 2459 while (1) {2460 mutex_lock(&c->ec_stripe_head_lock);2461 h = list_first_entry_or_null(&c->ec_stripe_head_list,2462 struct ec_stripe_head, list);2463 if (h)2464 list_del(&h->list);2465 mutex_unlock(&c->ec_stripe_head_lock);2466 if (!h)2467 break;2468 2469 if (h->s) {2470 for (i = 0; i < bkey_i_to_stripe(&h->s->new_stripe.key)->v.nr_blocks; i++)2471 BUG_ON(h->s->blocks[i]);2472 2473 kfree(h->s);2474 }2475 kfree(h);2476 }2477 2478 BUG_ON(!list_empty(&c->ec_stripe_new_list));2479 2480 free_heap(&c->ec_stripes_heap);2481 genradix_free(&c->stripes);2482 bioset_exit(&c->ec_bioset);2483}2484 2485void bch2_fs_ec_init_early(struct bch_fs *c)2486{2487 spin_lock_init(&c->ec_stripes_new_lock);2488 mutex_init(&c->ec_stripes_heap_lock);2489 2490 INIT_LIST_HEAD(&c->ec_stripe_head_list);2491 mutex_init(&c->ec_stripe_head_lock);2492 2493 INIT_LIST_HEAD(&c->ec_stripe_new_list);2494 mutex_init(&c->ec_stripe_new_lock);2495 init_waitqueue_head(&c->ec_stripe_new_wait);2496 2497 INIT_WORK(&c->ec_stripe_create_work, ec_stripe_create_work);2498 INIT_WORK(&c->ec_stripe_delete_work, ec_stripe_delete_work);2499}2500 2501int bch2_fs_ec_init(struct bch_fs *c)2502{2503 return bioset_init(&c->ec_bioset, 1, offsetof(struct ec_bio, bio),2504 BIOSET_NEED_BVECS);2505}2506