274 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Moving/copying garbage collector4 *5 * Copyright 2012 Google, Inc.6 */7 8#include "bcache.h"9#include "btree.h"10#include "debug.h"11#include "request.h"12 13#include <trace/events/bcache.h>14 15struct moving_io {16 struct closure cl;17 struct keybuf_key *w;18 struct data_insert_op op;19 struct bbio bio;20};21 22static bool moving_pred(struct keybuf *buf, struct bkey *k)23{24 struct cache_set *c = container_of(buf, struct cache_set,25 moving_gc_keys);26 unsigned int i;27 28 for (i = 0; i < KEY_PTRS(k); i++)29 if (ptr_available(c, k, i) &&30 GC_MOVE(PTR_BUCKET(c, k, i)))31 return true;32 33 return false;34}35 36/* Moving GC - IO loop */37 38static CLOSURE_CALLBACK(moving_io_destructor)39{40 closure_type(io, struct moving_io, cl);41 42 kfree(io);43}44 45static CLOSURE_CALLBACK(write_moving_finish)46{47 closure_type(io, struct moving_io, cl);48 struct bio *bio = &io->bio.bio;49 50 bio_free_pages(bio);51 52 if (io->op.replace_collision)53 trace_bcache_gc_copy_collision(&io->w->key);54 55 bch_keybuf_del(&io->op.c->moving_gc_keys, io->w);56 57 up(&io->op.c->moving_in_flight);58 59 closure_return_with_destructor(cl, moving_io_destructor);60}61 62static void read_moving_endio(struct bio *bio)63{64 struct bbio *b = container_of(bio, struct bbio, bio);65 struct moving_io *io = container_of(bio->bi_private,66 struct moving_io, cl);67 68 if (bio->bi_status)69 io->op.status = bio->bi_status;70 else if (!KEY_DIRTY(&b->key) &&71 ptr_stale(io->op.c, &b->key, 0)) {72 io->op.status = BLK_STS_IOERR;73 }74 75 bch_bbio_endio(io->op.c, bio, bio->bi_status, "reading data to move");76}77 78static void moving_init(struct moving_io *io)79{80 struct bio *bio = &io->bio.bio;81 82 bio_init(bio, NULL, bio->bi_inline_vecs,83 DIV_ROUND_UP(KEY_SIZE(&io->w->key), PAGE_SECTORS), 0);84 bio_get(bio);85 bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));86 87 bio->bi_iter.bi_size = KEY_SIZE(&io->w->key) << 9;88 bio->bi_private = &io->cl;89 bch_bio_map(bio, NULL);90}91 92static CLOSURE_CALLBACK(write_moving)93{94 closure_type(io, struct moving_io, cl);95 struct data_insert_op *op = &io->op;96 97 if (!op->status) {98 moving_init(io);99 100 io->bio.bio.bi_iter.bi_sector = KEY_START(&io->w->key);101 op->write_prio = 1;102 op->bio = &io->bio.bio;103 104 op->writeback = KEY_DIRTY(&io->w->key);105 op->csum = KEY_CSUM(&io->w->key);106 107 bkey_copy(&op->replace_key, &io->w->key);108 op->replace = true;109 110 closure_call(&op->cl, bch_data_insert, NULL, cl);111 }112 113 continue_at(cl, write_moving_finish, op->wq);114}115 116static CLOSURE_CALLBACK(read_moving_submit)117{118 closure_type(io, struct moving_io, cl);119 struct bio *bio = &io->bio.bio;120 121 bch_submit_bbio(bio, io->op.c, &io->w->key, 0);122 123 continue_at(cl, write_moving, io->op.wq);124}125 126static void read_moving(struct cache_set *c)127{128 struct keybuf_key *w;129 struct moving_io *io;130 struct bio *bio;131 struct closure cl;132 133 closure_init_stack(&cl);134 135 /* XXX: if we error, background writeback could stall indefinitely */136 137 while (!test_bit(CACHE_SET_STOPPING, &c->flags)) {138 w = bch_keybuf_next_rescan(c, &c->moving_gc_keys,139 &MAX_KEY, moving_pred);140 if (!w)141 break;142 143 if (ptr_stale(c, &w->key, 0)) {144 bch_keybuf_del(&c->moving_gc_keys, w);145 continue;146 }147 148 io = kzalloc(struct_size(io, bio.bio.bi_inline_vecs,149 DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS)),150 GFP_KERNEL);151 if (!io)152 goto err;153 154 w->private = io;155 io->w = w;156 io->op.inode = KEY_INODE(&w->key);157 io->op.c = c;158 io->op.wq = c->moving_gc_wq;159 160 moving_init(io);161 bio = &io->bio.bio;162 163 bio->bi_opf = REQ_OP_READ;164 bio->bi_end_io = read_moving_endio;165 166 if (bch_bio_alloc_pages(bio, GFP_KERNEL))167 goto err;168 169 trace_bcache_gc_copy(&w->key);170 171 down(&c->moving_in_flight);172 closure_call(&io->cl, read_moving_submit, NULL, &cl);173 }174 175 if (0) {176err: if (!IS_ERR_OR_NULL(w->private))177 kfree(w->private);178 179 bch_keybuf_del(&c->moving_gc_keys, w);180 }181 182 closure_sync(&cl);183}184 185static bool new_bucket_cmp(const void *l, const void *r, void __always_unused *args)186{187 struct bucket **_l = (struct bucket **)l;188 struct bucket **_r = (struct bucket **)r;189 190 return GC_SECTORS_USED(*_l) >= GC_SECTORS_USED(*_r);191}192 193static void new_bucket_swap(void *l, void *r, void __always_unused *args)194{195 struct bucket **_l = l;196 struct bucket **_r = r;197 198 swap(*_l, *_r);199}200 201static unsigned int bucket_heap_top(struct cache *ca)202{203 struct bucket *b;204 205 return (b = min_heap_peek(&ca->heap)[0]) ? GC_SECTORS_USED(b) : 0;206}207 208void bch_moving_gc(struct cache_set *c)209{210 struct cache *ca = c->cache;211 struct bucket *b;212 unsigned long sectors_to_move, reserve_sectors;213 const struct min_heap_callbacks callbacks = {214 .less = new_bucket_cmp,215 .swp = new_bucket_swap,216 };217 218 if (!c->copy_gc_enabled)219 return;220 221 mutex_lock(&c->bucket_lock);222 223 sectors_to_move = 0;224 reserve_sectors = ca->sb.bucket_size *225 fifo_used(&ca->free[RESERVE_MOVINGGC]);226 227 ca->heap.nr = 0;228 229 for_each_bucket(b, ca) {230 if (GC_MARK(b) == GC_MARK_METADATA ||231 !GC_SECTORS_USED(b) ||232 GC_SECTORS_USED(b) == ca->sb.bucket_size ||233 atomic_read(&b->pin))234 continue;235 236 if (!min_heap_full(&ca->heap)) {237 sectors_to_move += GC_SECTORS_USED(b);238 min_heap_push(&ca->heap, &b, &callbacks, NULL);239 } else if (!new_bucket_cmp(&b, min_heap_peek(&ca->heap), ca)) {240 sectors_to_move -= bucket_heap_top(ca);241 sectors_to_move += GC_SECTORS_USED(b);242 243 ca->heap.data[0] = b;244 min_heap_sift_down(&ca->heap, 0, &callbacks, NULL);245 }246 }247 248 while (sectors_to_move > reserve_sectors) {249 if (ca->heap.nr) {250 b = min_heap_peek(&ca->heap)[0];251 min_heap_pop(&ca->heap, &callbacks, NULL);252 }253 sectors_to_move -= GC_SECTORS_USED(b);254 }255 256 while (ca->heap.nr) {257 b = min_heap_peek(&ca->heap)[0];258 min_heap_pop(&ca->heap, &callbacks, NULL);259 SET_GC_MOVE(b, 1);260 }261 262 mutex_unlock(&c->bucket_lock);263 264 c->moving_gc_keys.last_scanned = ZERO_KEY;265 266 read_moving(c);267}268 269void bch_moving_init_cache_set(struct cache_set *c)270{271 bch_keybuf_init(&c->moving_gc_keys);272 sema_init(&c->moving_in_flight, 64);273}274