2924 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * bcache setup/teardown code, and some metadata io - read a superblock and4 * figure out what to do with it.5 *6 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>7 * Copyright 2012 Google, Inc.8 */9 10#include "bcache.h"11#include "btree.h"12#include "debug.h"13#include "extents.h"14#include "request.h"15#include "writeback.h"16#include "features.h"17 18#include <linux/blkdev.h>19#include <linux/pagemap.h>20#include <linux/debugfs.h>21#include <linux/idr.h>22#include <linux/kthread.h>23#include <linux/workqueue.h>24#include <linux/module.h>25#include <linux/random.h>26#include <linux/reboot.h>27#include <linux/sysfs.h>28 29unsigned int bch_cutoff_writeback;30unsigned int bch_cutoff_writeback_sync;31 32static const char bcache_magic[] = {33 0xc6, 0x85, 0x73, 0xf6, 0x4e, 0x1a, 0x45, 0xca,34 0x82, 0x65, 0xf5, 0x7f, 0x48, 0xba, 0x6d, 0x8135};36 37static const char invalid_uuid[] = {38 0xa0, 0x3e, 0xf8, 0xed, 0x3e, 0xe1, 0xb8, 0x78,39 0xc8, 0x50, 0xfc, 0x5e, 0xcb, 0x16, 0xcd, 0x9940};41 42static struct kobject *bcache_kobj;43struct mutex bch_register_lock;44bool bcache_is_reboot;45LIST_HEAD(bch_cache_sets);46static LIST_HEAD(uncached_devices);47 48static int bcache_major;49static DEFINE_IDA(bcache_device_idx);50static wait_queue_head_t unregister_wait;51struct workqueue_struct *bcache_wq;52struct workqueue_struct *bch_flush_wq;53struct workqueue_struct *bch_journal_wq;54 55 56#define BTREE_MAX_PAGES (256 * 1024 / PAGE_SIZE)57/* limitation of partitions number on single bcache device */58#define BCACHE_MINORS 12859/* limitation of bcache devices number on single system */60#define BCACHE_DEVICE_IDX_MAX ((1U << MINORBITS)/BCACHE_MINORS)61 62/* Superblock */63 64static unsigned int get_bucket_size(struct cache_sb *sb, struct cache_sb_disk *s)65{66 unsigned int bucket_size = le16_to_cpu(s->bucket_size);67 68 if (sb->version >= BCACHE_SB_VERSION_CDEV_WITH_FEATURES) {69 if (bch_has_feature_large_bucket(sb)) {70 unsigned int max, order;71 72 max = sizeof(unsigned int) * BITS_PER_BYTE - 1;73 order = le16_to_cpu(s->bucket_size);74 /*75 * bcache tool will make sure the overflow won't76 * happen, an error message here is enough.77 */78 if (order > max)79 pr_err("Bucket size (1 << %u) overflows\n",80 order);81 bucket_size = 1 << order;82 } else if (bch_has_feature_obso_large_bucket(sb)) {83 bucket_size +=84 le16_to_cpu(s->obso_bucket_size_hi) << 16;85 }86 }87 88 return bucket_size;89}90 91static const char *read_super_common(struct cache_sb *sb, struct block_device *bdev,92 struct cache_sb_disk *s)93{94 const char *err;95 unsigned int i;96 97 sb->first_bucket= le16_to_cpu(s->first_bucket);98 sb->nbuckets = le64_to_cpu(s->nbuckets);99 sb->bucket_size = get_bucket_size(sb, s);100 101 sb->nr_in_set = le16_to_cpu(s->nr_in_set);102 sb->nr_this_dev = le16_to_cpu(s->nr_this_dev);103 104 err = "Too many journal buckets";105 if (sb->keys > SB_JOURNAL_BUCKETS)106 goto err;107 108 err = "Too many buckets";109 if (sb->nbuckets > LONG_MAX)110 goto err;111 112 err = "Not enough buckets";113 if (sb->nbuckets < 1 << 7)114 goto err;115 116 err = "Bad block size (not power of 2)";117 if (!is_power_of_2(sb->block_size))118 goto err;119 120 err = "Bad block size (larger than page size)";121 if (sb->block_size > PAGE_SECTORS)122 goto err;123 124 err = "Bad bucket size (not power of 2)";125 if (!is_power_of_2(sb->bucket_size))126 goto err;127 128 err = "Bad bucket size (smaller than page size)";129 if (sb->bucket_size < PAGE_SECTORS)130 goto err;131 132 err = "Invalid superblock: device too small";133 if (get_capacity(bdev->bd_disk) <134 sb->bucket_size * sb->nbuckets)135 goto err;136 137 err = "Bad UUID";138 if (bch_is_zero(sb->set_uuid, 16))139 goto err;140 141 err = "Bad cache device number in set";142 if (!sb->nr_in_set ||143 sb->nr_in_set <= sb->nr_this_dev ||144 sb->nr_in_set > MAX_CACHES_PER_SET)145 goto err;146 147 err = "Journal buckets not sequential";148 for (i = 0; i < sb->keys; i++)149 if (sb->d[i] != sb->first_bucket + i)150 goto err;151 152 err = "Too many journal buckets";153 if (sb->first_bucket + sb->keys > sb->nbuckets)154 goto err;155 156 err = "Invalid superblock: first bucket comes before end of super";157 if (sb->first_bucket * sb->bucket_size < 16)158 goto err;159 160 err = NULL;161err:162 return err;163}164 165 166static const char *read_super(struct cache_sb *sb, struct block_device *bdev,167 struct cache_sb_disk **res)168{169 const char *err;170 struct cache_sb_disk *s;171 struct page *page;172 unsigned int i;173 174 page = read_cache_page_gfp(bdev->bd_mapping,175 SB_OFFSET >> PAGE_SHIFT, GFP_KERNEL);176 if (IS_ERR(page))177 return "IO error";178 s = page_address(page) + offset_in_page(SB_OFFSET);179 180 sb->offset = le64_to_cpu(s->offset);181 sb->version = le64_to_cpu(s->version);182 183 memcpy(sb->magic, s->magic, 16);184 memcpy(sb->uuid, s->uuid, 16);185 memcpy(sb->set_uuid, s->set_uuid, 16);186 memcpy(sb->label, s->label, SB_LABEL_SIZE);187 188 sb->flags = le64_to_cpu(s->flags);189 sb->seq = le64_to_cpu(s->seq);190 sb->last_mount = le32_to_cpu(s->last_mount);191 sb->keys = le16_to_cpu(s->keys);192 193 for (i = 0; i < SB_JOURNAL_BUCKETS; i++)194 sb->d[i] = le64_to_cpu(s->d[i]);195 196 pr_debug("read sb version %llu, flags %llu, seq %llu, journal size %u\n",197 sb->version, sb->flags, sb->seq, sb->keys);198 199 err = "Not a bcache superblock (bad offset)";200 if (sb->offset != SB_SECTOR)201 goto err;202 203 err = "Not a bcache superblock (bad magic)";204 if (memcmp(sb->magic, bcache_magic, 16))205 goto err;206 207 err = "Bad checksum";208 if (s->csum != csum_set(s))209 goto err;210 211 err = "Bad UUID";212 if (bch_is_zero(sb->uuid, 16))213 goto err;214 215 sb->block_size = le16_to_cpu(s->block_size);216 217 err = "Superblock block size smaller than device block size";218 if (sb->block_size << 9 < bdev_logical_block_size(bdev))219 goto err;220 221 switch (sb->version) {222 case BCACHE_SB_VERSION_BDEV:223 sb->data_offset = BDEV_DATA_START_DEFAULT;224 break;225 case BCACHE_SB_VERSION_BDEV_WITH_OFFSET:226 case BCACHE_SB_VERSION_BDEV_WITH_FEATURES:227 sb->data_offset = le64_to_cpu(s->data_offset);228 229 err = "Bad data offset";230 if (sb->data_offset < BDEV_DATA_START_DEFAULT)231 goto err;232 233 break;234 case BCACHE_SB_VERSION_CDEV:235 case BCACHE_SB_VERSION_CDEV_WITH_UUID:236 err = read_super_common(sb, bdev, s);237 if (err)238 goto err;239 break;240 case BCACHE_SB_VERSION_CDEV_WITH_FEATURES:241 /*242 * Feature bits are needed in read_super_common(),243 * convert them firstly.244 */245 sb->feature_compat = le64_to_cpu(s->feature_compat);246 sb->feature_incompat = le64_to_cpu(s->feature_incompat);247 sb->feature_ro_compat = le64_to_cpu(s->feature_ro_compat);248 249 /* Check incompatible features */250 err = "Unsupported compatible feature found";251 if (bch_has_unknown_compat_features(sb))252 goto err;253 254 err = "Unsupported read-only compatible feature found";255 if (bch_has_unknown_ro_compat_features(sb))256 goto err;257 258 err = "Unsupported incompatible feature found";259 if (bch_has_unknown_incompat_features(sb))260 goto err;261 262 err = read_super_common(sb, bdev, s);263 if (err)264 goto err;265 break;266 default:267 err = "Unsupported superblock version";268 goto err;269 }270 271 sb->last_mount = (u32)ktime_get_real_seconds();272 *res = s;273 return NULL;274err:275 put_page(page);276 return err;277}278 279static void write_bdev_super_endio(struct bio *bio)280{281 struct cached_dev *dc = bio->bi_private;282 283 if (bio->bi_status)284 bch_count_backing_io_errors(dc, bio);285 286 closure_put(&dc->sb_write);287}288 289static void __write_super(struct cache_sb *sb, struct cache_sb_disk *out,290 struct bio *bio)291{292 unsigned int i;293 294 bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_META;295 bio->bi_iter.bi_sector = SB_SECTOR;296 __bio_add_page(bio, virt_to_page(out), SB_SIZE,297 offset_in_page(out));298 299 out->offset = cpu_to_le64(sb->offset);300 301 memcpy(out->uuid, sb->uuid, 16);302 memcpy(out->set_uuid, sb->set_uuid, 16);303 memcpy(out->label, sb->label, SB_LABEL_SIZE);304 305 out->flags = cpu_to_le64(sb->flags);306 out->seq = cpu_to_le64(sb->seq);307 308 out->last_mount = cpu_to_le32(sb->last_mount);309 out->first_bucket = cpu_to_le16(sb->first_bucket);310 out->keys = cpu_to_le16(sb->keys);311 312 for (i = 0; i < sb->keys; i++)313 out->d[i] = cpu_to_le64(sb->d[i]);314 315 if (sb->version >= BCACHE_SB_VERSION_CDEV_WITH_FEATURES) {316 out->feature_compat = cpu_to_le64(sb->feature_compat);317 out->feature_incompat = cpu_to_le64(sb->feature_incompat);318 out->feature_ro_compat = cpu_to_le64(sb->feature_ro_compat);319 }320 321 out->version = cpu_to_le64(sb->version);322 out->csum = csum_set(out);323 324 pr_debug("ver %llu, flags %llu, seq %llu\n",325 sb->version, sb->flags, sb->seq);326 327 submit_bio(bio);328}329 330static CLOSURE_CALLBACK(bch_write_bdev_super_unlock)331{332 closure_type(dc, struct cached_dev, sb_write);333 334 up(&dc->sb_write_mutex);335}336 337void bch_write_bdev_super(struct cached_dev *dc, struct closure *parent)338{339 struct closure *cl = &dc->sb_write;340 struct bio *bio = &dc->sb_bio;341 342 down(&dc->sb_write_mutex);343 closure_init(cl, parent);344 345 bio_init(bio, dc->bdev, dc->sb_bv, 1, 0);346 bio->bi_end_io = write_bdev_super_endio;347 bio->bi_private = dc;348 349 closure_get(cl);350 /* I/O request sent to backing device */351 __write_super(&dc->sb, dc->sb_disk, bio);352 353 closure_return_with_destructor(cl, bch_write_bdev_super_unlock);354}355 356static void write_super_endio(struct bio *bio)357{358 struct cache *ca = bio->bi_private;359 360 /* is_read = 0 */361 bch_count_io_errors(ca, bio->bi_status, 0,362 "writing superblock");363 closure_put(&ca->set->sb_write);364}365 366static CLOSURE_CALLBACK(bcache_write_super_unlock)367{368 closure_type(c, struct cache_set, sb_write);369 370 up(&c->sb_write_mutex);371}372 373void bcache_write_super(struct cache_set *c)374{375 struct closure *cl = &c->sb_write;376 struct cache *ca = c->cache;377 struct bio *bio = &ca->sb_bio;378 unsigned int version = BCACHE_SB_VERSION_CDEV_WITH_UUID;379 380 down(&c->sb_write_mutex);381 closure_init(cl, &c->cl);382 383 ca->sb.seq++;384 385 if (ca->sb.version < version)386 ca->sb.version = version;387 388 bio_init(bio, ca->bdev, ca->sb_bv, 1, 0);389 bio->bi_end_io = write_super_endio;390 bio->bi_private = ca;391 392 closure_get(cl);393 __write_super(&ca->sb, ca->sb_disk, bio);394 395 closure_return_with_destructor(cl, bcache_write_super_unlock);396}397 398/* UUID io */399 400static void uuid_endio(struct bio *bio)401{402 struct closure *cl = bio->bi_private;403 struct cache_set *c = container_of(cl, struct cache_set, uuid_write);404 405 cache_set_err_on(bio->bi_status, c, "accessing uuids");406 bch_bbio_free(bio, c);407 closure_put(cl);408}409 410static CLOSURE_CALLBACK(uuid_io_unlock)411{412 closure_type(c, struct cache_set, uuid_write);413 414 up(&c->uuid_write_mutex);415}416 417static void uuid_io(struct cache_set *c, blk_opf_t opf, struct bkey *k,418 struct closure *parent)419{420 struct closure *cl = &c->uuid_write;421 struct uuid_entry *u;422 unsigned int i;423 char buf[80];424 425 BUG_ON(!parent);426 down(&c->uuid_write_mutex);427 closure_init(cl, parent);428 429 for (i = 0; i < KEY_PTRS(k); i++) {430 struct bio *bio = bch_bbio_alloc(c);431 432 bio->bi_opf = opf | REQ_SYNC | REQ_META;433 bio->bi_iter.bi_size = KEY_SIZE(k) << 9;434 435 bio->bi_end_io = uuid_endio;436 bio->bi_private = cl;437 bch_bio_map(bio, c->uuids);438 439 bch_submit_bbio(bio, c, k, i);440 441 if ((opf & REQ_OP_MASK) != REQ_OP_WRITE)442 break;443 }444 445 bch_extent_to_text(buf, sizeof(buf), k);446 pr_debug("%s UUIDs at %s\n", (opf & REQ_OP_MASK) == REQ_OP_WRITE ?447 "wrote" : "read", buf);448 449 for (u = c->uuids; u < c->uuids + c->nr_uuids; u++)450 if (!bch_is_zero(u->uuid, 16))451 pr_debug("Slot %zi: %pU: %s: 1st: %u last: %u inv: %u\n",452 u - c->uuids, u->uuid, u->label,453 u->first_reg, u->last_reg, u->invalidated);454 455 closure_return_with_destructor(cl, uuid_io_unlock);456}457 458static char *uuid_read(struct cache_set *c, struct jset *j, struct closure *cl)459{460 struct bkey *k = &j->uuid_bucket;461 462 if (__bch_btree_ptr_invalid(c, k))463 return "bad uuid pointer";464 465 bkey_copy(&c->uuid_bucket, k);466 uuid_io(c, REQ_OP_READ, k, cl);467 468 if (j->version < BCACHE_JSET_VERSION_UUIDv1) {469 struct uuid_entry_v0 *u0 = (void *) c->uuids;470 struct uuid_entry *u1 = (void *) c->uuids;471 int i;472 473 closure_sync(cl);474 475 /*476 * Since the new uuid entry is bigger than the old, we have to477 * convert starting at the highest memory address and work down478 * in order to do it in place479 */480 481 for (i = c->nr_uuids - 1;482 i >= 0;483 --i) {484 memcpy(u1[i].uuid, u0[i].uuid, 16);485 memcpy(u1[i].label, u0[i].label, 32);486 487 u1[i].first_reg = u0[i].first_reg;488 u1[i].last_reg = u0[i].last_reg;489 u1[i].invalidated = u0[i].invalidated;490 491 u1[i].flags = 0;492 u1[i].sectors = 0;493 }494 }495 496 return NULL;497}498 499static int __uuid_write(struct cache_set *c)500{501 BKEY_PADDED(key) k;502 struct closure cl;503 struct cache *ca = c->cache;504 unsigned int size;505 506 closure_init_stack(&cl);507 lockdep_assert_held(&bch_register_lock);508 509 if (bch_bucket_alloc_set(c, RESERVE_BTREE, &k.key, true))510 return 1;511 512 size = meta_bucket_pages(&ca->sb) * PAGE_SECTORS;513 SET_KEY_SIZE(&k.key, size);514 uuid_io(c, REQ_OP_WRITE, &k.key, &cl);515 closure_sync(&cl);516 517 /* Only one bucket used for uuid write */518 atomic_long_add(ca->sb.bucket_size, &ca->meta_sectors_written);519 520 bkey_copy(&c->uuid_bucket, &k.key);521 bkey_put(c, &k.key);522 return 0;523}524 525int bch_uuid_write(struct cache_set *c)526{527 int ret = __uuid_write(c);528 529 if (!ret)530 bch_journal_meta(c, NULL);531 532 return ret;533}534 535static struct uuid_entry *uuid_find(struct cache_set *c, const char *uuid)536{537 struct uuid_entry *u;538 539 for (u = c->uuids;540 u < c->uuids + c->nr_uuids; u++)541 if (!memcmp(u->uuid, uuid, 16))542 return u;543 544 return NULL;545}546 547static struct uuid_entry *uuid_find_empty(struct cache_set *c)548{549 static const char zero_uuid[16] = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";550 551 return uuid_find(c, zero_uuid);552}553 554/*555 * Bucket priorities/gens:556 *557 * For each bucket, we store on disk its558 * 8 bit gen559 * 16 bit priority560 *561 * See alloc.c for an explanation of the gen. The priority is used to implement562 * lru (and in the future other) cache replacement policies; for most purposes563 * it's just an opaque integer.564 *565 * The gens and the priorities don't have a whole lot to do with each other, and566 * it's actually the gens that must be written out at specific times - it's no567 * big deal if the priorities don't get written, if we lose them we just reuse568 * buckets in suboptimal order.569 *570 * On disk they're stored in a packed array, and in as many buckets are required571 * to fit them all. The buckets we use to store them form a list; the journal572 * header points to the first bucket, the first bucket points to the second573 * bucket, et cetera.574 *575 * This code is used by the allocation code; periodically (whenever it runs out576 * of buckets to allocate from) the allocation code will invalidate some577 * buckets, but it can't use those buckets until their new gens are safely on578 * disk.579 */580 581static void prio_endio(struct bio *bio)582{583 struct cache *ca = bio->bi_private;584 585 cache_set_err_on(bio->bi_status, ca->set, "accessing priorities");586 bch_bbio_free(bio, ca->set);587 closure_put(&ca->prio);588}589 590static void prio_io(struct cache *ca, uint64_t bucket, blk_opf_t opf)591{592 struct closure *cl = &ca->prio;593 struct bio *bio = bch_bbio_alloc(ca->set);594 595 closure_init_stack(cl);596 597 bio->bi_iter.bi_sector = bucket * ca->sb.bucket_size;598 bio_set_dev(bio, ca->bdev);599 bio->bi_iter.bi_size = meta_bucket_bytes(&ca->sb);600 601 bio->bi_end_io = prio_endio;602 bio->bi_private = ca;603 bio->bi_opf = opf | REQ_SYNC | REQ_META;604 bch_bio_map(bio, ca->disk_buckets);605 606 closure_bio_submit(ca->set, bio, &ca->prio);607 closure_sync(cl);608}609 610int bch_prio_write(struct cache *ca, bool wait)611{612 int i;613 struct bucket *b;614 struct closure cl;615 616 pr_debug("free_prio=%zu, free_none=%zu, free_inc=%zu\n",617 fifo_used(&ca->free[RESERVE_PRIO]),618 fifo_used(&ca->free[RESERVE_NONE]),619 fifo_used(&ca->free_inc));620 621 /*622 * Pre-check if there are enough free buckets. In the non-blocking623 * scenario it's better to fail early rather than starting to allocate624 * buckets and do a cleanup later in case of failure.625 */626 if (!wait) {627 size_t avail = fifo_used(&ca->free[RESERVE_PRIO]) +628 fifo_used(&ca->free[RESERVE_NONE]);629 if (prio_buckets(ca) > avail)630 return -ENOMEM;631 }632 633 closure_init_stack(&cl);634 635 lockdep_assert_held(&ca->set->bucket_lock);636 637 ca->disk_buckets->seq++;638 639 atomic_long_add(ca->sb.bucket_size * prio_buckets(ca),640 &ca->meta_sectors_written);641 642 for (i = prio_buckets(ca) - 1; i >= 0; --i) {643 long bucket;644 struct prio_set *p = ca->disk_buckets;645 struct bucket_disk *d = p->data;646 struct bucket_disk *end = d + prios_per_bucket(ca);647 648 for (b = ca->buckets + i * prios_per_bucket(ca);649 b < ca->buckets + ca->sb.nbuckets && d < end;650 b++, d++) {651 d->prio = cpu_to_le16(b->prio);652 d->gen = b->gen;653 }654 655 p->next_bucket = ca->prio_buckets[i + 1];656 p->magic = pset_magic(&ca->sb);657 p->csum = bch_crc64(&p->magic, meta_bucket_bytes(&ca->sb) - 8);658 659 bucket = bch_bucket_alloc(ca, RESERVE_PRIO, wait);660 BUG_ON(bucket == -1);661 662 mutex_unlock(&ca->set->bucket_lock);663 prio_io(ca, bucket, REQ_OP_WRITE);664 mutex_lock(&ca->set->bucket_lock);665 666 ca->prio_buckets[i] = bucket;667 atomic_dec_bug(&ca->buckets[bucket].pin);668 }669 670 mutex_unlock(&ca->set->bucket_lock);671 672 bch_journal_meta(ca->set, &cl);673 closure_sync(&cl);674 675 mutex_lock(&ca->set->bucket_lock);676 677 /*678 * Don't want the old priorities to get garbage collected until after we679 * finish writing the new ones, and they're journalled680 */681 for (i = 0; i < prio_buckets(ca); i++) {682 if (ca->prio_last_buckets[i])683 __bch_bucket_free(ca,684 &ca->buckets[ca->prio_last_buckets[i]]);685 686 ca->prio_last_buckets[i] = ca->prio_buckets[i];687 }688 return 0;689}690 691static int prio_read(struct cache *ca, uint64_t bucket)692{693 struct prio_set *p = ca->disk_buckets;694 struct bucket_disk *d = p->data + prios_per_bucket(ca), *end = d;695 struct bucket *b;696 unsigned int bucket_nr = 0;697 int ret = -EIO;698 699 for (b = ca->buckets;700 b < ca->buckets + ca->sb.nbuckets;701 b++, d++) {702 if (d == end) {703 ca->prio_buckets[bucket_nr] = bucket;704 ca->prio_last_buckets[bucket_nr] = bucket;705 bucket_nr++;706 707 prio_io(ca, bucket, REQ_OP_READ);708 709 if (p->csum !=710 bch_crc64(&p->magic, meta_bucket_bytes(&ca->sb) - 8)) {711 pr_warn("bad csum reading priorities\n");712 goto out;713 }714 715 if (p->magic != pset_magic(&ca->sb)) {716 pr_warn("bad magic reading priorities\n");717 goto out;718 }719 720 bucket = p->next_bucket;721 d = p->data;722 }723 724 b->prio = le16_to_cpu(d->prio);725 b->gen = b->last_gc = d->gen;726 }727 728 ret = 0;729out:730 return ret;731}732 733/* Bcache device */734 735static int open_dev(struct gendisk *disk, blk_mode_t mode)736{737 struct bcache_device *d = disk->private_data;738 739 if (test_bit(BCACHE_DEV_CLOSING, &d->flags))740 return -ENXIO;741 742 closure_get(&d->cl);743 return 0;744}745 746static void release_dev(struct gendisk *b)747{748 struct bcache_device *d = b->private_data;749 750 closure_put(&d->cl);751}752 753static int ioctl_dev(struct block_device *b, blk_mode_t mode,754 unsigned int cmd, unsigned long arg)755{756 struct bcache_device *d = b->bd_disk->private_data;757 758 return d->ioctl(d, mode, cmd, arg);759}760 761static const struct block_device_operations bcache_cached_ops = {762 .submit_bio = cached_dev_submit_bio,763 .open = open_dev,764 .release = release_dev,765 .ioctl = ioctl_dev,766 .owner = THIS_MODULE,767};768 769static const struct block_device_operations bcache_flash_ops = {770 .submit_bio = flash_dev_submit_bio,771 .open = open_dev,772 .release = release_dev,773 .ioctl = ioctl_dev,774 .owner = THIS_MODULE,775};776 777void bcache_device_stop(struct bcache_device *d)778{779 if (!test_and_set_bit(BCACHE_DEV_CLOSING, &d->flags))780 /*781 * closure_fn set to782 * - cached device: cached_dev_flush()783 * - flash dev: flash_dev_flush()784 */785 closure_queue(&d->cl);786}787 788static void bcache_device_unlink(struct bcache_device *d)789{790 lockdep_assert_held(&bch_register_lock);791 792 if (d->c && !test_and_set_bit(BCACHE_DEV_UNLINK_DONE, &d->flags)) {793 struct cache *ca = d->c->cache;794 795 sysfs_remove_link(&d->c->kobj, d->name);796 sysfs_remove_link(&d->kobj, "cache");797 798 bd_unlink_disk_holder(ca->bdev, d->disk);799 }800}801 802static void bcache_device_link(struct bcache_device *d, struct cache_set *c,803 const char *name)804{805 struct cache *ca = c->cache;806 int ret;807 808 bd_link_disk_holder(ca->bdev, d->disk);809 810 snprintf(d->name, BCACHEDEVNAME_SIZE,811 "%s%u", name, d->id);812 813 ret = sysfs_create_link(&d->kobj, &c->kobj, "cache");814 if (ret < 0)815 pr_err("Couldn't create device -> cache set symlink\n");816 817 ret = sysfs_create_link(&c->kobj, &d->kobj, d->name);818 if (ret < 0)819 pr_err("Couldn't create cache set -> device symlink\n");820 821 clear_bit(BCACHE_DEV_UNLINK_DONE, &d->flags);822}823 824static void bcache_device_detach(struct bcache_device *d)825{826 lockdep_assert_held(&bch_register_lock);827 828 atomic_dec(&d->c->attached_dev_nr);829 830 if (test_bit(BCACHE_DEV_DETACHING, &d->flags)) {831 struct uuid_entry *u = d->c->uuids + d->id;832 833 SET_UUID_FLASH_ONLY(u, 0);834 memcpy(u->uuid, invalid_uuid, 16);835 u->invalidated = cpu_to_le32((u32)ktime_get_real_seconds());836 bch_uuid_write(d->c);837 }838 839 bcache_device_unlink(d);840 841 d->c->devices[d->id] = NULL;842 closure_put(&d->c->caching);843 d->c = NULL;844}845 846static void bcache_device_attach(struct bcache_device *d, struct cache_set *c,847 unsigned int id)848{849 d->id = id;850 d->c = c;851 c->devices[id] = d;852 853 if (id >= c->devices_max_used)854 c->devices_max_used = id + 1;855 856 closure_get(&c->caching);857}858 859static inline int first_minor_to_idx(int first_minor)860{861 return (first_minor/BCACHE_MINORS);862}863 864static inline int idx_to_first_minor(int idx)865{866 return (idx * BCACHE_MINORS);867}868 869static void bcache_device_free(struct bcache_device *d)870{871 struct gendisk *disk = d->disk;872 873 lockdep_assert_held(&bch_register_lock);874 875 if (disk)876 pr_info("%s stopped\n", disk->disk_name);877 else878 pr_err("bcache device (NULL gendisk) stopped\n");879 880 if (d->c)881 bcache_device_detach(d);882 883 if (disk) {884 ida_free(&bcache_device_idx,885 first_minor_to_idx(disk->first_minor));886 put_disk(disk);887 }888 889 bioset_exit(&d->bio_split);890 kvfree(d->full_dirty_stripes);891 kvfree(d->stripe_sectors_dirty);892 893 closure_debug_destroy(&d->cl);894}895 896static int bcache_device_init(struct bcache_device *d, unsigned int block_size,897 sector_t sectors, struct block_device *cached_bdev,898 const struct block_device_operations *ops)899{900 const size_t max_stripes = min_t(size_t, INT_MAX,901 SIZE_MAX / sizeof(atomic_t));902 struct queue_limits lim = {903 .max_hw_sectors = UINT_MAX,904 .max_sectors = UINT_MAX,905 .max_segment_size = UINT_MAX,906 .max_segments = BIO_MAX_VECS,907 .max_hw_discard_sectors = UINT_MAX,908 .io_min = block_size,909 .logical_block_size = block_size,910 .physical_block_size = block_size,911 .features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA,912 };913 uint64_t n;914 int idx;915 916 if (cached_bdev) {917 d->stripe_size = bdev_io_opt(cached_bdev) >> SECTOR_SHIFT;918 lim.io_opt = umax(block_size, bdev_io_opt(cached_bdev));919 }920 if (!d->stripe_size)921 d->stripe_size = 1 << 31;922 else if (d->stripe_size < BCH_MIN_STRIPE_SZ)923 d->stripe_size = roundup(BCH_MIN_STRIPE_SZ, d->stripe_size);924 925 n = DIV_ROUND_UP_ULL(sectors, d->stripe_size);926 if (!n || n > max_stripes) {927 pr_err("nr_stripes too large or invalid: %llu (start sector beyond end of disk?)\n",928 n);929 return -ENOMEM;930 }931 d->nr_stripes = n;932 933 n = d->nr_stripes * sizeof(atomic_t);934 d->stripe_sectors_dirty = kvzalloc(n, GFP_KERNEL);935 if (!d->stripe_sectors_dirty)936 return -ENOMEM;937 938 n = BITS_TO_LONGS(d->nr_stripes) * sizeof(unsigned long);939 d->full_dirty_stripes = kvzalloc(n, GFP_KERNEL);940 if (!d->full_dirty_stripes)941 goto out_free_stripe_sectors_dirty;942 943 idx = ida_alloc_max(&bcache_device_idx, BCACHE_DEVICE_IDX_MAX - 1,944 GFP_KERNEL);945 if (idx < 0)946 goto out_free_full_dirty_stripes;947 948 if (bioset_init(&d->bio_split, 4, offsetof(struct bbio, bio),949 BIOSET_NEED_BVECS|BIOSET_NEED_RESCUER))950 goto out_ida_remove;951 952 if (lim.logical_block_size > PAGE_SIZE && cached_bdev) {953 /*954 * This should only happen with BCACHE_SB_VERSION_BDEV.955 * Block/page size is checked for BCACHE_SB_VERSION_CDEV.956 */957 pr_info("bcache%i: sb/logical block size (%u) greater than page size (%lu) falling back to device logical block size (%u)\n",958 idx, lim.logical_block_size,959 PAGE_SIZE, bdev_logical_block_size(cached_bdev));960 961 /* This also adjusts physical block size/min io size if needed */962 lim.logical_block_size = bdev_logical_block_size(cached_bdev);963 }964 965 d->disk = blk_alloc_disk(&lim, NUMA_NO_NODE);966 if (IS_ERR(d->disk))967 goto out_bioset_exit;968 969 set_capacity(d->disk, sectors);970 snprintf(d->disk->disk_name, DISK_NAME_LEN, "bcache%i", idx);971 972 d->disk->major = bcache_major;973 d->disk->first_minor = idx_to_first_minor(idx);974 d->disk->minors = BCACHE_MINORS;975 d->disk->fops = ops;976 d->disk->private_data = d;977 return 0;978 979out_bioset_exit:980 bioset_exit(&d->bio_split);981out_ida_remove:982 ida_free(&bcache_device_idx, idx);983out_free_full_dirty_stripes:984 kvfree(d->full_dirty_stripes);985out_free_stripe_sectors_dirty:986 kvfree(d->stripe_sectors_dirty);987 return -ENOMEM;988 989}990 991/* Cached device */992 993static void calc_cached_dev_sectors(struct cache_set *c)994{995 uint64_t sectors = 0;996 struct cached_dev *dc;997 998 list_for_each_entry(dc, &c->cached_devs, list)999 sectors += bdev_nr_sectors(dc->bdev);1000 1001 c->cached_dev_sectors = sectors;1002}1003 1004#define BACKING_DEV_OFFLINE_TIMEOUT 51005static int cached_dev_status_update(void *arg)1006{1007 struct cached_dev *dc = arg;1008 struct request_queue *q;1009 1010 /*1011 * If this delayed worker is stopping outside, directly quit here.1012 * dc->io_disable might be set via sysfs interface, so check it1013 * here too.1014 */1015 while (!kthread_should_stop() && !dc->io_disable) {1016 q = bdev_get_queue(dc->bdev);1017 if (blk_queue_dying(q))1018 dc->offline_seconds++;1019 else1020 dc->offline_seconds = 0;1021 1022 if (dc->offline_seconds >= BACKING_DEV_OFFLINE_TIMEOUT) {1023 pr_err("%pg: device offline for %d seconds\n",1024 dc->bdev,1025 BACKING_DEV_OFFLINE_TIMEOUT);1026 pr_err("%s: disable I/O request due to backing device offline\n",1027 dc->disk.name);1028 dc->io_disable = true;1029 /* let others know earlier that io_disable is true */1030 smp_mb();1031 bcache_device_stop(&dc->disk);1032 break;1033 }1034 schedule_timeout_interruptible(HZ);1035 }1036 1037 wait_for_kthread_stop();1038 return 0;1039}1040 1041 1042int bch_cached_dev_run(struct cached_dev *dc)1043{1044 int ret = 0;1045 struct bcache_device *d = &dc->disk;1046 char *buf = kmemdup_nul(dc->sb.label, SB_LABEL_SIZE, GFP_KERNEL);1047 char *env[] = {1048 "DRIVER=bcache",1049 kasprintf(GFP_KERNEL, "CACHED_UUID=%pU", dc->sb.uuid),1050 kasprintf(GFP_KERNEL, "CACHED_LABEL=%s", buf ? : ""),1051 NULL,1052 };1053 1054 if (dc->io_disable) {1055 pr_err("I/O disabled on cached dev %pg\n", dc->bdev);1056 ret = -EIO;1057 goto out;1058 }1059 1060 if (atomic_xchg(&dc->running, 1)) {1061 pr_info("cached dev %pg is running already\n", dc->bdev);1062 ret = -EBUSY;1063 goto out;1064 }1065 1066 if (!d->c &&1067 BDEV_STATE(&dc->sb) != BDEV_STATE_NONE) {1068 struct closure cl;1069 1070 closure_init_stack(&cl);1071 1072 SET_BDEV_STATE(&dc->sb, BDEV_STATE_STALE);1073 bch_write_bdev_super(dc, &cl);1074 closure_sync(&cl);1075 }1076 1077 ret = add_disk(d->disk);1078 if (ret)1079 goto out;1080 bd_link_disk_holder(dc->bdev, dc->disk.disk);1081 /*1082 * won't show up in the uevent file, use udevadm monitor -e instead1083 * only class / kset properties are persistent1084 */1085 kobject_uevent_env(&disk_to_dev(d->disk)->kobj, KOBJ_CHANGE, env);1086 1087 if (sysfs_create_link(&d->kobj, &disk_to_dev(d->disk)->kobj, "dev") ||1088 sysfs_create_link(&disk_to_dev(d->disk)->kobj,1089 &d->kobj, "bcache")) {1090 pr_err("Couldn't create bcache dev <-> disk sysfs symlinks\n");1091 ret = -ENOMEM;1092 goto out;1093 }1094 1095 dc->status_update_thread = kthread_run(cached_dev_status_update,1096 dc, "bcache_status_update");1097 if (IS_ERR(dc->status_update_thread)) {1098 pr_warn("failed to create bcache_status_update kthread, continue to run without monitoring backing device status\n");1099 }1100 1101out:1102 kfree(env[1]);1103 kfree(env[2]);1104 kfree(buf);1105 return ret;1106}1107 1108/*1109 * If BCACHE_DEV_RATE_DW_RUNNING is set, it means routine of the delayed1110 * work dc->writeback_rate_update is running. Wait until the routine1111 * quits (BCACHE_DEV_RATE_DW_RUNNING is clear), then continue to1112 * cancel it. If BCACHE_DEV_RATE_DW_RUNNING is not clear after time_out1113 * seconds, give up waiting here and continue to cancel it too.1114 */1115static void cancel_writeback_rate_update_dwork(struct cached_dev *dc)1116{1117 int time_out = WRITEBACK_RATE_UPDATE_SECS_MAX * HZ;1118 1119 do {1120 if (!test_bit(BCACHE_DEV_RATE_DW_RUNNING,1121 &dc->disk.flags))1122 break;1123 time_out--;1124 schedule_timeout_interruptible(1);1125 } while (time_out > 0);1126 1127 if (time_out == 0)1128 pr_warn("give up waiting for dc->writeback_write_update to quit\n");1129 1130 cancel_delayed_work_sync(&dc->writeback_rate_update);1131}1132 1133static void cached_dev_detach_finish(struct work_struct *w)1134{1135 struct cached_dev *dc = container_of(w, struct cached_dev, detach);1136 struct cache_set *c = dc->disk.c;1137 1138 BUG_ON(!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags));1139 BUG_ON(refcount_read(&dc->count));1140 1141 1142 if (test_and_clear_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags))1143 cancel_writeback_rate_update_dwork(dc);1144 1145 if (!IS_ERR_OR_NULL(dc->writeback_thread)) {1146 kthread_stop(dc->writeback_thread);1147 dc->writeback_thread = NULL;1148 }1149 1150 mutex_lock(&bch_register_lock);1151 1152 bcache_device_detach(&dc->disk);1153 list_move(&dc->list, &uncached_devices);1154 calc_cached_dev_sectors(c);1155 1156 clear_bit(BCACHE_DEV_DETACHING, &dc->disk.flags);1157 clear_bit(BCACHE_DEV_UNLINK_DONE, &dc->disk.flags);1158 1159 mutex_unlock(&bch_register_lock);1160 1161 pr_info("Caching disabled for %pg\n", dc->bdev);1162 1163 /* Drop ref we took in cached_dev_detach() */1164 closure_put(&dc->disk.cl);1165}1166 1167void bch_cached_dev_detach(struct cached_dev *dc)1168{1169 lockdep_assert_held(&bch_register_lock);1170 1171 if (test_bit(BCACHE_DEV_CLOSING, &dc->disk.flags))1172 return;1173 1174 if (test_and_set_bit(BCACHE_DEV_DETACHING, &dc->disk.flags))1175 return;1176 1177 /*1178 * Block the device from being closed and freed until we're finished1179 * detaching1180 */1181 closure_get(&dc->disk.cl);1182 1183 bch_writeback_queue(dc);1184 1185 cached_dev_put(dc);1186}1187 1188int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c,1189 uint8_t *set_uuid)1190{1191 uint32_t rtime = cpu_to_le32((u32)ktime_get_real_seconds());1192 struct uuid_entry *u;1193 struct cached_dev *exist_dc, *t;1194 int ret = 0;1195 1196 if ((set_uuid && memcmp(set_uuid, c->set_uuid, 16)) ||1197 (!set_uuid && memcmp(dc->sb.set_uuid, c->set_uuid, 16)))1198 return -ENOENT;1199 1200 if (dc->disk.c) {1201 pr_err("Can't attach %pg: already attached\n", dc->bdev);1202 return -EINVAL;1203 }1204 1205 if (test_bit(CACHE_SET_STOPPING, &c->flags)) {1206 pr_err("Can't attach %pg: shutting down\n", dc->bdev);1207 return -EINVAL;1208 }1209 1210 if (dc->sb.block_size < c->cache->sb.block_size) {1211 /* Will die */1212 pr_err("Couldn't attach %pg: block size less than set's block size\n",1213 dc->bdev);1214 return -EINVAL;1215 }1216 1217 /* Check whether already attached */1218 list_for_each_entry_safe(exist_dc, t, &c->cached_devs, list) {1219 if (!memcmp(dc->sb.uuid, exist_dc->sb.uuid, 16)) {1220 pr_err("Tried to attach %pg but duplicate UUID already attached\n",1221 dc->bdev);1222 1223 return -EINVAL;1224 }1225 }1226 1227 u = uuid_find(c, dc->sb.uuid);1228 1229 if (u &&1230 (BDEV_STATE(&dc->sb) == BDEV_STATE_STALE ||1231 BDEV_STATE(&dc->sb) == BDEV_STATE_NONE)) {1232 memcpy(u->uuid, invalid_uuid, 16);1233 u->invalidated = cpu_to_le32((u32)ktime_get_real_seconds());1234 u = NULL;1235 }1236 1237 if (!u) {1238 if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) {1239 pr_err("Couldn't find uuid for %pg in set\n", dc->bdev);1240 return -ENOENT;1241 }1242 1243 u = uuid_find_empty(c);1244 if (!u) {1245 pr_err("Not caching %pg, no room for UUID\n", dc->bdev);1246 return -EINVAL;1247 }1248 }1249 1250 /*1251 * Deadlocks since we're called via sysfs...1252 * sysfs_remove_file(&dc->kobj, &sysfs_attach);1253 */1254 1255 if (bch_is_zero(u->uuid, 16)) {1256 struct closure cl;1257 1258 closure_init_stack(&cl);1259 1260 memcpy(u->uuid, dc->sb.uuid, 16);1261 memcpy(u->label, dc->sb.label, SB_LABEL_SIZE);1262 u->first_reg = u->last_reg = rtime;1263 bch_uuid_write(c);1264 1265 memcpy(dc->sb.set_uuid, c->set_uuid, 16);1266 SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN);1267 1268 bch_write_bdev_super(dc, &cl);1269 closure_sync(&cl);1270 } else {1271 u->last_reg = rtime;1272 bch_uuid_write(c);1273 }1274 1275 bcache_device_attach(&dc->disk, c, u - c->uuids);1276 list_move(&dc->list, &c->cached_devs);1277 calc_cached_dev_sectors(c);1278 1279 /*1280 * dc->c must be set before dc->count != 0 - paired with the mb in1281 * cached_dev_get()1282 */1283 smp_wmb();1284 refcount_set(&dc->count, 1);1285 1286 /* Block writeback thread, but spawn it */1287 down_write(&dc->writeback_lock);1288 if (bch_cached_dev_writeback_start(dc)) {1289 up_write(&dc->writeback_lock);1290 pr_err("Couldn't start writeback facilities for %s\n",1291 dc->disk.disk->disk_name);1292 return -ENOMEM;1293 }1294 1295 if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) {1296 atomic_set(&dc->has_dirty, 1);1297 bch_writeback_queue(dc);1298 }1299 1300 bch_sectors_dirty_init(&dc->disk);1301 1302 ret = bch_cached_dev_run(dc);1303 if (ret && (ret != -EBUSY)) {1304 up_write(&dc->writeback_lock);1305 /*1306 * bch_register_lock is held, bcache_device_stop() is not1307 * able to be directly called. The kthread and kworker1308 * created previously in bch_cached_dev_writeback_start()1309 * have to be stopped manually here.1310 */1311 kthread_stop(dc->writeback_thread);1312 cancel_writeback_rate_update_dwork(dc);1313 pr_err("Couldn't run cached device %pg\n", dc->bdev);1314 return ret;1315 }1316 1317 bcache_device_link(&dc->disk, c, "bdev");1318 atomic_inc(&c->attached_dev_nr);1319 1320 if (bch_has_feature_obso_large_bucket(&(c->cache->sb))) {1321 pr_err("The obsoleted large bucket layout is unsupported, set the bcache device into read-only\n");1322 pr_err("Please update to the latest bcache-tools to create the cache device\n");1323 set_disk_ro(dc->disk.disk, 1);1324 }1325 1326 /* Allow the writeback thread to proceed */1327 up_write(&dc->writeback_lock);1328 1329 pr_info("Caching %pg as %s on set %pU\n",1330 dc->bdev,1331 dc->disk.disk->disk_name,1332 dc->disk.c->set_uuid);1333 return 0;1334}1335 1336/* when dc->disk.kobj released */1337void bch_cached_dev_release(struct kobject *kobj)1338{1339 struct cached_dev *dc = container_of(kobj, struct cached_dev,1340 disk.kobj);1341 kfree(dc);1342 module_put(THIS_MODULE);1343}1344 1345static CLOSURE_CALLBACK(cached_dev_free)1346{1347 closure_type(dc, struct cached_dev, disk.cl);1348 1349 if (test_and_clear_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags))1350 cancel_writeback_rate_update_dwork(dc);1351 1352 if (!IS_ERR_OR_NULL(dc->writeback_thread))1353 kthread_stop(dc->writeback_thread);1354 if (!IS_ERR_OR_NULL(dc->status_update_thread))1355 kthread_stop(dc->status_update_thread);1356 1357 mutex_lock(&bch_register_lock);1358 1359 if (atomic_read(&dc->running)) {1360 bd_unlink_disk_holder(dc->bdev, dc->disk.disk);1361 del_gendisk(dc->disk.disk);1362 }1363 bcache_device_free(&dc->disk);1364 list_del(&dc->list);1365 1366 mutex_unlock(&bch_register_lock);1367 1368 if (dc->sb_disk)1369 put_page(virt_to_page(dc->sb_disk));1370 1371 if (dc->bdev_file)1372 fput(dc->bdev_file);1373 1374 wake_up(&unregister_wait);1375 1376 kobject_put(&dc->disk.kobj);1377}1378 1379static CLOSURE_CALLBACK(cached_dev_flush)1380{1381 closure_type(dc, struct cached_dev, disk.cl);1382 struct bcache_device *d = &dc->disk;1383 1384 mutex_lock(&bch_register_lock);1385 bcache_device_unlink(d);1386 mutex_unlock(&bch_register_lock);1387 1388 bch_cache_accounting_destroy(&dc->accounting);1389 kobject_del(&d->kobj);1390 1391 continue_at(cl, cached_dev_free, system_wq);1392}1393 1394static int cached_dev_init(struct cached_dev *dc, unsigned int block_size)1395{1396 int ret;1397 struct io *io;1398 struct request_queue *q = bdev_get_queue(dc->bdev);1399 1400 __module_get(THIS_MODULE);1401 INIT_LIST_HEAD(&dc->list);1402 closure_init(&dc->disk.cl, NULL);1403 set_closure_fn(&dc->disk.cl, cached_dev_flush, system_wq);1404 kobject_init(&dc->disk.kobj, &bch_cached_dev_ktype);1405 INIT_WORK(&dc->detach, cached_dev_detach_finish);1406 sema_init(&dc->sb_write_mutex, 1);1407 INIT_LIST_HEAD(&dc->io_lru);1408 spin_lock_init(&dc->io_lock);1409 bch_cache_accounting_init(&dc->accounting, &dc->disk.cl);1410 1411 dc->sequential_cutoff = 4 << 20;1412 1413 for (io = dc->io; io < dc->io + RECENT_IO; io++) {1414 list_add(&io->lru, &dc->io_lru);1415 hlist_add_head(&io->hash, dc->io_hash + RECENT_IO);1416 }1417 1418 if (bdev_io_opt(dc->bdev))1419 dc->partial_stripes_expensive = !!(q->limits.features &1420 BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE);1421 1422 ret = bcache_device_init(&dc->disk, block_size,1423 bdev_nr_sectors(dc->bdev) - dc->sb.data_offset,1424 dc->bdev, &bcache_cached_ops);1425 if (ret)1426 return ret;1427 1428 atomic_set(&dc->io_errors, 0);1429 dc->io_disable = false;1430 dc->error_limit = DEFAULT_CACHED_DEV_ERROR_LIMIT;1431 /* default to auto */1432 dc->stop_when_cache_set_failed = BCH_CACHED_DEV_STOP_AUTO;1433 1434 bch_cached_dev_request_init(dc);1435 bch_cached_dev_writeback_init(dc);1436 return 0;1437}1438 1439/* Cached device - bcache superblock */1440 1441static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk,1442 struct file *bdev_file,1443 struct cached_dev *dc)1444{1445 const char *err = "cannot allocate memory";1446 struct cache_set *c;1447 int ret = -ENOMEM;1448 1449 memcpy(&dc->sb, sb, sizeof(struct cache_sb));1450 dc->bdev_file = bdev_file;1451 dc->bdev = file_bdev(bdev_file);1452 dc->sb_disk = sb_disk;1453 1454 if (cached_dev_init(dc, sb->block_size << 9))1455 goto err;1456 1457 err = "error creating kobject";1458 if (kobject_add(&dc->disk.kobj, bdev_kobj(dc->bdev), "bcache"))1459 goto err;1460 if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj))1461 goto err;1462 1463 pr_info("registered backing device %pg\n", dc->bdev);1464 1465 list_add(&dc->list, &uncached_devices);1466 /* attach to a matched cache set if it exists */1467 list_for_each_entry(c, &bch_cache_sets, list)1468 bch_cached_dev_attach(dc, c, NULL);1469 1470 if (BDEV_STATE(&dc->sb) == BDEV_STATE_NONE ||1471 BDEV_STATE(&dc->sb) == BDEV_STATE_STALE) {1472 err = "failed to run cached device";1473 ret = bch_cached_dev_run(dc);1474 if (ret)1475 goto err;1476 }1477 1478 return 0;1479err:1480 pr_notice("error %pg: %s\n", dc->bdev, err);1481 bcache_device_stop(&dc->disk);1482 return ret;1483}1484 1485/* Flash only volumes */1486 1487/* When d->kobj released */1488void bch_flash_dev_release(struct kobject *kobj)1489{1490 struct bcache_device *d = container_of(kobj, struct bcache_device,1491 kobj);1492 kfree(d);1493}1494 1495static CLOSURE_CALLBACK(flash_dev_free)1496{1497 closure_type(d, struct bcache_device, cl);1498 1499 mutex_lock(&bch_register_lock);1500 atomic_long_sub(bcache_dev_sectors_dirty(d),1501 &d->c->flash_dev_dirty_sectors);1502 del_gendisk(d->disk);1503 bcache_device_free(d);1504 mutex_unlock(&bch_register_lock);1505 kobject_put(&d->kobj);1506}1507 1508static CLOSURE_CALLBACK(flash_dev_flush)1509{1510 closure_type(d, struct bcache_device, cl);1511 1512 mutex_lock(&bch_register_lock);1513 bcache_device_unlink(d);1514 mutex_unlock(&bch_register_lock);1515 kobject_del(&d->kobj);1516 continue_at(cl, flash_dev_free, system_wq);1517}1518 1519static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)1520{1521 int err = -ENOMEM;1522 struct bcache_device *d = kzalloc(sizeof(struct bcache_device),1523 GFP_KERNEL);1524 if (!d)1525 goto err_ret;1526 1527 closure_init(&d->cl, NULL);1528 set_closure_fn(&d->cl, flash_dev_flush, system_wq);1529 1530 kobject_init(&d->kobj, &bch_flash_dev_ktype);1531 1532 if (bcache_device_init(d, block_bytes(c->cache), u->sectors,1533 NULL, &bcache_flash_ops))1534 goto err;1535 1536 bcache_device_attach(d, c, u - c->uuids);1537 bch_sectors_dirty_init(d);1538 bch_flash_dev_request_init(d);1539 err = add_disk(d->disk);1540 if (err)1541 goto err;1542 1543 err = kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache");1544 if (err)1545 goto err;1546 1547 bcache_device_link(d, c, "volume");1548 1549 if (bch_has_feature_obso_large_bucket(&c->cache->sb)) {1550 pr_err("The obsoleted large bucket layout is unsupported, set the bcache device into read-only\n");1551 pr_err("Please update to the latest bcache-tools to create the cache device\n");1552 set_disk_ro(d->disk, 1);1553 }1554 1555 return 0;1556err:1557 kobject_put(&d->kobj);1558err_ret:1559 return err;1560}1561 1562static int flash_devs_run(struct cache_set *c)1563{1564 int ret = 0;1565 struct uuid_entry *u;1566 1567 for (u = c->uuids;1568 u < c->uuids + c->nr_uuids && !ret;1569 u++)1570 if (UUID_FLASH_ONLY(u))1571 ret = flash_dev_run(c, u);1572 1573 return ret;1574}1575 1576int bch_flash_dev_create(struct cache_set *c, uint64_t size)1577{1578 struct uuid_entry *u;1579 1580 if (test_bit(CACHE_SET_STOPPING, &c->flags))1581 return -EINTR;1582 1583 if (!test_bit(CACHE_SET_RUNNING, &c->flags))1584 return -EPERM;1585 1586 u = uuid_find_empty(c);1587 if (!u) {1588 pr_err("Can't create volume, no room for UUID\n");1589 return -EINVAL;1590 }1591 1592 get_random_bytes(u->uuid, 16);1593 memset(u->label, 0, 32);1594 u->first_reg = u->last_reg = cpu_to_le32((u32)ktime_get_real_seconds());1595 1596 SET_UUID_FLASH_ONLY(u, 1);1597 u->sectors = size >> 9;1598 1599 bch_uuid_write(c);1600 1601 return flash_dev_run(c, u);1602}1603 1604bool bch_cached_dev_error(struct cached_dev *dc)1605{1606 if (!dc || test_bit(BCACHE_DEV_CLOSING, &dc->disk.flags))1607 return false;1608 1609 dc->io_disable = true;1610 /* make others know io_disable is true earlier */1611 smp_mb();1612 1613 pr_err("stop %s: too many IO errors on backing device %pg\n",1614 dc->disk.disk->disk_name, dc->bdev);1615 1616 bcache_device_stop(&dc->disk);1617 return true;1618}1619 1620/* Cache set */1621 1622__printf(2, 3)1623bool bch_cache_set_error(struct cache_set *c, const char *fmt, ...)1624{1625 struct va_format vaf;1626 va_list args;1627 1628 if (c->on_error != ON_ERROR_PANIC &&1629 test_bit(CACHE_SET_STOPPING, &c->flags))1630 return false;1631 1632 if (test_and_set_bit(CACHE_SET_IO_DISABLE, &c->flags))1633 pr_info("CACHE_SET_IO_DISABLE already set\n");1634 1635 /*1636 * XXX: we can be called from atomic context1637 * acquire_console_sem();1638 */1639 1640 va_start(args, fmt);1641 1642 vaf.fmt = fmt;1643 vaf.va = &args;1644 1645 pr_err("error on %pU: %pV, disabling caching\n",1646 c->set_uuid, &vaf);1647 1648 va_end(args);1649 1650 if (c->on_error == ON_ERROR_PANIC)1651 panic("panic forced after error\n");1652 1653 bch_cache_set_unregister(c);1654 return true;1655}1656 1657/* When c->kobj released */1658void bch_cache_set_release(struct kobject *kobj)1659{1660 struct cache_set *c = container_of(kobj, struct cache_set, kobj);1661 1662 kfree(c);1663 module_put(THIS_MODULE);1664}1665 1666static CLOSURE_CALLBACK(cache_set_free)1667{1668 closure_type(c, struct cache_set, cl);1669 struct cache *ca;1670 1671 debugfs_remove(c->debug);1672 1673 bch_open_buckets_free(c);1674 bch_btree_cache_free(c);1675 bch_journal_free(c);1676 1677 mutex_lock(&bch_register_lock);1678 bch_bset_sort_state_free(&c->sort);1679 free_pages((unsigned long) c->uuids, ilog2(meta_bucket_pages(&c->cache->sb)));1680 1681 ca = c->cache;1682 if (ca) {1683 ca->set = NULL;1684 c->cache = NULL;1685 kobject_put(&ca->kobj);1686 }1687 1688 1689 if (c->moving_gc_wq)1690 destroy_workqueue(c->moving_gc_wq);1691 bioset_exit(&c->bio_split);1692 mempool_exit(&c->fill_iter);1693 mempool_exit(&c->bio_meta);1694 mempool_exit(&c->search);1695 kfree(c->devices);1696 1697 list_del(&c->list);1698 mutex_unlock(&bch_register_lock);1699 1700 pr_info("Cache set %pU unregistered\n", c->set_uuid);1701 wake_up(&unregister_wait);1702 1703 closure_debug_destroy(&c->cl);1704 kobject_put(&c->kobj);1705}1706 1707static CLOSURE_CALLBACK(cache_set_flush)1708{1709 closure_type(c, struct cache_set, caching);1710 struct cache *ca = c->cache;1711 struct btree *b;1712 1713 bch_cache_accounting_destroy(&c->accounting);1714 1715 kobject_put(&c->internal);1716 kobject_del(&c->kobj);1717 1718 if (!IS_ERR_OR_NULL(c->gc_thread))1719 kthread_stop(c->gc_thread);1720 1721 if (!IS_ERR(c->root))1722 list_add(&c->root->list, &c->btree_cache);1723 1724 /*1725 * Avoid flushing cached nodes if cache set is retiring1726 * due to too many I/O errors detected.1727 */1728 if (!test_bit(CACHE_SET_IO_DISABLE, &c->flags))1729 list_for_each_entry(b, &c->btree_cache, list) {1730 mutex_lock(&b->write_lock);1731 if (btree_node_dirty(b))1732 __bch_btree_node_write(b, NULL);1733 mutex_unlock(&b->write_lock);1734 }1735 1736 if (ca->alloc_thread)1737 kthread_stop(ca->alloc_thread);1738 1739 if (c->journal.cur) {1740 cancel_delayed_work_sync(&c->journal.work);1741 /* flush last journal entry if needed */1742 c->journal.work.work.func(&c->journal.work.work);1743 }1744 1745 closure_return(cl);1746}1747 1748/*1749 * This function is only called when CACHE_SET_IO_DISABLE is set, which means1750 * cache set is unregistering due to too many I/O errors. In this condition,1751 * the bcache device might be stopped, it depends on stop_when_cache_set_failed1752 * value and whether the broken cache has dirty data:1753 *1754 * dc->stop_when_cache_set_failed dc->has_dirty stop bcache device1755 * BCH_CACHED_STOP_AUTO 0 NO1756 * BCH_CACHED_STOP_AUTO 1 YES1757 * BCH_CACHED_DEV_STOP_ALWAYS 0 YES1758 * BCH_CACHED_DEV_STOP_ALWAYS 1 YES1759 *1760 * The expected behavior is, if stop_when_cache_set_failed is configured to1761 * "auto" via sysfs interface, the bcache device will not be stopped if the1762 * backing device is clean on the broken cache device.1763 */1764static void conditional_stop_bcache_device(struct cache_set *c,1765 struct bcache_device *d,1766 struct cached_dev *dc)1767{1768 if (dc->stop_when_cache_set_failed == BCH_CACHED_DEV_STOP_ALWAYS) {1769 pr_warn("stop_when_cache_set_failed of %s is \"always\", stop it for failed cache set %pU.\n",1770 d->disk->disk_name, c->set_uuid);1771 bcache_device_stop(d);1772 } else if (atomic_read(&dc->has_dirty)) {1773 /*1774 * dc->stop_when_cache_set_failed == BCH_CACHED_STOP_AUTO1775 * and dc->has_dirty == 11776 */1777 pr_warn("stop_when_cache_set_failed of %s is \"auto\" and cache is dirty, stop it to avoid potential data corruption.\n",1778 d->disk->disk_name);1779 /*1780 * There might be a small time gap that cache set is1781 * released but bcache device is not. Inside this time1782 * gap, regular I/O requests will directly go into1783 * backing device as no cache set attached to. This1784 * behavior may also introduce potential inconsistence1785 * data in writeback mode while cache is dirty.1786 * Therefore before calling bcache_device_stop() due1787 * to a broken cache device, dc->io_disable should be1788 * explicitly set to true.1789 */1790 dc->io_disable = true;1791 /* make others know io_disable is true earlier */1792 smp_mb();1793 bcache_device_stop(d);1794 } else {1795 /*1796 * dc->stop_when_cache_set_failed == BCH_CACHED_STOP_AUTO1797 * and dc->has_dirty == 01798 */1799 pr_warn("stop_when_cache_set_failed of %s is \"auto\" and cache is clean, keep it alive.\n",1800 d->disk->disk_name);1801 }1802}1803 1804static CLOSURE_CALLBACK(__cache_set_unregister)1805{1806 closure_type(c, struct cache_set, caching);1807 struct cached_dev *dc;1808 struct bcache_device *d;1809 size_t i;1810 1811 mutex_lock(&bch_register_lock);1812 1813 for (i = 0; i < c->devices_max_used; i++) {1814 d = c->devices[i];1815 if (!d)1816 continue;1817 1818 if (!UUID_FLASH_ONLY(&c->uuids[i]) &&1819 test_bit(CACHE_SET_UNREGISTERING, &c->flags)) {1820 dc = container_of(d, struct cached_dev, disk);1821 bch_cached_dev_detach(dc);1822 if (test_bit(CACHE_SET_IO_DISABLE, &c->flags))1823 conditional_stop_bcache_device(c, d, dc);1824 } else {1825 bcache_device_stop(d);1826 }1827 }1828 1829 mutex_unlock(&bch_register_lock);1830 1831 continue_at(cl, cache_set_flush, system_wq);1832}1833 1834void bch_cache_set_stop(struct cache_set *c)1835{1836 if (!test_and_set_bit(CACHE_SET_STOPPING, &c->flags))1837 /* closure_fn set to __cache_set_unregister() */1838 closure_queue(&c->caching);1839}1840 1841void bch_cache_set_unregister(struct cache_set *c)1842{1843 set_bit(CACHE_SET_UNREGISTERING, &c->flags);1844 bch_cache_set_stop(c);1845}1846 1847#define alloc_meta_bucket_pages(gfp, sb) \1848 ((void *) __get_free_pages(__GFP_ZERO|__GFP_COMP|gfp, ilog2(meta_bucket_pages(sb))))1849 1850struct cache_set *bch_cache_set_alloc(struct cache_sb *sb)1851{1852 int iter_size;1853 struct cache *ca = container_of(sb, struct cache, sb);1854 struct cache_set *c = kzalloc(sizeof(struct cache_set), GFP_KERNEL);1855 1856 if (!c)1857 return NULL;1858 1859 __module_get(THIS_MODULE);1860 closure_init(&c->cl, NULL);1861 set_closure_fn(&c->cl, cache_set_free, system_wq);1862 1863 closure_init(&c->caching, &c->cl);1864 set_closure_fn(&c->caching, __cache_set_unregister, system_wq);1865 1866 /* Maybe create continue_at_noreturn() and use it here? */1867 closure_set_stopped(&c->cl);1868 closure_put(&c->cl);1869 1870 kobject_init(&c->kobj, &bch_cache_set_ktype);1871 kobject_init(&c->internal, &bch_cache_set_internal_ktype);1872 1873 bch_cache_accounting_init(&c->accounting, &c->cl);1874 1875 memcpy(c->set_uuid, sb->set_uuid, 16);1876 1877 c->cache = ca;1878 c->cache->set = c;1879 c->bucket_bits = ilog2(sb->bucket_size);1880 c->block_bits = ilog2(sb->block_size);1881 c->nr_uuids = meta_bucket_bytes(sb) / sizeof(struct uuid_entry);1882 c->devices_max_used = 0;1883 atomic_set(&c->attached_dev_nr, 0);1884 c->btree_pages = meta_bucket_pages(sb);1885 if (c->btree_pages > BTREE_MAX_PAGES)1886 c->btree_pages = max_t(int, c->btree_pages / 4,1887 BTREE_MAX_PAGES);1888 1889 sema_init(&c->sb_write_mutex, 1);1890 mutex_init(&c->bucket_lock);1891 init_waitqueue_head(&c->btree_cache_wait);1892 spin_lock_init(&c->btree_cannibalize_lock);1893 init_waitqueue_head(&c->bucket_wait);1894 init_waitqueue_head(&c->gc_wait);1895 sema_init(&c->uuid_write_mutex, 1);1896 1897 spin_lock_init(&c->btree_gc_time.lock);1898 spin_lock_init(&c->btree_split_time.lock);1899 spin_lock_init(&c->btree_read_time.lock);1900 1901 bch_moving_init_cache_set(c);1902 1903 INIT_LIST_HEAD(&c->list);1904 INIT_LIST_HEAD(&c->cached_devs);1905 INIT_LIST_HEAD(&c->btree_cache);1906 INIT_LIST_HEAD(&c->btree_cache_freeable);1907 INIT_LIST_HEAD(&c->btree_cache_freed);1908 INIT_LIST_HEAD(&c->data_buckets);1909 1910 iter_size = ((meta_bucket_pages(sb) * PAGE_SECTORS) / sb->block_size) *1911 sizeof(struct btree_iter_set);1912 1913 c->devices = kcalloc(c->nr_uuids, sizeof(void *), GFP_KERNEL);1914 if (!c->devices)1915 goto err;1916 1917 if (mempool_init_slab_pool(&c->search, 32, bch_search_cache))1918 goto err;1919 1920 if (mempool_init_kmalloc_pool(&c->bio_meta, 2,1921 sizeof(struct bbio) +1922 sizeof(struct bio_vec) * meta_bucket_pages(sb)))1923 goto err;1924 1925 if (mempool_init_kmalloc_pool(&c->fill_iter, 1, iter_size))1926 goto err;1927 1928 if (bioset_init(&c->bio_split, 4, offsetof(struct bbio, bio),1929 BIOSET_NEED_RESCUER))1930 goto err;1931 1932 c->uuids = alloc_meta_bucket_pages(GFP_KERNEL, sb);1933 if (!c->uuids)1934 goto err;1935 1936 c->moving_gc_wq = alloc_workqueue("bcache_gc", WQ_MEM_RECLAIM, 0);1937 if (!c->moving_gc_wq)1938 goto err;1939 1940 if (bch_journal_alloc(c))1941 goto err;1942 1943 if (bch_btree_cache_alloc(c))1944 goto err;1945 1946 if (bch_open_buckets_alloc(c))1947 goto err;1948 1949 if (bch_bset_sort_state_init(&c->sort, ilog2(c->btree_pages)))1950 goto err;1951 1952 c->congested_read_threshold_us = 2000;1953 c->congested_write_threshold_us = 20000;1954 c->error_limit = DEFAULT_IO_ERROR_LIMIT;1955 c->idle_max_writeback_rate_enabled = 1;1956 WARN_ON(test_and_clear_bit(CACHE_SET_IO_DISABLE, &c->flags));1957 1958 return c;1959err:1960 bch_cache_set_unregister(c);1961 return NULL;1962}1963 1964static int run_cache_set(struct cache_set *c)1965{1966 const char *err = "cannot allocate memory";1967 struct cached_dev *dc, *t;1968 struct cache *ca = c->cache;1969 struct closure cl;1970 LIST_HEAD(journal);1971 struct journal_replay *l;1972 1973 closure_init_stack(&cl);1974 1975 c->nbuckets = ca->sb.nbuckets;1976 set_gc_sectors(c);1977 1978 if (CACHE_SYNC(&c->cache->sb)) {1979 struct bkey *k;1980 struct jset *j;1981 1982 err = "cannot allocate memory for journal";1983 if (bch_journal_read(c, &journal))1984 goto err;1985 1986 pr_debug("btree_journal_read() done\n");1987 1988 err = "no journal entries found";1989 if (list_empty(&journal))1990 goto err;1991 1992 j = &list_entry(journal.prev, struct journal_replay, list)->j;1993 1994 err = "IO error reading priorities";1995 if (prio_read(ca, j->prio_bucket[ca->sb.nr_this_dev]))1996 goto err;1997 1998 /*1999 * If prio_read() fails it'll call cache_set_error and we'll2000 * tear everything down right away, but if we perhaps checked2001 * sooner we could avoid journal replay.2002 */2003 2004 k = &j->btree_root;2005 2006 err = "bad btree root";2007 if (__bch_btree_ptr_invalid(c, k))2008 goto err;2009 2010 err = "error reading btree root";2011 c->root = bch_btree_node_get(c, NULL, k,2012 j->btree_level,2013 true, NULL);2014 if (IS_ERR(c->root))2015 goto err;2016 2017 list_del_init(&c->root->list);2018 rw_unlock(true, c->root);2019 2020 err = uuid_read(c, j, &cl);2021 if (err)2022 goto err;2023 2024 err = "error in recovery";2025 if (bch_btree_check(c))2026 goto err;2027 2028 bch_journal_mark(c, &journal);2029 bch_initial_gc_finish(c);2030 pr_debug("btree_check() done\n");2031 2032 /*2033 * bcache_journal_next() can't happen sooner, or2034 * btree_gc_finish() will give spurious errors about last_gc >2035 * gc_gen - this is a hack but oh well.2036 */2037 bch_journal_next(&c->journal);2038 2039 err = "error starting allocator thread";2040 if (bch_cache_allocator_start(ca))2041 goto err;2042 2043 /*2044 * First place it's safe to allocate: btree_check() and2045 * btree_gc_finish() have to run before we have buckets to2046 * allocate, and bch_bucket_alloc_set() might cause a journal2047 * entry to be written so bcache_journal_next() has to be called2048 * first.2049 *2050 * If the uuids were in the old format we have to rewrite them2051 * before the next journal entry is written:2052 */2053 if (j->version < BCACHE_JSET_VERSION_UUID)2054 __uuid_write(c);2055 2056 err = "bcache: replay journal failed";2057 if (bch_journal_replay(c, &journal))2058 goto err;2059 } else {2060 unsigned int j;2061 2062 pr_notice("invalidating existing data\n");2063 ca->sb.keys = clamp_t(int, ca->sb.nbuckets >> 7,2064 2, SB_JOURNAL_BUCKETS);2065 2066 for (j = 0; j < ca->sb.keys; j++)2067 ca->sb.d[j] = ca->sb.first_bucket + j;2068 2069 bch_initial_gc_finish(c);2070 2071 err = "error starting allocator thread";2072 if (bch_cache_allocator_start(ca))2073 goto err;2074 2075 mutex_lock(&c->bucket_lock);2076 bch_prio_write(ca, true);2077 mutex_unlock(&c->bucket_lock);2078 2079 err = "cannot allocate new UUID bucket";2080 if (__uuid_write(c))2081 goto err;2082 2083 err = "cannot allocate new btree root";2084 c->root = __bch_btree_node_alloc(c, NULL, 0, true, NULL);2085 if (IS_ERR(c->root))2086 goto err;2087 2088 mutex_lock(&c->root->write_lock);2089 bkey_copy_key(&c->root->key, &MAX_KEY);2090 bch_btree_node_write(c->root, &cl);2091 mutex_unlock(&c->root->write_lock);2092 2093 bch_btree_set_root(c->root);2094 rw_unlock(true, c->root);2095 2096 /*2097 * We don't want to write the first journal entry until2098 * everything is set up - fortunately journal entries won't be2099 * written until the SET_CACHE_SYNC() here:2100 */2101 SET_CACHE_SYNC(&c->cache->sb, true);2102 2103 bch_journal_next(&c->journal);2104 bch_journal_meta(c, &cl);2105 }2106 2107 err = "error starting gc thread";2108 if (bch_gc_thread_start(c))2109 goto err;2110 2111 closure_sync(&cl);2112 c->cache->sb.last_mount = (u32)ktime_get_real_seconds();2113 bcache_write_super(c);2114 2115 if (bch_has_feature_obso_large_bucket(&c->cache->sb))2116 pr_err("Detect obsoleted large bucket layout, all attached bcache device will be read-only\n");2117 2118 list_for_each_entry_safe(dc, t, &uncached_devices, list)2119 bch_cached_dev_attach(dc, c, NULL);2120 2121 flash_devs_run(c);2122 2123 bch_journal_space_reserve(&c->journal);2124 set_bit(CACHE_SET_RUNNING, &c->flags);2125 return 0;2126err:2127 while (!list_empty(&journal)) {2128 l = list_first_entry(&journal, struct journal_replay, list);2129 list_del(&l->list);2130 kfree(l);2131 }2132 2133 closure_sync(&cl);2134 2135 bch_cache_set_error(c, "%s", err);2136 2137 return -EIO;2138}2139 2140static const char *register_cache_set(struct cache *ca)2141{2142 char buf[12];2143 const char *err = "cannot allocate memory";2144 struct cache_set *c;2145 2146 list_for_each_entry(c, &bch_cache_sets, list)2147 if (!memcmp(c->set_uuid, ca->sb.set_uuid, 16)) {2148 if (c->cache)2149 return "duplicate cache set member";2150 2151 goto found;2152 }2153 2154 c = bch_cache_set_alloc(&ca->sb);2155 if (!c)2156 return err;2157 2158 err = "error creating kobject";2159 if (kobject_add(&c->kobj, bcache_kobj, "%pU", c->set_uuid) ||2160 kobject_add(&c->internal, &c->kobj, "internal"))2161 goto err;2162 2163 if (bch_cache_accounting_add_kobjs(&c->accounting, &c->kobj))2164 goto err;2165 2166 bch_debug_init_cache_set(c);2167 2168 list_add(&c->list, &bch_cache_sets);2169found:2170 sprintf(buf, "cache%i", ca->sb.nr_this_dev);2171 if (sysfs_create_link(&ca->kobj, &c->kobj, "set") ||2172 sysfs_create_link(&c->kobj, &ca->kobj, buf))2173 goto err;2174 2175 kobject_get(&ca->kobj);2176 ca->set = c;2177 ca->set->cache = ca;2178 2179 err = "failed to run cache set";2180 if (run_cache_set(c) < 0)2181 goto err;2182 2183 return NULL;2184err:2185 bch_cache_set_unregister(c);2186 return err;2187}2188 2189/* Cache device */2190 2191/* When ca->kobj released */2192void bch_cache_release(struct kobject *kobj)2193{2194 struct cache *ca = container_of(kobj, struct cache, kobj);2195 unsigned int i;2196 2197 if (ca->set) {2198 BUG_ON(ca->set->cache != ca);2199 ca->set->cache = NULL;2200 }2201 2202 free_pages((unsigned long) ca->disk_buckets, ilog2(meta_bucket_pages(&ca->sb)));2203 kfree(ca->prio_buckets);2204 vfree(ca->buckets);2205 2206 free_heap(&ca->heap);2207 free_fifo(&ca->free_inc);2208 2209 for (i = 0; i < RESERVE_NR; i++)2210 free_fifo(&ca->free[i]);2211 2212 if (ca->sb_disk)2213 put_page(virt_to_page(ca->sb_disk));2214 2215 if (ca->bdev_file)2216 fput(ca->bdev_file);2217 2218 kfree(ca);2219 module_put(THIS_MODULE);2220}2221 2222static int cache_alloc(struct cache *ca)2223{2224 size_t free;2225 size_t btree_buckets;2226 struct bucket *b;2227 int ret = -ENOMEM;2228 const char *err = NULL;2229 2230 __module_get(THIS_MODULE);2231 kobject_init(&ca->kobj, &bch_cache_ktype);2232 2233 bio_init(&ca->journal.bio, NULL, ca->journal.bio.bi_inline_vecs, 8, 0);2234 2235 /*2236 * when ca->sb.njournal_buckets is not zero, journal exists,2237 * and in bch_journal_replay(), tree node may split,2238 * so bucket of RESERVE_BTREE type is needed,2239 * the worst situation is all journal buckets are valid journal,2240 * and all the keys need to replay,2241 * so the number of RESERVE_BTREE type buckets should be as much2242 * as journal buckets2243 */2244 btree_buckets = ca->sb.njournal_buckets ?: 8;2245 free = roundup_pow_of_two(ca->sb.nbuckets) >> 10;2246 if (!free) {2247 ret = -EPERM;2248 err = "ca->sb.nbuckets is too small";2249 goto err_free;2250 }2251 2252 if (!init_fifo(&ca->free[RESERVE_BTREE], btree_buckets,2253 GFP_KERNEL)) {2254 err = "ca->free[RESERVE_BTREE] alloc failed";2255 goto err_btree_alloc;2256 }2257 2258 if (!init_fifo_exact(&ca->free[RESERVE_PRIO], prio_buckets(ca),2259 GFP_KERNEL)) {2260 err = "ca->free[RESERVE_PRIO] alloc failed";2261 goto err_prio_alloc;2262 }2263 2264 if (!init_fifo(&ca->free[RESERVE_MOVINGGC], free, GFP_KERNEL)) {2265 err = "ca->free[RESERVE_MOVINGGC] alloc failed";2266 goto err_movinggc_alloc;2267 }2268 2269 if (!init_fifo(&ca->free[RESERVE_NONE], free, GFP_KERNEL)) {2270 err = "ca->free[RESERVE_NONE] alloc failed";2271 goto err_none_alloc;2272 }2273 2274 if (!init_fifo(&ca->free_inc, free << 2, GFP_KERNEL)) {2275 err = "ca->free_inc alloc failed";2276 goto err_free_inc_alloc;2277 }2278 2279 if (!init_heap(&ca->heap, free << 3, GFP_KERNEL)) {2280 err = "ca->heap alloc failed";2281 goto err_heap_alloc;2282 }2283 2284 ca->buckets = vzalloc(array_size(sizeof(struct bucket),2285 ca->sb.nbuckets));2286 if (!ca->buckets) {2287 err = "ca->buckets alloc failed";2288 goto err_buckets_alloc;2289 }2290 2291 ca->prio_buckets = kzalloc(array3_size(sizeof(uint64_t),2292 prio_buckets(ca), 2),2293 GFP_KERNEL);2294 if (!ca->prio_buckets) {2295 err = "ca->prio_buckets alloc failed";2296 goto err_prio_buckets_alloc;2297 }2298 2299 ca->disk_buckets = alloc_meta_bucket_pages(GFP_KERNEL, &ca->sb);2300 if (!ca->disk_buckets) {2301 err = "ca->disk_buckets alloc failed";2302 goto err_disk_buckets_alloc;2303 }2304 2305 ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca);2306 2307 for_each_bucket(b, ca)2308 atomic_set(&b->pin, 0);2309 return 0;2310 2311err_disk_buckets_alloc:2312 kfree(ca->prio_buckets);2313err_prio_buckets_alloc:2314 vfree(ca->buckets);2315err_buckets_alloc:2316 free_heap(&ca->heap);2317err_heap_alloc:2318 free_fifo(&ca->free_inc);2319err_free_inc_alloc:2320 free_fifo(&ca->free[RESERVE_NONE]);2321err_none_alloc:2322 free_fifo(&ca->free[RESERVE_MOVINGGC]);2323err_movinggc_alloc:2324 free_fifo(&ca->free[RESERVE_PRIO]);2325err_prio_alloc:2326 free_fifo(&ca->free[RESERVE_BTREE]);2327err_btree_alloc:2328err_free:2329 module_put(THIS_MODULE);2330 if (err)2331 pr_notice("error %pg: %s\n", ca->bdev, err);2332 return ret;2333}2334 2335static int register_cache(struct cache_sb *sb, struct cache_sb_disk *sb_disk,2336 struct file *bdev_file,2337 struct cache *ca)2338{2339 const char *err = NULL; /* must be set for any error case */2340 int ret = 0;2341 2342 memcpy(&ca->sb, sb, sizeof(struct cache_sb));2343 ca->bdev_file = bdev_file;2344 ca->bdev = file_bdev(bdev_file);2345 ca->sb_disk = sb_disk;2346 2347 if (bdev_max_discard_sectors(file_bdev(bdev_file)))2348 ca->discard = CACHE_DISCARD(&ca->sb);2349 2350 ret = cache_alloc(ca);2351 if (ret != 0) {2352 if (ret == -ENOMEM)2353 err = "cache_alloc(): -ENOMEM";2354 else if (ret == -EPERM)2355 err = "cache_alloc(): cache device is too small";2356 else2357 err = "cache_alloc(): unknown error";2358 pr_notice("error %pg: %s\n", file_bdev(bdev_file), err);2359 /*2360 * If we failed here, it means ca->kobj is not initialized yet,2361 * kobject_put() won't be called and there is no chance to2362 * call fput() to bdev in bch_cache_release(). So2363 * we explicitly call fput() on the block device here.2364 */2365 fput(bdev_file);2366 return ret;2367 }2368 2369 if (kobject_add(&ca->kobj, bdev_kobj(file_bdev(bdev_file)), "bcache")) {2370 pr_notice("error %pg: error calling kobject_add\n",2371 file_bdev(bdev_file));2372 ret = -ENOMEM;2373 goto out;2374 }2375 2376 mutex_lock(&bch_register_lock);2377 err = register_cache_set(ca);2378 mutex_unlock(&bch_register_lock);2379 2380 if (err) {2381 ret = -ENODEV;2382 goto out;2383 }2384 2385 pr_info("registered cache device %pg\n", file_bdev(ca->bdev_file));2386 2387out:2388 kobject_put(&ca->kobj);2389 return ret;2390}2391 2392/* Global interfaces/init */2393 2394static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,2395 const char *buffer, size_t size);2396static ssize_t bch_pending_bdevs_cleanup(struct kobject *k,2397 struct kobj_attribute *attr,2398 const char *buffer, size_t size);2399 2400kobj_attribute_write(register, register_bcache);2401kobj_attribute_write(register_quiet, register_bcache);2402kobj_attribute_write(pendings_cleanup, bch_pending_bdevs_cleanup);2403 2404static bool bch_is_open_backing(dev_t dev)2405{2406 struct cache_set *c, *tc;2407 struct cached_dev *dc, *t;2408 2409 list_for_each_entry_safe(c, tc, &bch_cache_sets, list)2410 list_for_each_entry_safe(dc, t, &c->cached_devs, list)2411 if (dc->bdev->bd_dev == dev)2412 return true;2413 list_for_each_entry_safe(dc, t, &uncached_devices, list)2414 if (dc->bdev->bd_dev == dev)2415 return true;2416 return false;2417}2418 2419static bool bch_is_open_cache(dev_t dev)2420{2421 struct cache_set *c, *tc;2422 2423 list_for_each_entry_safe(c, tc, &bch_cache_sets, list) {2424 struct cache *ca = c->cache;2425 2426 if (ca->bdev->bd_dev == dev)2427 return true;2428 }2429 2430 return false;2431}2432 2433static bool bch_is_open(dev_t dev)2434{2435 return bch_is_open_cache(dev) || bch_is_open_backing(dev);2436}2437 2438struct async_reg_args {2439 struct delayed_work reg_work;2440 char *path;2441 struct cache_sb *sb;2442 struct cache_sb_disk *sb_disk;2443 struct file *bdev_file;2444 void *holder;2445};2446 2447static void register_bdev_worker(struct work_struct *work)2448{2449 int fail = false;2450 struct async_reg_args *args =2451 container_of(work, struct async_reg_args, reg_work.work);2452 2453 mutex_lock(&bch_register_lock);2454 if (register_bdev(args->sb, args->sb_disk, args->bdev_file,2455 args->holder) < 0)2456 fail = true;2457 mutex_unlock(&bch_register_lock);2458 2459 if (fail)2460 pr_info("error %s: fail to register backing device\n",2461 args->path);2462 kfree(args->sb);2463 kfree(args->path);2464 kfree(args);2465 module_put(THIS_MODULE);2466}2467 2468static void register_cache_worker(struct work_struct *work)2469{2470 int fail = false;2471 struct async_reg_args *args =2472 container_of(work, struct async_reg_args, reg_work.work);2473 2474 /* blkdev_put() will be called in bch_cache_release() */2475 if (register_cache(args->sb, args->sb_disk, args->bdev_file,2476 args->holder))2477 fail = true;2478 2479 if (fail)2480 pr_info("error %s: fail to register cache device\n",2481 args->path);2482 kfree(args->sb);2483 kfree(args->path);2484 kfree(args);2485 module_put(THIS_MODULE);2486}2487 2488static void register_device_async(struct async_reg_args *args)2489{2490 if (SB_IS_BDEV(args->sb))2491 INIT_DELAYED_WORK(&args->reg_work, register_bdev_worker);2492 else2493 INIT_DELAYED_WORK(&args->reg_work, register_cache_worker);2494 2495 /* 10 jiffies is enough for a delay */2496 queue_delayed_work(system_wq, &args->reg_work, 10);2497}2498 2499static void *alloc_holder_object(struct cache_sb *sb)2500{2501 if (SB_IS_BDEV(sb))2502 return kzalloc(sizeof(struct cached_dev), GFP_KERNEL);2503 return kzalloc(sizeof(struct cache), GFP_KERNEL);2504}2505 2506static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,2507 const char *buffer, size_t size)2508{2509 const char *err;2510 char *path = NULL;2511 struct cache_sb *sb;2512 struct cache_sb_disk *sb_disk;2513 struct file *bdev_file, *bdev_file2;2514 void *holder = NULL;2515 ssize_t ret;2516 bool async_registration = false;2517 bool quiet = false;2518 2519#ifdef CONFIG_BCACHE_ASYNC_REGISTRATION2520 async_registration = true;2521#endif2522 2523 ret = -EBUSY;2524 err = "failed to reference bcache module";2525 if (!try_module_get(THIS_MODULE))2526 goto out;2527 2528 /* For latest state of bcache_is_reboot */2529 smp_mb();2530 err = "bcache is in reboot";2531 if (bcache_is_reboot)2532 goto out_module_put;2533 2534 ret = -ENOMEM;2535 err = "cannot allocate memory";2536 path = kstrndup(buffer, size, GFP_KERNEL);2537 if (!path)2538 goto out_module_put;2539 2540 sb = kmalloc(sizeof(struct cache_sb), GFP_KERNEL);2541 if (!sb)2542 goto out_free_path;2543 2544 ret = -EINVAL;2545 err = "failed to open device";2546 bdev_file = bdev_file_open_by_path(strim(path), BLK_OPEN_READ, NULL, NULL);2547 if (IS_ERR(bdev_file))2548 goto out_free_sb;2549 2550 err = read_super(sb, file_bdev(bdev_file), &sb_disk);2551 if (err)2552 goto out_blkdev_put;2553 2554 holder = alloc_holder_object(sb);2555 if (!holder) {2556 ret = -ENOMEM;2557 err = "cannot allocate memory";2558 goto out_put_sb_page;2559 }2560 2561 /* Now reopen in exclusive mode with proper holder */2562 bdev_file2 = bdev_file_open_by_dev(file_bdev(bdev_file)->bd_dev,2563 BLK_OPEN_READ | BLK_OPEN_WRITE, holder, NULL);2564 fput(bdev_file);2565 bdev_file = bdev_file2;2566 if (IS_ERR(bdev_file)) {2567 ret = PTR_ERR(bdev_file);2568 bdev_file = NULL;2569 if (ret == -EBUSY) {2570 dev_t dev;2571 2572 mutex_lock(&bch_register_lock);2573 if (lookup_bdev(strim(path), &dev) == 0 &&2574 bch_is_open(dev))2575 err = "device already registered";2576 else2577 err = "device busy";2578 mutex_unlock(&bch_register_lock);2579 if (attr == &ksysfs_register_quiet) {2580 quiet = true;2581 ret = size;2582 }2583 }2584 goto out_free_holder;2585 }2586 2587 err = "failed to register device";2588 2589 if (async_registration) {2590 /* register in asynchronous way */2591 struct async_reg_args *args =2592 kzalloc(sizeof(struct async_reg_args), GFP_KERNEL);2593 2594 if (!args) {2595 ret = -ENOMEM;2596 err = "cannot allocate memory";2597 goto out_free_holder;2598 }2599 2600 args->path = path;2601 args->sb = sb;2602 args->sb_disk = sb_disk;2603 args->bdev_file = bdev_file;2604 args->holder = holder;2605 register_device_async(args);2606 /* No wait and returns to user space */2607 goto async_done;2608 }2609 2610 if (SB_IS_BDEV(sb)) {2611 mutex_lock(&bch_register_lock);2612 ret = register_bdev(sb, sb_disk, bdev_file, holder);2613 mutex_unlock(&bch_register_lock);2614 /* blkdev_put() will be called in cached_dev_free() */2615 if (ret < 0)2616 goto out_free_sb;2617 } else {2618 /* blkdev_put() will be called in bch_cache_release() */2619 ret = register_cache(sb, sb_disk, bdev_file, holder);2620 if (ret)2621 goto out_free_sb;2622 }2623 2624 kfree(sb);2625 kfree(path);2626 module_put(THIS_MODULE);2627async_done:2628 return size;2629 2630out_free_holder:2631 kfree(holder);2632out_put_sb_page:2633 put_page(virt_to_page(sb_disk));2634out_blkdev_put:2635 if (bdev_file)2636 fput(bdev_file);2637out_free_sb:2638 kfree(sb);2639out_free_path:2640 kfree(path);2641 path = NULL;2642out_module_put:2643 module_put(THIS_MODULE);2644out:2645 if (!quiet)2646 pr_info("error %s: %s\n", path?path:"", err);2647 return ret;2648}2649 2650 2651struct pdev {2652 struct list_head list;2653 struct cached_dev *dc;2654};2655 2656static ssize_t bch_pending_bdevs_cleanup(struct kobject *k,2657 struct kobj_attribute *attr,2658 const char *buffer,2659 size_t size)2660{2661 LIST_HEAD(pending_devs);2662 ssize_t ret = size;2663 struct cached_dev *dc, *tdc;2664 struct pdev *pdev, *tpdev;2665 struct cache_set *c, *tc;2666 2667 mutex_lock(&bch_register_lock);2668 list_for_each_entry_safe(dc, tdc, &uncached_devices, list) {2669 pdev = kmalloc(sizeof(struct pdev), GFP_KERNEL);2670 if (!pdev)2671 break;2672 pdev->dc = dc;2673 list_add(&pdev->list, &pending_devs);2674 }2675 2676 list_for_each_entry_safe(pdev, tpdev, &pending_devs, list) {2677 char *pdev_set_uuid = pdev->dc->sb.set_uuid;2678 list_for_each_entry_safe(c, tc, &bch_cache_sets, list) {2679 char *set_uuid = c->set_uuid;2680 2681 if (!memcmp(pdev_set_uuid, set_uuid, 16)) {2682 list_del(&pdev->list);2683 kfree(pdev);2684 break;2685 }2686 }2687 }2688 mutex_unlock(&bch_register_lock);2689 2690 list_for_each_entry_safe(pdev, tpdev, &pending_devs, list) {2691 pr_info("delete pdev %p\n", pdev);2692 list_del(&pdev->list);2693 bcache_device_stop(&pdev->dc->disk);2694 kfree(pdev);2695 }2696 2697 return ret;2698}2699 2700static int bcache_reboot(struct notifier_block *n, unsigned long code, void *x)2701{2702 if (bcache_is_reboot)2703 return NOTIFY_DONE;2704 2705 if (code == SYS_DOWN ||2706 code == SYS_HALT ||2707 code == SYS_POWER_OFF) {2708 DEFINE_WAIT(wait);2709 unsigned long start = jiffies;2710 bool stopped = false;2711 2712 struct cache_set *c, *tc;2713 struct cached_dev *dc, *tdc;2714 2715 mutex_lock(&bch_register_lock);2716 2717 if (bcache_is_reboot)2718 goto out;2719 2720 /* New registration is rejected since now */2721 bcache_is_reboot = true;2722 /*2723 * Make registering caller (if there is) on other CPU2724 * core know bcache_is_reboot set to true earlier2725 */2726 smp_mb();2727 2728 if (list_empty(&bch_cache_sets) &&2729 list_empty(&uncached_devices))2730 goto out;2731 2732 mutex_unlock(&bch_register_lock);2733 2734 pr_info("Stopping all devices:\n");2735 2736 /*2737 * The reason bch_register_lock is not held to call2738 * bch_cache_set_stop() and bcache_device_stop() is to2739 * avoid potential deadlock during reboot, because cache2740 * set or bcache device stopping process will acquire2741 * bch_register_lock too.2742 *2743 * We are safe here because bcache_is_reboot sets to2744 * true already, register_bcache() will reject new2745 * registration now. bcache_is_reboot also makes sure2746 * bcache_reboot() won't be re-entered on by other thread,2747 * so there is no race in following list iteration by2748 * list_for_each_entry_safe().2749 */2750 list_for_each_entry_safe(c, tc, &bch_cache_sets, list)2751 bch_cache_set_stop(c);2752 2753 list_for_each_entry_safe(dc, tdc, &uncached_devices, list)2754 bcache_device_stop(&dc->disk);2755 2756 2757 /*2758 * Give an early chance for other kthreads and2759 * kworkers to stop themselves2760 */2761 schedule();2762 2763 /* What's a condition variable? */2764 while (1) {2765 long timeout = start + 10 * HZ - jiffies;2766 2767 mutex_lock(&bch_register_lock);2768 stopped = list_empty(&bch_cache_sets) &&2769 list_empty(&uncached_devices);2770 2771 if (timeout < 0 || stopped)2772 break;2773 2774 prepare_to_wait(&unregister_wait, &wait,2775 TASK_UNINTERRUPTIBLE);2776 2777 mutex_unlock(&bch_register_lock);2778 schedule_timeout(timeout);2779 }2780 2781 finish_wait(&unregister_wait, &wait);2782 2783 if (stopped)2784 pr_info("All devices stopped\n");2785 else2786 pr_notice("Timeout waiting for devices to be closed\n");2787out:2788 mutex_unlock(&bch_register_lock);2789 }2790 2791 return NOTIFY_DONE;2792}2793 2794static struct notifier_block reboot = {2795 .notifier_call = bcache_reboot,2796 .priority = INT_MAX, /* before any real devices */2797};2798 2799static void bcache_exit(void)2800{2801 bch_debug_exit();2802 bch_request_exit();2803 if (bcache_kobj)2804 kobject_put(bcache_kobj);2805 if (bcache_wq)2806 destroy_workqueue(bcache_wq);2807 if (bch_journal_wq)2808 destroy_workqueue(bch_journal_wq);2809 if (bch_flush_wq)2810 destroy_workqueue(bch_flush_wq);2811 bch_btree_exit();2812 2813 if (bcache_major)2814 unregister_blkdev(bcache_major, "bcache");2815 unregister_reboot_notifier(&reboot);2816 mutex_destroy(&bch_register_lock);2817}2818 2819/* Check and fixup module parameters */2820static void check_module_parameters(void)2821{2822 if (bch_cutoff_writeback_sync == 0)2823 bch_cutoff_writeback_sync = CUTOFF_WRITEBACK_SYNC;2824 else if (bch_cutoff_writeback_sync > CUTOFF_WRITEBACK_SYNC_MAX) {2825 pr_warn("set bch_cutoff_writeback_sync (%u) to max value %u\n",2826 bch_cutoff_writeback_sync, CUTOFF_WRITEBACK_SYNC_MAX);2827 bch_cutoff_writeback_sync = CUTOFF_WRITEBACK_SYNC_MAX;2828 }2829 2830 if (bch_cutoff_writeback == 0)2831 bch_cutoff_writeback = CUTOFF_WRITEBACK;2832 else if (bch_cutoff_writeback > CUTOFF_WRITEBACK_MAX) {2833 pr_warn("set bch_cutoff_writeback (%u) to max value %u\n",2834 bch_cutoff_writeback, CUTOFF_WRITEBACK_MAX);2835 bch_cutoff_writeback = CUTOFF_WRITEBACK_MAX;2836 }2837 2838 if (bch_cutoff_writeback > bch_cutoff_writeback_sync) {2839 pr_warn("set bch_cutoff_writeback (%u) to %u\n",2840 bch_cutoff_writeback, bch_cutoff_writeback_sync);2841 bch_cutoff_writeback = bch_cutoff_writeback_sync;2842 }2843}2844 2845static int __init bcache_init(void)2846{2847 static const struct attribute *files[] = {2848 &ksysfs_register.attr,2849 &ksysfs_register_quiet.attr,2850 &ksysfs_pendings_cleanup.attr,2851 NULL2852 };2853 2854 check_module_parameters();2855 2856 mutex_init(&bch_register_lock);2857 init_waitqueue_head(&unregister_wait);2858 register_reboot_notifier(&reboot);2859 2860 bcache_major = register_blkdev(0, "bcache");2861 if (bcache_major < 0) {2862 unregister_reboot_notifier(&reboot);2863 mutex_destroy(&bch_register_lock);2864 return bcache_major;2865 }2866 2867 if (bch_btree_init())2868 goto err;2869 2870 bcache_wq = alloc_workqueue("bcache", WQ_MEM_RECLAIM, 0);2871 if (!bcache_wq)2872 goto err;2873 2874 /*2875 * Let's not make this `WQ_MEM_RECLAIM` for the following reasons:2876 *2877 * 1. It used `system_wq` before which also does no memory reclaim.2878 * 2. With `WQ_MEM_RECLAIM` desktop stalls, increased boot times, and2879 * reduced throughput can be observed.2880 *2881 * We still want to user our own queue to not congest the `system_wq`.2882 */2883 bch_flush_wq = alloc_workqueue("bch_flush", 0, 0);2884 if (!bch_flush_wq)2885 goto err;2886 2887 bch_journal_wq = alloc_workqueue("bch_journal", WQ_MEM_RECLAIM, 0);2888 if (!bch_journal_wq)2889 goto err;2890 2891 bcache_kobj = kobject_create_and_add("bcache", fs_kobj);2892 if (!bcache_kobj)2893 goto err;2894 2895 if (bch_request_init() ||2896 sysfs_create_files(bcache_kobj, files))2897 goto err;2898 2899 bch_debug_init();2900 2901 bcache_is_reboot = false;2902 2903 return 0;2904err:2905 bcache_exit();2906 return -ENOMEM;2907}2908 2909/*2910 * Module hooks2911 */2912module_exit(bcache_exit);2913module_init(bcache_init);2914 2915module_param(bch_cutoff_writeback, uint, 0);2916MODULE_PARM_DESC(bch_cutoff_writeback, "threshold to cutoff writeback");2917 2918module_param(bch_cutoff_writeback_sync, uint, 0);2919MODULE_PARM_DESC(bch_cutoff_writeback_sync, "hard threshold to cutoff writeback");2920 2921MODULE_DESCRIPTION("Bcache: a Linux block layer cache");2922MODULE_AUTHOR("Kent Overstreet <kent.overstreet@gmail.com>");2923MODULE_LICENSE("GPL");2924