4563 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2011-2012 Red Hat UK.4 *5 * This file is released under the GPL.6 */7 8#include "dm-thin-metadata.h"9#include "dm-bio-prison-v1.h"10#include "dm.h"11 12#include <linux/device-mapper.h>13#include <linux/dm-io.h>14#include <linux/dm-kcopyd.h>15#include <linux/jiffies.h>16#include <linux/log2.h>17#include <linux/list.h>18#include <linux/rculist.h>19#include <linux/init.h>20#include <linux/module.h>21#include <linux/slab.h>22#include <linux/vmalloc.h>23#include <linux/sort.h>24#include <linux/rbtree.h>25 26#define DM_MSG_PREFIX "thin"27 28/*29 * Tunable constants30 */31#define ENDIO_HOOK_POOL_SIZE 102432#define MAPPING_POOL_SIZE 102433#define COMMIT_PERIOD HZ34#define NO_SPACE_TIMEOUT_SECS 6035 36static unsigned int no_space_timeout_secs = NO_SPACE_TIMEOUT_SECS;37 38DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,39 "A percentage of time allocated for copy on write");40 41/*42 * The block size of the device holding pool data must be43 * between 64KB and 1GB.44 */45#define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT)46#define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)47 48/*49 * Device id is restricted to 24 bits.50 */51#define MAX_DEV_ID ((1 << 24) - 1)52 53/*54 * How do we handle breaking sharing of data blocks?55 * =================================================56 *57 * We use a standard copy-on-write btree to store the mappings for the58 * devices (note I'm talking about copy-on-write of the metadata here, not59 * the data). When you take an internal snapshot you clone the root node60 * of the origin btree. After this there is no concept of an origin or a61 * snapshot. They are just two device trees that happen to point to the62 * same data blocks.63 *64 * When we get a write in we decide if it's to a shared data block using65 * some timestamp magic. If it is, we have to break sharing.66 *67 * Let's say we write to a shared block in what was the origin. The68 * steps are:69 *70 * i) plug io further to this physical block. (see bio_prison code).71 *72 * ii) quiesce any read io to that shared data block. Obviously73 * including all devices that share this block. (see dm_deferred_set code)74 *75 * iii) copy the data block to a newly allocate block. This step can be76 * missed out if the io covers the block. (schedule_copy).77 *78 * iv) insert the new mapping into the origin's btree79 * (process_prepared_mapping). This act of inserting breaks some80 * sharing of btree nodes between the two devices. Breaking sharing only81 * effects the btree of that specific device. Btrees for the other82 * devices that share the block never change. The btree for the origin83 * device as it was after the last commit is untouched, ie. we're using84 * persistent data structures in the functional programming sense.85 *86 * v) unplug io to this physical block, including the io that triggered87 * the breaking of sharing.88 *89 * Steps (ii) and (iii) occur in parallel.90 *91 * The metadata _doesn't_ need to be committed before the io continues. We92 * get away with this because the io is always written to a _new_ block.93 * If there's a crash, then:94 *95 * - The origin mapping will point to the old origin block (the shared96 * one). This will contain the data as it was before the io that triggered97 * the breaking of sharing came in.98 *99 * - The snap mapping still points to the old block. As it would after100 * the commit.101 *102 * The downside of this scheme is the timestamp magic isn't perfect, and103 * will continue to think that data block in the snapshot device is shared104 * even after the write to the origin has broken sharing. I suspect data105 * blocks will typically be shared by many different devices, so we're106 * breaking sharing n + 1 times, rather than n, where n is the number of107 * devices that reference this data block. At the moment I think the108 * benefits far, far outweigh the disadvantages.109 */110 111/*----------------------------------------------------------------*/112 113/*114 * Key building.115 */116enum lock_space {117 VIRTUAL,118 PHYSICAL119};120 121static bool build_key(struct dm_thin_device *td, enum lock_space ls,122 dm_block_t b, dm_block_t e, struct dm_cell_key *key)123{124 key->virtual = (ls == VIRTUAL);125 key->dev = dm_thin_dev_id(td);126 key->block_begin = b;127 key->block_end = e;128 129 return dm_cell_key_has_valid_range(key);130}131 132static void build_data_key(struct dm_thin_device *td, dm_block_t b,133 struct dm_cell_key *key)134{135 (void) build_key(td, PHYSICAL, b, b + 1llu, key);136}137 138static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,139 struct dm_cell_key *key)140{141 (void) build_key(td, VIRTUAL, b, b + 1llu, key);142}143 144/*----------------------------------------------------------------*/145 146#define THROTTLE_THRESHOLD (1 * HZ)147 148struct throttle {149 struct rw_semaphore lock;150 unsigned long threshold;151 bool throttle_applied;152};153 154static void throttle_init(struct throttle *t)155{156 init_rwsem(&t->lock);157 t->throttle_applied = false;158}159 160static void throttle_work_start(struct throttle *t)161{162 t->threshold = jiffies + THROTTLE_THRESHOLD;163}164 165static void throttle_work_update(struct throttle *t)166{167 if (!t->throttle_applied && time_is_before_jiffies(t->threshold)) {168 down_write(&t->lock);169 t->throttle_applied = true;170 }171}172 173static void throttle_work_complete(struct throttle *t)174{175 if (t->throttle_applied) {176 t->throttle_applied = false;177 up_write(&t->lock);178 }179}180 181static void throttle_lock(struct throttle *t)182{183 down_read(&t->lock);184}185 186static void throttle_unlock(struct throttle *t)187{188 up_read(&t->lock);189}190 191/*----------------------------------------------------------------*/192 193/*194 * A pool device ties together a metadata device and a data device. It195 * also provides the interface for creating and destroying internal196 * devices.197 */198struct dm_thin_new_mapping;199 200/*201 * The pool runs in various modes. Ordered in degraded order for comparisons.202 */203enum pool_mode {204 PM_WRITE, /* metadata may be changed */205 PM_OUT_OF_DATA_SPACE, /* metadata may be changed, though data may not be allocated */206 207 /*208 * Like READ_ONLY, except may switch back to WRITE on metadata resize. Reported as READ_ONLY.209 */210 PM_OUT_OF_METADATA_SPACE,211 PM_READ_ONLY, /* metadata may not be changed */212 213 PM_FAIL, /* all I/O fails */214};215 216struct pool_features {217 enum pool_mode mode;218 219 bool zero_new_blocks:1;220 bool discard_enabled:1;221 bool discard_passdown:1;222 bool error_if_no_space:1;223};224 225struct thin_c;226typedef void (*process_bio_fn)(struct thin_c *tc, struct bio *bio);227typedef void (*process_cell_fn)(struct thin_c *tc, struct dm_bio_prison_cell *cell);228typedef void (*process_mapping_fn)(struct dm_thin_new_mapping *m);229 230#define CELL_SORT_ARRAY_SIZE 8192231 232struct pool {233 struct list_head list;234 struct dm_target *ti; /* Only set if a pool target is bound */235 236 struct mapped_device *pool_md;237 struct block_device *data_dev;238 struct block_device *md_dev;239 struct dm_pool_metadata *pmd;240 241 dm_block_t low_water_blocks;242 uint32_t sectors_per_block;243 int sectors_per_block_shift;244 245 struct pool_features pf;246 bool low_water_triggered:1; /* A dm event has been sent */247 bool suspended:1;248 bool out_of_data_space:1;249 250 struct dm_bio_prison *prison;251 struct dm_kcopyd_client *copier;252 253 struct work_struct worker;254 struct workqueue_struct *wq;255 struct throttle throttle;256 struct delayed_work waker;257 struct delayed_work no_space_timeout;258 259 unsigned long last_commit_jiffies;260 unsigned int ref_count;261 262 spinlock_t lock;263 struct bio_list deferred_flush_bios;264 struct bio_list deferred_flush_completions;265 struct list_head prepared_mappings;266 struct list_head prepared_discards;267 struct list_head prepared_discards_pt2;268 struct list_head active_thins;269 270 struct dm_deferred_set *shared_read_ds;271 struct dm_deferred_set *all_io_ds;272 273 struct dm_thin_new_mapping *next_mapping;274 275 process_bio_fn process_bio;276 process_bio_fn process_discard;277 278 process_cell_fn process_cell;279 process_cell_fn process_discard_cell;280 281 process_mapping_fn process_prepared_mapping;282 process_mapping_fn process_prepared_discard;283 process_mapping_fn process_prepared_discard_pt2;284 285 struct dm_bio_prison_cell **cell_sort_array;286 287 mempool_t mapping_pool;288};289 290static void metadata_operation_failed(struct pool *pool, const char *op, int r);291 292static enum pool_mode get_pool_mode(struct pool *pool)293{294 return pool->pf.mode;295}296 297static void notify_of_pool_mode_change(struct pool *pool)298{299 static const char *descs[] = {300 "write",301 "out-of-data-space",302 "read-only",303 "read-only",304 "fail"305 };306 const char *extra_desc = NULL;307 enum pool_mode mode = get_pool_mode(pool);308 309 if (mode == PM_OUT_OF_DATA_SPACE) {310 if (!pool->pf.error_if_no_space)311 extra_desc = " (queue IO)";312 else313 extra_desc = " (error IO)";314 }315 316 dm_table_event(pool->ti->table);317 DMINFO("%s: switching pool to %s%s mode",318 dm_device_name(pool->pool_md),319 descs[(int)mode], extra_desc ? : "");320}321 322/*323 * Target context for a pool.324 */325struct pool_c {326 struct dm_target *ti;327 struct pool *pool;328 struct dm_dev *data_dev;329 struct dm_dev *metadata_dev;330 331 dm_block_t low_water_blocks;332 struct pool_features requested_pf; /* Features requested during table load */333 struct pool_features adjusted_pf; /* Features used after adjusting for constituent devices */334};335 336/*337 * Target context for a thin.338 */339struct thin_c {340 struct list_head list;341 struct dm_dev *pool_dev;342 struct dm_dev *origin_dev;343 sector_t origin_size;344 dm_thin_id dev_id;345 346 struct pool *pool;347 struct dm_thin_device *td;348 struct mapped_device *thin_md;349 350 bool requeue_mode:1;351 spinlock_t lock;352 struct list_head deferred_cells;353 struct bio_list deferred_bio_list;354 struct bio_list retry_on_resume_list;355 struct rb_root sort_bio_list; /* sorted list of deferred bios */356 357 /*358 * Ensures the thin is not destroyed until the worker has finished359 * iterating the active_thins list.360 */361 refcount_t refcount;362 struct completion can_destroy;363};364 365/*----------------------------------------------------------------*/366 367static bool block_size_is_power_of_two(struct pool *pool)368{369 return pool->sectors_per_block_shift >= 0;370}371 372static sector_t block_to_sectors(struct pool *pool, dm_block_t b)373{374 return block_size_is_power_of_two(pool) ?375 (b << pool->sectors_per_block_shift) :376 (b * pool->sectors_per_block);377}378 379/*----------------------------------------------------------------*/380 381struct discard_op {382 struct thin_c *tc;383 struct blk_plug plug;384 struct bio *parent_bio;385 struct bio *bio;386};387 388static void begin_discard(struct discard_op *op, struct thin_c *tc, struct bio *parent)389{390 BUG_ON(!parent);391 392 op->tc = tc;393 blk_start_plug(&op->plug);394 op->parent_bio = parent;395 op->bio = NULL;396}397 398static int issue_discard(struct discard_op *op, dm_block_t data_b, dm_block_t data_e)399{400 struct thin_c *tc = op->tc;401 sector_t s = block_to_sectors(tc->pool, data_b);402 sector_t len = block_to_sectors(tc->pool, data_e - data_b);403 404 return __blkdev_issue_discard(tc->pool_dev->bdev, s, len, GFP_NOIO, &op->bio);405}406 407static void end_discard(struct discard_op *op, int r)408{409 if (op->bio) {410 /*411 * Even if one of the calls to issue_discard failed, we412 * need to wait for the chain to complete.413 */414 bio_chain(op->bio, op->parent_bio);415 op->bio->bi_opf = REQ_OP_DISCARD;416 submit_bio(op->bio);417 }418 419 blk_finish_plug(&op->plug);420 421 /*422 * Even if r is set, there could be sub discards in flight that we423 * need to wait for.424 */425 if (r && !op->parent_bio->bi_status)426 op->parent_bio->bi_status = errno_to_blk_status(r);427 bio_endio(op->parent_bio);428}429 430/*----------------------------------------------------------------*/431 432/*433 * wake_worker() is used when new work is queued and when pool_resume is434 * ready to continue deferred IO processing.435 */436static void wake_worker(struct pool *pool)437{438 queue_work(pool->wq, &pool->worker);439}440 441/*----------------------------------------------------------------*/442 443static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bio,444 struct dm_bio_prison_cell **cell_result)445{446 int r;447 struct dm_bio_prison_cell *cell_prealloc;448 449 /*450 * Allocate a cell from the prison's mempool.451 * This might block but it can't fail.452 */453 cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO);454 455 r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result);456 if (r) {457 /*458 * We reused an old cell; we can get rid of459 * the new one.460 */461 dm_bio_prison_free_cell(pool->prison, cell_prealloc);462 }463 464 return r;465}466 467static void cell_release(struct pool *pool,468 struct dm_bio_prison_cell *cell,469 struct bio_list *bios)470{471 dm_cell_release(pool->prison, cell, bios);472 dm_bio_prison_free_cell(pool->prison, cell);473}474 475static void cell_visit_release(struct pool *pool,476 void (*fn)(void *, struct dm_bio_prison_cell *),477 void *context,478 struct dm_bio_prison_cell *cell)479{480 dm_cell_visit_release(pool->prison, fn, context, cell);481 dm_bio_prison_free_cell(pool->prison, cell);482}483 484static void cell_release_no_holder(struct pool *pool,485 struct dm_bio_prison_cell *cell,486 struct bio_list *bios)487{488 dm_cell_release_no_holder(pool->prison, cell, bios);489 dm_bio_prison_free_cell(pool->prison, cell);490}491 492static void cell_error_with_code(struct pool *pool,493 struct dm_bio_prison_cell *cell, blk_status_t error_code)494{495 dm_cell_error(pool->prison, cell, error_code);496 dm_bio_prison_free_cell(pool->prison, cell);497}498 499static blk_status_t get_pool_io_error_code(struct pool *pool)500{501 return pool->out_of_data_space ? BLK_STS_NOSPC : BLK_STS_IOERR;502}503 504static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell)505{506 cell_error_with_code(pool, cell, get_pool_io_error_code(pool));507}508 509static void cell_success(struct pool *pool, struct dm_bio_prison_cell *cell)510{511 cell_error_with_code(pool, cell, 0);512}513 514static void cell_requeue(struct pool *pool, struct dm_bio_prison_cell *cell)515{516 cell_error_with_code(pool, cell, BLK_STS_DM_REQUEUE);517}518 519/*----------------------------------------------------------------*/520 521/*522 * A global list of pools that uses a struct mapped_device as a key.523 */524static struct dm_thin_pool_table {525 struct mutex mutex;526 struct list_head pools;527} dm_thin_pool_table;528 529static void pool_table_init(void)530{531 mutex_init(&dm_thin_pool_table.mutex);532 INIT_LIST_HEAD(&dm_thin_pool_table.pools);533}534 535static void pool_table_exit(void)536{537 mutex_destroy(&dm_thin_pool_table.mutex);538}539 540static void __pool_table_insert(struct pool *pool)541{542 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));543 list_add(&pool->list, &dm_thin_pool_table.pools);544}545 546static void __pool_table_remove(struct pool *pool)547{548 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));549 list_del(&pool->list);550}551 552static struct pool *__pool_table_lookup(struct mapped_device *md)553{554 struct pool *pool = NULL, *tmp;555 556 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));557 558 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {559 if (tmp->pool_md == md) {560 pool = tmp;561 break;562 }563 }564 565 return pool;566}567 568static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev)569{570 struct pool *pool = NULL, *tmp;571 572 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));573 574 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {575 if (tmp->md_dev == md_dev) {576 pool = tmp;577 break;578 }579 }580 581 return pool;582}583 584/*----------------------------------------------------------------*/585 586struct dm_thin_endio_hook {587 struct thin_c *tc;588 struct dm_deferred_entry *shared_read_entry;589 struct dm_deferred_entry *all_io_entry;590 struct dm_thin_new_mapping *overwrite_mapping;591 struct rb_node rb_node;592 struct dm_bio_prison_cell *cell;593};594 595static void error_bio_list(struct bio_list *bios, blk_status_t error)596{597 struct bio *bio;598 599 while ((bio = bio_list_pop(bios))) {600 bio->bi_status = error;601 bio_endio(bio);602 }603}604 605static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master,606 blk_status_t error)607{608 struct bio_list bios;609 610 bio_list_init(&bios);611 612 spin_lock_irq(&tc->lock);613 bio_list_merge_init(&bios, master);614 spin_unlock_irq(&tc->lock);615 616 error_bio_list(&bios, error);617}618 619static void requeue_deferred_cells(struct thin_c *tc)620{621 struct pool *pool = tc->pool;622 struct list_head cells;623 struct dm_bio_prison_cell *cell, *tmp;624 625 INIT_LIST_HEAD(&cells);626 627 spin_lock_irq(&tc->lock);628 list_splice_init(&tc->deferred_cells, &cells);629 spin_unlock_irq(&tc->lock);630 631 list_for_each_entry_safe(cell, tmp, &cells, user_list)632 cell_requeue(pool, cell);633}634 635static void requeue_io(struct thin_c *tc)636{637 struct bio_list bios;638 639 bio_list_init(&bios);640 641 spin_lock_irq(&tc->lock);642 bio_list_merge_init(&bios, &tc->deferred_bio_list);643 bio_list_merge_init(&bios, &tc->retry_on_resume_list);644 spin_unlock_irq(&tc->lock);645 646 error_bio_list(&bios, BLK_STS_DM_REQUEUE);647 requeue_deferred_cells(tc);648}649 650static void error_retry_list_with_code(struct pool *pool, blk_status_t error)651{652 struct thin_c *tc;653 654 rcu_read_lock();655 list_for_each_entry_rcu(tc, &pool->active_thins, list)656 error_thin_bio_list(tc, &tc->retry_on_resume_list, error);657 rcu_read_unlock();658}659 660static void error_retry_list(struct pool *pool)661{662 error_retry_list_with_code(pool, get_pool_io_error_code(pool));663}664 665/*666 * This section of code contains the logic for processing a thin device's IO.667 * Much of the code depends on pool object resources (lists, workqueues, etc)668 * but most is exclusively called from the thin target rather than the thin-pool669 * target.670 */671 672static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio)673{674 struct pool *pool = tc->pool;675 sector_t block_nr = bio->bi_iter.bi_sector;676 677 if (block_size_is_power_of_two(pool))678 block_nr >>= pool->sectors_per_block_shift;679 else680 (void) sector_div(block_nr, pool->sectors_per_block);681 682 return block_nr;683}684 685/*686 * Returns the _complete_ blocks that this bio covers.687 */688static void get_bio_block_range(struct thin_c *tc, struct bio *bio,689 dm_block_t *begin, dm_block_t *end)690{691 struct pool *pool = tc->pool;692 sector_t b = bio->bi_iter.bi_sector;693 sector_t e = b + (bio->bi_iter.bi_size >> SECTOR_SHIFT);694 695 b += pool->sectors_per_block - 1ull; /* so we round up */696 697 if (block_size_is_power_of_two(pool)) {698 b >>= pool->sectors_per_block_shift;699 e >>= pool->sectors_per_block_shift;700 } else {701 (void) sector_div(b, pool->sectors_per_block);702 (void) sector_div(e, pool->sectors_per_block);703 }704 705 if (e < b) {706 /* Can happen if the bio is within a single block. */707 e = b;708 }709 710 *begin = b;711 *end = e;712}713 714static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)715{716 struct pool *pool = tc->pool;717 sector_t bi_sector = bio->bi_iter.bi_sector;718 719 bio_set_dev(bio, tc->pool_dev->bdev);720 if (block_size_is_power_of_two(pool)) {721 bio->bi_iter.bi_sector =722 (block << pool->sectors_per_block_shift) |723 (bi_sector & (pool->sectors_per_block - 1));724 } else {725 bio->bi_iter.bi_sector = (block * pool->sectors_per_block) +726 sector_div(bi_sector, pool->sectors_per_block);727 }728}729 730static void remap_to_origin(struct thin_c *tc, struct bio *bio)731{732 bio_set_dev(bio, tc->origin_dev->bdev);733}734 735static int bio_triggers_commit(struct thin_c *tc, struct bio *bio)736{737 return op_is_flush(bio->bi_opf) &&738 dm_thin_changed_this_transaction(tc->td);739}740 741static void inc_all_io_entry(struct pool *pool, struct bio *bio)742{743 struct dm_thin_endio_hook *h;744 745 if (bio_op(bio) == REQ_OP_DISCARD)746 return;747 748 h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));749 h->all_io_entry = dm_deferred_entry_inc(pool->all_io_ds);750}751 752static void issue(struct thin_c *tc, struct bio *bio)753{754 struct pool *pool = tc->pool;755 756 if (!bio_triggers_commit(tc, bio)) {757 dm_submit_bio_remap(bio, NULL);758 return;759 }760 761 /*762 * Complete bio with an error if earlier I/O caused changes to763 * the metadata that can't be committed e.g, due to I/O errors764 * on the metadata device.765 */766 if (dm_thin_aborted_changes(tc->td)) {767 bio_io_error(bio);768 return;769 }770 771 /*772 * Batch together any bios that trigger commits and then issue a773 * single commit for them in process_deferred_bios().774 */775 spin_lock_irq(&pool->lock);776 bio_list_add(&pool->deferred_flush_bios, bio);777 spin_unlock_irq(&pool->lock);778}779 780static void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio)781{782 remap_to_origin(tc, bio);783 issue(tc, bio);784}785 786static void remap_and_issue(struct thin_c *tc, struct bio *bio,787 dm_block_t block)788{789 remap(tc, bio, block);790 issue(tc, bio);791}792 793/*----------------------------------------------------------------*/794 795/*796 * Bio endio functions.797 */798struct dm_thin_new_mapping {799 struct list_head list;800 801 bool pass_discard:1;802 bool maybe_shared:1;803 804 /*805 * Track quiescing, copying and zeroing preparation actions. When this806 * counter hits zero the block is prepared and can be inserted into the807 * btree.808 */809 atomic_t prepare_actions;810 811 blk_status_t status;812 struct thin_c *tc;813 dm_block_t virt_begin, virt_end;814 dm_block_t data_block;815 struct dm_bio_prison_cell *cell;816 817 /*818 * If the bio covers the whole area of a block then we can avoid819 * zeroing or copying. Instead this bio is hooked. The bio will820 * still be in the cell, so care has to be taken to avoid issuing821 * the bio twice.822 */823 struct bio *bio;824 bio_end_io_t *saved_bi_end_io;825};826 827static void __complete_mapping_preparation(struct dm_thin_new_mapping *m)828{829 struct pool *pool = m->tc->pool;830 831 if (atomic_dec_and_test(&m->prepare_actions)) {832 list_add_tail(&m->list, &pool->prepared_mappings);833 wake_worker(pool);834 }835}836 837static void complete_mapping_preparation(struct dm_thin_new_mapping *m)838{839 unsigned long flags;840 struct pool *pool = m->tc->pool;841 842 spin_lock_irqsave(&pool->lock, flags);843 __complete_mapping_preparation(m);844 spin_unlock_irqrestore(&pool->lock, flags);845}846 847static void copy_complete(int read_err, unsigned long write_err, void *context)848{849 struct dm_thin_new_mapping *m = context;850 851 m->status = read_err || write_err ? BLK_STS_IOERR : 0;852 complete_mapping_preparation(m);853}854 855static void overwrite_endio(struct bio *bio)856{857 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));858 struct dm_thin_new_mapping *m = h->overwrite_mapping;859 860 bio->bi_end_io = m->saved_bi_end_io;861 862 m->status = bio->bi_status;863 complete_mapping_preparation(m);864}865 866/*----------------------------------------------------------------*/867 868/*869 * Workqueue.870 */871 872/*873 * Prepared mapping jobs.874 */875 876/*877 * This sends the bios in the cell, except the original holder, back878 * to the deferred_bios list.879 */880static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell)881{882 struct pool *pool = tc->pool;883 unsigned long flags;884 struct bio_list bios;885 886 bio_list_init(&bios);887 cell_release_no_holder(pool, cell, &bios);888 889 if (!bio_list_empty(&bios)) {890 spin_lock_irqsave(&tc->lock, flags);891 bio_list_merge(&tc->deferred_bio_list, &bios);892 spin_unlock_irqrestore(&tc->lock, flags);893 wake_worker(pool);894 }895}896 897static void thin_defer_bio(struct thin_c *tc, struct bio *bio);898 899struct remap_info {900 struct thin_c *tc;901 struct bio_list defer_bios;902 struct bio_list issue_bios;903};904 905static void __inc_remap_and_issue_cell(void *context,906 struct dm_bio_prison_cell *cell)907{908 struct remap_info *info = context;909 struct bio *bio;910 911 while ((bio = bio_list_pop(&cell->bios))) {912 if (op_is_flush(bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD)913 bio_list_add(&info->defer_bios, bio);914 else {915 inc_all_io_entry(info->tc->pool, bio);916 917 /*918 * We can't issue the bios with the bio prison lock919 * held, so we add them to a list to issue on920 * return from this function.921 */922 bio_list_add(&info->issue_bios, bio);923 }924 }925}926 927static void inc_remap_and_issue_cell(struct thin_c *tc,928 struct dm_bio_prison_cell *cell,929 dm_block_t block)930{931 struct bio *bio;932 struct remap_info info;933 934 info.tc = tc;935 bio_list_init(&info.defer_bios);936 bio_list_init(&info.issue_bios);937 938 /*939 * We have to be careful to inc any bios we're about to issue940 * before the cell is released, and avoid a race with new bios941 * being added to the cell.942 */943 cell_visit_release(tc->pool, __inc_remap_and_issue_cell,944 &info, cell);945 946 while ((bio = bio_list_pop(&info.defer_bios)))947 thin_defer_bio(tc, bio);948 949 while ((bio = bio_list_pop(&info.issue_bios)))950 remap_and_issue(info.tc, bio, block);951}952 953static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)954{955 cell_error(m->tc->pool, m->cell);956 list_del(&m->list);957 mempool_free(m, &m->tc->pool->mapping_pool);958}959 960static void complete_overwrite_bio(struct thin_c *tc, struct bio *bio)961{962 struct pool *pool = tc->pool;963 964 /*965 * If the bio has the REQ_FUA flag set we must commit the metadata966 * before signaling its completion.967 */968 if (!bio_triggers_commit(tc, bio)) {969 bio_endio(bio);970 return;971 }972 973 /*974 * Complete bio with an error if earlier I/O caused changes to the975 * metadata that can't be committed, e.g, due to I/O errors on the976 * metadata device.977 */978 if (dm_thin_aborted_changes(tc->td)) {979 bio_io_error(bio);980 return;981 }982 983 /*984 * Batch together any bios that trigger commits and then issue a985 * single commit for them in process_deferred_bios().986 */987 spin_lock_irq(&pool->lock);988 bio_list_add(&pool->deferred_flush_completions, bio);989 spin_unlock_irq(&pool->lock);990}991 992static void process_prepared_mapping(struct dm_thin_new_mapping *m)993{994 struct thin_c *tc = m->tc;995 struct pool *pool = tc->pool;996 struct bio *bio = m->bio;997 int r;998 999 if (m->status) {1000 cell_error(pool, m->cell);1001 goto out;1002 }1003 1004 /*1005 * Commit the prepared block into the mapping btree.1006 * Any I/O for this block arriving after this point will get1007 * remapped to it directly.1008 */1009 r = dm_thin_insert_block(tc->td, m->virt_begin, m->data_block);1010 if (r) {1011 metadata_operation_failed(pool, "dm_thin_insert_block", r);1012 cell_error(pool, m->cell);1013 goto out;1014 }1015 1016 /*1017 * Release any bios held while the block was being provisioned.1018 * If we are processing a write bio that completely covers the block,1019 * we already processed it so can ignore it now when processing1020 * the bios in the cell.1021 */1022 if (bio) {1023 inc_remap_and_issue_cell(tc, m->cell, m->data_block);1024 complete_overwrite_bio(tc, bio);1025 } else {1026 inc_all_io_entry(tc->pool, m->cell->holder);1027 remap_and_issue(tc, m->cell->holder, m->data_block);1028 inc_remap_and_issue_cell(tc, m->cell, m->data_block);1029 }1030 1031out:1032 list_del(&m->list);1033 mempool_free(m, &pool->mapping_pool);1034}1035 1036/*----------------------------------------------------------------*/1037 1038static void free_discard_mapping(struct dm_thin_new_mapping *m)1039{1040 struct thin_c *tc = m->tc;1041 1042 if (m->cell)1043 cell_defer_no_holder(tc, m->cell);1044 mempool_free(m, &tc->pool->mapping_pool);1045}1046 1047static void process_prepared_discard_fail(struct dm_thin_new_mapping *m)1048{1049 bio_io_error(m->bio);1050 free_discard_mapping(m);1051}1052 1053static void process_prepared_discard_success(struct dm_thin_new_mapping *m)1054{1055 bio_endio(m->bio);1056 free_discard_mapping(m);1057}1058 1059static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m)1060{1061 int r;1062 struct thin_c *tc = m->tc;1063 1064 r = dm_thin_remove_range(tc->td, m->cell->key.block_begin, m->cell->key.block_end);1065 if (r) {1066 metadata_operation_failed(tc->pool, "dm_thin_remove_range", r);1067 bio_io_error(m->bio);1068 } else1069 bio_endio(m->bio);1070 1071 cell_defer_no_holder(tc, m->cell);1072 mempool_free(m, &tc->pool->mapping_pool);1073}1074 1075/*----------------------------------------------------------------*/1076 1077static void passdown_double_checking_shared_status(struct dm_thin_new_mapping *m,1078 struct bio *discard_parent)1079{1080 /*1081 * We've already unmapped this range of blocks, but before we1082 * passdown we have to check that these blocks are now unused.1083 */1084 int r = 0;1085 bool shared = true;1086 struct thin_c *tc = m->tc;1087 struct pool *pool = tc->pool;1088 dm_block_t b = m->data_block, e, end = m->data_block + m->virt_end - m->virt_begin;1089 struct discard_op op;1090 1091 begin_discard(&op, tc, discard_parent);1092 while (b != end) {1093 /* find start of unmapped run */1094 for (; b < end; b++) {1095 r = dm_pool_block_is_shared(pool->pmd, b, &shared);1096 if (r)1097 goto out;1098 1099 if (!shared)1100 break;1101 }1102 1103 if (b == end)1104 break;1105 1106 /* find end of run */1107 for (e = b + 1; e != end; e++) {1108 r = dm_pool_block_is_shared(pool->pmd, e, &shared);1109 if (r)1110 goto out;1111 1112 if (shared)1113 break;1114 }1115 1116 r = issue_discard(&op, b, e);1117 if (r)1118 goto out;1119 1120 b = e;1121 }1122out:1123 end_discard(&op, r);1124}1125 1126static void queue_passdown_pt2(struct dm_thin_new_mapping *m)1127{1128 unsigned long flags;1129 struct pool *pool = m->tc->pool;1130 1131 spin_lock_irqsave(&pool->lock, flags);1132 list_add_tail(&m->list, &pool->prepared_discards_pt2);1133 spin_unlock_irqrestore(&pool->lock, flags);1134 wake_worker(pool);1135}1136 1137static void passdown_endio(struct bio *bio)1138{1139 /*1140 * It doesn't matter if the passdown discard failed, we still want1141 * to unmap (we ignore err).1142 */1143 queue_passdown_pt2(bio->bi_private);1144 bio_put(bio);1145}1146 1147static void process_prepared_discard_passdown_pt1(struct dm_thin_new_mapping *m)1148{1149 int r;1150 struct thin_c *tc = m->tc;1151 struct pool *pool = tc->pool;1152 struct bio *discard_parent;1153 dm_block_t data_end = m->data_block + (m->virt_end - m->virt_begin);1154 1155 /*1156 * Only this thread allocates blocks, so we can be sure that the1157 * newly unmapped blocks will not be allocated before the end of1158 * the function.1159 */1160 r = dm_thin_remove_range(tc->td, m->virt_begin, m->virt_end);1161 if (r) {1162 metadata_operation_failed(pool, "dm_thin_remove_range", r);1163 bio_io_error(m->bio);1164 cell_defer_no_holder(tc, m->cell);1165 mempool_free(m, &pool->mapping_pool);1166 return;1167 }1168 1169 /*1170 * Increment the unmapped blocks. This prevents a race between the1171 * passdown io and reallocation of freed blocks.1172 */1173 r = dm_pool_inc_data_range(pool->pmd, m->data_block, data_end);1174 if (r) {1175 metadata_operation_failed(pool, "dm_pool_inc_data_range", r);1176 bio_io_error(m->bio);1177 cell_defer_no_holder(tc, m->cell);1178 mempool_free(m, &pool->mapping_pool);1179 return;1180 }1181 1182 discard_parent = bio_alloc(NULL, 1, 0, GFP_NOIO);1183 discard_parent->bi_end_io = passdown_endio;1184 discard_parent->bi_private = m;1185 if (m->maybe_shared)1186 passdown_double_checking_shared_status(m, discard_parent);1187 else {1188 struct discard_op op;1189 1190 begin_discard(&op, tc, discard_parent);1191 r = issue_discard(&op, m->data_block, data_end);1192 end_discard(&op, r);1193 }1194}1195 1196static void process_prepared_discard_passdown_pt2(struct dm_thin_new_mapping *m)1197{1198 int r;1199 struct thin_c *tc = m->tc;1200 struct pool *pool = tc->pool;1201 1202 /*1203 * The passdown has completed, so now we can decrement all those1204 * unmapped blocks.1205 */1206 r = dm_pool_dec_data_range(pool->pmd, m->data_block,1207 m->data_block + (m->virt_end - m->virt_begin));1208 if (r) {1209 metadata_operation_failed(pool, "dm_pool_dec_data_range", r);1210 bio_io_error(m->bio);1211 } else1212 bio_endio(m->bio);1213 1214 cell_defer_no_holder(tc, m->cell);1215 mempool_free(m, &pool->mapping_pool);1216}1217 1218static void process_prepared(struct pool *pool, struct list_head *head,1219 process_mapping_fn *fn)1220{1221 struct list_head maps;1222 struct dm_thin_new_mapping *m, *tmp;1223 1224 INIT_LIST_HEAD(&maps);1225 spin_lock_irq(&pool->lock);1226 list_splice_init(head, &maps);1227 spin_unlock_irq(&pool->lock);1228 1229 list_for_each_entry_safe(m, tmp, &maps, list)1230 (*fn)(m);1231}1232 1233/*1234 * Deferred bio jobs.1235 */1236static int io_overlaps_block(struct pool *pool, struct bio *bio)1237{1238 return bio->bi_iter.bi_size ==1239 (pool->sectors_per_block << SECTOR_SHIFT);1240}1241 1242static int io_overwrites_block(struct pool *pool, struct bio *bio)1243{1244 return (bio_data_dir(bio) == WRITE) &&1245 io_overlaps_block(pool, bio);1246}1247 1248static void save_and_set_endio(struct bio *bio, bio_end_io_t **save,1249 bio_end_io_t *fn)1250{1251 *save = bio->bi_end_io;1252 bio->bi_end_io = fn;1253}1254 1255static int ensure_next_mapping(struct pool *pool)1256{1257 if (pool->next_mapping)1258 return 0;1259 1260 pool->next_mapping = mempool_alloc(&pool->mapping_pool, GFP_ATOMIC);1261 1262 return pool->next_mapping ? 0 : -ENOMEM;1263}1264 1265static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool)1266{1267 struct dm_thin_new_mapping *m = pool->next_mapping;1268 1269 BUG_ON(!pool->next_mapping);1270 1271 memset(m, 0, sizeof(struct dm_thin_new_mapping));1272 INIT_LIST_HEAD(&m->list);1273 m->bio = NULL;1274 1275 pool->next_mapping = NULL;1276 1277 return m;1278}1279 1280static void ll_zero(struct thin_c *tc, struct dm_thin_new_mapping *m,1281 sector_t begin, sector_t end)1282{1283 struct dm_io_region to;1284 1285 to.bdev = tc->pool_dev->bdev;1286 to.sector = begin;1287 to.count = end - begin;1288 1289 dm_kcopyd_zero(tc->pool->copier, 1, &to, 0, copy_complete, m);1290}1291 1292static void remap_and_issue_overwrite(struct thin_c *tc, struct bio *bio,1293 dm_block_t data_begin,1294 struct dm_thin_new_mapping *m)1295{1296 struct pool *pool = tc->pool;1297 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));1298 1299 h->overwrite_mapping = m;1300 m->bio = bio;1301 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);1302 inc_all_io_entry(pool, bio);1303 remap_and_issue(tc, bio, data_begin);1304}1305 1306/*1307 * A partial copy also needs to zero the uncopied region.1308 */1309static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,1310 struct dm_dev *origin, dm_block_t data_origin,1311 dm_block_t data_dest,1312 struct dm_bio_prison_cell *cell, struct bio *bio,1313 sector_t len)1314{1315 struct pool *pool = tc->pool;1316 struct dm_thin_new_mapping *m = get_next_mapping(pool);1317 1318 m->tc = tc;1319 m->virt_begin = virt_block;1320 m->virt_end = virt_block + 1u;1321 m->data_block = data_dest;1322 m->cell = cell;1323 1324 /*1325 * quiesce action + copy action + an extra reference held for the1326 * duration of this function (we may need to inc later for a1327 * partial zero).1328 */1329 atomic_set(&m->prepare_actions, 3);1330 1331 if (!dm_deferred_set_add_work(pool->shared_read_ds, &m->list))1332 complete_mapping_preparation(m); /* already quiesced */1333 1334 /*1335 * IO to pool_dev remaps to the pool target's data_dev.1336 *1337 * If the whole block of data is being overwritten, we can issue the1338 * bio immediately. Otherwise we use kcopyd to clone the data first.1339 */1340 if (io_overwrites_block(pool, bio))1341 remap_and_issue_overwrite(tc, bio, data_dest, m);1342 else {1343 struct dm_io_region from, to;1344 1345 from.bdev = origin->bdev;1346 from.sector = data_origin * pool->sectors_per_block;1347 from.count = len;1348 1349 to.bdev = tc->pool_dev->bdev;1350 to.sector = data_dest * pool->sectors_per_block;1351 to.count = len;1352 1353 dm_kcopyd_copy(pool->copier, &from, 1, &to,1354 0, copy_complete, m);1355 1356 /*1357 * Do we need to zero a tail region?1358 */1359 if (len < pool->sectors_per_block && pool->pf.zero_new_blocks) {1360 atomic_inc(&m->prepare_actions);1361 ll_zero(tc, m,1362 data_dest * pool->sectors_per_block + len,1363 (data_dest + 1) * pool->sectors_per_block);1364 }1365 }1366 1367 complete_mapping_preparation(m); /* drop our ref */1368}1369 1370static void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block,1371 dm_block_t data_origin, dm_block_t data_dest,1372 struct dm_bio_prison_cell *cell, struct bio *bio)1373{1374 schedule_copy(tc, virt_block, tc->pool_dev,1375 data_origin, data_dest, cell, bio,1376 tc->pool->sectors_per_block);1377}1378 1379static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,1380 dm_block_t data_block, struct dm_bio_prison_cell *cell,1381 struct bio *bio)1382{1383 struct pool *pool = tc->pool;1384 struct dm_thin_new_mapping *m = get_next_mapping(pool);1385 1386 atomic_set(&m->prepare_actions, 1); /* no need to quiesce */1387 m->tc = tc;1388 m->virt_begin = virt_block;1389 m->virt_end = virt_block + 1u;1390 m->data_block = data_block;1391 m->cell = cell;1392 1393 /*1394 * If the whole block of data is being overwritten or we are not1395 * zeroing pre-existing data, we can issue the bio immediately.1396 * Otherwise we use kcopyd to zero the data first.1397 */1398 if (pool->pf.zero_new_blocks) {1399 if (io_overwrites_block(pool, bio))1400 remap_and_issue_overwrite(tc, bio, data_block, m);1401 else {1402 ll_zero(tc, m, data_block * pool->sectors_per_block,1403 (data_block + 1) * pool->sectors_per_block);1404 }1405 } else1406 process_prepared_mapping(m);1407}1408 1409static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block,1410 dm_block_t data_dest,1411 struct dm_bio_prison_cell *cell, struct bio *bio)1412{1413 struct pool *pool = tc->pool;1414 sector_t virt_block_begin = virt_block * pool->sectors_per_block;1415 sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block;1416 1417 if (virt_block_end <= tc->origin_size) {1418 schedule_copy(tc, virt_block, tc->origin_dev,1419 virt_block, data_dest, cell, bio,1420 pool->sectors_per_block);1421 1422 } else if (virt_block_begin < tc->origin_size) {1423 schedule_copy(tc, virt_block, tc->origin_dev,1424 virt_block, data_dest, cell, bio,1425 tc->origin_size - virt_block_begin);1426 1427 } else1428 schedule_zero(tc, virt_block, data_dest, cell, bio);1429}1430 1431static void set_pool_mode(struct pool *pool, enum pool_mode new_mode);1432 1433static void requeue_bios(struct pool *pool);1434 1435static bool is_read_only_pool_mode(enum pool_mode mode)1436{1437 return (mode == PM_OUT_OF_METADATA_SPACE || mode == PM_READ_ONLY);1438}1439 1440static bool is_read_only(struct pool *pool)1441{1442 return is_read_only_pool_mode(get_pool_mode(pool));1443}1444 1445static void check_for_metadata_space(struct pool *pool)1446{1447 int r;1448 const char *ooms_reason = NULL;1449 dm_block_t nr_free;1450 1451 r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free);1452 if (r)1453 ooms_reason = "Could not get free metadata blocks";1454 else if (!nr_free)1455 ooms_reason = "No free metadata blocks";1456 1457 if (ooms_reason && !is_read_only(pool)) {1458 DMERR("%s", ooms_reason);1459 set_pool_mode(pool, PM_OUT_OF_METADATA_SPACE);1460 }1461}1462 1463static void check_for_data_space(struct pool *pool)1464{1465 int r;1466 dm_block_t nr_free;1467 1468 if (get_pool_mode(pool) != PM_OUT_OF_DATA_SPACE)1469 return;1470 1471 r = dm_pool_get_free_block_count(pool->pmd, &nr_free);1472 if (r)1473 return;1474 1475 if (nr_free) {1476 set_pool_mode(pool, PM_WRITE);1477 requeue_bios(pool);1478 }1479}1480 1481/*1482 * A non-zero return indicates read_only or fail_io mode.1483 * Many callers don't care about the return value.1484 */1485static int commit(struct pool *pool)1486{1487 int r;1488 1489 if (get_pool_mode(pool) >= PM_OUT_OF_METADATA_SPACE)1490 return -EINVAL;1491 1492 r = dm_pool_commit_metadata(pool->pmd);1493 if (r)1494 metadata_operation_failed(pool, "dm_pool_commit_metadata", r);1495 else {1496 check_for_metadata_space(pool);1497 check_for_data_space(pool);1498 }1499 1500 return r;1501}1502 1503static void check_low_water_mark(struct pool *pool, dm_block_t free_blocks)1504{1505 if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) {1506 DMWARN("%s: reached low water mark for data device: sending event.",1507 dm_device_name(pool->pool_md));1508 spin_lock_irq(&pool->lock);1509 pool->low_water_triggered = true;1510 spin_unlock_irq(&pool->lock);1511 dm_table_event(pool->ti->table);1512 }1513}1514 1515static int alloc_data_block(struct thin_c *tc, dm_block_t *result)1516{1517 int r;1518 dm_block_t free_blocks;1519 struct pool *pool = tc->pool;1520 1521 if (WARN_ON(get_pool_mode(pool) != PM_WRITE))1522 return -EINVAL;1523 1524 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);1525 if (r) {1526 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);1527 return r;1528 }1529 1530 check_low_water_mark(pool, free_blocks);1531 1532 if (!free_blocks) {1533 /*1534 * Try to commit to see if that will free up some1535 * more space.1536 */1537 r = commit(pool);1538 if (r)1539 return r;1540 1541 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);1542 if (r) {1543 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);1544 return r;1545 }1546 1547 if (!free_blocks) {1548 set_pool_mode(pool, PM_OUT_OF_DATA_SPACE);1549 return -ENOSPC;1550 }1551 }1552 1553 r = dm_pool_alloc_data_block(pool->pmd, result);1554 if (r) {1555 if (r == -ENOSPC)1556 set_pool_mode(pool, PM_OUT_OF_DATA_SPACE);1557 else1558 metadata_operation_failed(pool, "dm_pool_alloc_data_block", r);1559 return r;1560 }1561 1562 r = dm_pool_get_free_metadata_block_count(pool->pmd, &free_blocks);1563 if (r) {1564 metadata_operation_failed(pool, "dm_pool_get_free_metadata_block_count", r);1565 return r;1566 }1567 1568 if (!free_blocks) {1569 /* Let's commit before we use up the metadata reserve. */1570 r = commit(pool);1571 if (r)1572 return r;1573 }1574 1575 return 0;1576}1577 1578/*1579 * If we have run out of space, queue bios until the device is1580 * resumed, presumably after having been reloaded with more space.1581 */1582static void retry_on_resume(struct bio *bio)1583{1584 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));1585 struct thin_c *tc = h->tc;1586 1587 spin_lock_irq(&tc->lock);1588 bio_list_add(&tc->retry_on_resume_list, bio);1589 spin_unlock_irq(&tc->lock);1590}1591 1592static blk_status_t should_error_unserviceable_bio(struct pool *pool)1593{1594 enum pool_mode m = get_pool_mode(pool);1595 1596 switch (m) {1597 case PM_WRITE:1598 /* Shouldn't get here */1599 DMERR_LIMIT("bio unserviceable, yet pool is in PM_WRITE mode");1600 return BLK_STS_IOERR;1601 1602 case PM_OUT_OF_DATA_SPACE:1603 return pool->pf.error_if_no_space ? BLK_STS_NOSPC : 0;1604 1605 case PM_OUT_OF_METADATA_SPACE:1606 case PM_READ_ONLY:1607 case PM_FAIL:1608 return BLK_STS_IOERR;1609 default:1610 /* Shouldn't get here */1611 DMERR_LIMIT("bio unserviceable, yet pool has an unknown mode");1612 return BLK_STS_IOERR;1613 }1614}1615 1616static void handle_unserviceable_bio(struct pool *pool, struct bio *bio)1617{1618 blk_status_t error = should_error_unserviceable_bio(pool);1619 1620 if (error) {1621 bio->bi_status = error;1622 bio_endio(bio);1623 } else1624 retry_on_resume(bio);1625}1626 1627static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell)1628{1629 struct bio *bio;1630 struct bio_list bios;1631 blk_status_t error;1632 1633 error = should_error_unserviceable_bio(pool);1634 if (error) {1635 cell_error_with_code(pool, cell, error);1636 return;1637 }1638 1639 bio_list_init(&bios);1640 cell_release(pool, cell, &bios);1641 1642 while ((bio = bio_list_pop(&bios)))1643 retry_on_resume(bio);1644}1645 1646static void process_discard_cell_no_passdown(struct thin_c *tc,1647 struct dm_bio_prison_cell *virt_cell)1648{1649 struct pool *pool = tc->pool;1650 struct dm_thin_new_mapping *m = get_next_mapping(pool);1651 1652 /*1653 * We don't need to lock the data blocks, since there's no1654 * passdown. We only lock data blocks for allocation and breaking sharing.1655 */1656 m->tc = tc;1657 m->virt_begin = virt_cell->key.block_begin;1658 m->virt_end = virt_cell->key.block_end;1659 m->cell = virt_cell;1660 m->bio = virt_cell->holder;1661 1662 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))1663 pool->process_prepared_discard(m);1664}1665 1666static void break_up_discard_bio(struct thin_c *tc, dm_block_t begin, dm_block_t end,1667 struct bio *bio)1668{1669 struct pool *pool = tc->pool;1670 1671 int r;1672 bool maybe_shared;1673 struct dm_cell_key data_key;1674 struct dm_bio_prison_cell *data_cell;1675 struct dm_thin_new_mapping *m;1676 dm_block_t virt_begin, virt_end, data_begin, data_end;1677 dm_block_t len, next_boundary;1678 1679 while (begin != end) {1680 r = dm_thin_find_mapped_range(tc->td, begin, end, &virt_begin, &virt_end,1681 &data_begin, &maybe_shared);1682 if (r) {1683 /*1684 * Silently fail, letting any mappings we've1685 * created complete.1686 */1687 break;1688 }1689 1690 data_end = data_begin + (virt_end - virt_begin);1691 1692 /*1693 * Make sure the data region obeys the bio prison restrictions.1694 */1695 while (data_begin < data_end) {1696 r = ensure_next_mapping(pool);1697 if (r)1698 return; /* we did our best */1699 1700 next_boundary = ((data_begin >> BIO_PRISON_MAX_RANGE_SHIFT) + 1)1701 << BIO_PRISON_MAX_RANGE_SHIFT;1702 len = min_t(sector_t, data_end - data_begin, next_boundary - data_begin);1703 1704 /* This key is certainly within range given the above splitting */1705 (void) build_key(tc->td, PHYSICAL, data_begin, data_begin + len, &data_key);1706 if (bio_detain(tc->pool, &data_key, NULL, &data_cell)) {1707 /* contention, we'll give up with this range */1708 data_begin += len;1709 continue;1710 }1711 1712 /*1713 * IO may still be going to the destination block. We must1714 * quiesce before we can do the removal.1715 */1716 m = get_next_mapping(pool);1717 m->tc = tc;1718 m->maybe_shared = maybe_shared;1719 m->virt_begin = virt_begin;1720 m->virt_end = virt_begin + len;1721 m->data_block = data_begin;1722 m->cell = data_cell;1723 m->bio = bio;1724 1725 /*1726 * The parent bio must not complete before sub discard bios are1727 * chained to it (see end_discard's bio_chain)!1728 *1729 * This per-mapping bi_remaining increment is paired with1730 * the implicit decrement that occurs via bio_endio() in1731 * end_discard().1732 */1733 bio_inc_remaining(bio);1734 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))1735 pool->process_prepared_discard(m);1736 1737 virt_begin += len;1738 data_begin += len;1739 }1740 1741 begin = virt_end;1742 }1743}1744 1745static void process_discard_cell_passdown(struct thin_c *tc, struct dm_bio_prison_cell *virt_cell)1746{1747 struct bio *bio = virt_cell->holder;1748 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));1749 1750 /*1751 * The virt_cell will only get freed once the origin bio completes.1752 * This means it will remain locked while all the individual1753 * passdown bios are in flight.1754 */1755 h->cell = virt_cell;1756 break_up_discard_bio(tc, virt_cell->key.block_begin, virt_cell->key.block_end, bio);1757 1758 /*1759 * We complete the bio now, knowing that the bi_remaining field1760 * will prevent completion until the sub range discards have1761 * completed.1762 */1763 bio_endio(bio);1764}1765 1766static void process_discard_bio(struct thin_c *tc, struct bio *bio)1767{1768 dm_block_t begin, end;1769 struct dm_cell_key virt_key;1770 struct dm_bio_prison_cell *virt_cell;1771 1772 get_bio_block_range(tc, bio, &begin, &end);1773 if (begin == end) {1774 /*1775 * The discard covers less than a block.1776 */1777 bio_endio(bio);1778 return;1779 }1780 1781 if (unlikely(!build_key(tc->td, VIRTUAL, begin, end, &virt_key))) {1782 DMERR_LIMIT("Discard doesn't respect bio prison limits");1783 bio_endio(bio);1784 return;1785 }1786 1787 if (bio_detain(tc->pool, &virt_key, bio, &virt_cell)) {1788 /*1789 * Potential starvation issue: We're relying on the1790 * fs/application being well behaved, and not trying to1791 * send IO to a region at the same time as discarding it.1792 * If they do this persistently then it's possible this1793 * cell will never be granted.1794 */1795 return;1796 }1797 1798 tc->pool->process_discard_cell(tc, virt_cell);1799}1800 1801static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block,1802 struct dm_cell_key *key,1803 struct dm_thin_lookup_result *lookup_result,1804 struct dm_bio_prison_cell *cell)1805{1806 int r;1807 dm_block_t data_block;1808 struct pool *pool = tc->pool;1809 1810 r = alloc_data_block(tc, &data_block);1811 switch (r) {1812 case 0:1813 schedule_internal_copy(tc, block, lookup_result->block,1814 data_block, cell, bio);1815 break;1816 1817 case -ENOSPC:1818 retry_bios_on_resume(pool, cell);1819 break;1820 1821 default:1822 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",1823 __func__, r);1824 cell_error(pool, cell);1825 break;1826 }1827}1828 1829static void __remap_and_issue_shared_cell(void *context,1830 struct dm_bio_prison_cell *cell)1831{1832 struct remap_info *info = context;1833 struct bio *bio;1834 1835 while ((bio = bio_list_pop(&cell->bios))) {1836 if (bio_data_dir(bio) == WRITE || op_is_flush(bio->bi_opf) ||1837 bio_op(bio) == REQ_OP_DISCARD)1838 bio_list_add(&info->defer_bios, bio);1839 else {1840 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));1841 1842 h->shared_read_entry = dm_deferred_entry_inc(info->tc->pool->shared_read_ds);1843 inc_all_io_entry(info->tc->pool, bio);1844 bio_list_add(&info->issue_bios, bio);1845 }1846 }1847}1848 1849static void remap_and_issue_shared_cell(struct thin_c *tc,1850 struct dm_bio_prison_cell *cell,1851 dm_block_t block)1852{1853 struct bio *bio;1854 struct remap_info info;1855 1856 info.tc = tc;1857 bio_list_init(&info.defer_bios);1858 bio_list_init(&info.issue_bios);1859 1860 cell_visit_release(tc->pool, __remap_and_issue_shared_cell,1861 &info, cell);1862 1863 while ((bio = bio_list_pop(&info.defer_bios)))1864 thin_defer_bio(tc, bio);1865 1866 while ((bio = bio_list_pop(&info.issue_bios)))1867 remap_and_issue(tc, bio, block);1868}1869 1870static void process_shared_bio(struct thin_c *tc, struct bio *bio,1871 dm_block_t block,1872 struct dm_thin_lookup_result *lookup_result,1873 struct dm_bio_prison_cell *virt_cell)1874{1875 struct dm_bio_prison_cell *data_cell;1876 struct pool *pool = tc->pool;1877 struct dm_cell_key key;1878 1879 /*1880 * If cell is already occupied, then sharing is already in the process1881 * of being broken so we have nothing further to do here.1882 */1883 build_data_key(tc->td, lookup_result->block, &key);1884 if (bio_detain(pool, &key, bio, &data_cell)) {1885 cell_defer_no_holder(tc, virt_cell);1886 return;1887 }1888 1889 if (bio_data_dir(bio) == WRITE && bio->bi_iter.bi_size) {1890 break_sharing(tc, bio, block, &key, lookup_result, data_cell);1891 cell_defer_no_holder(tc, virt_cell);1892 } else {1893 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));1894 1895 h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds);1896 inc_all_io_entry(pool, bio);1897 remap_and_issue(tc, bio, lookup_result->block);1898 1899 remap_and_issue_shared_cell(tc, data_cell, lookup_result->block);1900 remap_and_issue_shared_cell(tc, virt_cell, lookup_result->block);1901 }1902}1903 1904static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block,1905 struct dm_bio_prison_cell *cell)1906{1907 int r;1908 dm_block_t data_block;1909 struct pool *pool = tc->pool;1910 1911 /*1912 * Remap empty bios (flushes) immediately, without provisioning.1913 */1914 if (!bio->bi_iter.bi_size) {1915 inc_all_io_entry(pool, bio);1916 cell_defer_no_holder(tc, cell);1917 1918 remap_and_issue(tc, bio, 0);1919 return;1920 }1921 1922 /*1923 * Fill read bios with zeroes and complete them immediately.1924 */1925 if (bio_data_dir(bio) == READ) {1926 zero_fill_bio(bio);1927 cell_defer_no_holder(tc, cell);1928 bio_endio(bio);1929 return;1930 }1931 1932 r = alloc_data_block(tc, &data_block);1933 switch (r) {1934 case 0:1935 if (tc->origin_dev)1936 schedule_external_copy(tc, block, data_block, cell, bio);1937 else1938 schedule_zero(tc, block, data_block, cell, bio);1939 break;1940 1941 case -ENOSPC:1942 retry_bios_on_resume(pool, cell);1943 break;1944 1945 default:1946 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",1947 __func__, r);1948 cell_error(pool, cell);1949 break;1950 }1951}1952 1953static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)1954{1955 int r;1956 struct pool *pool = tc->pool;1957 struct bio *bio = cell->holder;1958 dm_block_t block = get_bio_block(tc, bio);1959 struct dm_thin_lookup_result lookup_result;1960 1961 if (tc->requeue_mode) {1962 cell_requeue(pool, cell);1963 return;1964 }1965 1966 r = dm_thin_find_block(tc->td, block, 1, &lookup_result);1967 switch (r) {1968 case 0:1969 if (lookup_result.shared)1970 process_shared_bio(tc, bio, block, &lookup_result, cell);1971 else {1972 inc_all_io_entry(pool, bio);1973 remap_and_issue(tc, bio, lookup_result.block);1974 inc_remap_and_issue_cell(tc, cell, lookup_result.block);1975 }1976 break;1977 1978 case -ENODATA:1979 if (bio_data_dir(bio) == READ && tc->origin_dev) {1980 inc_all_io_entry(pool, bio);1981 cell_defer_no_holder(tc, cell);1982 1983 if (bio_end_sector(bio) <= tc->origin_size)1984 remap_to_origin_and_issue(tc, bio);1985 1986 else if (bio->bi_iter.bi_sector < tc->origin_size) {1987 zero_fill_bio(bio);1988 bio->bi_iter.bi_size = (tc->origin_size - bio->bi_iter.bi_sector) << SECTOR_SHIFT;1989 remap_to_origin_and_issue(tc, bio);1990 1991 } else {1992 zero_fill_bio(bio);1993 bio_endio(bio);1994 }1995 } else1996 provision_block(tc, bio, block, cell);1997 break;1998 1999 default:2000 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",2001 __func__, r);2002 cell_defer_no_holder(tc, cell);2003 bio_io_error(bio);2004 break;2005 }2006}2007 2008static void process_bio(struct thin_c *tc, struct bio *bio)2009{2010 struct pool *pool = tc->pool;2011 dm_block_t block = get_bio_block(tc, bio);2012 struct dm_bio_prison_cell *cell;2013 struct dm_cell_key key;2014 2015 /*2016 * If cell is already occupied, then the block is already2017 * being provisioned so we have nothing further to do here.2018 */2019 build_virtual_key(tc->td, block, &key);2020 if (bio_detain(pool, &key, bio, &cell))2021 return;2022 2023 process_cell(tc, cell);2024}2025 2026static void __process_bio_read_only(struct thin_c *tc, struct bio *bio,2027 struct dm_bio_prison_cell *cell)2028{2029 int r;2030 int rw = bio_data_dir(bio);2031 dm_block_t block = get_bio_block(tc, bio);2032 struct dm_thin_lookup_result lookup_result;2033 2034 r = dm_thin_find_block(tc->td, block, 1, &lookup_result);2035 switch (r) {2036 case 0:2037 if (lookup_result.shared && (rw == WRITE) && bio->bi_iter.bi_size) {2038 handle_unserviceable_bio(tc->pool, bio);2039 if (cell)2040 cell_defer_no_holder(tc, cell);2041 } else {2042 inc_all_io_entry(tc->pool, bio);2043 remap_and_issue(tc, bio, lookup_result.block);2044 if (cell)2045 inc_remap_and_issue_cell(tc, cell, lookup_result.block);2046 }2047 break;2048 2049 case -ENODATA:2050 if (cell)2051 cell_defer_no_holder(tc, cell);2052 if (rw != READ) {2053 handle_unserviceable_bio(tc->pool, bio);2054 break;2055 }2056 2057 if (tc->origin_dev) {2058 inc_all_io_entry(tc->pool, bio);2059 remap_to_origin_and_issue(tc, bio);2060 break;2061 }2062 2063 zero_fill_bio(bio);2064 bio_endio(bio);2065 break;2066 2067 default:2068 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",2069 __func__, r);2070 if (cell)2071 cell_defer_no_holder(tc, cell);2072 bio_io_error(bio);2073 break;2074 }2075}2076 2077static void process_bio_read_only(struct thin_c *tc, struct bio *bio)2078{2079 __process_bio_read_only(tc, bio, NULL);2080}2081 2082static void process_cell_read_only(struct thin_c *tc, struct dm_bio_prison_cell *cell)2083{2084 __process_bio_read_only(tc, cell->holder, cell);2085}2086 2087static void process_bio_success(struct thin_c *tc, struct bio *bio)2088{2089 bio_endio(bio);2090}2091 2092static void process_bio_fail(struct thin_c *tc, struct bio *bio)2093{2094 bio_io_error(bio);2095}2096 2097static void process_cell_success(struct thin_c *tc, struct dm_bio_prison_cell *cell)2098{2099 cell_success(tc->pool, cell);2100}2101 2102static void process_cell_fail(struct thin_c *tc, struct dm_bio_prison_cell *cell)2103{2104 cell_error(tc->pool, cell);2105}2106 2107/*2108 * FIXME: should we also commit due to size of transaction, measured in2109 * metadata blocks?2110 */2111static int need_commit_due_to_time(struct pool *pool)2112{2113 return !time_in_range(jiffies, pool->last_commit_jiffies,2114 pool->last_commit_jiffies + COMMIT_PERIOD);2115}2116 2117#define thin_pbd(node) rb_entry((node), struct dm_thin_endio_hook, rb_node)2118#define thin_bio(pbd) dm_bio_from_per_bio_data((pbd), sizeof(struct dm_thin_endio_hook))2119 2120static void __thin_bio_rb_add(struct thin_c *tc, struct bio *bio)2121{2122 struct rb_node **rbp, *parent;2123 struct dm_thin_endio_hook *pbd;2124 sector_t bi_sector = bio->bi_iter.bi_sector;2125 2126 rbp = &tc->sort_bio_list.rb_node;2127 parent = NULL;2128 while (*rbp) {2129 parent = *rbp;2130 pbd = thin_pbd(parent);2131 2132 if (bi_sector < thin_bio(pbd)->bi_iter.bi_sector)2133 rbp = &(*rbp)->rb_left;2134 else2135 rbp = &(*rbp)->rb_right;2136 }2137 2138 pbd = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));2139 rb_link_node(&pbd->rb_node, parent, rbp);2140 rb_insert_color(&pbd->rb_node, &tc->sort_bio_list);2141}2142 2143static void __extract_sorted_bios(struct thin_c *tc)2144{2145 struct rb_node *node;2146 struct dm_thin_endio_hook *pbd;2147 struct bio *bio;2148 2149 for (node = rb_first(&tc->sort_bio_list); node; node = rb_next(node)) {2150 pbd = thin_pbd(node);2151 bio = thin_bio(pbd);2152 2153 bio_list_add(&tc->deferred_bio_list, bio);2154 rb_erase(&pbd->rb_node, &tc->sort_bio_list);2155 }2156 2157 WARN_ON(!RB_EMPTY_ROOT(&tc->sort_bio_list));2158}2159 2160static void __sort_thin_deferred_bios(struct thin_c *tc)2161{2162 struct bio *bio;2163 struct bio_list bios;2164 2165 bio_list_init(&bios);2166 bio_list_merge(&bios, &tc->deferred_bio_list);2167 bio_list_init(&tc->deferred_bio_list);2168 2169 /* Sort deferred_bio_list using rb-tree */2170 while ((bio = bio_list_pop(&bios)))2171 __thin_bio_rb_add(tc, bio);2172 2173 /*2174 * Transfer the sorted bios in sort_bio_list back to2175 * deferred_bio_list to allow lockless submission of2176 * all bios.2177 */2178 __extract_sorted_bios(tc);2179}2180 2181static void process_thin_deferred_bios(struct thin_c *tc)2182{2183 struct pool *pool = tc->pool;2184 struct bio *bio;2185 struct bio_list bios;2186 struct blk_plug plug;2187 unsigned int count = 0;2188 2189 if (tc->requeue_mode) {2190 error_thin_bio_list(tc, &tc->deferred_bio_list,2191 BLK_STS_DM_REQUEUE);2192 return;2193 }2194 2195 bio_list_init(&bios);2196 2197 spin_lock_irq(&tc->lock);2198 2199 if (bio_list_empty(&tc->deferred_bio_list)) {2200 spin_unlock_irq(&tc->lock);2201 return;2202 }2203 2204 __sort_thin_deferred_bios(tc);2205 2206 bio_list_merge(&bios, &tc->deferred_bio_list);2207 bio_list_init(&tc->deferred_bio_list);2208 2209 spin_unlock_irq(&tc->lock);2210 2211 blk_start_plug(&plug);2212 while ((bio = bio_list_pop(&bios))) {2213 /*2214 * If we've got no free new_mapping structs, and processing2215 * this bio might require one, we pause until there are some2216 * prepared mappings to process.2217 */2218 if (ensure_next_mapping(pool)) {2219 spin_lock_irq(&tc->lock);2220 bio_list_add(&tc->deferred_bio_list, bio);2221 bio_list_merge(&tc->deferred_bio_list, &bios);2222 spin_unlock_irq(&tc->lock);2223 break;2224 }2225 2226 if (bio_op(bio) == REQ_OP_DISCARD)2227 pool->process_discard(tc, bio);2228 else2229 pool->process_bio(tc, bio);2230 2231 if ((count++ & 127) == 0) {2232 throttle_work_update(&pool->throttle);2233 dm_pool_issue_prefetches(pool->pmd);2234 }2235 cond_resched();2236 }2237 blk_finish_plug(&plug);2238}2239 2240static int cmp_cells(const void *lhs, const void *rhs)2241{2242 struct dm_bio_prison_cell *lhs_cell = *((struct dm_bio_prison_cell **) lhs);2243 struct dm_bio_prison_cell *rhs_cell = *((struct dm_bio_prison_cell **) rhs);2244 2245 BUG_ON(!lhs_cell->holder);2246 BUG_ON(!rhs_cell->holder);2247 2248 if (lhs_cell->holder->bi_iter.bi_sector < rhs_cell->holder->bi_iter.bi_sector)2249 return -1;2250 2251 if (lhs_cell->holder->bi_iter.bi_sector > rhs_cell->holder->bi_iter.bi_sector)2252 return 1;2253 2254 return 0;2255}2256 2257static unsigned int sort_cells(struct pool *pool, struct list_head *cells)2258{2259 unsigned int count = 0;2260 struct dm_bio_prison_cell *cell, *tmp;2261 2262 list_for_each_entry_safe(cell, tmp, cells, user_list) {2263 if (count >= CELL_SORT_ARRAY_SIZE)2264 break;2265 2266 pool->cell_sort_array[count++] = cell;2267 list_del(&cell->user_list);2268 }2269 2270 sort(pool->cell_sort_array, count, sizeof(cell), cmp_cells, NULL);2271 2272 return count;2273}2274 2275static void process_thin_deferred_cells(struct thin_c *tc)2276{2277 struct pool *pool = tc->pool;2278 struct list_head cells;2279 struct dm_bio_prison_cell *cell;2280 unsigned int i, j, count;2281 2282 INIT_LIST_HEAD(&cells);2283 2284 spin_lock_irq(&tc->lock);2285 list_splice_init(&tc->deferred_cells, &cells);2286 spin_unlock_irq(&tc->lock);2287 2288 if (list_empty(&cells))2289 return;2290 2291 do {2292 count = sort_cells(tc->pool, &cells);2293 2294 for (i = 0; i < count; i++) {2295 cell = pool->cell_sort_array[i];2296 BUG_ON(!cell->holder);2297 2298 /*2299 * If we've got no free new_mapping structs, and processing2300 * this bio might require one, we pause until there are some2301 * prepared mappings to process.2302 */2303 if (ensure_next_mapping(pool)) {2304 for (j = i; j < count; j++)2305 list_add(&pool->cell_sort_array[j]->user_list, &cells);2306 2307 spin_lock_irq(&tc->lock);2308 list_splice(&cells, &tc->deferred_cells);2309 spin_unlock_irq(&tc->lock);2310 return;2311 }2312 2313 if (bio_op(cell->holder) == REQ_OP_DISCARD)2314 pool->process_discard_cell(tc, cell);2315 else2316 pool->process_cell(tc, cell);2317 }2318 cond_resched();2319 } while (!list_empty(&cells));2320}2321 2322static void thin_get(struct thin_c *tc);2323static void thin_put(struct thin_c *tc);2324 2325/*2326 * We can't hold rcu_read_lock() around code that can block. So we2327 * find a thin with the rcu lock held; bump a refcount; then drop2328 * the lock.2329 */2330static struct thin_c *get_first_thin(struct pool *pool)2331{2332 struct thin_c *tc = NULL;2333 2334 rcu_read_lock();2335 if (!list_empty(&pool->active_thins)) {2336 tc = list_entry_rcu(pool->active_thins.next, struct thin_c, list);2337 thin_get(tc);2338 }2339 rcu_read_unlock();2340 2341 return tc;2342}2343 2344static struct thin_c *get_next_thin(struct pool *pool, struct thin_c *tc)2345{2346 struct thin_c *old_tc = tc;2347 2348 rcu_read_lock();2349 list_for_each_entry_continue_rcu(tc, &pool->active_thins, list) {2350 thin_get(tc);2351 thin_put(old_tc);2352 rcu_read_unlock();2353 return tc;2354 }2355 thin_put(old_tc);2356 rcu_read_unlock();2357 2358 return NULL;2359}2360 2361static void process_deferred_bios(struct pool *pool)2362{2363 struct bio *bio;2364 struct bio_list bios, bio_completions;2365 struct thin_c *tc;2366 2367 tc = get_first_thin(pool);2368 while (tc) {2369 process_thin_deferred_cells(tc);2370 process_thin_deferred_bios(tc);2371 tc = get_next_thin(pool, tc);2372 }2373 2374 /*2375 * If there are any deferred flush bios, we must commit the metadata2376 * before issuing them or signaling their completion.2377 */2378 bio_list_init(&bios);2379 bio_list_init(&bio_completions);2380 2381 spin_lock_irq(&pool->lock);2382 bio_list_merge(&bios, &pool->deferred_flush_bios);2383 bio_list_init(&pool->deferred_flush_bios);2384 2385 bio_list_merge(&bio_completions, &pool->deferred_flush_completions);2386 bio_list_init(&pool->deferred_flush_completions);2387 spin_unlock_irq(&pool->lock);2388 2389 if (bio_list_empty(&bios) && bio_list_empty(&bio_completions) &&2390 !(dm_pool_changed_this_transaction(pool->pmd) && need_commit_due_to_time(pool)))2391 return;2392 2393 if (commit(pool)) {2394 bio_list_merge(&bios, &bio_completions);2395 2396 while ((bio = bio_list_pop(&bios)))2397 bio_io_error(bio);2398 return;2399 }2400 pool->last_commit_jiffies = jiffies;2401 2402 while ((bio = bio_list_pop(&bio_completions)))2403 bio_endio(bio);2404 2405 while ((bio = bio_list_pop(&bios))) {2406 /*2407 * The data device was flushed as part of metadata commit,2408 * so complete redundant flushes immediately.2409 */2410 if (bio->bi_opf & REQ_PREFLUSH)2411 bio_endio(bio);2412 else2413 dm_submit_bio_remap(bio, NULL);2414 }2415}2416 2417static void do_worker(struct work_struct *ws)2418{2419 struct pool *pool = container_of(ws, struct pool, worker);2420 2421 throttle_work_start(&pool->throttle);2422 dm_pool_issue_prefetches(pool->pmd);2423 throttle_work_update(&pool->throttle);2424 process_prepared(pool, &pool->prepared_mappings, &pool->process_prepared_mapping);2425 throttle_work_update(&pool->throttle);2426 process_prepared(pool, &pool->prepared_discards, &pool->process_prepared_discard);2427 throttle_work_update(&pool->throttle);2428 process_prepared(pool, &pool->prepared_discards_pt2, &pool->process_prepared_discard_pt2);2429 throttle_work_update(&pool->throttle);2430 process_deferred_bios(pool);2431 throttle_work_complete(&pool->throttle);2432}2433 2434/*2435 * We want to commit periodically so that not too much2436 * unwritten data builds up.2437 */2438static void do_waker(struct work_struct *ws)2439{2440 struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker);2441 2442 wake_worker(pool);2443 queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD);2444}2445 2446/*2447 * We're holding onto IO to allow userland time to react. After the2448 * timeout either the pool will have been resized (and thus back in2449 * PM_WRITE mode), or we degrade to PM_OUT_OF_DATA_SPACE w/ error_if_no_space.2450 */2451static void do_no_space_timeout(struct work_struct *ws)2452{2453 struct pool *pool = container_of(to_delayed_work(ws), struct pool,2454 no_space_timeout);2455 2456 if (get_pool_mode(pool) == PM_OUT_OF_DATA_SPACE && !pool->pf.error_if_no_space) {2457 pool->pf.error_if_no_space = true;2458 notify_of_pool_mode_change(pool);2459 error_retry_list_with_code(pool, BLK_STS_NOSPC);2460 }2461}2462 2463/*----------------------------------------------------------------*/2464 2465struct pool_work {2466 struct work_struct worker;2467 struct completion complete;2468};2469 2470static struct pool_work *to_pool_work(struct work_struct *ws)2471{2472 return container_of(ws, struct pool_work, worker);2473}2474 2475static void pool_work_complete(struct pool_work *pw)2476{2477 complete(&pw->complete);2478}2479 2480static void pool_work_wait(struct pool_work *pw, struct pool *pool,2481 void (*fn)(struct work_struct *))2482{2483 INIT_WORK_ONSTACK(&pw->worker, fn);2484 init_completion(&pw->complete);2485 queue_work(pool->wq, &pw->worker);2486 wait_for_completion(&pw->complete);2487}2488 2489/*----------------------------------------------------------------*/2490 2491struct noflush_work {2492 struct pool_work pw;2493 struct thin_c *tc;2494};2495 2496static struct noflush_work *to_noflush(struct work_struct *ws)2497{2498 return container_of(to_pool_work(ws), struct noflush_work, pw);2499}2500 2501static void do_noflush_start(struct work_struct *ws)2502{2503 struct noflush_work *w = to_noflush(ws);2504 2505 w->tc->requeue_mode = true;2506 requeue_io(w->tc);2507 pool_work_complete(&w->pw);2508}2509 2510static void do_noflush_stop(struct work_struct *ws)2511{2512 struct noflush_work *w = to_noflush(ws);2513 2514 w->tc->requeue_mode = false;2515 pool_work_complete(&w->pw);2516}2517 2518static void noflush_work(struct thin_c *tc, void (*fn)(struct work_struct *))2519{2520 struct noflush_work w;2521 2522 w.tc = tc;2523 pool_work_wait(&w.pw, tc->pool, fn);2524}2525 2526/*----------------------------------------------------------------*/2527 2528static void set_discard_callbacks(struct pool *pool)2529{2530 struct pool_c *pt = pool->ti->private;2531 2532 if (pt->adjusted_pf.discard_passdown) {2533 pool->process_discard_cell = process_discard_cell_passdown;2534 pool->process_prepared_discard = process_prepared_discard_passdown_pt1;2535 pool->process_prepared_discard_pt2 = process_prepared_discard_passdown_pt2;2536 } else {2537 pool->process_discard_cell = process_discard_cell_no_passdown;2538 pool->process_prepared_discard = process_prepared_discard_no_passdown;2539 }2540}2541 2542static void set_pool_mode(struct pool *pool, enum pool_mode new_mode)2543{2544 struct pool_c *pt = pool->ti->private;2545 bool needs_check = dm_pool_metadata_needs_check(pool->pmd);2546 enum pool_mode old_mode = get_pool_mode(pool);2547 unsigned long no_space_timeout = READ_ONCE(no_space_timeout_secs) * HZ;2548 2549 /*2550 * Never allow the pool to transition to PM_WRITE mode if user2551 * intervention is required to verify metadata and data consistency.2552 */2553 if (new_mode == PM_WRITE && needs_check) {2554 DMERR("%s: unable to switch pool to write mode until repaired.",2555 dm_device_name(pool->pool_md));2556 if (old_mode != new_mode)2557 new_mode = old_mode;2558 else2559 new_mode = PM_READ_ONLY;2560 }2561 /*2562 * If we were in PM_FAIL mode, rollback of metadata failed. We're2563 * not going to recover without a thin_repair. So we never let the2564 * pool move out of the old mode.2565 */2566 if (old_mode == PM_FAIL)2567 new_mode = old_mode;2568 2569 switch (new_mode) {2570 case PM_FAIL:2571 dm_pool_metadata_read_only(pool->pmd);2572 pool->process_bio = process_bio_fail;2573 pool->process_discard = process_bio_fail;2574 pool->process_cell = process_cell_fail;2575 pool->process_discard_cell = process_cell_fail;2576 pool->process_prepared_mapping = process_prepared_mapping_fail;2577 pool->process_prepared_discard = process_prepared_discard_fail;2578 2579 error_retry_list(pool);2580 break;2581 2582 case PM_OUT_OF_METADATA_SPACE:2583 case PM_READ_ONLY:2584 dm_pool_metadata_read_only(pool->pmd);2585 pool->process_bio = process_bio_read_only;2586 pool->process_discard = process_bio_success;2587 pool->process_cell = process_cell_read_only;2588 pool->process_discard_cell = process_cell_success;2589 pool->process_prepared_mapping = process_prepared_mapping_fail;2590 pool->process_prepared_discard = process_prepared_discard_success;2591 2592 error_retry_list(pool);2593 break;2594 2595 case PM_OUT_OF_DATA_SPACE:2596 /*2597 * Ideally we'd never hit this state; the low water mark2598 * would trigger userland to extend the pool before we2599 * completely run out of data space. However, many small2600 * IOs to unprovisioned space can consume data space at an2601 * alarming rate. Adjust your low water mark if you're2602 * frequently seeing this mode.2603 */2604 pool->out_of_data_space = true;2605 pool->process_bio = process_bio_read_only;2606 pool->process_discard = process_discard_bio;2607 pool->process_cell = process_cell_read_only;2608 pool->process_prepared_mapping = process_prepared_mapping;2609 set_discard_callbacks(pool);2610 2611 if (!pool->pf.error_if_no_space && no_space_timeout)2612 queue_delayed_work(pool->wq, &pool->no_space_timeout, no_space_timeout);2613 break;2614 2615 case PM_WRITE:2616 if (old_mode == PM_OUT_OF_DATA_SPACE)2617 cancel_delayed_work_sync(&pool->no_space_timeout);2618 pool->out_of_data_space = false;2619 pool->pf.error_if_no_space = pt->requested_pf.error_if_no_space;2620 dm_pool_metadata_read_write(pool->pmd);2621 pool->process_bio = process_bio;2622 pool->process_discard = process_discard_bio;2623 pool->process_cell = process_cell;2624 pool->process_prepared_mapping = process_prepared_mapping;2625 set_discard_callbacks(pool);2626 break;2627 }2628 2629 pool->pf.mode = new_mode;2630 /*2631 * The pool mode may have changed, sync it so bind_control_target()2632 * doesn't cause an unexpected mode transition on resume.2633 */2634 pt->adjusted_pf.mode = new_mode;2635 2636 if (old_mode != new_mode)2637 notify_of_pool_mode_change(pool);2638}2639 2640static void abort_transaction(struct pool *pool)2641{2642 const char *dev_name = dm_device_name(pool->pool_md);2643 2644 DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);2645 if (dm_pool_abort_metadata(pool->pmd)) {2646 DMERR("%s: failed to abort metadata transaction", dev_name);2647 set_pool_mode(pool, PM_FAIL);2648 }2649 2650 if (dm_pool_metadata_set_needs_check(pool->pmd)) {2651 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);2652 set_pool_mode(pool, PM_FAIL);2653 }2654}2655 2656static void metadata_operation_failed(struct pool *pool, const char *op, int r)2657{2658 DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",2659 dm_device_name(pool->pool_md), op, r);2660 2661 abort_transaction(pool);2662 set_pool_mode(pool, PM_READ_ONLY);2663}2664 2665/*----------------------------------------------------------------*/2666 2667/*2668 * Mapping functions.2669 */2670 2671/*2672 * Called only while mapping a thin bio to hand it over to the workqueue.2673 */2674static void thin_defer_bio(struct thin_c *tc, struct bio *bio)2675{2676 struct pool *pool = tc->pool;2677 2678 spin_lock_irq(&tc->lock);2679 bio_list_add(&tc->deferred_bio_list, bio);2680 spin_unlock_irq(&tc->lock);2681 2682 wake_worker(pool);2683}2684 2685static void thin_defer_bio_with_throttle(struct thin_c *tc, struct bio *bio)2686{2687 struct pool *pool = tc->pool;2688 2689 throttle_lock(&pool->throttle);2690 thin_defer_bio(tc, bio);2691 throttle_unlock(&pool->throttle);2692}2693 2694static void thin_defer_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)2695{2696 struct pool *pool = tc->pool;2697 2698 throttle_lock(&pool->throttle);2699 spin_lock_irq(&tc->lock);2700 list_add_tail(&cell->user_list, &tc->deferred_cells);2701 spin_unlock_irq(&tc->lock);2702 throttle_unlock(&pool->throttle);2703 2704 wake_worker(pool);2705}2706 2707static void thin_hook_bio(struct thin_c *tc, struct bio *bio)2708{2709 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));2710 2711 h->tc = tc;2712 h->shared_read_entry = NULL;2713 h->all_io_entry = NULL;2714 h->overwrite_mapping = NULL;2715 h->cell = NULL;2716}2717 2718/*2719 * Non-blocking function called from the thin target's map function.2720 */2721static int thin_bio_map(struct dm_target *ti, struct bio *bio)2722{2723 int r;2724 struct thin_c *tc = ti->private;2725 dm_block_t block = get_bio_block(tc, bio);2726 struct dm_thin_device *td = tc->td;2727 struct dm_thin_lookup_result result;2728 struct dm_bio_prison_cell *virt_cell, *data_cell;2729 struct dm_cell_key key;2730 2731 thin_hook_bio(tc, bio);2732 2733 if (tc->requeue_mode) {2734 bio->bi_status = BLK_STS_DM_REQUEUE;2735 bio_endio(bio);2736 return DM_MAPIO_SUBMITTED;2737 }2738 2739 if (get_pool_mode(tc->pool) == PM_FAIL) {2740 bio_io_error(bio);2741 return DM_MAPIO_SUBMITTED;2742 }2743 2744 if (op_is_flush(bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD) {2745 thin_defer_bio_with_throttle(tc, bio);2746 return DM_MAPIO_SUBMITTED;2747 }2748 2749 /*2750 * We must hold the virtual cell before doing the lookup, otherwise2751 * there's a race with discard.2752 */2753 build_virtual_key(tc->td, block, &key);2754 if (bio_detain(tc->pool, &key, bio, &virt_cell))2755 return DM_MAPIO_SUBMITTED;2756 2757 r = dm_thin_find_block(td, block, 0, &result);2758 2759 /*2760 * Note that we defer readahead too.2761 */2762 switch (r) {2763 case 0:2764 if (unlikely(result.shared)) {2765 /*2766 * We have a race condition here between the2767 * result.shared value returned by the lookup and2768 * snapshot creation, which may cause new2769 * sharing.2770 *2771 * To avoid this always quiesce the origin before2772 * taking the snap. You want to do this anyway to2773 * ensure a consistent application view2774 * (i.e. lockfs).2775 *2776 * More distant ancestors are irrelevant. The2777 * shared flag will be set in their case.2778 */2779 thin_defer_cell(tc, virt_cell);2780 return DM_MAPIO_SUBMITTED;2781 }2782 2783 build_data_key(tc->td, result.block, &key);2784 if (bio_detain(tc->pool, &key, bio, &data_cell)) {2785 cell_defer_no_holder(tc, virt_cell);2786 return DM_MAPIO_SUBMITTED;2787 }2788 2789 inc_all_io_entry(tc->pool, bio);2790 cell_defer_no_holder(tc, data_cell);2791 cell_defer_no_holder(tc, virt_cell);2792 2793 remap(tc, bio, result.block);2794 return DM_MAPIO_REMAPPED;2795 2796 case -ENODATA:2797 case -EWOULDBLOCK:2798 thin_defer_cell(tc, virt_cell);2799 return DM_MAPIO_SUBMITTED;2800 2801 default:2802 /*2803 * Must always call bio_io_error on failure.2804 * dm_thin_find_block can fail with -EINVAL if the2805 * pool is switched to fail-io mode.2806 */2807 bio_io_error(bio);2808 cell_defer_no_holder(tc, virt_cell);2809 return DM_MAPIO_SUBMITTED;2810 }2811}2812 2813static void requeue_bios(struct pool *pool)2814{2815 struct thin_c *tc;2816 2817 rcu_read_lock();2818 list_for_each_entry_rcu(tc, &pool->active_thins, list) {2819 spin_lock_irq(&tc->lock);2820 bio_list_merge(&tc->deferred_bio_list, &tc->retry_on_resume_list);2821 bio_list_init(&tc->retry_on_resume_list);2822 spin_unlock_irq(&tc->lock);2823 }2824 rcu_read_unlock();2825}2826 2827/*2828 *--------------------------------------------------------------2829 * Binding of control targets to a pool object2830 *--------------------------------------------------------------2831 */2832static bool is_factor(sector_t block_size, uint32_t n)2833{2834 return !sector_div(block_size, n);2835}2836 2837/*2838 * If discard_passdown was enabled verify that the data device2839 * supports discards. Disable discard_passdown if not.2840 */2841static void disable_discard_passdown_if_not_supported(struct pool_c *pt)2842{2843 struct pool *pool = pt->pool;2844 struct block_device *data_bdev = pt->data_dev->bdev;2845 struct queue_limits *data_limits = &bdev_get_queue(data_bdev)->limits;2846 const char *reason = NULL;2847 2848 if (!pt->adjusted_pf.discard_passdown)2849 return;2850 2851 if (!bdev_max_discard_sectors(pt->data_dev->bdev))2852 reason = "discard unsupported";2853 2854 else if (data_limits->max_discard_sectors < pool->sectors_per_block)2855 reason = "max discard sectors smaller than a block";2856 2857 if (reason) {2858 DMWARN("Data device (%pg) %s: Disabling discard passdown.", data_bdev, reason);2859 pt->adjusted_pf.discard_passdown = false;2860 }2861}2862 2863static int bind_control_target(struct pool *pool, struct dm_target *ti)2864{2865 struct pool_c *pt = ti->private;2866 2867 /*2868 * We want to make sure that a pool in PM_FAIL mode is never upgraded.2869 */2870 enum pool_mode old_mode = get_pool_mode(pool);2871 enum pool_mode new_mode = pt->adjusted_pf.mode;2872 2873 /*2874 * Don't change the pool's mode until set_pool_mode() below.2875 * Otherwise the pool's process_* function pointers may2876 * not match the desired pool mode.2877 */2878 pt->adjusted_pf.mode = old_mode;2879 2880 pool->ti = ti;2881 pool->pf = pt->adjusted_pf;2882 pool->low_water_blocks = pt->low_water_blocks;2883 2884 set_pool_mode(pool, new_mode);2885 2886 return 0;2887}2888 2889static void unbind_control_target(struct pool *pool, struct dm_target *ti)2890{2891 if (pool->ti == ti)2892 pool->ti = NULL;2893}2894 2895/*2896 *--------------------------------------------------------------2897 * Pool creation2898 *--------------------------------------------------------------2899 */2900/* Initialize pool features. */2901static void pool_features_init(struct pool_features *pf)2902{2903 pf->mode = PM_WRITE;2904 pf->zero_new_blocks = true;2905 pf->discard_enabled = true;2906 pf->discard_passdown = true;2907 pf->error_if_no_space = false;2908}2909 2910static void __pool_destroy(struct pool *pool)2911{2912 __pool_table_remove(pool);2913 2914 vfree(pool->cell_sort_array);2915 if (dm_pool_metadata_close(pool->pmd) < 0)2916 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);2917 2918 dm_bio_prison_destroy(pool->prison);2919 dm_kcopyd_client_destroy(pool->copier);2920 2921 cancel_delayed_work_sync(&pool->waker);2922 cancel_delayed_work_sync(&pool->no_space_timeout);2923 if (pool->wq)2924 destroy_workqueue(pool->wq);2925 2926 if (pool->next_mapping)2927 mempool_free(pool->next_mapping, &pool->mapping_pool);2928 mempool_exit(&pool->mapping_pool);2929 dm_deferred_set_destroy(pool->shared_read_ds);2930 dm_deferred_set_destroy(pool->all_io_ds);2931 kfree(pool);2932}2933 2934static struct kmem_cache *_new_mapping_cache;2935 2936static struct pool *pool_create(struct mapped_device *pool_md,2937 struct block_device *metadata_dev,2938 struct block_device *data_dev,2939 unsigned long block_size,2940 int read_only, char **error)2941{2942 int r;2943 void *err_p;2944 struct pool *pool;2945 struct dm_pool_metadata *pmd;2946 bool format_device = read_only ? false : true;2947 2948 pmd = dm_pool_metadata_open(metadata_dev, block_size, format_device);2949 if (IS_ERR(pmd)) {2950 *error = "Error creating metadata object";2951 return ERR_CAST(pmd);2952 }2953 2954 pool = kzalloc(sizeof(*pool), GFP_KERNEL);2955 if (!pool) {2956 *error = "Error allocating memory for pool";2957 err_p = ERR_PTR(-ENOMEM);2958 goto bad_pool;2959 }2960 2961 pool->pmd = pmd;2962 pool->sectors_per_block = block_size;2963 if (block_size & (block_size - 1))2964 pool->sectors_per_block_shift = -1;2965 else2966 pool->sectors_per_block_shift = __ffs(block_size);2967 pool->low_water_blocks = 0;2968 pool_features_init(&pool->pf);2969 pool->prison = dm_bio_prison_create();2970 if (!pool->prison) {2971 *error = "Error creating pool's bio prison";2972 err_p = ERR_PTR(-ENOMEM);2973 goto bad_prison;2974 }2975 2976 pool->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);2977 if (IS_ERR(pool->copier)) {2978 r = PTR_ERR(pool->copier);2979 *error = "Error creating pool's kcopyd client";2980 err_p = ERR_PTR(r);2981 goto bad_kcopyd_client;2982 }2983 2984 /*2985 * Create singlethreaded workqueue that will service all devices2986 * that use this metadata.2987 */2988 pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);2989 if (!pool->wq) {2990 *error = "Error creating pool's workqueue";2991 err_p = ERR_PTR(-ENOMEM);2992 goto bad_wq;2993 }2994 2995 throttle_init(&pool->throttle);2996 INIT_WORK(&pool->worker, do_worker);2997 INIT_DELAYED_WORK(&pool->waker, do_waker);2998 INIT_DELAYED_WORK(&pool->no_space_timeout, do_no_space_timeout);2999 spin_lock_init(&pool->lock);3000 bio_list_init(&pool->deferred_flush_bios);3001 bio_list_init(&pool->deferred_flush_completions);3002 INIT_LIST_HEAD(&pool->prepared_mappings);3003 INIT_LIST_HEAD(&pool->prepared_discards);3004 INIT_LIST_HEAD(&pool->prepared_discards_pt2);3005 INIT_LIST_HEAD(&pool->active_thins);3006 pool->low_water_triggered = false;3007 pool->suspended = true;3008 pool->out_of_data_space = false;3009 3010 pool->shared_read_ds = dm_deferred_set_create();3011 if (!pool->shared_read_ds) {3012 *error = "Error creating pool's shared read deferred set";3013 err_p = ERR_PTR(-ENOMEM);3014 goto bad_shared_read_ds;3015 }3016 3017 pool->all_io_ds = dm_deferred_set_create();3018 if (!pool->all_io_ds) {3019 *error = "Error creating pool's all io deferred set";3020 err_p = ERR_PTR(-ENOMEM);3021 goto bad_all_io_ds;3022 }3023 3024 pool->next_mapping = NULL;3025 r = mempool_init_slab_pool(&pool->mapping_pool, MAPPING_POOL_SIZE,3026 _new_mapping_cache);3027 if (r) {3028 *error = "Error creating pool's mapping mempool";3029 err_p = ERR_PTR(r);3030 goto bad_mapping_pool;3031 }3032 3033 pool->cell_sort_array =3034 vmalloc(array_size(CELL_SORT_ARRAY_SIZE,3035 sizeof(*pool->cell_sort_array)));3036 if (!pool->cell_sort_array) {3037 *error = "Error allocating cell sort array";3038 err_p = ERR_PTR(-ENOMEM);3039 goto bad_sort_array;3040 }3041 3042 pool->ref_count = 1;3043 pool->last_commit_jiffies = jiffies;3044 pool->pool_md = pool_md;3045 pool->md_dev = metadata_dev;3046 pool->data_dev = data_dev;3047 __pool_table_insert(pool);3048 3049 return pool;3050 3051bad_sort_array:3052 mempool_exit(&pool->mapping_pool);3053bad_mapping_pool:3054 dm_deferred_set_destroy(pool->all_io_ds);3055bad_all_io_ds:3056 dm_deferred_set_destroy(pool->shared_read_ds);3057bad_shared_read_ds:3058 destroy_workqueue(pool->wq);3059bad_wq:3060 dm_kcopyd_client_destroy(pool->copier);3061bad_kcopyd_client:3062 dm_bio_prison_destroy(pool->prison);3063bad_prison:3064 kfree(pool);3065bad_pool:3066 if (dm_pool_metadata_close(pmd))3067 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);3068 3069 return err_p;3070}3071 3072static void __pool_inc(struct pool *pool)3073{3074 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));3075 pool->ref_count++;3076}3077 3078static void __pool_dec(struct pool *pool)3079{3080 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));3081 BUG_ON(!pool->ref_count);3082 if (!--pool->ref_count)3083 __pool_destroy(pool);3084}3085 3086static struct pool *__pool_find(struct mapped_device *pool_md,3087 struct block_device *metadata_dev,3088 struct block_device *data_dev,3089 unsigned long block_size, int read_only,3090 char **error, int *created)3091{3092 struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev);3093 3094 if (pool) {3095 if (pool->pool_md != pool_md) {3096 *error = "metadata device already in use by a pool";3097 return ERR_PTR(-EBUSY);3098 }3099 if (pool->data_dev != data_dev) {3100 *error = "data device already in use by a pool";3101 return ERR_PTR(-EBUSY);3102 }3103 __pool_inc(pool);3104 3105 } else {3106 pool = __pool_table_lookup(pool_md);3107 if (pool) {3108 if (pool->md_dev != metadata_dev || pool->data_dev != data_dev) {3109 *error = "different pool cannot replace a pool";3110 return ERR_PTR(-EINVAL);3111 }3112 __pool_inc(pool);3113 3114 } else {3115 pool = pool_create(pool_md, metadata_dev, data_dev, block_size, read_only, error);3116 *created = 1;3117 }3118 }3119 3120 return pool;3121}3122 3123/*3124 *--------------------------------------------------------------3125 * Pool target methods3126 *--------------------------------------------------------------3127 */3128static void pool_dtr(struct dm_target *ti)3129{3130 struct pool_c *pt = ti->private;3131 3132 mutex_lock(&dm_thin_pool_table.mutex);3133 3134 unbind_control_target(pt->pool, ti);3135 __pool_dec(pt->pool);3136 dm_put_device(ti, pt->metadata_dev);3137 dm_put_device(ti, pt->data_dev);3138 kfree(pt);3139 3140 mutex_unlock(&dm_thin_pool_table.mutex);3141}3142 3143static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf,3144 struct dm_target *ti)3145{3146 int r;3147 unsigned int argc;3148 const char *arg_name;3149 3150 static const struct dm_arg _args[] = {3151 {0, 4, "Invalid number of pool feature arguments"},3152 };3153 3154 /*3155 * No feature arguments supplied.3156 */3157 if (!as->argc)3158 return 0;3159 3160 r = dm_read_arg_group(_args, as, &argc, &ti->error);3161 if (r)3162 return -EINVAL;3163 3164 while (argc && !r) {3165 arg_name = dm_shift_arg(as);3166 argc--;3167 3168 if (!strcasecmp(arg_name, "skip_block_zeroing"))3169 pf->zero_new_blocks = false;3170 3171 else if (!strcasecmp(arg_name, "ignore_discard"))3172 pf->discard_enabled = false;3173 3174 else if (!strcasecmp(arg_name, "no_discard_passdown"))3175 pf->discard_passdown = false;3176 3177 else if (!strcasecmp(arg_name, "read_only"))3178 pf->mode = PM_READ_ONLY;3179 3180 else if (!strcasecmp(arg_name, "error_if_no_space"))3181 pf->error_if_no_space = true;3182 3183 else {3184 ti->error = "Unrecognised pool feature requested";3185 r = -EINVAL;3186 break;3187 }3188 }3189 3190 return r;3191}3192 3193static void metadata_low_callback(void *context)3194{3195 struct pool *pool = context;3196 3197 DMWARN("%s: reached low water mark for metadata device: sending event.",3198 dm_device_name(pool->pool_md));3199 3200 dm_table_event(pool->ti->table);3201}3202 3203/*3204 * We need to flush the data device **before** committing the metadata.3205 *3206 * This ensures that the data blocks of any newly inserted mappings are3207 * properly written to non-volatile storage and won't be lost in case of a3208 * crash.3209 *3210 * Failure to do so can result in data corruption in the case of internal or3211 * external snapshots and in the case of newly provisioned blocks, when block3212 * zeroing is enabled.3213 */3214static int metadata_pre_commit_callback(void *context)3215{3216 struct pool *pool = context;3217 3218 return blkdev_issue_flush(pool->data_dev);3219}3220 3221static sector_t get_dev_size(struct block_device *bdev)3222{3223 return bdev_nr_sectors(bdev);3224}3225 3226static void warn_if_metadata_device_too_big(struct block_device *bdev)3227{3228 sector_t metadata_dev_size = get_dev_size(bdev);3229 3230 if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING)3231 DMWARN("Metadata device %pg is larger than %u sectors: excess space will not be used.",3232 bdev, THIN_METADATA_MAX_SECTORS);3233}3234 3235static sector_t get_metadata_dev_size(struct block_device *bdev)3236{3237 sector_t metadata_dev_size = get_dev_size(bdev);3238 3239 if (metadata_dev_size > THIN_METADATA_MAX_SECTORS)3240 metadata_dev_size = THIN_METADATA_MAX_SECTORS;3241 3242 return metadata_dev_size;3243}3244 3245static dm_block_t get_metadata_dev_size_in_blocks(struct block_device *bdev)3246{3247 sector_t metadata_dev_size = get_metadata_dev_size(bdev);3248 3249 sector_div(metadata_dev_size, THIN_METADATA_BLOCK_SIZE);3250 3251 return metadata_dev_size;3252}3253 3254/*3255 * When a metadata threshold is crossed a dm event is triggered, and3256 * userland should respond by growing the metadata device. We could let3257 * userland set the threshold, like we do with the data threshold, but I'm3258 * not sure they know enough to do this well.3259 */3260static dm_block_t calc_metadata_threshold(struct pool_c *pt)3261{3262 /*3263 * 4M is ample for all ops with the possible exception of thin3264 * device deletion which is harmless if it fails (just retry the3265 * delete after you've grown the device).3266 */3267 dm_block_t quarter = get_metadata_dev_size_in_blocks(pt->metadata_dev->bdev) / 4;3268 3269 return min((dm_block_t)1024ULL /* 4M */, quarter);3270}3271 3272/*3273 * thin-pool <metadata dev> <data dev>3274 * <data block size (sectors)>3275 * <low water mark (blocks)>3276 * [<#feature args> [<arg>]*]3277 *3278 * Optional feature arguments are:3279 * skip_block_zeroing: skips the zeroing of newly-provisioned blocks.3280 * ignore_discard: disable discard3281 * no_discard_passdown: don't pass discards down to the data device3282 * read_only: Don't allow any changes to be made to the pool metadata.3283 * error_if_no_space: error IOs, instead of queueing, if no space.3284 */3285static int pool_ctr(struct dm_target *ti, unsigned int argc, char **argv)3286{3287 int r, pool_created = 0;3288 struct pool_c *pt;3289 struct pool *pool;3290 struct pool_features pf;3291 struct dm_arg_set as;3292 struct dm_dev *data_dev;3293 unsigned long block_size;3294 dm_block_t low_water_blocks;3295 struct dm_dev *metadata_dev;3296 blk_mode_t metadata_mode;3297 3298 /*3299 * FIXME Remove validation from scope of lock.3300 */3301 mutex_lock(&dm_thin_pool_table.mutex);3302 3303 if (argc < 4) {3304 ti->error = "Invalid argument count";3305 r = -EINVAL;3306 goto out_unlock;3307 }3308 3309 as.argc = argc;3310 as.argv = argv;3311 3312 /* make sure metadata and data are different devices */3313 if (!strcmp(argv[0], argv[1])) {3314 ti->error = "Error setting metadata or data device";3315 r = -EINVAL;3316 goto out_unlock;3317 }3318 3319 /*3320 * Set default pool features.3321 */3322 pool_features_init(&pf);3323 3324 dm_consume_args(&as, 4);3325 r = parse_pool_features(&as, &pf, ti);3326 if (r)3327 goto out_unlock;3328 3329 metadata_mode = BLK_OPEN_READ |3330 ((pf.mode == PM_READ_ONLY) ? 0 : BLK_OPEN_WRITE);3331 r = dm_get_device(ti, argv[0], metadata_mode, &metadata_dev);3332 if (r) {3333 ti->error = "Error opening metadata block device";3334 goto out_unlock;3335 }3336 warn_if_metadata_device_too_big(metadata_dev->bdev);3337 3338 r = dm_get_device(ti, argv[1], BLK_OPEN_READ | BLK_OPEN_WRITE, &data_dev);3339 if (r) {3340 ti->error = "Error getting data device";3341 goto out_metadata;3342 }3343 3344 if (kstrtoul(argv[2], 10, &block_size) || !block_size ||3345 block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||3346 block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||3347 block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {3348 ti->error = "Invalid block size";3349 r = -EINVAL;3350 goto out;3351 }3352 3353 if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) {3354 ti->error = "Invalid low water mark";3355 r = -EINVAL;3356 goto out;3357 }3358 3359 pt = kzalloc(sizeof(*pt), GFP_KERNEL);3360 if (!pt) {3361 r = -ENOMEM;3362 goto out;3363 }3364 3365 pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev, data_dev->bdev,3366 block_size, pf.mode == PM_READ_ONLY, &ti->error, &pool_created);3367 if (IS_ERR(pool)) {3368 r = PTR_ERR(pool);3369 goto out_free_pt;3370 }3371 3372 /*3373 * 'pool_created' reflects whether this is the first table load.3374 * Top level discard support is not allowed to be changed after3375 * initial load. This would require a pool reload to trigger thin3376 * device changes.3377 */3378 if (!pool_created && pf.discard_enabled != pool->pf.discard_enabled) {3379 ti->error = "Discard support cannot be disabled once enabled";3380 r = -EINVAL;3381 goto out_flags_changed;3382 }3383 3384 pt->pool = pool;3385 pt->ti = ti;3386 pt->metadata_dev = metadata_dev;3387 pt->data_dev = data_dev;3388 pt->low_water_blocks = low_water_blocks;3389 pt->adjusted_pf = pt->requested_pf = pf;3390 ti->num_flush_bios = 1;3391 ti->limit_swap_bios = true;3392 3393 /*3394 * Only need to enable discards if the pool should pass3395 * them down to the data device. The thin device's discard3396 * processing will cause mappings to be removed from the btree.3397 */3398 if (pf.discard_enabled && pf.discard_passdown) {3399 ti->num_discard_bios = 1;3400 /*3401 * Setting 'discards_supported' circumvents the normal3402 * stacking of discard limits (this keeps the pool and3403 * thin devices' discard limits consistent).3404 */3405 ti->discards_supported = true;3406 ti->max_discard_granularity = true;3407 }3408 ti->private = pt;3409 3410 r = dm_pool_register_metadata_threshold(pt->pool->pmd,3411 calc_metadata_threshold(pt),3412 metadata_low_callback,3413 pool);3414 if (r) {3415 ti->error = "Error registering metadata threshold";3416 goto out_flags_changed;3417 }3418 3419 dm_pool_register_pre_commit_callback(pool->pmd,3420 metadata_pre_commit_callback, pool);3421 3422 mutex_unlock(&dm_thin_pool_table.mutex);3423 3424 return 0;3425 3426out_flags_changed:3427 __pool_dec(pool);3428out_free_pt:3429 kfree(pt);3430out:3431 dm_put_device(ti, data_dev);3432out_metadata:3433 dm_put_device(ti, metadata_dev);3434out_unlock:3435 mutex_unlock(&dm_thin_pool_table.mutex);3436 3437 return r;3438}3439 3440static int pool_map(struct dm_target *ti, struct bio *bio)3441{3442 struct pool_c *pt = ti->private;3443 struct pool *pool = pt->pool;3444 3445 /*3446 * As this is a singleton target, ti->begin is always zero.3447 */3448 spin_lock_irq(&pool->lock);3449 bio_set_dev(bio, pt->data_dev->bdev);3450 spin_unlock_irq(&pool->lock);3451 3452 return DM_MAPIO_REMAPPED;3453}3454 3455static int maybe_resize_data_dev(struct dm_target *ti, bool *need_commit)3456{3457 int r;3458 struct pool_c *pt = ti->private;3459 struct pool *pool = pt->pool;3460 sector_t data_size = ti->len;3461 dm_block_t sb_data_size;3462 3463 *need_commit = false;3464 3465 (void) sector_div(data_size, pool->sectors_per_block);3466 3467 r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size);3468 if (r) {3469 DMERR("%s: failed to retrieve data device size",3470 dm_device_name(pool->pool_md));3471 return r;3472 }3473 3474 if (data_size < sb_data_size) {3475 DMERR("%s: pool target (%llu blocks) too small: expected %llu",3476 dm_device_name(pool->pool_md),3477 (unsigned long long)data_size, sb_data_size);3478 return -EINVAL;3479 3480 } else if (data_size > sb_data_size) {3481 if (dm_pool_metadata_needs_check(pool->pmd)) {3482 DMERR("%s: unable to grow the data device until repaired.",3483 dm_device_name(pool->pool_md));3484 return 0;3485 }3486 3487 if (sb_data_size)3488 DMINFO("%s: growing the data device from %llu to %llu blocks",3489 dm_device_name(pool->pool_md),3490 sb_data_size, (unsigned long long)data_size);3491 r = dm_pool_resize_data_dev(pool->pmd, data_size);3492 if (r) {3493 metadata_operation_failed(pool, "dm_pool_resize_data_dev", r);3494 return r;3495 }3496 3497 *need_commit = true;3498 }3499 3500 return 0;3501}3502 3503static int maybe_resize_metadata_dev(struct dm_target *ti, bool *need_commit)3504{3505 int r;3506 struct pool_c *pt = ti->private;3507 struct pool *pool = pt->pool;3508 dm_block_t metadata_dev_size, sb_metadata_dev_size;3509 3510 *need_commit = false;3511 3512 metadata_dev_size = get_metadata_dev_size_in_blocks(pool->md_dev);3513 3514 r = dm_pool_get_metadata_dev_size(pool->pmd, &sb_metadata_dev_size);3515 if (r) {3516 DMERR("%s: failed to retrieve metadata device size",3517 dm_device_name(pool->pool_md));3518 return r;3519 }3520 3521 if (metadata_dev_size < sb_metadata_dev_size) {3522 DMERR("%s: metadata device (%llu blocks) too small: expected %llu",3523 dm_device_name(pool->pool_md),3524 metadata_dev_size, sb_metadata_dev_size);3525 return -EINVAL;3526 3527 } else if (metadata_dev_size > sb_metadata_dev_size) {3528 if (dm_pool_metadata_needs_check(pool->pmd)) {3529 DMERR("%s: unable to grow the metadata device until repaired.",3530 dm_device_name(pool->pool_md));3531 return 0;3532 }3533 3534 warn_if_metadata_device_too_big(pool->md_dev);3535 DMINFO("%s: growing the metadata device from %llu to %llu blocks",3536 dm_device_name(pool->pool_md),3537 sb_metadata_dev_size, metadata_dev_size);3538 3539 if (get_pool_mode(pool) == PM_OUT_OF_METADATA_SPACE)3540 set_pool_mode(pool, PM_WRITE);3541 3542 r = dm_pool_resize_metadata_dev(pool->pmd, metadata_dev_size);3543 if (r) {3544 metadata_operation_failed(pool, "dm_pool_resize_metadata_dev", r);3545 return r;3546 }3547 3548 *need_commit = true;3549 }3550 3551 return 0;3552}3553 3554/*3555 * Retrieves the number of blocks of the data device from3556 * the superblock and compares it to the actual device size,3557 * thus resizing the data device in case it has grown.3558 *3559 * This both copes with opening preallocated data devices in the ctr3560 * being followed by a resume3561 * -and-3562 * calling the resume method individually after userspace has3563 * grown the data device in reaction to a table event.3564 */3565static int pool_preresume(struct dm_target *ti)3566{3567 int r;3568 bool need_commit1, need_commit2;3569 struct pool_c *pt = ti->private;3570 struct pool *pool = pt->pool;3571 3572 /*3573 * Take control of the pool object.3574 */3575 r = bind_control_target(pool, ti);3576 if (r)3577 goto out;3578 3579 r = maybe_resize_data_dev(ti, &need_commit1);3580 if (r)3581 goto out;3582 3583 r = maybe_resize_metadata_dev(ti, &need_commit2);3584 if (r)3585 goto out;3586 3587 if (need_commit1 || need_commit2)3588 (void) commit(pool);3589out:3590 /*3591 * When a thin-pool is PM_FAIL, it cannot be rebuilt if3592 * bio is in deferred list. Therefore need to return 03593 * to allow pool_resume() to flush IO.3594 */3595 if (r && get_pool_mode(pool) == PM_FAIL)3596 r = 0;3597 3598 return r;3599}3600 3601static void pool_suspend_active_thins(struct pool *pool)3602{3603 struct thin_c *tc;3604 3605 /* Suspend all active thin devices */3606 tc = get_first_thin(pool);3607 while (tc) {3608 dm_internal_suspend_noflush(tc->thin_md);3609 tc = get_next_thin(pool, tc);3610 }3611}3612 3613static void pool_resume_active_thins(struct pool *pool)3614{3615 struct thin_c *tc;3616 3617 /* Resume all active thin devices */3618 tc = get_first_thin(pool);3619 while (tc) {3620 dm_internal_resume(tc->thin_md);3621 tc = get_next_thin(pool, tc);3622 }3623}3624 3625static void pool_resume(struct dm_target *ti)3626{3627 struct pool_c *pt = ti->private;3628 struct pool *pool = pt->pool;3629 3630 /*3631 * Must requeue active_thins' bios and then resume3632 * active_thins _before_ clearing 'suspend' flag.3633 */3634 requeue_bios(pool);3635 pool_resume_active_thins(pool);3636 3637 spin_lock_irq(&pool->lock);3638 pool->low_water_triggered = false;3639 pool->suspended = false;3640 spin_unlock_irq(&pool->lock);3641 3642 do_waker(&pool->waker.work);3643}3644 3645static void pool_presuspend(struct dm_target *ti)3646{3647 struct pool_c *pt = ti->private;3648 struct pool *pool = pt->pool;3649 3650 spin_lock_irq(&pool->lock);3651 pool->suspended = true;3652 spin_unlock_irq(&pool->lock);3653 3654 pool_suspend_active_thins(pool);3655}3656 3657static void pool_presuspend_undo(struct dm_target *ti)3658{3659 struct pool_c *pt = ti->private;3660 struct pool *pool = pt->pool;3661 3662 pool_resume_active_thins(pool);3663 3664 spin_lock_irq(&pool->lock);3665 pool->suspended = false;3666 spin_unlock_irq(&pool->lock);3667}3668 3669static void pool_postsuspend(struct dm_target *ti)3670{3671 struct pool_c *pt = ti->private;3672 struct pool *pool = pt->pool;3673 3674 cancel_delayed_work_sync(&pool->waker);3675 cancel_delayed_work_sync(&pool->no_space_timeout);3676 flush_workqueue(pool->wq);3677 (void) commit(pool);3678}3679 3680static int check_arg_count(unsigned int argc, unsigned int args_required)3681{3682 if (argc != args_required) {3683 DMWARN("Message received with %u arguments instead of %u.",3684 argc, args_required);3685 return -EINVAL;3686 }3687 3688 return 0;3689}3690 3691static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning)3692{3693 if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) &&3694 *dev_id <= MAX_DEV_ID)3695 return 0;3696 3697 if (warning)3698 DMWARN("Message received with invalid device id: %s", arg);3699 3700 return -EINVAL;3701}3702 3703static int process_create_thin_mesg(unsigned int argc, char **argv, struct pool *pool)3704{3705 dm_thin_id dev_id;3706 int r;3707 3708 r = check_arg_count(argc, 2);3709 if (r)3710 return r;3711 3712 r = read_dev_id(argv[1], &dev_id, 1);3713 if (r)3714 return r;3715 3716 r = dm_pool_create_thin(pool->pmd, dev_id);3717 if (r) {3718 DMWARN("Creation of new thinly-provisioned device with id %s failed.",3719 argv[1]);3720 return r;3721 }3722 3723 return 0;3724}3725 3726static int process_create_snap_mesg(unsigned int argc, char **argv, struct pool *pool)3727{3728 dm_thin_id dev_id;3729 dm_thin_id origin_dev_id;3730 int r;3731 3732 r = check_arg_count(argc, 3);3733 if (r)3734 return r;3735 3736 r = read_dev_id(argv[1], &dev_id, 1);3737 if (r)3738 return r;3739 3740 r = read_dev_id(argv[2], &origin_dev_id, 1);3741 if (r)3742 return r;3743 3744 r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id);3745 if (r) {3746 DMWARN("Creation of new snapshot %s of device %s failed.",3747 argv[1], argv[2]);3748 return r;3749 }3750 3751 return 0;3752}3753 3754static int process_delete_mesg(unsigned int argc, char **argv, struct pool *pool)3755{3756 dm_thin_id dev_id;3757 int r;3758 3759 r = check_arg_count(argc, 2);3760 if (r)3761 return r;3762 3763 r = read_dev_id(argv[1], &dev_id, 1);3764 if (r)3765 return r;3766 3767 r = dm_pool_delete_thin_device(pool->pmd, dev_id);3768 if (r)3769 DMWARN("Deletion of thin device %s failed.", argv[1]);3770 3771 return r;3772}3773 3774static int process_set_transaction_id_mesg(unsigned int argc, char **argv, struct pool *pool)3775{3776 dm_thin_id old_id, new_id;3777 int r;3778 3779 r = check_arg_count(argc, 3);3780 if (r)3781 return r;3782 3783 if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) {3784 DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]);3785 return -EINVAL;3786 }3787 3788 if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) {3789 DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]);3790 return -EINVAL;3791 }3792 3793 r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id);3794 if (r) {3795 DMWARN("Failed to change transaction id from %s to %s.",3796 argv[1], argv[2]);3797 return r;3798 }3799 3800 return 0;3801}3802 3803static int process_reserve_metadata_snap_mesg(unsigned int argc, char **argv, struct pool *pool)3804{3805 int r;3806 3807 r = check_arg_count(argc, 1);3808 if (r)3809 return r;3810 3811 (void) commit(pool);3812 3813 r = dm_pool_reserve_metadata_snap(pool->pmd);3814 if (r)3815 DMWARN("reserve_metadata_snap message failed.");3816 3817 return r;3818}3819 3820static int process_release_metadata_snap_mesg(unsigned int argc, char **argv, struct pool *pool)3821{3822 int r;3823 3824 r = check_arg_count(argc, 1);3825 if (r)3826 return r;3827 3828 r = dm_pool_release_metadata_snap(pool->pmd);3829 if (r)3830 DMWARN("release_metadata_snap message failed.");3831 3832 return r;3833}3834 3835/*3836 * Messages supported:3837 * create_thin <dev_id>3838 * create_snap <dev_id> <origin_id>3839 * delete <dev_id>3840 * set_transaction_id <current_trans_id> <new_trans_id>3841 * reserve_metadata_snap3842 * release_metadata_snap3843 */3844static int pool_message(struct dm_target *ti, unsigned int argc, char **argv,3845 char *result, unsigned int maxlen)3846{3847 int r = -EINVAL;3848 struct pool_c *pt = ti->private;3849 struct pool *pool = pt->pool;3850 3851 if (get_pool_mode(pool) >= PM_OUT_OF_METADATA_SPACE) {3852 DMERR("%s: unable to service pool target messages in READ_ONLY or FAIL mode",3853 dm_device_name(pool->pool_md));3854 return -EOPNOTSUPP;3855 }3856 3857 if (!strcasecmp(argv[0], "create_thin"))3858 r = process_create_thin_mesg(argc, argv, pool);3859 3860 else if (!strcasecmp(argv[0], "create_snap"))3861 r = process_create_snap_mesg(argc, argv, pool);3862 3863 else if (!strcasecmp(argv[0], "delete"))3864 r = process_delete_mesg(argc, argv, pool);3865 3866 else if (!strcasecmp(argv[0], "set_transaction_id"))3867 r = process_set_transaction_id_mesg(argc, argv, pool);3868 3869 else if (!strcasecmp(argv[0], "reserve_metadata_snap"))3870 r = process_reserve_metadata_snap_mesg(argc, argv, pool);3871 3872 else if (!strcasecmp(argv[0], "release_metadata_snap"))3873 r = process_release_metadata_snap_mesg(argc, argv, pool);3874 3875 else3876 DMWARN("Unrecognised thin pool target message received: %s", argv[0]);3877 3878 if (!r)3879 (void) commit(pool);3880 3881 return r;3882}3883 3884static void emit_flags(struct pool_features *pf, char *result,3885 unsigned int sz, unsigned int maxlen)3886{3887 unsigned int count = !pf->zero_new_blocks + !pf->discard_enabled +3888 !pf->discard_passdown + (pf->mode == PM_READ_ONLY) +3889 pf->error_if_no_space;3890 DMEMIT("%u ", count);3891 3892 if (!pf->zero_new_blocks)3893 DMEMIT("skip_block_zeroing ");3894 3895 if (!pf->discard_enabled)3896 DMEMIT("ignore_discard ");3897 3898 if (!pf->discard_passdown)3899 DMEMIT("no_discard_passdown ");3900 3901 if (pf->mode == PM_READ_ONLY)3902 DMEMIT("read_only ");3903 3904 if (pf->error_if_no_space)3905 DMEMIT("error_if_no_space ");3906}3907 3908/*3909 * Status line is:3910 * <transaction id> <used metadata sectors>/<total metadata sectors>3911 * <used data sectors>/<total data sectors> <held metadata root>3912 * <pool mode> <discard config> <no space config> <needs_check>3913 */3914static void pool_status(struct dm_target *ti, status_type_t type,3915 unsigned int status_flags, char *result, unsigned int maxlen)3916{3917 int r;3918 unsigned int sz = 0;3919 uint64_t transaction_id;3920 dm_block_t nr_free_blocks_data;3921 dm_block_t nr_free_blocks_metadata;3922 dm_block_t nr_blocks_data;3923 dm_block_t nr_blocks_metadata;3924 dm_block_t held_root;3925 enum pool_mode mode;3926 char buf[BDEVNAME_SIZE];3927 char buf2[BDEVNAME_SIZE];3928 struct pool_c *pt = ti->private;3929 struct pool *pool = pt->pool;3930 3931 switch (type) {3932 case STATUSTYPE_INFO:3933 if (get_pool_mode(pool) == PM_FAIL) {3934 DMEMIT("Fail");3935 break;3936 }3937 3938 /* Commit to ensure statistics aren't out-of-date */3939 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))3940 (void) commit(pool);3941 3942 r = dm_pool_get_metadata_transaction_id(pool->pmd, &transaction_id);3943 if (r) {3944 DMERR("%s: dm_pool_get_metadata_transaction_id returned %d",3945 dm_device_name(pool->pool_md), r);3946 goto err;3947 }3948 3949 r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free_blocks_metadata);3950 if (r) {3951 DMERR("%s: dm_pool_get_free_metadata_block_count returned %d",3952 dm_device_name(pool->pool_md), r);3953 goto err;3954 }3955 3956 r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata);3957 if (r) {3958 DMERR("%s: dm_pool_get_metadata_dev_size returned %d",3959 dm_device_name(pool->pool_md), r);3960 goto err;3961 }3962 3963 r = dm_pool_get_free_block_count(pool->pmd, &nr_free_blocks_data);3964 if (r) {3965 DMERR("%s: dm_pool_get_free_block_count returned %d",3966 dm_device_name(pool->pool_md), r);3967 goto err;3968 }3969 3970 r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data);3971 if (r) {3972 DMERR("%s: dm_pool_get_data_dev_size returned %d",3973 dm_device_name(pool->pool_md), r);3974 goto err;3975 }3976 3977 r = dm_pool_get_metadata_snap(pool->pmd, &held_root);3978 if (r) {3979 DMERR("%s: dm_pool_get_metadata_snap returned %d",3980 dm_device_name(pool->pool_md), r);3981 goto err;3982 }3983 3984 DMEMIT("%llu %llu/%llu %llu/%llu ",3985 (unsigned long long)transaction_id,3986 (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),3987 (unsigned long long)nr_blocks_metadata,3988 (unsigned long long)(nr_blocks_data - nr_free_blocks_data),3989 (unsigned long long)nr_blocks_data);3990 3991 if (held_root)3992 DMEMIT("%llu ", held_root);3993 else3994 DMEMIT("- ");3995 3996 mode = get_pool_mode(pool);3997 if (mode == PM_OUT_OF_DATA_SPACE)3998 DMEMIT("out_of_data_space ");3999 else if (is_read_only_pool_mode(mode))4000 DMEMIT("ro ");4001 else4002 DMEMIT("rw ");4003 4004 if (!pool->pf.discard_enabled)4005 DMEMIT("ignore_discard ");4006 else if (pool->pf.discard_passdown)4007 DMEMIT("discard_passdown ");4008 else4009 DMEMIT("no_discard_passdown ");4010 4011 if (pool->pf.error_if_no_space)4012 DMEMIT("error_if_no_space ");4013 else4014 DMEMIT("queue_if_no_space ");4015 4016 if (dm_pool_metadata_needs_check(pool->pmd))4017 DMEMIT("needs_check ");4018 else4019 DMEMIT("- ");4020 4021 DMEMIT("%llu ", (unsigned long long)calc_metadata_threshold(pt));4022 4023 break;4024 4025 case STATUSTYPE_TABLE:4026 DMEMIT("%s %s %lu %llu ",4027 format_dev_t(buf, pt->metadata_dev->bdev->bd_dev),4028 format_dev_t(buf2, pt->data_dev->bdev->bd_dev),4029 (unsigned long)pool->sectors_per_block,4030 (unsigned long long)pt->low_water_blocks);4031 emit_flags(&pt->requested_pf, result, sz, maxlen);4032 break;4033 4034 case STATUSTYPE_IMA:4035 *result = '\0';4036 break;4037 }4038 return;4039 4040err:4041 DMEMIT("Error");4042}4043 4044static int pool_iterate_devices(struct dm_target *ti,4045 iterate_devices_callout_fn fn, void *data)4046{4047 struct pool_c *pt = ti->private;4048 4049 return fn(ti, pt->data_dev, 0, ti->len, data);4050}4051 4052static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits)4053{4054 struct pool_c *pt = ti->private;4055 struct pool *pool = pt->pool;4056 sector_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;4057 4058 /*4059 * If max_sectors is smaller than pool->sectors_per_block adjust it4060 * to the highest possible power-of-2 factor of pool->sectors_per_block.4061 * This is especially beneficial when the pool's data device is a RAID4062 * device that has a full stripe width that matches pool->sectors_per_block4063 * -- because even though partial RAID stripe-sized IOs will be issued to a4064 * single RAID stripe; when aggregated they will end on a full RAID stripe4065 * boundary.. which avoids additional partial RAID stripe writes cascading4066 */4067 if (limits->max_sectors < pool->sectors_per_block) {4068 while (!is_factor(pool->sectors_per_block, limits->max_sectors)) {4069 if ((limits->max_sectors & (limits->max_sectors - 1)) == 0)4070 limits->max_sectors--;4071 limits->max_sectors = rounddown_pow_of_two(limits->max_sectors);4072 }4073 }4074 4075 /*4076 * If the system-determined stacked limits are compatible with the4077 * pool's blocksize (io_opt is a factor) do not override them.4078 */4079 if (io_opt_sectors < pool->sectors_per_block ||4080 !is_factor(io_opt_sectors, pool->sectors_per_block)) {4081 if (is_factor(pool->sectors_per_block, limits->max_sectors))4082 limits->io_min = limits->max_sectors << SECTOR_SHIFT;4083 else4084 limits->io_min = pool->sectors_per_block << SECTOR_SHIFT;4085 limits->io_opt = pool->sectors_per_block << SECTOR_SHIFT;4086 }4087 4088 /*4089 * pt->adjusted_pf is a staging area for the actual features to use.4090 * They get transferred to the live pool in bind_control_target()4091 * called from pool_preresume().4092 */4093 4094 if (pt->adjusted_pf.discard_enabled) {4095 disable_discard_passdown_if_not_supported(pt);4096 if (!pt->adjusted_pf.discard_passdown)4097 limits->max_hw_discard_sectors = 0;4098 /*4099 * The pool uses the same discard limits as the underlying data4100 * device. DM core has already set this up.4101 */4102 } else {4103 /*4104 * Must explicitly disallow stacking discard limits otherwise the4105 * block layer will stack them if pool's data device has support.4106 */4107 limits->discard_granularity = 0;4108 }4109}4110 4111static struct target_type pool_target = {4112 .name = "thin-pool",4113 .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE |4114 DM_TARGET_IMMUTABLE,4115 .version = {1, 23, 0},4116 .module = THIS_MODULE,4117 .ctr = pool_ctr,4118 .dtr = pool_dtr,4119 .map = pool_map,4120 .presuspend = pool_presuspend,4121 .presuspend_undo = pool_presuspend_undo,4122 .postsuspend = pool_postsuspend,4123 .preresume = pool_preresume,4124 .resume = pool_resume,4125 .message = pool_message,4126 .status = pool_status,4127 .iterate_devices = pool_iterate_devices,4128 .io_hints = pool_io_hints,4129};4130 4131/*4132 *--------------------------------------------------------------4133 * Thin target methods4134 *--------------------------------------------------------------4135 */4136static void thin_get(struct thin_c *tc)4137{4138 refcount_inc(&tc->refcount);4139}4140 4141static void thin_put(struct thin_c *tc)4142{4143 if (refcount_dec_and_test(&tc->refcount))4144 complete(&tc->can_destroy);4145}4146 4147static void thin_dtr(struct dm_target *ti)4148{4149 struct thin_c *tc = ti->private;4150 4151 spin_lock_irq(&tc->pool->lock);4152 list_del_rcu(&tc->list);4153 spin_unlock_irq(&tc->pool->lock);4154 synchronize_rcu();4155 4156 thin_put(tc);4157 wait_for_completion(&tc->can_destroy);4158 4159 mutex_lock(&dm_thin_pool_table.mutex);4160 4161 __pool_dec(tc->pool);4162 dm_pool_close_thin_device(tc->td);4163 dm_put_device(ti, tc->pool_dev);4164 if (tc->origin_dev)4165 dm_put_device(ti, tc->origin_dev);4166 kfree(tc);4167 4168 mutex_unlock(&dm_thin_pool_table.mutex);4169}4170 4171/*4172 * Thin target parameters:4173 *4174 * <pool_dev> <dev_id> [origin_dev]4175 *4176 * pool_dev: the path to the pool (eg, /dev/mapper/my_pool)4177 * dev_id: the internal device identifier4178 * origin_dev: a device external to the pool that should act as the origin4179 *4180 * If the pool device has discards disabled, they get disabled for the thin4181 * device as well.4182 */4183static int thin_ctr(struct dm_target *ti, unsigned int argc, char **argv)4184{4185 int r;4186 struct thin_c *tc;4187 struct dm_dev *pool_dev, *origin_dev;4188 struct mapped_device *pool_md;4189 4190 mutex_lock(&dm_thin_pool_table.mutex);4191 4192 if (argc != 2 && argc != 3) {4193 ti->error = "Invalid argument count";4194 r = -EINVAL;4195 goto out_unlock;4196 }4197 4198 tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL);4199 if (!tc) {4200 ti->error = "Out of memory";4201 r = -ENOMEM;4202 goto out_unlock;4203 }4204 tc->thin_md = dm_table_get_md(ti->table);4205 spin_lock_init(&tc->lock);4206 INIT_LIST_HEAD(&tc->deferred_cells);4207 bio_list_init(&tc->deferred_bio_list);4208 bio_list_init(&tc->retry_on_resume_list);4209 tc->sort_bio_list = RB_ROOT;4210 4211 if (argc == 3) {4212 if (!strcmp(argv[0], argv[2])) {4213 ti->error = "Error setting origin device";4214 r = -EINVAL;4215 goto bad_origin_dev;4216 }4217 4218 r = dm_get_device(ti, argv[2], BLK_OPEN_READ, &origin_dev);4219 if (r) {4220 ti->error = "Error opening origin device";4221 goto bad_origin_dev;4222 }4223 tc->origin_dev = origin_dev;4224 }4225 4226 r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev);4227 if (r) {4228 ti->error = "Error opening pool device";4229 goto bad_pool_dev;4230 }4231 tc->pool_dev = pool_dev;4232 4233 if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) {4234 ti->error = "Invalid device id";4235 r = -EINVAL;4236 goto bad_common;4237 }4238 4239 pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev);4240 if (!pool_md) {4241 ti->error = "Couldn't get pool mapped device";4242 r = -EINVAL;4243 goto bad_common;4244 }4245 4246 tc->pool = __pool_table_lookup(pool_md);4247 if (!tc->pool) {4248 ti->error = "Couldn't find pool object";4249 r = -EINVAL;4250 goto bad_pool_lookup;4251 }4252 __pool_inc(tc->pool);4253 4254 if (get_pool_mode(tc->pool) == PM_FAIL) {4255 ti->error = "Couldn't open thin device, Pool is in fail mode";4256 r = -EINVAL;4257 goto bad_pool;4258 }4259 4260 r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td);4261 if (r) {4262 ti->error = "Couldn't open thin internal device";4263 goto bad_pool;4264 }4265 4266 r = dm_set_target_max_io_len(ti, tc->pool->sectors_per_block);4267 if (r)4268 goto bad;4269 4270 ti->num_flush_bios = 1;4271 ti->limit_swap_bios = true;4272 ti->flush_supported = true;4273 ti->accounts_remapped_io = true;4274 ti->per_io_data_size = sizeof(struct dm_thin_endio_hook);4275 4276 /* In case the pool supports discards, pass them on. */4277 if (tc->pool->pf.discard_enabled) {4278 ti->discards_supported = true;4279 ti->num_discard_bios = 1;4280 ti->max_discard_granularity = true;4281 }4282 4283 mutex_unlock(&dm_thin_pool_table.mutex);4284 4285 spin_lock_irq(&tc->pool->lock);4286 if (tc->pool->suspended) {4287 spin_unlock_irq(&tc->pool->lock);4288 mutex_lock(&dm_thin_pool_table.mutex); /* reacquire for __pool_dec */4289 ti->error = "Unable to activate thin device while pool is suspended";4290 r = -EINVAL;4291 goto bad;4292 }4293 refcount_set(&tc->refcount, 1);4294 init_completion(&tc->can_destroy);4295 list_add_tail_rcu(&tc->list, &tc->pool->active_thins);4296 spin_unlock_irq(&tc->pool->lock);4297 /*4298 * This synchronize_rcu() call is needed here otherwise we risk a4299 * wake_worker() call finding no bios to process (because the newly4300 * added tc isn't yet visible). So this reduces latency since we4301 * aren't then dependent on the periodic commit to wake_worker().4302 */4303 synchronize_rcu();4304 4305 dm_put(pool_md);4306 4307 return 0;4308 4309bad:4310 dm_pool_close_thin_device(tc->td);4311bad_pool:4312 __pool_dec(tc->pool);4313bad_pool_lookup:4314 dm_put(pool_md);4315bad_common:4316 dm_put_device(ti, tc->pool_dev);4317bad_pool_dev:4318 if (tc->origin_dev)4319 dm_put_device(ti, tc->origin_dev);4320bad_origin_dev:4321 kfree(tc);4322out_unlock:4323 mutex_unlock(&dm_thin_pool_table.mutex);4324 4325 return r;4326}4327 4328static int thin_map(struct dm_target *ti, struct bio *bio)4329{4330 bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);4331 4332 return thin_bio_map(ti, bio);4333}4334 4335static int thin_endio(struct dm_target *ti, struct bio *bio,4336 blk_status_t *err)4337{4338 unsigned long flags;4339 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));4340 struct list_head work;4341 struct dm_thin_new_mapping *m, *tmp;4342 struct pool *pool = h->tc->pool;4343 4344 if (h->shared_read_entry) {4345 INIT_LIST_HEAD(&work);4346 dm_deferred_entry_dec(h->shared_read_entry, &work);4347 4348 spin_lock_irqsave(&pool->lock, flags);4349 list_for_each_entry_safe(m, tmp, &work, list) {4350 list_del(&m->list);4351 __complete_mapping_preparation(m);4352 }4353 spin_unlock_irqrestore(&pool->lock, flags);4354 }4355 4356 if (h->all_io_entry) {4357 INIT_LIST_HEAD(&work);4358 dm_deferred_entry_dec(h->all_io_entry, &work);4359 if (!list_empty(&work)) {4360 spin_lock_irqsave(&pool->lock, flags);4361 list_for_each_entry_safe(m, tmp, &work, list)4362 list_add_tail(&m->list, &pool->prepared_discards);4363 spin_unlock_irqrestore(&pool->lock, flags);4364 wake_worker(pool);4365 }4366 }4367 4368 if (h->cell)4369 cell_defer_no_holder(h->tc, h->cell);4370 4371 return DM_ENDIO_DONE;4372}4373 4374static void thin_presuspend(struct dm_target *ti)4375{4376 struct thin_c *tc = ti->private;4377 4378 if (dm_noflush_suspending(ti))4379 noflush_work(tc, do_noflush_start);4380}4381 4382static void thin_postsuspend(struct dm_target *ti)4383{4384 struct thin_c *tc = ti->private;4385 4386 /*4387 * The dm_noflush_suspending flag has been cleared by now, so4388 * unfortunately we must always run this.4389 */4390 noflush_work(tc, do_noflush_stop);4391}4392 4393static int thin_preresume(struct dm_target *ti)4394{4395 struct thin_c *tc = ti->private;4396 4397 if (tc->origin_dev)4398 tc->origin_size = get_dev_size(tc->origin_dev->bdev);4399 4400 return 0;4401}4402 4403/*4404 * <nr mapped sectors> <highest mapped sector>4405 */4406static void thin_status(struct dm_target *ti, status_type_t type,4407 unsigned int status_flags, char *result, unsigned int maxlen)4408{4409 int r;4410 ssize_t sz = 0;4411 dm_block_t mapped, highest;4412 char buf[BDEVNAME_SIZE];4413 struct thin_c *tc = ti->private;4414 4415 if (get_pool_mode(tc->pool) == PM_FAIL) {4416 DMEMIT("Fail");4417 return;4418 }4419 4420 if (!tc->td)4421 DMEMIT("-");4422 else {4423 switch (type) {4424 case STATUSTYPE_INFO:4425 r = dm_thin_get_mapped_count(tc->td, &mapped);4426 if (r) {4427 DMERR("dm_thin_get_mapped_count returned %d", r);4428 goto err;4429 }4430 4431 r = dm_thin_get_highest_mapped_block(tc->td, &highest);4432 if (r < 0) {4433 DMERR("dm_thin_get_highest_mapped_block returned %d", r);4434 goto err;4435 }4436 4437 DMEMIT("%llu ", mapped * tc->pool->sectors_per_block);4438 if (r)4439 DMEMIT("%llu", ((highest + 1) *4440 tc->pool->sectors_per_block) - 1);4441 else4442 DMEMIT("-");4443 break;4444 4445 case STATUSTYPE_TABLE:4446 DMEMIT("%s %lu",4447 format_dev_t(buf, tc->pool_dev->bdev->bd_dev),4448 (unsigned long) tc->dev_id);4449 if (tc->origin_dev)4450 DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev));4451 break;4452 4453 case STATUSTYPE_IMA:4454 *result = '\0';4455 break;4456 }4457 }4458 4459 return;4460 4461err:4462 DMEMIT("Error");4463}4464 4465static int thin_iterate_devices(struct dm_target *ti,4466 iterate_devices_callout_fn fn, void *data)4467{4468 sector_t blocks;4469 struct thin_c *tc = ti->private;4470 struct pool *pool = tc->pool;4471 4472 /*4473 * We can't call dm_pool_get_data_dev_size() since that blocks. So4474 * we follow a more convoluted path through to the pool's target.4475 */4476 if (!pool->ti)4477 return 0; /* nothing is bound */4478 4479 blocks = pool->ti->len;4480 (void) sector_div(blocks, pool->sectors_per_block);4481 if (blocks)4482 return fn(ti, tc->pool_dev, 0, pool->sectors_per_block * blocks, data);4483 4484 return 0;4485}4486 4487static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits)4488{4489 struct thin_c *tc = ti->private;4490 struct pool *pool = tc->pool;4491 4492 if (pool->pf.discard_enabled) {4493 limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT;4494 limits->max_hw_discard_sectors = pool->sectors_per_block * BIO_PRISON_MAX_RANGE;4495 }4496}4497 4498static struct target_type thin_target = {4499 .name = "thin",4500 .version = {1, 23, 0},4501 .module = THIS_MODULE,4502 .ctr = thin_ctr,4503 .dtr = thin_dtr,4504 .map = thin_map,4505 .end_io = thin_endio,4506 .preresume = thin_preresume,4507 .presuspend = thin_presuspend,4508 .postsuspend = thin_postsuspend,4509 .status = thin_status,4510 .iterate_devices = thin_iterate_devices,4511 .io_hints = thin_io_hints,4512};4513 4514/*----------------------------------------------------------------*/4515 4516static int __init dm_thin_init(void)4517{4518 int r = -ENOMEM;4519 4520 pool_table_init();4521 4522 _new_mapping_cache = KMEM_CACHE(dm_thin_new_mapping, 0);4523 if (!_new_mapping_cache)4524 return r;4525 4526 r = dm_register_target(&thin_target);4527 if (r)4528 goto bad_new_mapping_cache;4529 4530 r = dm_register_target(&pool_target);4531 if (r)4532 goto bad_thin_target;4533 4534 return 0;4535 4536bad_thin_target:4537 dm_unregister_target(&thin_target);4538bad_new_mapping_cache:4539 kmem_cache_destroy(_new_mapping_cache);4540 4541 return r;4542}4543 4544static void dm_thin_exit(void)4545{4546 dm_unregister_target(&thin_target);4547 dm_unregister_target(&pool_target);4548 4549 kmem_cache_destroy(_new_mapping_cache);4550 4551 pool_table_exit();4552}4553 4554module_init(dm_thin_init);4555module_exit(dm_thin_exit);4556 4557module_param_named(no_space_timeout, no_space_timeout_secs, uint, 0644);4558MODULE_PARM_DESC(no_space_timeout, "Out of data space queue IO timeout in seconds");4559 4560MODULE_DESCRIPTION(DM_NAME " thin provisioning target");4561MODULE_AUTHOR("Joe Thornber <dm-devel@lists.linux.dev>");4562MODULE_LICENSE("GPL");4563