1999 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2001 Sistina Software (UK) Limited.4 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.5 *6 * This file is released under the GPL.7 */8 9#include "dm-core.h"10#include "dm-rq.h"11 12#include <linux/module.h>13#include <linux/vmalloc.h>14#include <linux/blkdev.h>15#include <linux/blk-integrity.h>16#include <linux/namei.h>17#include <linux/ctype.h>18#include <linux/string.h>19#include <linux/slab.h>20#include <linux/interrupt.h>21#include <linux/mutex.h>22#include <linux/delay.h>23#include <linux/atomic.h>24#include <linux/blk-mq.h>25#include <linux/mount.h>26#include <linux/dax.h>27 28#define DM_MSG_PREFIX "table"29 30#define NODE_SIZE L1_CACHE_BYTES31#define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t))32#define CHILDREN_PER_NODE (KEYS_PER_NODE + 1)33 34/*35 * Similar to ceiling(log_size(n))36 */37static unsigned int int_log(unsigned int n, unsigned int base)38{39 int result = 0;40 41 while (n > 1) {42 n = dm_div_up(n, base);43 result++;44 }45 46 return result;47}48 49/*50 * Calculate the index of the child node of the n'th node k'th key.51 */52static inline unsigned int get_child(unsigned int n, unsigned int k)53{54 return (n * CHILDREN_PER_NODE) + k;55}56 57/*58 * Return the n'th node of level l from table t.59 */60static inline sector_t *get_node(struct dm_table *t,61 unsigned int l, unsigned int n)62{63 return t->index[l] + (n * KEYS_PER_NODE);64}65 66/*67 * Return the highest key that you could lookup from the n'th68 * node on level l of the btree.69 */70static sector_t high(struct dm_table *t, unsigned int l, unsigned int n)71{72 for (; l < t->depth - 1; l++)73 n = get_child(n, CHILDREN_PER_NODE - 1);74 75 if (n >= t->counts[l])76 return (sector_t) -1;77 78 return get_node(t, l, n)[KEYS_PER_NODE - 1];79}80 81/*82 * Fills in a level of the btree based on the highs of the level83 * below it.84 */85static int setup_btree_index(unsigned int l, struct dm_table *t)86{87 unsigned int n, k;88 sector_t *node;89 90 for (n = 0U; n < t->counts[l]; n++) {91 node = get_node(t, l, n);92 93 for (k = 0U; k < KEYS_PER_NODE; k++)94 node[k] = high(t, l + 1, get_child(n, k));95 }96 97 return 0;98}99 100/*101 * highs, and targets are managed as dynamic arrays during a102 * table load.103 */104static int alloc_targets(struct dm_table *t, unsigned int num)105{106 sector_t *n_highs;107 struct dm_target *n_targets;108 109 /*110 * Allocate both the target array and offset array at once.111 */112 n_highs = kvcalloc(num, sizeof(struct dm_target) + sizeof(sector_t),113 GFP_KERNEL);114 if (!n_highs)115 return -ENOMEM;116 117 n_targets = (struct dm_target *) (n_highs + num);118 119 memset(n_highs, -1, sizeof(*n_highs) * num);120 kvfree(t->highs);121 122 t->num_allocated = num;123 t->highs = n_highs;124 t->targets = n_targets;125 126 return 0;127}128 129int dm_table_create(struct dm_table **result, blk_mode_t mode,130 unsigned int num_targets, struct mapped_device *md)131{132 struct dm_table *t;133 134 if (num_targets > DM_MAX_TARGETS)135 return -EOVERFLOW;136 137 t = kzalloc(sizeof(*t), GFP_KERNEL);138 139 if (!t)140 return -ENOMEM;141 142 INIT_LIST_HEAD(&t->devices);143 init_rwsem(&t->devices_lock);144 145 if (!num_targets)146 num_targets = KEYS_PER_NODE;147 148 num_targets = dm_round_up(num_targets, KEYS_PER_NODE);149 150 if (!num_targets) {151 kfree(t);152 return -EOVERFLOW;153 }154 155 if (alloc_targets(t, num_targets)) {156 kfree(t);157 return -ENOMEM;158 }159 160 t->type = DM_TYPE_NONE;161 t->mode = mode;162 t->md = md;163 t->flush_bypasses_map = true;164 *result = t;165 return 0;166}167 168static void free_devices(struct list_head *devices, struct mapped_device *md)169{170 struct list_head *tmp, *next;171 172 list_for_each_safe(tmp, next, devices) {173 struct dm_dev_internal *dd =174 list_entry(tmp, struct dm_dev_internal, list);175 DMWARN("%s: dm_table_destroy: dm_put_device call missing for %s",176 dm_device_name(md), dd->dm_dev->name);177 dm_put_table_device(md, dd->dm_dev);178 kfree(dd);179 }180}181 182static void dm_table_destroy_crypto_profile(struct dm_table *t);183 184void dm_table_destroy(struct dm_table *t)185{186 if (!t)187 return;188 189 /* free the indexes */190 if (t->depth >= 2)191 kvfree(t->index[t->depth - 2]);192 193 /* free the targets */194 for (unsigned int i = 0; i < t->num_targets; i++) {195 struct dm_target *ti = dm_table_get_target(t, i);196 197 if (ti->type->dtr)198 ti->type->dtr(ti);199 200 dm_put_target_type(ti->type);201 }202 203 kvfree(t->highs);204 205 /* free the device list */206 free_devices(&t->devices, t->md);207 208 dm_free_md_mempools(t->mempools);209 210 dm_table_destroy_crypto_profile(t);211 212 kfree(t);213}214 215/*216 * See if we've already got a device in the list.217 */218static struct dm_dev_internal *find_device(struct list_head *l, dev_t dev)219{220 struct dm_dev_internal *dd;221 222 list_for_each_entry(dd, l, list)223 if (dd->dm_dev->bdev->bd_dev == dev)224 return dd;225 226 return NULL;227}228 229/*230 * If possible, this checks an area of a destination device is invalid.231 */232static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev,233 sector_t start, sector_t len, void *data)234{235 struct queue_limits *limits = data;236 struct block_device *bdev = dev->bdev;237 sector_t dev_size = bdev_nr_sectors(bdev);238 unsigned short logical_block_size_sectors =239 limits->logical_block_size >> SECTOR_SHIFT;240 241 if (!dev_size)242 return 0;243 244 if ((start >= dev_size) || (start + len > dev_size)) {245 DMERR("%s: %pg too small for target: start=%llu, len=%llu, dev_size=%llu",246 dm_device_name(ti->table->md), bdev,247 (unsigned long long)start,248 (unsigned long long)len,249 (unsigned long long)dev_size);250 return 1;251 }252 253 /*254 * If the target is mapped to zoned block device(s), check255 * that the zones are not partially mapped.256 */257 if (bdev_is_zoned(bdev)) {258 unsigned int zone_sectors = bdev_zone_sectors(bdev);259 260 if (start & (zone_sectors - 1)) {261 DMERR("%s: start=%llu not aligned to h/w zone size %u of %pg",262 dm_device_name(ti->table->md),263 (unsigned long long)start,264 zone_sectors, bdev);265 return 1;266 }267 268 /*269 * Note: The last zone of a zoned block device may be smaller270 * than other zones. So for a target mapping the end of a271 * zoned block device with such a zone, len would not be zone272 * aligned. We do not allow such last smaller zone to be part273 * of the mapping here to ensure that mappings with multiple274 * devices do not end up with a smaller zone in the middle of275 * the sector range.276 */277 if (len & (zone_sectors - 1)) {278 DMERR("%s: len=%llu not aligned to h/w zone size %u of %pg",279 dm_device_name(ti->table->md),280 (unsigned long long)len,281 zone_sectors, bdev);282 return 1;283 }284 }285 286 if (logical_block_size_sectors <= 1)287 return 0;288 289 if (start & (logical_block_size_sectors - 1)) {290 DMERR("%s: start=%llu not aligned to h/w logical block size %u of %pg",291 dm_device_name(ti->table->md),292 (unsigned long long)start,293 limits->logical_block_size, bdev);294 return 1;295 }296 297 if (len & (logical_block_size_sectors - 1)) {298 DMERR("%s: len=%llu not aligned to h/w logical block size %u of %pg",299 dm_device_name(ti->table->md),300 (unsigned long long)len,301 limits->logical_block_size, bdev);302 return 1;303 }304 305 return 0;306}307 308/*309 * This upgrades the mode on an already open dm_dev, being310 * careful to leave things as they were if we fail to reopen the311 * device and not to touch the existing bdev field in case312 * it is accessed concurrently.313 */314static int upgrade_mode(struct dm_dev_internal *dd, blk_mode_t new_mode,315 struct mapped_device *md)316{317 int r;318 struct dm_dev *old_dev, *new_dev;319 320 old_dev = dd->dm_dev;321 322 r = dm_get_table_device(md, dd->dm_dev->bdev->bd_dev,323 dd->dm_dev->mode | new_mode, &new_dev);324 if (r)325 return r;326 327 dd->dm_dev = new_dev;328 dm_put_table_device(md, old_dev);329 330 return 0;331}332 333/*334 * Note: the __ref annotation is because this function can call the __init335 * marked early_lookup_bdev when called during early boot code from dm-init.c.336 */337int __ref dm_devt_from_path(const char *path, dev_t *dev_p)338{339 int r;340 dev_t dev;341 unsigned int major, minor;342 char dummy;343 344 if (sscanf(path, "%u:%u%c", &major, &minor, &dummy) == 2) {345 /* Extract the major/minor numbers */346 dev = MKDEV(major, minor);347 if (MAJOR(dev) != major || MINOR(dev) != minor)348 return -EOVERFLOW;349 } else {350 r = lookup_bdev(path, &dev);351#ifndef MODULE352 if (r && system_state < SYSTEM_RUNNING)353 r = early_lookup_bdev(path, &dev);354#endif355 if (r)356 return r;357 }358 *dev_p = dev;359 return 0;360}361EXPORT_SYMBOL(dm_devt_from_path);362 363/*364 * Add a device to the list, or just increment the usage count if365 * it's already present.366 */367int dm_get_device(struct dm_target *ti, const char *path, blk_mode_t mode,368 struct dm_dev **result)369{370 int r;371 dev_t dev;372 struct dm_dev_internal *dd;373 struct dm_table *t = ti->table;374 375 BUG_ON(!t);376 377 r = dm_devt_from_path(path, &dev);378 if (r)379 return r;380 381 if (dev == disk_devt(t->md->disk))382 return -EINVAL;383 384 down_write(&t->devices_lock);385 386 dd = find_device(&t->devices, dev);387 if (!dd) {388 dd = kmalloc(sizeof(*dd), GFP_KERNEL);389 if (!dd) {390 r = -ENOMEM;391 goto unlock_ret_r;392 }393 394 r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev);395 if (r) {396 kfree(dd);397 goto unlock_ret_r;398 }399 400 refcount_set(&dd->count, 1);401 list_add(&dd->list, &t->devices);402 goto out;403 404 } else if (dd->dm_dev->mode != (mode | dd->dm_dev->mode)) {405 r = upgrade_mode(dd, mode, t->md);406 if (r)407 goto unlock_ret_r;408 }409 refcount_inc(&dd->count);410out:411 up_write(&t->devices_lock);412 *result = dd->dm_dev;413 return 0;414 415unlock_ret_r:416 up_write(&t->devices_lock);417 return r;418}419EXPORT_SYMBOL(dm_get_device);420 421static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev,422 sector_t start, sector_t len, void *data)423{424 struct queue_limits *limits = data;425 struct block_device *bdev = dev->bdev;426 struct request_queue *q = bdev_get_queue(bdev);427 428 if (unlikely(!q)) {429 DMWARN("%s: Cannot set limits for nonexistent device %pg",430 dm_device_name(ti->table->md), bdev);431 return 0;432 }433 434 if (blk_stack_limits(limits, &q->limits,435 get_start_sect(bdev) + start) < 0)436 DMWARN("%s: adding target device %pg caused an alignment inconsistency: "437 "physical_block_size=%u, logical_block_size=%u, "438 "alignment_offset=%u, start=%llu",439 dm_device_name(ti->table->md), bdev,440 q->limits.physical_block_size,441 q->limits.logical_block_size,442 q->limits.alignment_offset,443 (unsigned long long) start << SECTOR_SHIFT);444 445 /*446 * Only stack the integrity profile if the target doesn't have native447 * integrity support.448 */449 if (!dm_target_has_integrity(ti->type))450 queue_limits_stack_integrity_bdev(limits, bdev);451 return 0;452}453 454/*455 * Decrement a device's use count and remove it if necessary.456 */457void dm_put_device(struct dm_target *ti, struct dm_dev *d)458{459 int found = 0;460 struct dm_table *t = ti->table;461 struct list_head *devices = &t->devices;462 struct dm_dev_internal *dd;463 464 down_write(&t->devices_lock);465 466 list_for_each_entry(dd, devices, list) {467 if (dd->dm_dev == d) {468 found = 1;469 break;470 }471 }472 if (!found) {473 DMERR("%s: device %s not in table devices list",474 dm_device_name(t->md), d->name);475 goto unlock_ret;476 }477 if (refcount_dec_and_test(&dd->count)) {478 dm_put_table_device(t->md, d);479 list_del(&dd->list);480 kfree(dd);481 }482 483unlock_ret:484 up_write(&t->devices_lock);485}486EXPORT_SYMBOL(dm_put_device);487 488/*489 * Checks to see if the target joins onto the end of the table.490 */491static int adjoin(struct dm_table *t, struct dm_target *ti)492{493 struct dm_target *prev;494 495 if (!t->num_targets)496 return !ti->begin;497 498 prev = &t->targets[t->num_targets - 1];499 return (ti->begin == (prev->begin + prev->len));500}501 502/*503 * Used to dynamically allocate the arg array.504 *505 * We do first allocation with GFP_NOIO because dm-mpath and dm-thin must506 * process messages even if some device is suspended. These messages have a507 * small fixed number of arguments.508 *509 * On the other hand, dm-switch needs to process bulk data using messages and510 * excessive use of GFP_NOIO could cause trouble.511 */512static char **realloc_argv(unsigned int *size, char **old_argv)513{514 char **argv;515 unsigned int new_size;516 gfp_t gfp;517 518 if (*size) {519 new_size = *size * 2;520 gfp = GFP_KERNEL;521 } else {522 new_size = 8;523 gfp = GFP_NOIO;524 }525 argv = kmalloc_array(new_size, sizeof(*argv), gfp);526 if (argv && old_argv) {527 memcpy(argv, old_argv, *size * sizeof(*argv));528 *size = new_size;529 }530 531 kfree(old_argv);532 return argv;533}534 535/*536 * Destructively splits up the argument list to pass to ctr.537 */538int dm_split_args(int *argc, char ***argvp, char *input)539{540 char *start, *end = input, *out, **argv = NULL;541 unsigned int array_size = 0;542 543 *argc = 0;544 545 if (!input) {546 *argvp = NULL;547 return 0;548 }549 550 argv = realloc_argv(&array_size, argv);551 if (!argv)552 return -ENOMEM;553 554 while (1) {555 /* Skip whitespace */556 start = skip_spaces(end);557 558 if (!*start)559 break; /* success, we hit the end */560 561 /* 'out' is used to remove any back-quotes */562 end = out = start;563 while (*end) {564 /* Everything apart from '\0' can be quoted */565 if (*end == '\\' && *(end + 1)) {566 *out++ = *(end + 1);567 end += 2;568 continue;569 }570 571 if (isspace(*end))572 break; /* end of token */573 574 *out++ = *end++;575 }576 577 /* have we already filled the array ? */578 if ((*argc + 1) > array_size) {579 argv = realloc_argv(&array_size, argv);580 if (!argv)581 return -ENOMEM;582 }583 584 /* we know this is whitespace */585 if (*end)586 end++;587 588 /* terminate the string and put it in the array */589 *out = '\0';590 argv[*argc] = start;591 (*argc)++;592 }593 594 *argvp = argv;595 return 0;596}597 598static void dm_set_stacking_limits(struct queue_limits *limits)599{600 blk_set_stacking_limits(limits);601 limits->features |= BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT | BLK_FEAT_POLL;602}603 604/*605 * Impose necessary and sufficient conditions on a devices's table such606 * that any incoming bio which respects its logical_block_size can be607 * processed successfully. If it falls across the boundary between608 * two or more targets, the size of each piece it gets split into must609 * be compatible with the logical_block_size of the target processing it.610 */611static int validate_hardware_logical_block_alignment(struct dm_table *t,612 struct queue_limits *limits)613{614 /*615 * This function uses arithmetic modulo the logical_block_size616 * (in units of 512-byte sectors).617 */618 unsigned short device_logical_block_size_sects =619 limits->logical_block_size >> SECTOR_SHIFT;620 621 /*622 * Offset of the start of the next table entry, mod logical_block_size.623 */624 unsigned short next_target_start = 0;625 626 /*627 * Given an aligned bio that extends beyond the end of a628 * target, how many sectors must the next target handle?629 */630 unsigned short remaining = 0;631 632 struct dm_target *ti;633 struct queue_limits ti_limits;634 unsigned int i;635 636 /*637 * Check each entry in the table in turn.638 */639 for (i = 0; i < t->num_targets; i++) {640 ti = dm_table_get_target(t, i);641 642 dm_set_stacking_limits(&ti_limits);643 644 /* combine all target devices' limits */645 if (ti->type->iterate_devices)646 ti->type->iterate_devices(ti, dm_set_device_limits,647 &ti_limits);648 649 /*650 * If the remaining sectors fall entirely within this651 * table entry are they compatible with its logical_block_size?652 */653 if (remaining < ti->len &&654 remaining & ((ti_limits.logical_block_size >>655 SECTOR_SHIFT) - 1))656 break; /* Error */657 658 next_target_start =659 (unsigned short) ((next_target_start + ti->len) &660 (device_logical_block_size_sects - 1));661 remaining = next_target_start ?662 device_logical_block_size_sects - next_target_start : 0;663 }664 665 if (remaining) {666 DMERR("%s: table line %u (start sect %llu len %llu) "667 "not aligned to h/w logical block size %u",668 dm_device_name(t->md), i,669 (unsigned long long) ti->begin,670 (unsigned long long) ti->len,671 limits->logical_block_size);672 return -EINVAL;673 }674 675 return 0;676}677 678int dm_table_add_target(struct dm_table *t, const char *type,679 sector_t start, sector_t len, char *params)680{681 int r = -EINVAL, argc;682 char **argv;683 struct dm_target *ti;684 685 if (t->singleton) {686 DMERR("%s: target type %s must appear alone in table",687 dm_device_name(t->md), t->targets->type->name);688 return -EINVAL;689 }690 691 BUG_ON(t->num_targets >= t->num_allocated);692 693 ti = t->targets + t->num_targets;694 memset(ti, 0, sizeof(*ti));695 696 if (!len) {697 DMERR("%s: zero-length target", dm_device_name(t->md));698 return -EINVAL;699 }700 701 ti->type = dm_get_target_type(type);702 if (!ti->type) {703 DMERR("%s: %s: unknown target type", dm_device_name(t->md), type);704 return -EINVAL;705 }706 707 if (dm_target_needs_singleton(ti->type)) {708 if (t->num_targets) {709 ti->error = "singleton target type must appear alone in table";710 goto bad;711 }712 t->singleton = true;713 }714 715 if (dm_target_always_writeable(ti->type) &&716 !(t->mode & BLK_OPEN_WRITE)) {717 ti->error = "target type may not be included in a read-only table";718 goto bad;719 }720 721 if (t->immutable_target_type) {722 if (t->immutable_target_type != ti->type) {723 ti->error = "immutable target type cannot be mixed with other target types";724 goto bad;725 }726 } else if (dm_target_is_immutable(ti->type)) {727 if (t->num_targets) {728 ti->error = "immutable target type cannot be mixed with other target types";729 goto bad;730 }731 t->immutable_target_type = ti->type;732 }733 734 ti->table = t;735 ti->begin = start;736 ti->len = len;737 ti->error = "Unknown error";738 739 /*740 * Does this target adjoin the previous one ?741 */742 if (!adjoin(t, ti)) {743 ti->error = "Gap in table";744 goto bad;745 }746 747 r = dm_split_args(&argc, &argv, params);748 if (r) {749 ti->error = "couldn't split parameters";750 goto bad;751 }752 753 r = ti->type->ctr(ti, argc, argv);754 kfree(argv);755 if (r)756 goto bad;757 758 t->highs[t->num_targets++] = ti->begin + ti->len - 1;759 760 if (!ti->num_discard_bios && ti->discards_supported)761 DMWARN("%s: %s: ignoring discards_supported because num_discard_bios is zero.",762 dm_device_name(t->md), type);763 764 if (ti->limit_swap_bios && !static_key_enabled(&swap_bios_enabled.key))765 static_branch_enable(&swap_bios_enabled);766 767 if (!ti->flush_bypasses_map)768 t->flush_bypasses_map = false;769 770 return 0;771 772 bad:773 DMERR("%s: %s: %s (%pe)", dm_device_name(t->md), type, ti->error, ERR_PTR(r));774 dm_put_target_type(ti->type);775 return r;776}777 778/*779 * Target argument parsing helpers.780 */781static int validate_next_arg(const struct dm_arg *arg, struct dm_arg_set *arg_set,782 unsigned int *value, char **error, unsigned int grouped)783{784 const char *arg_str = dm_shift_arg(arg_set);785 char dummy;786 787 if (!arg_str ||788 (sscanf(arg_str, "%u%c", value, &dummy) != 1) ||789 (*value < arg->min) ||790 (*value > arg->max) ||791 (grouped && arg_set->argc < *value)) {792 *error = arg->error;793 return -EINVAL;794 }795 796 return 0;797}798 799int dm_read_arg(const struct dm_arg *arg, struct dm_arg_set *arg_set,800 unsigned int *value, char **error)801{802 return validate_next_arg(arg, arg_set, value, error, 0);803}804EXPORT_SYMBOL(dm_read_arg);805 806int dm_read_arg_group(const struct dm_arg *arg, struct dm_arg_set *arg_set,807 unsigned int *value, char **error)808{809 return validate_next_arg(arg, arg_set, value, error, 1);810}811EXPORT_SYMBOL(dm_read_arg_group);812 813const char *dm_shift_arg(struct dm_arg_set *as)814{815 char *r;816 817 if (as->argc) {818 as->argc--;819 r = *as->argv;820 as->argv++;821 return r;822 }823 824 return NULL;825}826EXPORT_SYMBOL(dm_shift_arg);827 828void dm_consume_args(struct dm_arg_set *as, unsigned int num_args)829{830 BUG_ON(as->argc < num_args);831 as->argc -= num_args;832 as->argv += num_args;833}834EXPORT_SYMBOL(dm_consume_args);835 836static bool __table_type_bio_based(enum dm_queue_mode table_type)837{838 return (table_type == DM_TYPE_BIO_BASED ||839 table_type == DM_TYPE_DAX_BIO_BASED);840}841 842static bool __table_type_request_based(enum dm_queue_mode table_type)843{844 return table_type == DM_TYPE_REQUEST_BASED;845}846 847void dm_table_set_type(struct dm_table *t, enum dm_queue_mode type)848{849 t->type = type;850}851EXPORT_SYMBOL_GPL(dm_table_set_type);852 853/* validate the dax capability of the target device span */854static int device_not_dax_capable(struct dm_target *ti, struct dm_dev *dev,855 sector_t start, sector_t len, void *data)856{857 if (dev->dax_dev)858 return false;859 860 DMDEBUG("%pg: error: dax unsupported by block device", dev->bdev);861 return true;862}863 864/* Check devices support synchronous DAX */865static int device_not_dax_synchronous_capable(struct dm_target *ti, struct dm_dev *dev,866 sector_t start, sector_t len, void *data)867{868 return !dev->dax_dev || !dax_synchronous(dev->dax_dev);869}870 871static bool dm_table_supports_dax(struct dm_table *t,872 iterate_devices_callout_fn iterate_fn)873{874 /* Ensure that all targets support DAX. */875 for (unsigned int i = 0; i < t->num_targets; i++) {876 struct dm_target *ti = dm_table_get_target(t, i);877 878 if (!ti->type->direct_access)879 return false;880 881 if (dm_target_is_wildcard(ti->type) ||882 !ti->type->iterate_devices ||883 ti->type->iterate_devices(ti, iterate_fn, NULL))884 return false;885 }886 887 return true;888}889 890static int device_is_rq_stackable(struct dm_target *ti, struct dm_dev *dev,891 sector_t start, sector_t len, void *data)892{893 struct block_device *bdev = dev->bdev;894 struct request_queue *q = bdev_get_queue(bdev);895 896 /* request-based cannot stack on partitions! */897 if (bdev_is_partition(bdev))898 return false;899 900 return queue_is_mq(q);901}902 903static int dm_table_determine_type(struct dm_table *t)904{905 unsigned int bio_based = 0, request_based = 0, hybrid = 0;906 struct dm_target *ti;907 struct list_head *devices = dm_table_get_devices(t);908 enum dm_queue_mode live_md_type = dm_get_md_type(t->md);909 910 if (t->type != DM_TYPE_NONE) {911 /* target already set the table's type */912 if (t->type == DM_TYPE_BIO_BASED) {913 /* possibly upgrade to a variant of bio-based */914 goto verify_bio_based;915 }916 BUG_ON(t->type == DM_TYPE_DAX_BIO_BASED);917 goto verify_rq_based;918 }919 920 for (unsigned int i = 0; i < t->num_targets; i++) {921 ti = dm_table_get_target(t, i);922 if (dm_target_hybrid(ti))923 hybrid = 1;924 else if (dm_target_request_based(ti))925 request_based = 1;926 else927 bio_based = 1;928 929 if (bio_based && request_based) {930 DMERR("Inconsistent table: different target types can't be mixed up");931 return -EINVAL;932 }933 }934 935 if (hybrid && !bio_based && !request_based) {936 /*937 * The targets can work either way.938 * Determine the type from the live device.939 * Default to bio-based if device is new.940 */941 if (__table_type_request_based(live_md_type))942 request_based = 1;943 else944 bio_based = 1;945 }946 947 if (bio_based) {948verify_bio_based:949 /* We must use this table as bio-based */950 t->type = DM_TYPE_BIO_BASED;951 if (dm_table_supports_dax(t, device_not_dax_capable) ||952 (list_empty(devices) && live_md_type == DM_TYPE_DAX_BIO_BASED)) {953 t->type = DM_TYPE_DAX_BIO_BASED;954 }955 return 0;956 }957 958 BUG_ON(!request_based); /* No targets in this table */959 960 t->type = DM_TYPE_REQUEST_BASED;961 962verify_rq_based:963 /*964 * Request-based dm supports only tables that have a single target now.965 * To support multiple targets, request splitting support is needed,966 * and that needs lots of changes in the block-layer.967 * (e.g. request completion process for partial completion.)968 */969 if (t->num_targets > 1) {970 DMERR("request-based DM doesn't support multiple targets");971 return -EINVAL;972 }973 974 if (list_empty(devices)) {975 int srcu_idx;976 struct dm_table *live_table = dm_get_live_table(t->md, &srcu_idx);977 978 /* inherit live table's type */979 if (live_table)980 t->type = live_table->type;981 dm_put_live_table(t->md, srcu_idx);982 return 0;983 }984 985 ti = dm_table_get_immutable_target(t);986 if (!ti) {987 DMERR("table load rejected: immutable target is required");988 return -EINVAL;989 } else if (ti->max_io_len) {990 DMERR("table load rejected: immutable target that splits IO is not supported");991 return -EINVAL;992 }993 994 /* Non-request-stackable devices can't be used for request-based dm */995 if (!ti->type->iterate_devices ||996 !ti->type->iterate_devices(ti, device_is_rq_stackable, NULL)) {997 DMERR("table load rejected: including non-request-stackable devices");998 return -EINVAL;999 }1000 1001 return 0;1002}1003 1004enum dm_queue_mode dm_table_get_type(struct dm_table *t)1005{1006 return t->type;1007}1008 1009struct target_type *dm_table_get_immutable_target_type(struct dm_table *t)1010{1011 return t->immutable_target_type;1012}1013 1014struct dm_target *dm_table_get_immutable_target(struct dm_table *t)1015{1016 /* Immutable target is implicitly a singleton */1017 if (t->num_targets > 1 ||1018 !dm_target_is_immutable(t->targets[0].type))1019 return NULL;1020 1021 return t->targets;1022}1023 1024struct dm_target *dm_table_get_wildcard_target(struct dm_table *t)1025{1026 for (unsigned int i = 0; i < t->num_targets; i++) {1027 struct dm_target *ti = dm_table_get_target(t, i);1028 1029 if (dm_target_is_wildcard(ti->type))1030 return ti;1031 }1032 1033 return NULL;1034}1035 1036bool dm_table_bio_based(struct dm_table *t)1037{1038 return __table_type_bio_based(dm_table_get_type(t));1039}1040 1041bool dm_table_request_based(struct dm_table *t)1042{1043 return __table_type_request_based(dm_table_get_type(t));1044}1045 1046static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md)1047{1048 enum dm_queue_mode type = dm_table_get_type(t);1049 unsigned int per_io_data_size = 0, front_pad, io_front_pad;1050 unsigned int min_pool_size = 0, pool_size;1051 struct dm_md_mempools *pools;1052 unsigned int bioset_flags = 0;1053 bool mempool_needs_integrity = t->integrity_supported;1054 1055 if (unlikely(type == DM_TYPE_NONE)) {1056 DMERR("no table type is set, can't allocate mempools");1057 return -EINVAL;1058 }1059 1060 pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);1061 if (!pools)1062 return -ENOMEM;1063 1064 if (type == DM_TYPE_REQUEST_BASED) {1065 pool_size = dm_get_reserved_rq_based_ios();1066 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);1067 goto init_bs;1068 }1069 1070 if (md->queue->limits.features & BLK_FEAT_POLL)1071 bioset_flags |= BIOSET_PERCPU_CACHE;1072 1073 for (unsigned int i = 0; i < t->num_targets; i++) {1074 struct dm_target *ti = dm_table_get_target(t, i);1075 1076 per_io_data_size = max(per_io_data_size, ti->per_io_data_size);1077 min_pool_size = max(min_pool_size, ti->num_flush_bios);1078 1079 mempool_needs_integrity |= ti->mempool_needs_integrity;1080 }1081 pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);1082 front_pad = roundup(per_io_data_size,1083 __alignof__(struct dm_target_io)) + DM_TARGET_IO_BIO_OFFSET;1084 1085 io_front_pad = roundup(per_io_data_size,1086 __alignof__(struct dm_io)) + DM_IO_BIO_OFFSET;1087 if (bioset_init(&pools->io_bs, pool_size, io_front_pad, bioset_flags))1088 goto out_free_pools;1089 if (mempool_needs_integrity &&1090 bioset_integrity_create(&pools->io_bs, pool_size))1091 goto out_free_pools;1092init_bs:1093 if (bioset_init(&pools->bs, pool_size, front_pad, 0))1094 goto out_free_pools;1095 if (mempool_needs_integrity &&1096 bioset_integrity_create(&pools->bs, pool_size))1097 goto out_free_pools;1098 1099 t->mempools = pools;1100 return 0;1101 1102out_free_pools:1103 dm_free_md_mempools(pools);1104 return -ENOMEM;1105}1106 1107static int setup_indexes(struct dm_table *t)1108{1109 int i;1110 unsigned int total = 0;1111 sector_t *indexes;1112 1113 /* allocate the space for *all* the indexes */1114 for (i = t->depth - 2; i >= 0; i--) {1115 t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE);1116 total += t->counts[i];1117 }1118 1119 indexes = kvcalloc(total, NODE_SIZE, GFP_KERNEL);1120 if (!indexes)1121 return -ENOMEM;1122 1123 /* set up internal nodes, bottom-up */1124 for (i = t->depth - 2; i >= 0; i--) {1125 t->index[i] = indexes;1126 indexes += (KEYS_PER_NODE * t->counts[i]);1127 setup_btree_index(i, t);1128 }1129 1130 return 0;1131}1132 1133/*1134 * Builds the btree to index the map.1135 */1136static int dm_table_build_index(struct dm_table *t)1137{1138 int r = 0;1139 unsigned int leaf_nodes;1140 1141 /* how many indexes will the btree have ? */1142 leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE);1143 t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE);1144 1145 /* leaf layer has already been set up */1146 t->counts[t->depth - 1] = leaf_nodes;1147 t->index[t->depth - 1] = t->highs;1148 1149 if (t->depth >= 2)1150 r = setup_indexes(t);1151 1152 return r;1153}1154 1155#ifdef CONFIG_BLK_INLINE_ENCRYPTION1156 1157struct dm_crypto_profile {1158 struct blk_crypto_profile profile;1159 struct mapped_device *md;1160};1161 1162static int dm_keyslot_evict_callback(struct dm_target *ti, struct dm_dev *dev,1163 sector_t start, sector_t len, void *data)1164{1165 const struct blk_crypto_key *key = data;1166 1167 blk_crypto_evict_key(dev->bdev, key);1168 return 0;1169}1170 1171/*1172 * When an inline encryption key is evicted from a device-mapper device, evict1173 * it from all the underlying devices.1174 */1175static int dm_keyslot_evict(struct blk_crypto_profile *profile,1176 const struct blk_crypto_key *key, unsigned int slot)1177{1178 struct mapped_device *md =1179 container_of(profile, struct dm_crypto_profile, profile)->md;1180 struct dm_table *t;1181 int srcu_idx;1182 1183 t = dm_get_live_table(md, &srcu_idx);1184 if (!t)1185 return 0;1186 1187 for (unsigned int i = 0; i < t->num_targets; i++) {1188 struct dm_target *ti = dm_table_get_target(t, i);1189 1190 if (!ti->type->iterate_devices)1191 continue;1192 ti->type->iterate_devices(ti, dm_keyslot_evict_callback,1193 (void *)key);1194 }1195 1196 dm_put_live_table(md, srcu_idx);1197 return 0;1198}1199 1200static int1201device_intersect_crypto_capabilities(struct dm_target *ti, struct dm_dev *dev,1202 sector_t start, sector_t len, void *data)1203{1204 struct blk_crypto_profile *parent = data;1205 struct blk_crypto_profile *child =1206 bdev_get_queue(dev->bdev)->crypto_profile;1207 1208 blk_crypto_intersect_capabilities(parent, child);1209 return 0;1210}1211 1212void dm_destroy_crypto_profile(struct blk_crypto_profile *profile)1213{1214 struct dm_crypto_profile *dmcp = container_of(profile,1215 struct dm_crypto_profile,1216 profile);1217 1218 if (!profile)1219 return;1220 1221 blk_crypto_profile_destroy(profile);1222 kfree(dmcp);1223}1224 1225static void dm_table_destroy_crypto_profile(struct dm_table *t)1226{1227 dm_destroy_crypto_profile(t->crypto_profile);1228 t->crypto_profile = NULL;1229}1230 1231/*1232 * Constructs and initializes t->crypto_profile with a crypto profile that1233 * represents the common set of crypto capabilities of the devices described by1234 * the dm_table. However, if the constructed crypto profile doesn't support all1235 * crypto capabilities that are supported by the current mapped_device, it1236 * returns an error instead, since we don't support removing crypto capabilities1237 * on table changes. Finally, if the constructed crypto profile is "empty" (has1238 * no crypto capabilities at all), it just sets t->crypto_profile to NULL.1239 */1240static int dm_table_construct_crypto_profile(struct dm_table *t)1241{1242 struct dm_crypto_profile *dmcp;1243 struct blk_crypto_profile *profile;1244 unsigned int i;1245 bool empty_profile = true;1246 1247 dmcp = kmalloc(sizeof(*dmcp), GFP_KERNEL);1248 if (!dmcp)1249 return -ENOMEM;1250 dmcp->md = t->md;1251 1252 profile = &dmcp->profile;1253 blk_crypto_profile_init(profile, 0);1254 profile->ll_ops.keyslot_evict = dm_keyslot_evict;1255 profile->max_dun_bytes_supported = UINT_MAX;1256 memset(profile->modes_supported, 0xFF,1257 sizeof(profile->modes_supported));1258 1259 for (i = 0; i < t->num_targets; i++) {1260 struct dm_target *ti = dm_table_get_target(t, i);1261 1262 if (!dm_target_passes_crypto(ti->type)) {1263 blk_crypto_intersect_capabilities(profile, NULL);1264 break;1265 }1266 if (!ti->type->iterate_devices)1267 continue;1268 ti->type->iterate_devices(ti,1269 device_intersect_crypto_capabilities,1270 profile);1271 }1272 1273 if (t->md->queue &&1274 !blk_crypto_has_capabilities(profile,1275 t->md->queue->crypto_profile)) {1276 DMERR("Inline encryption capabilities of new DM table were more restrictive than the old table's. This is not supported!");1277 dm_destroy_crypto_profile(profile);1278 return -EINVAL;1279 }1280 1281 /*1282 * If the new profile doesn't actually support any crypto capabilities,1283 * we may as well represent it with a NULL profile.1284 */1285 for (i = 0; i < ARRAY_SIZE(profile->modes_supported); i++) {1286 if (profile->modes_supported[i]) {1287 empty_profile = false;1288 break;1289 }1290 }1291 1292 if (empty_profile) {1293 dm_destroy_crypto_profile(profile);1294 profile = NULL;1295 }1296 1297 /*1298 * t->crypto_profile is only set temporarily while the table is being1299 * set up, and it gets set to NULL after the profile has been1300 * transferred to the request_queue.1301 */1302 t->crypto_profile = profile;1303 1304 return 0;1305}1306 1307static void dm_update_crypto_profile(struct request_queue *q,1308 struct dm_table *t)1309{1310 if (!t->crypto_profile)1311 return;1312 1313 /* Make the crypto profile less restrictive. */1314 if (!q->crypto_profile) {1315 blk_crypto_register(t->crypto_profile, q);1316 } else {1317 blk_crypto_update_capabilities(q->crypto_profile,1318 t->crypto_profile);1319 dm_destroy_crypto_profile(t->crypto_profile);1320 }1321 t->crypto_profile = NULL;1322}1323 1324#else /* CONFIG_BLK_INLINE_ENCRYPTION */1325 1326static int dm_table_construct_crypto_profile(struct dm_table *t)1327{1328 return 0;1329}1330 1331void dm_destroy_crypto_profile(struct blk_crypto_profile *profile)1332{1333}1334 1335static void dm_table_destroy_crypto_profile(struct dm_table *t)1336{1337}1338 1339static void dm_update_crypto_profile(struct request_queue *q,1340 struct dm_table *t)1341{1342}1343 1344#endif /* !CONFIG_BLK_INLINE_ENCRYPTION */1345 1346/*1347 * Prepares the table for use by building the indices,1348 * setting the type, and allocating mempools.1349 */1350int dm_table_complete(struct dm_table *t)1351{1352 int r;1353 1354 r = dm_table_determine_type(t);1355 if (r) {1356 DMERR("unable to determine table type");1357 return r;1358 }1359 1360 r = dm_table_build_index(t);1361 if (r) {1362 DMERR("unable to build btrees");1363 return r;1364 }1365 1366 r = dm_table_construct_crypto_profile(t);1367 if (r) {1368 DMERR("could not construct crypto profile.");1369 return r;1370 }1371 1372 r = dm_table_alloc_md_mempools(t, t->md);1373 if (r)1374 DMERR("unable to allocate mempools");1375 1376 return r;1377}1378 1379static DEFINE_MUTEX(_event_lock);1380void dm_table_event_callback(struct dm_table *t,1381 void (*fn)(void *), void *context)1382{1383 mutex_lock(&_event_lock);1384 t->event_fn = fn;1385 t->event_context = context;1386 mutex_unlock(&_event_lock);1387}1388 1389void dm_table_event(struct dm_table *t)1390{1391 mutex_lock(&_event_lock);1392 if (t->event_fn)1393 t->event_fn(t->event_context);1394 mutex_unlock(&_event_lock);1395}1396EXPORT_SYMBOL(dm_table_event);1397 1398inline sector_t dm_table_get_size(struct dm_table *t)1399{1400 return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0;1401}1402EXPORT_SYMBOL(dm_table_get_size);1403 1404/*1405 * Search the btree for the correct target.1406 *1407 * Caller should check returned pointer for NULL1408 * to trap I/O beyond end of device.1409 */1410struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector)1411{1412 unsigned int l, n = 0, k = 0;1413 sector_t *node;1414 1415 if (unlikely(sector >= dm_table_get_size(t)))1416 return NULL;1417 1418 for (l = 0; l < t->depth; l++) {1419 n = get_child(n, k);1420 node = get_node(t, l, n);1421 1422 for (k = 0; k < KEYS_PER_NODE; k++)1423 if (node[k] >= sector)1424 break;1425 }1426 1427 return &t->targets[(KEYS_PER_NODE * n) + k];1428}1429 1430/*1431 * type->iterate_devices() should be called when the sanity check needs to1432 * iterate and check all underlying data devices. iterate_devices() will1433 * iterate all underlying data devices until it encounters a non-zero return1434 * code, returned by whether the input iterate_devices_callout_fn, or1435 * iterate_devices() itself internally.1436 *1437 * For some target type (e.g. dm-stripe), one call of iterate_devices() may1438 * iterate multiple underlying devices internally, in which case a non-zero1439 * return code returned by iterate_devices_callout_fn will stop the iteration1440 * in advance.1441 *1442 * Cases requiring _any_ underlying device supporting some kind of attribute,1443 * should use the iteration structure like dm_table_any_dev_attr(), or call1444 * it directly. @func should handle semantics of positive examples, e.g.1445 * capable of something.1446 *1447 * Cases requiring _all_ underlying devices supporting some kind of attribute,1448 * should use the iteration structure like dm_table_supports_nowait() or1449 * dm_table_supports_discards(). Or introduce dm_table_all_devs_attr() that1450 * uses an @anti_func that handle semantics of counter examples, e.g. not1451 * capable of something. So: return !dm_table_any_dev_attr(t, anti_func, data);1452 */1453static bool dm_table_any_dev_attr(struct dm_table *t,1454 iterate_devices_callout_fn func, void *data)1455{1456 for (unsigned int i = 0; i < t->num_targets; i++) {1457 struct dm_target *ti = dm_table_get_target(t, i);1458 1459 if (ti->type->iterate_devices &&1460 ti->type->iterate_devices(ti, func, data))1461 return true;1462 }1463 1464 return false;1465}1466 1467static int count_device(struct dm_target *ti, struct dm_dev *dev,1468 sector_t start, sector_t len, void *data)1469{1470 unsigned int *num_devices = data;1471 1472 (*num_devices)++;1473 1474 return 0;1475}1476 1477/*1478 * Check whether a table has no data devices attached using each1479 * target's iterate_devices method.1480 * Returns false if the result is unknown because a target doesn't1481 * support iterate_devices.1482 */1483bool dm_table_has_no_data_devices(struct dm_table *t)1484{1485 for (unsigned int i = 0; i < t->num_targets; i++) {1486 struct dm_target *ti = dm_table_get_target(t, i);1487 unsigned int num_devices = 0;1488 1489 if (!ti->type->iterate_devices)1490 return false;1491 1492 ti->type->iterate_devices(ti, count_device, &num_devices);1493 if (num_devices)1494 return false;1495 }1496 1497 return true;1498}1499 1500static int device_not_zoned(struct dm_target *ti, struct dm_dev *dev,1501 sector_t start, sector_t len, void *data)1502{1503 bool *zoned = data;1504 1505 return bdev_is_zoned(dev->bdev) != *zoned;1506}1507 1508static int device_is_zoned_model(struct dm_target *ti, struct dm_dev *dev,1509 sector_t start, sector_t len, void *data)1510{1511 return bdev_is_zoned(dev->bdev);1512}1513 1514/*1515 * Check the device zoned model based on the target feature flag. If the target1516 * has the DM_TARGET_ZONED_HM feature flag set, host-managed zoned devices are1517 * also accepted but all devices must have the same zoned model. If the target1518 * has the DM_TARGET_MIXED_ZONED_MODEL feature set, the devices can have any1519 * zoned model with all zoned devices having the same zone size.1520 */1521static bool dm_table_supports_zoned(struct dm_table *t, bool zoned)1522{1523 for (unsigned int i = 0; i < t->num_targets; i++) {1524 struct dm_target *ti = dm_table_get_target(t, i);1525 1526 /*1527 * For the wildcard target (dm-error), if we do not have a1528 * backing device, we must always return false. If we have a1529 * backing device, the result must depend on checking zoned1530 * model, like for any other target. So for this, check directly1531 * if the target backing device is zoned as we get "false" when1532 * dm-error was set without a backing device.1533 */1534 if (dm_target_is_wildcard(ti->type) &&1535 !ti->type->iterate_devices(ti, device_is_zoned_model, NULL))1536 return false;1537 1538 if (dm_target_supports_zoned_hm(ti->type)) {1539 if (!ti->type->iterate_devices ||1540 ti->type->iterate_devices(ti, device_not_zoned,1541 &zoned))1542 return false;1543 } else if (!dm_target_supports_mixed_zoned_model(ti->type)) {1544 if (zoned)1545 return false;1546 }1547 }1548 1549 return true;1550}1551 1552static int device_not_matches_zone_sectors(struct dm_target *ti, struct dm_dev *dev,1553 sector_t start, sector_t len, void *data)1554{1555 unsigned int *zone_sectors = data;1556 1557 if (!bdev_is_zoned(dev->bdev))1558 return 0;1559 return bdev_zone_sectors(dev->bdev) != *zone_sectors;1560}1561 1562/*1563 * Check consistency of zoned model and zone sectors across all targets. For1564 * zone sectors, if the destination device is a zoned block device, it shall1565 * have the specified zone_sectors.1566 */1567static int validate_hardware_zoned(struct dm_table *t, bool zoned,1568 unsigned int zone_sectors)1569{1570 if (!zoned)1571 return 0;1572 1573 if (!dm_table_supports_zoned(t, zoned)) {1574 DMERR("%s: zoned model is not consistent across all devices",1575 dm_device_name(t->md));1576 return -EINVAL;1577 }1578 1579 /* Check zone size validity and compatibility */1580 if (!zone_sectors || !is_power_of_2(zone_sectors))1581 return -EINVAL;1582 1583 if (dm_table_any_dev_attr(t, device_not_matches_zone_sectors, &zone_sectors)) {1584 DMERR("%s: zone sectors is not consistent across all zoned devices",1585 dm_device_name(t->md));1586 return -EINVAL;1587 }1588 1589 return 0;1590}1591 1592/*1593 * Establish the new table's queue_limits and validate them.1594 */1595int dm_calculate_queue_limits(struct dm_table *t,1596 struct queue_limits *limits)1597{1598 struct queue_limits ti_limits;1599 unsigned int zone_sectors = 0;1600 bool zoned = false;1601 1602 dm_set_stacking_limits(limits);1603 1604 t->integrity_supported = true;1605 for (unsigned int i = 0; i < t->num_targets; i++) {1606 struct dm_target *ti = dm_table_get_target(t, i);1607 1608 if (!dm_target_passes_integrity(ti->type))1609 t->integrity_supported = false;1610 }1611 1612 for (unsigned int i = 0; i < t->num_targets; i++) {1613 struct dm_target *ti = dm_table_get_target(t, i);1614 1615 dm_set_stacking_limits(&ti_limits);1616 1617 if (!ti->type->iterate_devices) {1618 /* Set I/O hints portion of queue limits */1619 if (ti->type->io_hints)1620 ti->type->io_hints(ti, &ti_limits);1621 goto combine_limits;1622 }1623 1624 /*1625 * Combine queue limits of all the devices this target uses.1626 */1627 ti->type->iterate_devices(ti, dm_set_device_limits,1628 &ti_limits);1629 1630 if (!zoned && (ti_limits.features & BLK_FEAT_ZONED)) {1631 /*1632 * After stacking all limits, validate all devices1633 * in table support this zoned model and zone sectors.1634 */1635 zoned = (ti_limits.features & BLK_FEAT_ZONED);1636 zone_sectors = ti_limits.chunk_sectors;1637 }1638 1639 /* Set I/O hints portion of queue limits */1640 if (ti->type->io_hints)1641 ti->type->io_hints(ti, &ti_limits);1642 1643 /*1644 * Check each device area is consistent with the target's1645 * overall queue limits.1646 */1647 if (ti->type->iterate_devices(ti, device_area_is_invalid,1648 &ti_limits))1649 return -EINVAL;1650 1651combine_limits:1652 /*1653 * Merge this target's queue limits into the overall limits1654 * for the table.1655 */1656 if (blk_stack_limits(limits, &ti_limits, 0) < 0)1657 DMWARN("%s: adding target device (start sect %llu len %llu) "1658 "caused an alignment inconsistency",1659 dm_device_name(t->md),1660 (unsigned long long) ti->begin,1661 (unsigned long long) ti->len);1662 1663 if (t->integrity_supported ||1664 dm_target_has_integrity(ti->type)) {1665 if (!queue_limits_stack_integrity(limits, &ti_limits)) {1666 DMWARN("%s: adding target device (start sect %llu len %llu) "1667 "disabled integrity support due to incompatibility",1668 dm_device_name(t->md),1669 (unsigned long long) ti->begin,1670 (unsigned long long) ti->len);1671 t->integrity_supported = false;1672 }1673 }1674 }1675 1676 /*1677 * Verify that the zoned model and zone sectors, as determined before1678 * any .io_hints override, are the same across all devices in the table.1679 * - this is especially relevant if .io_hints is emulating a disk-managed1680 * zoned model on host-managed zoned block devices.1681 * BUT...1682 */1683 if (limits->features & BLK_FEAT_ZONED) {1684 /*1685 * ...IF the above limits stacking determined a zoned model1686 * validate that all of the table's devices conform to it.1687 */1688 zoned = limits->features & BLK_FEAT_ZONED;1689 zone_sectors = limits->chunk_sectors;1690 }1691 if (validate_hardware_zoned(t, zoned, zone_sectors))1692 return -EINVAL;1693 1694 return validate_hardware_logical_block_alignment(t, limits);1695}1696 1697/*1698 * Check if a target requires flush support even if none of the underlying1699 * devices need it (e.g. to persist target-specific metadata).1700 */1701static bool dm_table_supports_flush(struct dm_table *t)1702{1703 for (unsigned int i = 0; i < t->num_targets; i++) {1704 struct dm_target *ti = dm_table_get_target(t, i);1705 1706 if (ti->num_flush_bios && ti->flush_supported)1707 return true;1708 }1709 1710 return false;1711}1712 1713static int device_dax_write_cache_enabled(struct dm_target *ti,1714 struct dm_dev *dev, sector_t start,1715 sector_t len, void *data)1716{1717 struct dax_device *dax_dev = dev->dax_dev;1718 1719 if (!dax_dev)1720 return false;1721 1722 if (dax_write_cache_enabled(dax_dev))1723 return true;1724 return false;1725}1726 1727static int device_not_write_zeroes_capable(struct dm_target *ti, struct dm_dev *dev,1728 sector_t start, sector_t len, void *data)1729{1730 struct request_queue *q = bdev_get_queue(dev->bdev);1731 1732 return !q->limits.max_write_zeroes_sectors;1733}1734 1735static bool dm_table_supports_write_zeroes(struct dm_table *t)1736{1737 for (unsigned int i = 0; i < t->num_targets; i++) {1738 struct dm_target *ti = dm_table_get_target(t, i);1739 1740 if (!ti->num_write_zeroes_bios)1741 return false;1742 1743 if (!ti->type->iterate_devices ||1744 ti->type->iterate_devices(ti, device_not_write_zeroes_capable, NULL))1745 return false;1746 }1747 1748 return true;1749}1750 1751static bool dm_table_supports_nowait(struct dm_table *t)1752{1753 for (unsigned int i = 0; i < t->num_targets; i++) {1754 struct dm_target *ti = dm_table_get_target(t, i);1755 1756 if (!dm_target_supports_nowait(ti->type))1757 return false;1758 }1759 1760 return true;1761}1762 1763static int device_not_discard_capable(struct dm_target *ti, struct dm_dev *dev,1764 sector_t start, sector_t len, void *data)1765{1766 return !bdev_max_discard_sectors(dev->bdev);1767}1768 1769static bool dm_table_supports_discards(struct dm_table *t)1770{1771 for (unsigned int i = 0; i < t->num_targets; i++) {1772 struct dm_target *ti = dm_table_get_target(t, i);1773 1774 if (!ti->num_discard_bios)1775 return false;1776 1777 /*1778 * Either the target provides discard support (as implied by setting1779 * 'discards_supported') or it relies on _all_ data devices having1780 * discard support.1781 */1782 if (!ti->discards_supported &&1783 (!ti->type->iterate_devices ||1784 ti->type->iterate_devices(ti, device_not_discard_capable, NULL)))1785 return false;1786 }1787 1788 return true;1789}1790 1791static int device_not_secure_erase_capable(struct dm_target *ti,1792 struct dm_dev *dev, sector_t start,1793 sector_t len, void *data)1794{1795 return !bdev_max_secure_erase_sectors(dev->bdev);1796}1797 1798static bool dm_table_supports_secure_erase(struct dm_table *t)1799{1800 for (unsigned int i = 0; i < t->num_targets; i++) {1801 struct dm_target *ti = dm_table_get_target(t, i);1802 1803 if (!ti->num_secure_erase_bios)1804 return false;1805 1806 if (!ti->type->iterate_devices ||1807 ti->type->iterate_devices(ti, device_not_secure_erase_capable, NULL))1808 return false;1809 }1810 1811 return true;1812}1813 1814int dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,1815 struct queue_limits *limits)1816{1817 int r;1818 1819 if (!dm_table_supports_nowait(t))1820 limits->features &= ~BLK_FEAT_NOWAIT;1821 1822 /*1823 * The current polling impementation does not support request based1824 * stacking.1825 */1826 if (!__table_type_bio_based(t->type))1827 limits->features &= ~BLK_FEAT_POLL;1828 1829 if (!dm_table_supports_discards(t)) {1830 limits->max_hw_discard_sectors = 0;1831 limits->discard_granularity = 0;1832 limits->discard_alignment = 0;1833 }1834 1835 if (!dm_table_supports_write_zeroes(t))1836 limits->max_write_zeroes_sectors = 0;1837 1838 if (!dm_table_supports_secure_erase(t))1839 limits->max_secure_erase_sectors = 0;1840 1841 if (dm_table_supports_flush(t))1842 limits->features |= BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA;1843 1844 if (dm_table_supports_dax(t, device_not_dax_capable)) {1845 limits->features |= BLK_FEAT_DAX;1846 if (dm_table_supports_dax(t, device_not_dax_synchronous_capable))1847 set_dax_synchronous(t->md->dax_dev);1848 } else1849 limits->features &= ~BLK_FEAT_DAX;1850 1851 if (dm_table_any_dev_attr(t, device_dax_write_cache_enabled, NULL))1852 dax_write_cache(t->md->dax_dev, true);1853 1854 /* For a zoned table, setup the zone related queue attributes. */1855 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&1856 (limits->features & BLK_FEAT_ZONED)) {1857 r = dm_set_zones_restrictions(t, q, limits);1858 if (r)1859 return r;1860 }1861 1862 r = queue_limits_set(q, limits);1863 if (r)1864 return r;1865 1866 /*1867 * Now that the limits are set, check the zones mapped by the table1868 * and setup the resources for zone append emulation if necessary.1869 */1870 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&1871 (limits->features & BLK_FEAT_ZONED)) {1872 r = dm_revalidate_zones(t, q);1873 if (r)1874 return r;1875 }1876 1877 dm_update_crypto_profile(q, t);1878 return 0;1879}1880 1881struct list_head *dm_table_get_devices(struct dm_table *t)1882{1883 return &t->devices;1884}1885 1886blk_mode_t dm_table_get_mode(struct dm_table *t)1887{1888 return t->mode;1889}1890EXPORT_SYMBOL(dm_table_get_mode);1891 1892enum suspend_mode {1893 PRESUSPEND,1894 PRESUSPEND_UNDO,1895 POSTSUSPEND,1896};1897 1898static void suspend_targets(struct dm_table *t, enum suspend_mode mode)1899{1900 lockdep_assert_held(&t->md->suspend_lock);1901 1902 for (unsigned int i = 0; i < t->num_targets; i++) {1903 struct dm_target *ti = dm_table_get_target(t, i);1904 1905 switch (mode) {1906 case PRESUSPEND:1907 if (ti->type->presuspend)1908 ti->type->presuspend(ti);1909 break;1910 case PRESUSPEND_UNDO:1911 if (ti->type->presuspend_undo)1912 ti->type->presuspend_undo(ti);1913 break;1914 case POSTSUSPEND:1915 if (ti->type->postsuspend)1916 ti->type->postsuspend(ti);1917 break;1918 }1919 }1920}1921 1922void dm_table_presuspend_targets(struct dm_table *t)1923{1924 if (!t)1925 return;1926 1927 suspend_targets(t, PRESUSPEND);1928}1929 1930void dm_table_presuspend_undo_targets(struct dm_table *t)1931{1932 if (!t)1933 return;1934 1935 suspend_targets(t, PRESUSPEND_UNDO);1936}1937 1938void dm_table_postsuspend_targets(struct dm_table *t)1939{1940 if (!t)1941 return;1942 1943 suspend_targets(t, POSTSUSPEND);1944}1945 1946int dm_table_resume_targets(struct dm_table *t)1947{1948 unsigned int i;1949 int r = 0;1950 1951 lockdep_assert_held(&t->md->suspend_lock);1952 1953 for (i = 0; i < t->num_targets; i++) {1954 struct dm_target *ti = dm_table_get_target(t, i);1955 1956 if (!ti->type->preresume)1957 continue;1958 1959 r = ti->type->preresume(ti);1960 if (r) {1961 DMERR("%s: %s: preresume failed, error = %d",1962 dm_device_name(t->md), ti->type->name, r);1963 return r;1964 }1965 }1966 1967 for (i = 0; i < t->num_targets; i++) {1968 struct dm_target *ti = dm_table_get_target(t, i);1969 1970 if (ti->type->resume)1971 ti->type->resume(ti);1972 }1973 1974 return 0;1975}1976 1977struct mapped_device *dm_table_get_md(struct dm_table *t)1978{1979 return t->md;1980}1981EXPORT_SYMBOL(dm_table_get_md);1982 1983const char *dm_table_device_name(struct dm_table *t)1984{1985 return dm_device_name(t->md);1986}1987EXPORT_SYMBOL_GPL(dm_table_device_name);1988 1989void dm_table_run_md_queue_async(struct dm_table *t)1990{1991 if (!dm_table_request_based(t))1992 return;1993 1994 if (t->md->queue)1995 blk_mq_run_hw_queues(t->md->queue, true);1996}1997EXPORT_SYMBOL(dm_table_run_md_queue_async);1998 1999