1860 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2012 Red Hat, Inc.4 *5 * This file is released under the GPL.6 */7 8#include "dm-cache-metadata.h"9 10#include "persistent-data/dm-array.h"11#include "persistent-data/dm-bitset.h"12#include "persistent-data/dm-space-map.h"13#include "persistent-data/dm-space-map-disk.h"14#include "persistent-data/dm-transaction-manager.h"15 16#include <linux/device-mapper.h>17#include <linux/refcount.h>18 19/*----------------------------------------------------------------*/20 21#define DM_MSG_PREFIX "cache metadata"22 23#define CACHE_SUPERBLOCK_MAGIC 0614200324#define CACHE_SUPERBLOCK_LOCATION 025 26/*27 * defines a range of metadata versions that this module can handle.28 */29#define MIN_CACHE_VERSION 130#define MAX_CACHE_VERSION 231 32/*33 * 3 for btree insert +34 * 2 for btree lookup used within space map35 */36#define CACHE_MAX_CONCURRENT_LOCKS 537#define SPACE_MAP_ROOT_SIZE 12838 39enum superblock_flag_bits {40 /* for spotting crashes that would invalidate the dirty bitset */41 CLEAN_SHUTDOWN,42 /* metadata must be checked using the tools */43 NEEDS_CHECK,44};45 46/*47 * Each mapping from cache block -> origin block carries a set of flags.48 */49enum mapping_bits {50 /*51 * A valid mapping. Because we're using an array we clear this52 * flag for an non existant mapping.53 */54 M_VALID = 1,55 56 /*57 * The data on the cache is different from that on the origin.58 * This flag is only used by metadata format 1.59 */60 M_DIRTY = 261};62 63struct cache_disk_superblock {64 __le32 csum;65 __le32 flags;66 __le64 blocknr;67 68 __u8 uuid[16];69 __le64 magic;70 __le32 version;71 72 __u8 policy_name[CACHE_POLICY_NAME_SIZE];73 __le32 policy_hint_size;74 75 __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];76 __le64 mapping_root;77 __le64 hint_root;78 79 __le64 discard_root;80 __le64 discard_block_size;81 __le64 discard_nr_blocks;82 83 __le32 data_block_size;84 __le32 metadata_block_size;85 __le32 cache_blocks;86 87 __le32 compat_flags;88 __le32 compat_ro_flags;89 __le32 incompat_flags;90 91 __le32 read_hits;92 __le32 read_misses;93 __le32 write_hits;94 __le32 write_misses;95 96 __le32 policy_version[CACHE_POLICY_VERSION_SIZE];97 98 /*99 * Metadata format 2 fields.100 */101 __le64 dirty_root;102} __packed;103 104struct dm_cache_metadata {105 refcount_t ref_count;106 struct list_head list;107 108 unsigned int version;109 struct block_device *bdev;110 struct dm_block_manager *bm;111 struct dm_space_map *metadata_sm;112 struct dm_transaction_manager *tm;113 114 struct dm_array_info info;115 struct dm_array_info hint_info;116 struct dm_disk_bitset discard_info;117 118 struct rw_semaphore root_lock;119 unsigned long flags;120 dm_block_t root;121 dm_block_t hint_root;122 dm_block_t discard_root;123 124 sector_t discard_block_size;125 dm_dblock_t discard_nr_blocks;126 127 sector_t data_block_size;128 dm_cblock_t cache_blocks;129 bool changed:1;130 bool clean_when_opened:1;131 132 char policy_name[CACHE_POLICY_NAME_SIZE];133 unsigned int policy_version[CACHE_POLICY_VERSION_SIZE];134 size_t policy_hint_size;135 struct dm_cache_statistics stats;136 137 /*138 * Reading the space map root can fail, so we read it into this139 * buffer before the superblock is locked and updated.140 */141 __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];142 143 /*144 * Set if a transaction has to be aborted but the attempt to roll145 * back to the previous (good) transaction failed. The only146 * metadata operation permissible in this state is the closing of147 * the device.148 */149 bool fail_io:1;150 151 /*152 * Metadata format 2 fields.153 */154 dm_block_t dirty_root;155 struct dm_disk_bitset dirty_info;156 157 /*158 * These structures are used when loading metadata. They're too159 * big to put on the stack.160 */161 struct dm_array_cursor mapping_cursor;162 struct dm_array_cursor hint_cursor;163 struct dm_bitset_cursor dirty_cursor;164};165 166/*167 *-----------------------------------------------------------------168 * superblock validator169 *-----------------------------------------------------------------170 */171#define SUPERBLOCK_CSUM_XOR 9031977172 173static void sb_prepare_for_write(const struct dm_block_validator *v,174 struct dm_block *b,175 size_t sb_block_size)176{177 struct cache_disk_superblock *disk_super = dm_block_data(b);178 179 disk_super->blocknr = cpu_to_le64(dm_block_location(b));180 disk_super->csum = cpu_to_le32(dm_bm_checksum(&disk_super->flags,181 sb_block_size - sizeof(__le32),182 SUPERBLOCK_CSUM_XOR));183}184 185static int check_metadata_version(struct cache_disk_superblock *disk_super)186{187 uint32_t metadata_version = le32_to_cpu(disk_super->version);188 189 if (metadata_version < MIN_CACHE_VERSION || metadata_version > MAX_CACHE_VERSION) {190 DMERR("Cache metadata version %u found, but only versions between %u and %u supported.",191 metadata_version, MIN_CACHE_VERSION, MAX_CACHE_VERSION);192 return -EINVAL;193 }194 195 return 0;196}197 198static int sb_check(const struct dm_block_validator *v,199 struct dm_block *b,200 size_t sb_block_size)201{202 struct cache_disk_superblock *disk_super = dm_block_data(b);203 __le32 csum_le;204 205 if (dm_block_location(b) != le64_to_cpu(disk_super->blocknr)) {206 DMERR("%s failed: blocknr %llu: wanted %llu",207 __func__, le64_to_cpu(disk_super->blocknr),208 (unsigned long long)dm_block_location(b));209 return -ENOTBLK;210 }211 212 if (le64_to_cpu(disk_super->magic) != CACHE_SUPERBLOCK_MAGIC) {213 DMERR("%s failed: magic %llu: wanted %llu",214 __func__, le64_to_cpu(disk_super->magic),215 (unsigned long long)CACHE_SUPERBLOCK_MAGIC);216 return -EILSEQ;217 }218 219 csum_le = cpu_to_le32(dm_bm_checksum(&disk_super->flags,220 sb_block_size - sizeof(__le32),221 SUPERBLOCK_CSUM_XOR));222 if (csum_le != disk_super->csum) {223 DMERR("%s failed: csum %u: wanted %u",224 __func__, le32_to_cpu(csum_le), le32_to_cpu(disk_super->csum));225 return -EILSEQ;226 }227 228 return check_metadata_version(disk_super);229}230 231static const struct dm_block_validator sb_validator = {232 .name = "superblock",233 .prepare_for_write = sb_prepare_for_write,234 .check = sb_check235};236 237/*----------------------------------------------------------------*/238 239static int superblock_read_lock(struct dm_cache_metadata *cmd,240 struct dm_block **sblock)241{242 return dm_bm_read_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,243 &sb_validator, sblock);244}245 246static int superblock_lock_zero(struct dm_cache_metadata *cmd,247 struct dm_block **sblock)248{249 return dm_bm_write_lock_zero(cmd->bm, CACHE_SUPERBLOCK_LOCATION,250 &sb_validator, sblock);251}252 253static int superblock_lock(struct dm_cache_metadata *cmd,254 struct dm_block **sblock)255{256 return dm_bm_write_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,257 &sb_validator, sblock);258}259 260/*----------------------------------------------------------------*/261 262static int __superblock_all_zeroes(struct dm_block_manager *bm, bool *result)263{264 int r;265 unsigned int i;266 struct dm_block *b;267 __le64 *data_le, zero = cpu_to_le64(0);268 unsigned int sb_block_size = dm_bm_block_size(bm) / sizeof(__le64);269 270 /*271 * We can't use a validator here - it may be all zeroes.272 */273 r = dm_bm_read_lock(bm, CACHE_SUPERBLOCK_LOCATION, NULL, &b);274 if (r)275 return r;276 277 data_le = dm_block_data(b);278 *result = true;279 for (i = 0; i < sb_block_size; i++) {280 if (data_le[i] != zero) {281 *result = false;282 break;283 }284 }285 286 dm_bm_unlock(b);287 288 return 0;289}290 291static void __setup_mapping_info(struct dm_cache_metadata *cmd)292{293 struct dm_btree_value_type vt;294 295 vt.context = NULL;296 vt.size = sizeof(__le64);297 vt.inc = NULL;298 vt.dec = NULL;299 vt.equal = NULL;300 dm_array_info_init(&cmd->info, cmd->tm, &vt);301 302 if (cmd->policy_hint_size) {303 vt.size = sizeof(__le32);304 dm_array_info_init(&cmd->hint_info, cmd->tm, &vt);305 }306}307 308static int __save_sm_root(struct dm_cache_metadata *cmd)309{310 int r;311 size_t metadata_len;312 313 r = dm_sm_root_size(cmd->metadata_sm, &metadata_len);314 if (r < 0)315 return r;316 317 return dm_sm_copy_root(cmd->metadata_sm, &cmd->metadata_space_map_root,318 metadata_len);319}320 321static void __copy_sm_root(struct dm_cache_metadata *cmd,322 struct cache_disk_superblock *disk_super)323{324 memcpy(&disk_super->metadata_space_map_root,325 &cmd->metadata_space_map_root,326 sizeof(cmd->metadata_space_map_root));327}328 329static bool separate_dirty_bits(struct dm_cache_metadata *cmd)330{331 return cmd->version >= 2;332}333 334static int __write_initial_superblock(struct dm_cache_metadata *cmd)335{336 int r;337 struct dm_block *sblock;338 struct cache_disk_superblock *disk_super;339 sector_t bdev_size = bdev_nr_sectors(cmd->bdev);340 341 /* FIXME: see if we can lose the max sectors limit */342 if (bdev_size > DM_CACHE_METADATA_MAX_SECTORS)343 bdev_size = DM_CACHE_METADATA_MAX_SECTORS;344 345 r = dm_tm_pre_commit(cmd->tm);346 if (r < 0)347 return r;348 349 /*350 * dm_sm_copy_root() can fail. So we need to do it before we start351 * updating the superblock.352 */353 r = __save_sm_root(cmd);354 if (r)355 return r;356 357 r = superblock_lock_zero(cmd, &sblock);358 if (r)359 return r;360 361 disk_super = dm_block_data(sblock);362 disk_super->flags = 0;363 memset(disk_super->uuid, 0, sizeof(disk_super->uuid));364 disk_super->magic = cpu_to_le64(CACHE_SUPERBLOCK_MAGIC);365 disk_super->version = cpu_to_le32(cmd->version);366 memset(disk_super->policy_name, 0, sizeof(disk_super->policy_name));367 memset(disk_super->policy_version, 0, sizeof(disk_super->policy_version));368 disk_super->policy_hint_size = cpu_to_le32(0);369 370 __copy_sm_root(cmd, disk_super);371 372 disk_super->mapping_root = cpu_to_le64(cmd->root);373 disk_super->hint_root = cpu_to_le64(cmd->hint_root);374 disk_super->discard_root = cpu_to_le64(cmd->discard_root);375 disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);376 disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));377 disk_super->metadata_block_size = cpu_to_le32(DM_CACHE_METADATA_BLOCK_SIZE);378 disk_super->data_block_size = cpu_to_le32(cmd->data_block_size);379 disk_super->cache_blocks = cpu_to_le32(0);380 381 disk_super->read_hits = cpu_to_le32(0);382 disk_super->read_misses = cpu_to_le32(0);383 disk_super->write_hits = cpu_to_le32(0);384 disk_super->write_misses = cpu_to_le32(0);385 386 if (separate_dirty_bits(cmd))387 disk_super->dirty_root = cpu_to_le64(cmd->dirty_root);388 389 return dm_tm_commit(cmd->tm, sblock);390}391 392static int __format_metadata(struct dm_cache_metadata *cmd)393{394 int r;395 396 r = dm_tm_create_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,397 &cmd->tm, &cmd->metadata_sm);398 if (r < 0) {399 DMERR("tm_create_with_sm failed");400 return r;401 }402 403 __setup_mapping_info(cmd);404 405 r = dm_array_empty(&cmd->info, &cmd->root);406 if (r < 0)407 goto bad;408 409 if (separate_dirty_bits(cmd)) {410 dm_disk_bitset_init(cmd->tm, &cmd->dirty_info);411 r = dm_bitset_empty(&cmd->dirty_info, &cmd->dirty_root);412 if (r < 0)413 goto bad;414 }415 416 dm_disk_bitset_init(cmd->tm, &cmd->discard_info);417 r = dm_bitset_empty(&cmd->discard_info, &cmd->discard_root);418 if (r < 0)419 goto bad;420 421 cmd->discard_block_size = 0;422 cmd->discard_nr_blocks = 0;423 424 r = __write_initial_superblock(cmd);425 if (r)426 goto bad;427 428 cmd->clean_when_opened = true;429 return 0;430 431bad:432 dm_tm_destroy(cmd->tm);433 dm_sm_destroy(cmd->metadata_sm);434 435 return r;436}437 438static int __check_incompat_features(struct cache_disk_superblock *disk_super,439 struct dm_cache_metadata *cmd)440{441 uint32_t incompat_flags, features;442 443 incompat_flags = le32_to_cpu(disk_super->incompat_flags);444 features = incompat_flags & ~DM_CACHE_FEATURE_INCOMPAT_SUPP;445 if (features) {446 DMERR("could not access metadata due to unsupported optional features (%lx).",447 (unsigned long)features);448 return -EINVAL;449 }450 451 /*452 * Check for read-only metadata to skip the following RDWR checks.453 */454 if (bdev_read_only(cmd->bdev))455 return 0;456 457 features = le32_to_cpu(disk_super->compat_ro_flags) & ~DM_CACHE_FEATURE_COMPAT_RO_SUPP;458 if (features) {459 DMERR("could not access metadata RDWR due to unsupported optional features (%lx).",460 (unsigned long)features);461 return -EINVAL;462 }463 464 return 0;465}466 467static int __open_metadata(struct dm_cache_metadata *cmd)468{469 int r;470 struct dm_block *sblock;471 struct cache_disk_superblock *disk_super;472 unsigned long sb_flags;473 474 r = superblock_read_lock(cmd, &sblock);475 if (r < 0) {476 DMERR("couldn't read lock superblock");477 return r;478 }479 480 disk_super = dm_block_data(sblock);481 482 /* Verify the data block size hasn't changed */483 if (le32_to_cpu(disk_super->data_block_size) != cmd->data_block_size) {484 DMERR("changing the data block size (from %u to %llu) is not supported",485 le32_to_cpu(disk_super->data_block_size),486 (unsigned long long)cmd->data_block_size);487 r = -EINVAL;488 goto bad;489 }490 491 r = __check_incompat_features(disk_super, cmd);492 if (r < 0)493 goto bad;494 495 r = dm_tm_open_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,496 disk_super->metadata_space_map_root,497 sizeof(disk_super->metadata_space_map_root),498 &cmd->tm, &cmd->metadata_sm);499 if (r < 0) {500 DMERR("tm_open_with_sm failed");501 goto bad;502 }503 504 __setup_mapping_info(cmd);505 dm_disk_bitset_init(cmd->tm, &cmd->dirty_info);506 dm_disk_bitset_init(cmd->tm, &cmd->discard_info);507 sb_flags = le32_to_cpu(disk_super->flags);508 cmd->clean_when_opened = test_bit(CLEAN_SHUTDOWN, &sb_flags);509 dm_bm_unlock(sblock);510 511 return 0;512 513bad:514 dm_bm_unlock(sblock);515 return r;516}517 518static int __open_or_format_metadata(struct dm_cache_metadata *cmd,519 bool format_device)520{521 int r;522 bool unformatted = false;523 524 r = __superblock_all_zeroes(cmd->bm, &unformatted);525 if (r)526 return r;527 528 if (unformatted)529 return format_device ? __format_metadata(cmd) : -EPERM;530 531 return __open_metadata(cmd);532}533 534static int __create_persistent_data_objects(struct dm_cache_metadata *cmd,535 bool may_format_device)536{537 int r;538 539 cmd->bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT,540 CACHE_MAX_CONCURRENT_LOCKS);541 if (IS_ERR(cmd->bm)) {542 DMERR("could not create block manager");543 r = PTR_ERR(cmd->bm);544 cmd->bm = NULL;545 return r;546 }547 548 r = __open_or_format_metadata(cmd, may_format_device);549 if (r) {550 dm_block_manager_destroy(cmd->bm);551 cmd->bm = NULL;552 }553 554 return r;555}556 557static void __destroy_persistent_data_objects(struct dm_cache_metadata *cmd,558 bool destroy_bm)559{560 dm_sm_destroy(cmd->metadata_sm);561 dm_tm_destroy(cmd->tm);562 if (destroy_bm)563 dm_block_manager_destroy(cmd->bm);564}565 566typedef unsigned long (*flags_mutator)(unsigned long);567 568static void update_flags(struct cache_disk_superblock *disk_super,569 flags_mutator mutator)570{571 uint32_t sb_flags = mutator(le32_to_cpu(disk_super->flags));572 573 disk_super->flags = cpu_to_le32(sb_flags);574}575 576static unsigned long set_clean_shutdown(unsigned long flags)577{578 set_bit(CLEAN_SHUTDOWN, &flags);579 return flags;580}581 582static unsigned long clear_clean_shutdown(unsigned long flags)583{584 clear_bit(CLEAN_SHUTDOWN, &flags);585 return flags;586}587 588static void read_superblock_fields(struct dm_cache_metadata *cmd,589 struct cache_disk_superblock *disk_super)590{591 cmd->version = le32_to_cpu(disk_super->version);592 cmd->flags = le32_to_cpu(disk_super->flags);593 cmd->root = le64_to_cpu(disk_super->mapping_root);594 cmd->hint_root = le64_to_cpu(disk_super->hint_root);595 cmd->discard_root = le64_to_cpu(disk_super->discard_root);596 cmd->discard_block_size = le64_to_cpu(disk_super->discard_block_size);597 cmd->discard_nr_blocks = to_dblock(le64_to_cpu(disk_super->discard_nr_blocks));598 cmd->data_block_size = le32_to_cpu(disk_super->data_block_size);599 cmd->cache_blocks = to_cblock(le32_to_cpu(disk_super->cache_blocks));600 strscpy(cmd->policy_name, disk_super->policy_name, sizeof(cmd->policy_name));601 cmd->policy_version[0] = le32_to_cpu(disk_super->policy_version[0]);602 cmd->policy_version[1] = le32_to_cpu(disk_super->policy_version[1]);603 cmd->policy_version[2] = le32_to_cpu(disk_super->policy_version[2]);604 cmd->policy_hint_size = le32_to_cpu(disk_super->policy_hint_size);605 606 cmd->stats.read_hits = le32_to_cpu(disk_super->read_hits);607 cmd->stats.read_misses = le32_to_cpu(disk_super->read_misses);608 cmd->stats.write_hits = le32_to_cpu(disk_super->write_hits);609 cmd->stats.write_misses = le32_to_cpu(disk_super->write_misses);610 611 if (separate_dirty_bits(cmd))612 cmd->dirty_root = le64_to_cpu(disk_super->dirty_root);613 614 cmd->changed = false;615}616 617/*618 * The mutator updates the superblock flags.619 */620static int __begin_transaction_flags(struct dm_cache_metadata *cmd,621 flags_mutator mutator)622{623 int r;624 struct cache_disk_superblock *disk_super;625 struct dm_block *sblock;626 627 r = superblock_lock(cmd, &sblock);628 if (r)629 return r;630 631 disk_super = dm_block_data(sblock);632 update_flags(disk_super, mutator);633 read_superblock_fields(cmd, disk_super);634 dm_bm_unlock(sblock);635 636 return dm_bm_flush(cmd->bm);637}638 639static int __begin_transaction(struct dm_cache_metadata *cmd)640{641 int r;642 struct cache_disk_superblock *disk_super;643 struct dm_block *sblock;644 645 /*646 * We re-read the superblock every time. Shouldn't need to do this647 * really.648 */649 r = superblock_read_lock(cmd, &sblock);650 if (r)651 return r;652 653 disk_super = dm_block_data(sblock);654 read_superblock_fields(cmd, disk_super);655 dm_bm_unlock(sblock);656 657 return 0;658}659 660static int __commit_transaction(struct dm_cache_metadata *cmd,661 flags_mutator mutator)662{663 int r;664 struct cache_disk_superblock *disk_super;665 struct dm_block *sblock;666 667 /*668 * We need to know if the cache_disk_superblock exceeds a 512-byte sector.669 */670 BUILD_BUG_ON(sizeof(struct cache_disk_superblock) > 512);671 672 if (separate_dirty_bits(cmd)) {673 r = dm_bitset_flush(&cmd->dirty_info, cmd->dirty_root,674 &cmd->dirty_root);675 if (r)676 return r;677 }678 679 r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root,680 &cmd->discard_root);681 if (r)682 return r;683 684 r = dm_tm_pre_commit(cmd->tm);685 if (r < 0)686 return r;687 688 r = __save_sm_root(cmd);689 if (r)690 return r;691 692 r = superblock_lock(cmd, &sblock);693 if (r)694 return r;695 696 disk_super = dm_block_data(sblock);697 698 disk_super->flags = cpu_to_le32(cmd->flags);699 if (mutator)700 update_flags(disk_super, mutator);701 702 disk_super->mapping_root = cpu_to_le64(cmd->root);703 if (separate_dirty_bits(cmd))704 disk_super->dirty_root = cpu_to_le64(cmd->dirty_root);705 disk_super->hint_root = cpu_to_le64(cmd->hint_root);706 disk_super->discard_root = cpu_to_le64(cmd->discard_root);707 disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);708 disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));709 disk_super->cache_blocks = cpu_to_le32(from_cblock(cmd->cache_blocks));710 strscpy(disk_super->policy_name, cmd->policy_name, sizeof(disk_super->policy_name));711 disk_super->policy_version[0] = cpu_to_le32(cmd->policy_version[0]);712 disk_super->policy_version[1] = cpu_to_le32(cmd->policy_version[1]);713 disk_super->policy_version[2] = cpu_to_le32(cmd->policy_version[2]);714 disk_super->policy_hint_size = cpu_to_le32(cmd->policy_hint_size);715 716 disk_super->read_hits = cpu_to_le32(cmd->stats.read_hits);717 disk_super->read_misses = cpu_to_le32(cmd->stats.read_misses);718 disk_super->write_hits = cpu_to_le32(cmd->stats.write_hits);719 disk_super->write_misses = cpu_to_le32(cmd->stats.write_misses);720 __copy_sm_root(cmd, disk_super);721 722 return dm_tm_commit(cmd->tm, sblock);723}724 725/*----------------------------------------------------------------*/726 727/*728 * The mappings are held in a dm-array that has 64-bit values stored in729 * little-endian format. The index is the cblock, the high 48bits of the730 * value are the oblock and the low 16 bit the flags.731 */732#define FLAGS_MASK ((1 << 16) - 1)733 734static __le64 pack_value(dm_oblock_t block, unsigned int flags)735{736 uint64_t value = from_oblock(block);737 738 value <<= 16;739 value = value | (flags & FLAGS_MASK);740 return cpu_to_le64(value);741}742 743static void unpack_value(__le64 value_le, dm_oblock_t *block, unsigned int *flags)744{745 uint64_t value = le64_to_cpu(value_le);746 uint64_t b = value >> 16;747 748 *block = to_oblock(b);749 *flags = value & FLAGS_MASK;750}751 752/*----------------------------------------------------------------*/753 754static struct dm_cache_metadata *metadata_open(struct block_device *bdev,755 sector_t data_block_size,756 bool may_format_device,757 size_t policy_hint_size,758 unsigned int metadata_version)759{760 int r;761 struct dm_cache_metadata *cmd;762 763 cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);764 if (!cmd) {765 DMERR("could not allocate metadata struct");766 return ERR_PTR(-ENOMEM);767 }768 769 cmd->version = metadata_version;770 refcount_set(&cmd->ref_count, 1);771 init_rwsem(&cmd->root_lock);772 cmd->bdev = bdev;773 cmd->data_block_size = data_block_size;774 cmd->cache_blocks = 0;775 cmd->policy_hint_size = policy_hint_size;776 cmd->changed = true;777 cmd->fail_io = false;778 779 r = __create_persistent_data_objects(cmd, may_format_device);780 if (r) {781 kfree(cmd);782 return ERR_PTR(r);783 }784 785 r = __begin_transaction_flags(cmd, clear_clean_shutdown);786 if (r < 0) {787 dm_cache_metadata_close(cmd);788 return ERR_PTR(r);789 }790 791 return cmd;792}793 794/*795 * We keep a little list of ref counted metadata objects to prevent two796 * different target instances creating separate bufio instances. This is797 * an issue if a table is reloaded before the suspend.798 */799static DEFINE_MUTEX(table_lock);800static LIST_HEAD(table);801 802static struct dm_cache_metadata *lookup(struct block_device *bdev)803{804 struct dm_cache_metadata *cmd;805 806 list_for_each_entry(cmd, &table, list)807 if (cmd->bdev == bdev) {808 refcount_inc(&cmd->ref_count);809 return cmd;810 }811 812 return NULL;813}814 815static struct dm_cache_metadata *lookup_or_open(struct block_device *bdev,816 sector_t data_block_size,817 bool may_format_device,818 size_t policy_hint_size,819 unsigned int metadata_version)820{821 struct dm_cache_metadata *cmd, *cmd2;822 823 mutex_lock(&table_lock);824 cmd = lookup(bdev);825 mutex_unlock(&table_lock);826 827 if (cmd)828 return cmd;829 830 cmd = metadata_open(bdev, data_block_size, may_format_device,831 policy_hint_size, metadata_version);832 if (!IS_ERR(cmd)) {833 mutex_lock(&table_lock);834 cmd2 = lookup(bdev);835 if (cmd2) {836 mutex_unlock(&table_lock);837 __destroy_persistent_data_objects(cmd, true);838 kfree(cmd);839 return cmd2;840 }841 list_add(&cmd->list, &table);842 mutex_unlock(&table_lock);843 }844 845 return cmd;846}847 848static bool same_params(struct dm_cache_metadata *cmd, sector_t data_block_size)849{850 if (cmd->data_block_size != data_block_size) {851 DMERR("data_block_size (%llu) different from that in metadata (%llu)",852 (unsigned long long) data_block_size,853 (unsigned long long) cmd->data_block_size);854 return false;855 }856 857 return true;858}859 860struct dm_cache_metadata *dm_cache_metadata_open(struct block_device *bdev,861 sector_t data_block_size,862 bool may_format_device,863 size_t policy_hint_size,864 unsigned int metadata_version)865{866 struct dm_cache_metadata *cmd = lookup_or_open(bdev, data_block_size, may_format_device,867 policy_hint_size, metadata_version);868 869 if (!IS_ERR(cmd) && !same_params(cmd, data_block_size)) {870 dm_cache_metadata_close(cmd);871 return ERR_PTR(-EINVAL);872 }873 874 return cmd;875}876 877void dm_cache_metadata_close(struct dm_cache_metadata *cmd)878{879 if (refcount_dec_and_test(&cmd->ref_count)) {880 mutex_lock(&table_lock);881 list_del(&cmd->list);882 mutex_unlock(&table_lock);883 884 if (!cmd->fail_io)885 __destroy_persistent_data_objects(cmd, true);886 kfree(cmd);887 }888}889 890/*891 * Checks that the given cache block is either unmapped or clean.892 */893static int block_clean_combined_dirty(struct dm_cache_metadata *cmd, dm_cblock_t b,894 bool *result)895{896 int r;897 __le64 value;898 dm_oblock_t ob;899 unsigned int flags;900 901 r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(b), &value);902 if (r)903 return r;904 905 unpack_value(value, &ob, &flags);906 *result = !((flags & M_VALID) && (flags & M_DIRTY));907 908 return 0;909}910 911static int blocks_are_clean_combined_dirty(struct dm_cache_metadata *cmd,912 dm_cblock_t begin, dm_cblock_t end,913 bool *result)914{915 int r;916 *result = true;917 918 while (begin != end) {919 r = block_clean_combined_dirty(cmd, begin, result);920 if (r) {921 DMERR("block_clean_combined_dirty failed");922 return r;923 }924 925 if (!*result) {926 DMERR("cache block %llu is dirty",927 (unsigned long long) from_cblock(begin));928 return 0;929 }930 931 begin = to_cblock(from_cblock(begin) + 1);932 }933 934 return 0;935}936 937static int blocks_are_clean_separate_dirty(struct dm_cache_metadata *cmd,938 dm_cblock_t begin, dm_cblock_t end,939 bool *result)940{941 int r;942 bool dirty_flag;943 *result = true;944 945 if (from_cblock(cmd->cache_blocks) == 0)946 /* Nothing to do */947 return 0;948 949 r = dm_bitset_cursor_begin(&cmd->dirty_info, cmd->dirty_root,950 from_cblock(cmd->cache_blocks), &cmd->dirty_cursor);951 if (r) {952 DMERR("%s: dm_bitset_cursor_begin for dirty failed", __func__);953 return r;954 }955 956 r = dm_bitset_cursor_skip(&cmd->dirty_cursor, from_cblock(begin));957 if (r) {958 DMERR("%s: dm_bitset_cursor_skip for dirty failed", __func__);959 dm_bitset_cursor_end(&cmd->dirty_cursor);960 return r;961 }962 963 while (begin != end) {964 /*965 * We assume that unmapped blocks have their dirty bit966 * cleared.967 */968 dirty_flag = dm_bitset_cursor_get_value(&cmd->dirty_cursor);969 if (dirty_flag) {970 DMERR("%s: cache block %llu is dirty", __func__,971 (unsigned long long) from_cblock(begin));972 dm_bitset_cursor_end(&cmd->dirty_cursor);973 *result = false;974 return 0;975 }976 977 begin = to_cblock(from_cblock(begin) + 1);978 if (begin == end)979 break;980 981 r = dm_bitset_cursor_next(&cmd->dirty_cursor);982 if (r) {983 DMERR("%s: dm_bitset_cursor_next for dirty failed", __func__);984 dm_bitset_cursor_end(&cmd->dirty_cursor);985 return r;986 }987 }988 989 dm_bitset_cursor_end(&cmd->dirty_cursor);990 991 return 0;992}993 994static int blocks_are_unmapped_or_clean(struct dm_cache_metadata *cmd,995 dm_cblock_t begin, dm_cblock_t end,996 bool *result)997{998 if (separate_dirty_bits(cmd))999 return blocks_are_clean_separate_dirty(cmd, begin, end, result);1000 else1001 return blocks_are_clean_combined_dirty(cmd, begin, end, result);1002}1003 1004static bool cmd_write_lock(struct dm_cache_metadata *cmd)1005{1006 down_write(&cmd->root_lock);1007 if (cmd->fail_io || dm_bm_is_read_only(cmd->bm)) {1008 up_write(&cmd->root_lock);1009 return false;1010 }1011 return true;1012}1013 1014#define WRITE_LOCK(cmd) \1015 do { \1016 if (!cmd_write_lock((cmd))) \1017 return -EINVAL; \1018 } while (0)1019 1020#define WRITE_LOCK_VOID(cmd) \1021 do { \1022 if (!cmd_write_lock((cmd))) \1023 return; \1024 } while (0)1025 1026#define WRITE_UNLOCK(cmd) \1027 up_write(&(cmd)->root_lock)1028 1029static bool cmd_read_lock(struct dm_cache_metadata *cmd)1030{1031 down_read(&cmd->root_lock);1032 if (cmd->fail_io) {1033 up_read(&cmd->root_lock);1034 return false;1035 }1036 return true;1037}1038 1039#define READ_LOCK(cmd) \1040 do { \1041 if (!cmd_read_lock((cmd))) \1042 return -EINVAL; \1043 } while (0)1044 1045#define READ_LOCK_VOID(cmd) \1046 do { \1047 if (!cmd_read_lock((cmd))) \1048 return; \1049 } while (0)1050 1051#define READ_UNLOCK(cmd) \1052 up_read(&(cmd)->root_lock)1053 1054int dm_cache_resize(struct dm_cache_metadata *cmd, dm_cblock_t new_cache_size)1055{1056 int r;1057 bool clean;1058 __le64 null_mapping = pack_value(0, 0);1059 1060 WRITE_LOCK(cmd);1061 __dm_bless_for_disk(&null_mapping);1062 1063 if (from_cblock(new_cache_size) < from_cblock(cmd->cache_blocks)) {1064 r = blocks_are_unmapped_or_clean(cmd, new_cache_size, cmd->cache_blocks, &clean);1065 if (r) {1066 __dm_unbless_for_disk(&null_mapping);1067 goto out;1068 }1069 1070 if (!clean) {1071 DMERR("unable to shrink cache due to dirty blocks");1072 r = -EINVAL;1073 __dm_unbless_for_disk(&null_mapping);1074 goto out;1075 }1076 }1077 1078 r = dm_array_resize(&cmd->info, cmd->root, from_cblock(cmd->cache_blocks),1079 from_cblock(new_cache_size),1080 &null_mapping, &cmd->root);1081 if (r)1082 goto out;1083 1084 if (separate_dirty_bits(cmd)) {1085 r = dm_bitset_resize(&cmd->dirty_info, cmd->dirty_root,1086 from_cblock(cmd->cache_blocks), from_cblock(new_cache_size),1087 false, &cmd->dirty_root);1088 if (r)1089 goto out;1090 }1091 1092 cmd->cache_blocks = new_cache_size;1093 cmd->changed = true;1094 1095out:1096 WRITE_UNLOCK(cmd);1097 1098 return r;1099}1100 1101int dm_cache_discard_bitset_resize(struct dm_cache_metadata *cmd,1102 sector_t discard_block_size,1103 dm_dblock_t new_nr_entries)1104{1105 int r;1106 1107 WRITE_LOCK(cmd);1108 r = dm_bitset_resize(&cmd->discard_info,1109 cmd->discard_root,1110 from_dblock(cmd->discard_nr_blocks),1111 from_dblock(new_nr_entries),1112 false, &cmd->discard_root);1113 if (!r) {1114 cmd->discard_block_size = discard_block_size;1115 cmd->discard_nr_blocks = new_nr_entries;1116 }1117 1118 cmd->changed = true;1119 WRITE_UNLOCK(cmd);1120 1121 return r;1122}1123 1124static int __set_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)1125{1126 return dm_bitset_set_bit(&cmd->discard_info, cmd->discard_root,1127 from_dblock(b), &cmd->discard_root);1128}1129 1130static int __clear_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)1131{1132 return dm_bitset_clear_bit(&cmd->discard_info, cmd->discard_root,1133 from_dblock(b), &cmd->discard_root);1134}1135 1136static int __discard(struct dm_cache_metadata *cmd,1137 dm_dblock_t dblock, bool discard)1138{1139 int r;1140 1141 r = (discard ? __set_discard : __clear_discard)(cmd, dblock);1142 if (r)1143 return r;1144 1145 cmd->changed = true;1146 return 0;1147}1148 1149int dm_cache_set_discard(struct dm_cache_metadata *cmd,1150 dm_dblock_t dblock, bool discard)1151{1152 int r;1153 1154 WRITE_LOCK(cmd);1155 r = __discard(cmd, dblock, discard);1156 WRITE_UNLOCK(cmd);1157 1158 return r;1159}1160 1161static int __load_discards(struct dm_cache_metadata *cmd,1162 load_discard_fn fn, void *context)1163{1164 int r = 0;1165 uint32_t b;1166 struct dm_bitset_cursor c;1167 1168 if (from_dblock(cmd->discard_nr_blocks) == 0)1169 /* nothing to do */1170 return 0;1171 1172 if (cmd->clean_when_opened) {1173 r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root, &cmd->discard_root);1174 if (r)1175 return r;1176 1177 r = dm_bitset_cursor_begin(&cmd->discard_info, cmd->discard_root,1178 from_dblock(cmd->discard_nr_blocks), &c);1179 if (r)1180 return r;1181 1182 for (b = 0; ; b++) {1183 r = fn(context, cmd->discard_block_size, to_dblock(b),1184 dm_bitset_cursor_get_value(&c));1185 if (r)1186 break;1187 1188 if (b >= (from_dblock(cmd->discard_nr_blocks) - 1))1189 break;1190 1191 r = dm_bitset_cursor_next(&c);1192 if (r)1193 break;1194 }1195 1196 dm_bitset_cursor_end(&c);1197 1198 } else {1199 for (b = 0; b < from_dblock(cmd->discard_nr_blocks); b++) {1200 r = fn(context, cmd->discard_block_size, to_dblock(b), false);1201 if (r)1202 return r;1203 }1204 }1205 1206 return r;1207}1208 1209int dm_cache_load_discards(struct dm_cache_metadata *cmd,1210 load_discard_fn fn, void *context)1211{1212 int r;1213 1214 READ_LOCK(cmd);1215 r = __load_discards(cmd, fn, context);1216 READ_UNLOCK(cmd);1217 1218 return r;1219}1220 1221int dm_cache_size(struct dm_cache_metadata *cmd, dm_cblock_t *result)1222{1223 READ_LOCK(cmd);1224 *result = cmd->cache_blocks;1225 READ_UNLOCK(cmd);1226 1227 return 0;1228}1229 1230static int __remove(struct dm_cache_metadata *cmd, dm_cblock_t cblock)1231{1232 int r;1233 __le64 value = pack_value(0, 0);1234 1235 __dm_bless_for_disk(&value);1236 r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),1237 &value, &cmd->root);1238 if (r)1239 return r;1240 1241 cmd->changed = true;1242 return 0;1243}1244 1245int dm_cache_remove_mapping(struct dm_cache_metadata *cmd, dm_cblock_t cblock)1246{1247 int r;1248 1249 WRITE_LOCK(cmd);1250 r = __remove(cmd, cblock);1251 WRITE_UNLOCK(cmd);1252 1253 return r;1254}1255 1256static int __insert(struct dm_cache_metadata *cmd,1257 dm_cblock_t cblock, dm_oblock_t oblock)1258{1259 int r;1260 __le64 value = pack_value(oblock, M_VALID);1261 1262 __dm_bless_for_disk(&value);1263 1264 r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),1265 &value, &cmd->root);1266 if (r)1267 return r;1268 1269 cmd->changed = true;1270 return 0;1271}1272 1273int dm_cache_insert_mapping(struct dm_cache_metadata *cmd,1274 dm_cblock_t cblock, dm_oblock_t oblock)1275{1276 int r;1277 1278 WRITE_LOCK(cmd);1279 r = __insert(cmd, cblock, oblock);1280 WRITE_UNLOCK(cmd);1281 1282 return r;1283}1284 1285static bool policy_unchanged(struct dm_cache_metadata *cmd,1286 struct dm_cache_policy *policy)1287{1288 const char *policy_name = dm_cache_policy_get_name(policy);1289 const unsigned int *policy_version = dm_cache_policy_get_version(policy);1290 size_t policy_hint_size = dm_cache_policy_get_hint_size(policy);1291 1292 /*1293 * Ensure policy names match.1294 */1295 if (strncmp(cmd->policy_name, policy_name, sizeof(cmd->policy_name)))1296 return false;1297 1298 /*1299 * Ensure policy major versions match.1300 */1301 if (cmd->policy_version[0] != policy_version[0])1302 return false;1303 1304 /*1305 * Ensure policy hint sizes match.1306 */1307 if (cmd->policy_hint_size != policy_hint_size)1308 return false;1309 1310 return true;1311}1312 1313static bool hints_array_initialized(struct dm_cache_metadata *cmd)1314{1315 return cmd->hint_root && cmd->policy_hint_size;1316}1317 1318static bool hints_array_available(struct dm_cache_metadata *cmd,1319 struct dm_cache_policy *policy)1320{1321 return cmd->clean_when_opened && policy_unchanged(cmd, policy) &&1322 hints_array_initialized(cmd);1323}1324 1325static int __load_mapping_v1(struct dm_cache_metadata *cmd,1326 uint64_t cb, bool hints_valid,1327 struct dm_array_cursor *mapping_cursor,1328 struct dm_array_cursor *hint_cursor,1329 load_mapping_fn fn, void *context)1330{1331 int r = 0;1332 1333 __le64 mapping;1334 __le32 hint = 0;1335 1336 __le64 *mapping_value_le;1337 __le32 *hint_value_le;1338 1339 dm_oblock_t oblock;1340 unsigned int flags;1341 bool dirty = true;1342 1343 dm_array_cursor_get_value(mapping_cursor, (void **) &mapping_value_le);1344 memcpy(&mapping, mapping_value_le, sizeof(mapping));1345 unpack_value(mapping, &oblock, &flags);1346 1347 if (flags & M_VALID) {1348 if (hints_valid) {1349 dm_array_cursor_get_value(hint_cursor, (void **) &hint_value_le);1350 memcpy(&hint, hint_value_le, sizeof(hint));1351 }1352 if (cmd->clean_when_opened)1353 dirty = flags & M_DIRTY;1354 1355 r = fn(context, oblock, to_cblock(cb), dirty,1356 le32_to_cpu(hint), hints_valid);1357 if (r) {1358 DMERR("policy couldn't load cache block %llu",1359 (unsigned long long) from_cblock(to_cblock(cb)));1360 }1361 }1362 1363 return r;1364}1365 1366static int __load_mapping_v2(struct dm_cache_metadata *cmd,1367 uint64_t cb, bool hints_valid,1368 struct dm_array_cursor *mapping_cursor,1369 struct dm_array_cursor *hint_cursor,1370 struct dm_bitset_cursor *dirty_cursor,1371 load_mapping_fn fn, void *context)1372{1373 int r = 0;1374 1375 __le64 mapping;1376 __le32 hint = 0;1377 1378 __le64 *mapping_value_le;1379 __le32 *hint_value_le;1380 1381 dm_oblock_t oblock;1382 unsigned int flags;1383 bool dirty = true;1384 1385 dm_array_cursor_get_value(mapping_cursor, (void **) &mapping_value_le);1386 memcpy(&mapping, mapping_value_le, sizeof(mapping));1387 unpack_value(mapping, &oblock, &flags);1388 1389 if (flags & M_VALID) {1390 if (hints_valid) {1391 dm_array_cursor_get_value(hint_cursor, (void **) &hint_value_le);1392 memcpy(&hint, hint_value_le, sizeof(hint));1393 }1394 if (cmd->clean_when_opened)1395 dirty = dm_bitset_cursor_get_value(dirty_cursor);1396 1397 r = fn(context, oblock, to_cblock(cb), dirty,1398 le32_to_cpu(hint), hints_valid);1399 if (r) {1400 DMERR("policy couldn't load cache block %llu",1401 (unsigned long long) from_cblock(to_cblock(cb)));1402 }1403 }1404 1405 return r;1406}1407 1408static int __load_mappings(struct dm_cache_metadata *cmd,1409 struct dm_cache_policy *policy,1410 load_mapping_fn fn, void *context)1411{1412 int r;1413 uint64_t cb;1414 1415 bool hints_valid = hints_array_available(cmd, policy);1416 1417 if (from_cblock(cmd->cache_blocks) == 0)1418 /* Nothing to do */1419 return 0;1420 1421 r = dm_array_cursor_begin(&cmd->info, cmd->root, &cmd->mapping_cursor);1422 if (r)1423 return r;1424 1425 if (hints_valid) {1426 r = dm_array_cursor_begin(&cmd->hint_info, cmd->hint_root, &cmd->hint_cursor);1427 if (r) {1428 dm_array_cursor_end(&cmd->mapping_cursor);1429 return r;1430 }1431 }1432 1433 if (separate_dirty_bits(cmd)) {1434 r = dm_bitset_cursor_begin(&cmd->dirty_info, cmd->dirty_root,1435 from_cblock(cmd->cache_blocks),1436 &cmd->dirty_cursor);1437 if (r) {1438 dm_array_cursor_end(&cmd->hint_cursor);1439 dm_array_cursor_end(&cmd->mapping_cursor);1440 return r;1441 }1442 }1443 1444 for (cb = 0; ; cb++) {1445 if (separate_dirty_bits(cmd))1446 r = __load_mapping_v2(cmd, cb, hints_valid,1447 &cmd->mapping_cursor,1448 &cmd->hint_cursor,1449 &cmd->dirty_cursor,1450 fn, context);1451 else1452 r = __load_mapping_v1(cmd, cb, hints_valid,1453 &cmd->mapping_cursor, &cmd->hint_cursor,1454 fn, context);1455 if (r)1456 goto out;1457 1458 /*1459 * We need to break out before we move the cursors.1460 */1461 if (cb >= (from_cblock(cmd->cache_blocks) - 1))1462 break;1463 1464 r = dm_array_cursor_next(&cmd->mapping_cursor);1465 if (r) {1466 DMERR("dm_array_cursor_next for mapping failed");1467 goto out;1468 }1469 1470 if (hints_valid) {1471 r = dm_array_cursor_next(&cmd->hint_cursor);1472 if (r) {1473 dm_array_cursor_end(&cmd->hint_cursor);1474 hints_valid = false;1475 }1476 }1477 1478 if (separate_dirty_bits(cmd)) {1479 r = dm_bitset_cursor_next(&cmd->dirty_cursor);1480 if (r) {1481 DMERR("dm_bitset_cursor_next for dirty failed");1482 goto out;1483 }1484 }1485 }1486out:1487 dm_array_cursor_end(&cmd->mapping_cursor);1488 if (hints_valid)1489 dm_array_cursor_end(&cmd->hint_cursor);1490 1491 if (separate_dirty_bits(cmd))1492 dm_bitset_cursor_end(&cmd->dirty_cursor);1493 1494 return r;1495}1496 1497int dm_cache_load_mappings(struct dm_cache_metadata *cmd,1498 struct dm_cache_policy *policy,1499 load_mapping_fn fn, void *context)1500{1501 int r;1502 1503 READ_LOCK(cmd);1504 r = __load_mappings(cmd, policy, fn, context);1505 READ_UNLOCK(cmd);1506 1507 return r;1508}1509 1510static int __dump_mapping(void *context, uint64_t cblock, void *leaf)1511{1512 __le64 value;1513 dm_oblock_t oblock;1514 unsigned int flags;1515 1516 memcpy(&value, leaf, sizeof(value));1517 unpack_value(value, &oblock, &flags);1518 1519 return 0;1520}1521 1522static int __dump_mappings(struct dm_cache_metadata *cmd)1523{1524 return dm_array_walk(&cmd->info, cmd->root, __dump_mapping, NULL);1525}1526 1527void dm_cache_dump(struct dm_cache_metadata *cmd)1528{1529 READ_LOCK_VOID(cmd);1530 __dump_mappings(cmd);1531 READ_UNLOCK(cmd);1532}1533 1534int dm_cache_changed_this_transaction(struct dm_cache_metadata *cmd)1535{1536 int r;1537 1538 READ_LOCK(cmd);1539 r = cmd->changed;1540 READ_UNLOCK(cmd);1541 1542 return r;1543}1544 1545static int __dirty(struct dm_cache_metadata *cmd, dm_cblock_t cblock, bool dirty)1546{1547 int r;1548 unsigned int flags;1549 dm_oblock_t oblock;1550 __le64 value;1551 1552 r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(cblock), &value);1553 if (r)1554 return r;1555 1556 unpack_value(value, &oblock, &flags);1557 1558 if (((flags & M_DIRTY) && dirty) || (!(flags & M_DIRTY) && !dirty))1559 /* nothing to be done */1560 return 0;1561 1562 value = pack_value(oblock, (flags & ~M_DIRTY) | (dirty ? M_DIRTY : 0));1563 __dm_bless_for_disk(&value);1564 1565 r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),1566 &value, &cmd->root);1567 if (r)1568 return r;1569 1570 cmd->changed = true;1571 return 0;1572 1573}1574 1575static int __set_dirty_bits_v1(struct dm_cache_metadata *cmd, unsigned int nr_bits, unsigned long *bits)1576{1577 int r;1578 unsigned int i;1579 1580 for (i = 0; i < nr_bits; i++) {1581 r = __dirty(cmd, to_cblock(i), test_bit(i, bits));1582 if (r)1583 return r;1584 }1585 1586 return 0;1587}1588 1589static int is_dirty_callback(uint32_t index, bool *value, void *context)1590{1591 unsigned long *bits = context;1592 *value = test_bit(index, bits);1593 return 0;1594}1595 1596static int __set_dirty_bits_v2(struct dm_cache_metadata *cmd, unsigned int nr_bits, unsigned long *bits)1597{1598 int r = 0;1599 1600 /* nr_bits is really just a sanity check */1601 if (nr_bits != from_cblock(cmd->cache_blocks)) {1602 DMERR("dirty bitset is wrong size");1603 return -EINVAL;1604 }1605 1606 r = dm_bitset_del(&cmd->dirty_info, cmd->dirty_root);1607 if (r)1608 return r;1609 1610 cmd->changed = true;1611 return dm_bitset_new(&cmd->dirty_info, &cmd->dirty_root, nr_bits, is_dirty_callback, bits);1612}1613 1614int dm_cache_set_dirty_bits(struct dm_cache_metadata *cmd,1615 unsigned int nr_bits,1616 unsigned long *bits)1617{1618 int r;1619 1620 WRITE_LOCK(cmd);1621 if (separate_dirty_bits(cmd))1622 r = __set_dirty_bits_v2(cmd, nr_bits, bits);1623 else1624 r = __set_dirty_bits_v1(cmd, nr_bits, bits);1625 WRITE_UNLOCK(cmd);1626 1627 return r;1628}1629 1630void dm_cache_metadata_get_stats(struct dm_cache_metadata *cmd,1631 struct dm_cache_statistics *stats)1632{1633 READ_LOCK_VOID(cmd);1634 *stats = cmd->stats;1635 READ_UNLOCK(cmd);1636}1637 1638void dm_cache_metadata_set_stats(struct dm_cache_metadata *cmd,1639 struct dm_cache_statistics *stats)1640{1641 WRITE_LOCK_VOID(cmd);1642 cmd->stats = *stats;1643 WRITE_UNLOCK(cmd);1644}1645 1646int dm_cache_commit(struct dm_cache_metadata *cmd, bool clean_shutdown)1647{1648 int r = -EINVAL;1649 flags_mutator mutator = (clean_shutdown ? set_clean_shutdown :1650 clear_clean_shutdown);1651 1652 WRITE_LOCK(cmd);1653 if (cmd->fail_io)1654 goto out;1655 1656 r = __commit_transaction(cmd, mutator);1657 if (r)1658 goto out;1659 1660 r = __begin_transaction(cmd);1661out:1662 WRITE_UNLOCK(cmd);1663 return r;1664}1665 1666int dm_cache_get_free_metadata_block_count(struct dm_cache_metadata *cmd,1667 dm_block_t *result)1668{1669 int r = -EINVAL;1670 1671 READ_LOCK(cmd);1672 if (!cmd->fail_io)1673 r = dm_sm_get_nr_free(cmd->metadata_sm, result);1674 READ_UNLOCK(cmd);1675 1676 return r;1677}1678 1679int dm_cache_get_metadata_dev_size(struct dm_cache_metadata *cmd,1680 dm_block_t *result)1681{1682 int r = -EINVAL;1683 1684 READ_LOCK(cmd);1685 if (!cmd->fail_io)1686 r = dm_sm_get_nr_blocks(cmd->metadata_sm, result);1687 READ_UNLOCK(cmd);1688 1689 return r;1690}1691 1692/*----------------------------------------------------------------*/1693 1694static int get_hint(uint32_t index, void *value_le, void *context)1695{1696 uint32_t value;1697 struct dm_cache_policy *policy = context;1698 1699 value = policy_get_hint(policy, to_cblock(index));1700 *((__le32 *) value_le) = cpu_to_le32(value);1701 1702 return 0;1703}1704 1705/*1706 * It's quicker to always delete the hint array, and recreate with1707 * dm_array_new().1708 */1709static int write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)1710{1711 int r;1712 size_t hint_size;1713 const char *policy_name = dm_cache_policy_get_name(policy);1714 const unsigned int *policy_version = dm_cache_policy_get_version(policy);1715 1716 if (!policy_name[0] ||1717 (strlen(policy_name) > sizeof(cmd->policy_name) - 1))1718 return -EINVAL;1719 1720 strscpy(cmd->policy_name, policy_name, sizeof(cmd->policy_name));1721 memcpy(cmd->policy_version, policy_version, sizeof(cmd->policy_version));1722 1723 hint_size = dm_cache_policy_get_hint_size(policy);1724 if (!hint_size)1725 return 0; /* short-circuit hints initialization */1726 cmd->policy_hint_size = hint_size;1727 1728 if (cmd->hint_root) {1729 r = dm_array_del(&cmd->hint_info, cmd->hint_root);1730 if (r)1731 return r;1732 }1733 1734 return dm_array_new(&cmd->hint_info, &cmd->hint_root,1735 from_cblock(cmd->cache_blocks),1736 get_hint, policy);1737}1738 1739int dm_cache_write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)1740{1741 int r;1742 1743 WRITE_LOCK(cmd);1744 r = write_hints(cmd, policy);1745 WRITE_UNLOCK(cmd);1746 1747 return r;1748}1749 1750int dm_cache_metadata_all_clean(struct dm_cache_metadata *cmd, bool *result)1751{1752 int r;1753 1754 READ_LOCK(cmd);1755 r = blocks_are_unmapped_or_clean(cmd, 0, cmd->cache_blocks, result);1756 READ_UNLOCK(cmd);1757 1758 return r;1759}1760 1761void dm_cache_metadata_set_read_only(struct dm_cache_metadata *cmd)1762{1763 WRITE_LOCK_VOID(cmd);1764 dm_bm_set_read_only(cmd->bm);1765 WRITE_UNLOCK(cmd);1766}1767 1768void dm_cache_metadata_set_read_write(struct dm_cache_metadata *cmd)1769{1770 WRITE_LOCK_VOID(cmd);1771 dm_bm_set_read_write(cmd->bm);1772 WRITE_UNLOCK(cmd);1773}1774 1775int dm_cache_metadata_set_needs_check(struct dm_cache_metadata *cmd)1776{1777 int r;1778 struct dm_block *sblock;1779 struct cache_disk_superblock *disk_super;1780 1781 WRITE_LOCK(cmd);1782 set_bit(NEEDS_CHECK, &cmd->flags);1783 1784 r = superblock_lock(cmd, &sblock);1785 if (r) {1786 DMERR("couldn't read superblock");1787 goto out;1788 }1789 1790 disk_super = dm_block_data(sblock);1791 disk_super->flags = cpu_to_le32(cmd->flags);1792 1793 dm_bm_unlock(sblock);1794 1795out:1796 WRITE_UNLOCK(cmd);1797 return r;1798}1799 1800int dm_cache_metadata_needs_check(struct dm_cache_metadata *cmd, bool *result)1801{1802 READ_LOCK(cmd);1803 *result = !!test_bit(NEEDS_CHECK, &cmd->flags);1804 READ_UNLOCK(cmd);1805 1806 return 0;1807}1808 1809int dm_cache_metadata_abort(struct dm_cache_metadata *cmd)1810{1811 int r = -EINVAL;1812 struct dm_block_manager *old_bm = NULL, *new_bm = NULL;1813 1814 /* fail_io is double-checked with cmd->root_lock held below */1815 if (unlikely(cmd->fail_io))1816 return r;1817 1818 /*1819 * Replacement block manager (new_bm) is created and old_bm destroyed outside of1820 * cmd root_lock to avoid ABBA deadlock that would result (due to life-cycle of1821 * shrinker associated with the block manager's bufio client vs cmd root_lock).1822 * - must take shrinker_mutex without holding cmd->root_lock1823 */1824 new_bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT,1825 CACHE_MAX_CONCURRENT_LOCKS);1826 1827 WRITE_LOCK(cmd);1828 if (cmd->fail_io) {1829 WRITE_UNLOCK(cmd);1830 goto out;1831 }1832 1833 __destroy_persistent_data_objects(cmd, false);1834 old_bm = cmd->bm;1835 if (IS_ERR(new_bm)) {1836 DMERR("could not create block manager during abort");1837 cmd->bm = NULL;1838 r = PTR_ERR(new_bm);1839 goto out_unlock;1840 }1841 1842 cmd->bm = new_bm;1843 r = __open_or_format_metadata(cmd, false);1844 if (r) {1845 cmd->bm = NULL;1846 goto out_unlock;1847 }1848 new_bm = NULL;1849out_unlock:1850 if (r)1851 cmd->fail_io = true;1852 WRITE_UNLOCK(cmd);1853 dm_block_manager_destroy(old_bm);1854out:1855 if (new_bm && !IS_ERR(new_bm))1856 dm_block_manager_destroy(new_bm);1857 1858 return r;1859}1860