165 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright 2022 Red Hat, Inc.4 */5 6#include <linux/bio.h>7#include <linux/blk-crypto.h>8#include <linux/blk-integrity.h>9 10#include "dm-core.h"11 12static inline bool dm_bvec_iter_rewind(const struct bio_vec *bv,13 struct bvec_iter *iter,14 unsigned int bytes)15{16 int idx;17 18 iter->bi_size += bytes;19 if (bytes <= iter->bi_bvec_done) {20 iter->bi_bvec_done -= bytes;21 return true;22 }23 24 bytes -= iter->bi_bvec_done;25 idx = iter->bi_idx - 1;26 27 while (idx >= 0 && bytes && bytes > bv[idx].bv_len) {28 bytes -= bv[idx].bv_len;29 idx--;30 }31 32 if (WARN_ONCE(idx < 0 && bytes,33 "Attempted to rewind iter beyond bvec's boundaries\n")) {34 iter->bi_size -= bytes;35 iter->bi_bvec_done = 0;36 iter->bi_idx = 0;37 return false;38 }39 40 iter->bi_idx = idx;41 iter->bi_bvec_done = bv[idx].bv_len - bytes;42 return true;43}44 45#if defined(CONFIG_BLK_DEV_INTEGRITY)46 47/**48 * dm_bio_integrity_rewind - Rewind integrity vector49 * @bio: bio whose integrity vector to update50 * @bytes_done: number of data bytes to rewind51 *52 * Description: This function calculates how many integrity bytes the53 * number of completed data bytes correspond to and rewind the54 * integrity vector accordingly.55 */56static void dm_bio_integrity_rewind(struct bio *bio, unsigned int bytes_done)57{58 struct bio_integrity_payload *bip = bio_integrity(bio);59 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);60 unsigned int bytes = bio_integrity_bytes(bi, bytes_done >> 9);61 62 bip->bip_iter.bi_sector -= bio_integrity_intervals(bi, bytes_done >> 9);63 dm_bvec_iter_rewind(bip->bip_vec, &bip->bip_iter, bytes);64}65 66#else /* CONFIG_BLK_DEV_INTEGRITY */67 68static inline void dm_bio_integrity_rewind(struct bio *bio,69 unsigned int bytes_done)70{71}72 73#endif74 75#if defined(CONFIG_BLK_INLINE_ENCRYPTION)76 77/* Decrements @dun by @dec, treating @dun as a multi-limb integer. */78static void dm_bio_crypt_dun_decrement(u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE],79 unsigned int dec)80{81 int i;82 83 for (i = 0; dec && i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++) {84 u64 prev = dun[i];85 86 dun[i] -= dec;87 if (dun[i] > prev)88 dec = 1;89 else90 dec = 0;91 }92}93 94static void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes)95{96 struct bio_crypt_ctx *bc = bio->bi_crypt_context;97 98 dm_bio_crypt_dun_decrement(bc->bc_dun,99 bytes >> bc->bc_key->data_unit_size_bits);100}101 102#else /* CONFIG_BLK_INLINE_ENCRYPTION */103 104static inline void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes)105{106}107 108#endif109 110static inline void dm_bio_rewind_iter(const struct bio *bio,111 struct bvec_iter *iter, unsigned int bytes)112{113 iter->bi_sector -= bytes >> 9;114 115 /* No advance means no rewind */116 if (bio_no_advance_iter(bio))117 iter->bi_size += bytes;118 else119 dm_bvec_iter_rewind(bio->bi_io_vec, iter, bytes);120}121 122/**123 * dm_bio_rewind - update ->bi_iter of @bio by rewinding @bytes.124 * @bio: bio to rewind125 * @bytes: how many bytes to rewind126 *127 * WARNING:128 * Caller must ensure that @bio has a fixed end sector, to allow129 * rewinding from end of bio and restoring its original position.130 * Caller is also responsibile for restoring bio's size.131 */132static void dm_bio_rewind(struct bio *bio, unsigned int bytes)133{134 if (bio_integrity(bio))135 dm_bio_integrity_rewind(bio, bytes);136 137 if (bio_has_crypt_ctx(bio))138 dm_bio_crypt_rewind(bio, bytes);139 140 dm_bio_rewind_iter(bio, &bio->bi_iter, bytes);141}142 143void dm_io_rewind(struct dm_io *io, struct bio_set *bs)144{145 struct bio *orig = io->orig_bio;146 struct bio *new_orig = bio_alloc_clone(orig->bi_bdev, orig,147 GFP_NOIO, bs);148 /*149 * dm_bio_rewind can restore to previous position since the150 * end sector is fixed for original bio, but we still need151 * to restore bio's size manually (using io->sectors).152 */153 dm_bio_rewind(new_orig, ((io->sector_offset << 9) -154 orig->bi_iter.bi_size));155 bio_trim(new_orig, 0, io->sectors);156 157 bio_chain(new_orig, orig);158 /*159 * __bi_remaining was increased (by dm_split_and_process_bio),160 * so must drop the one added in bio_chain.161 */162 atomic_dec(&orig->__bi_remaining);163 io->orig_bio = new_orig;164}165