brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2003 Jana Saout <jana@saout.de>4 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>5 * Copyright (C) 2006-2020 Red Hat, Inc. All rights reserved.6 * Copyright (C) 2013-2020 Milan Broz <gmazyland@gmail.com>7 *8 * This file is released under the GPL.9 */10 11#include <linux/completion.h>12#include <linux/err.h>13#include <linux/module.h>14#include <linux/init.h>15#include <linux/kernel.h>16#include <linux/key.h>17#include <linux/bio.h>18#include <linux/blkdev.h>19#include <linux/blk-integrity.h>20#include <linux/mempool.h>21#include <linux/slab.h>22#include <linux/crypto.h>23#include <linux/workqueue.h>24#include <linux/kthread.h>25#include <linux/backing-dev.h>26#include <linux/atomic.h>27#include <linux/scatterlist.h>28#include <linux/rbtree.h>29#include <linux/ctype.h>30#include <asm/page.h>31#include <linux/unaligned.h>32#include <crypto/hash.h>33#include <crypto/md5.h>34#include <crypto/skcipher.h>35#include <crypto/aead.h>36#include <crypto/authenc.h>37#include <crypto/utils.h>38#include <linux/rtnetlink.h> /* for struct rtattr and RTA macros only */39#include <linux/key-type.h>40#include <keys/user-type.h>41#include <keys/encrypted-type.h>42#include <keys/trusted-type.h>43 44#include <linux/device-mapper.h>45 46#include "dm-audit.h"47 48#define DM_MSG_PREFIX "crypt"49 50static DEFINE_IDA(workqueue_ida);51 52/*53 * context holding the current state of a multi-part conversion54 */55struct convert_context {56	struct completion restart;57	struct bio *bio_in;58	struct bvec_iter iter_in;59	struct bio *bio_out;60	struct bvec_iter iter_out;61	atomic_t cc_pending;62	u64 cc_sector;63	union {64		struct skcipher_request *req;65		struct aead_request *req_aead;66	} r;67	bool aead_recheck;68	bool aead_failed;69 70};71 72/*73 * per bio private data74 */75struct dm_crypt_io {76	struct crypt_config *cc;77	struct bio *base_bio;78	u8 *integrity_metadata;79	bool integrity_metadata_from_pool:1;80 81	struct work_struct work;82 83	struct convert_context ctx;84 85	atomic_t io_pending;86	blk_status_t error;87	sector_t sector;88 89	struct bvec_iter saved_bi_iter;90 91	struct rb_node rb_node;92} CRYPTO_MINALIGN_ATTR;93 94struct dm_crypt_request {95	struct convert_context *ctx;96	struct scatterlist sg_in[4];97	struct scatterlist sg_out[4];98	u64 iv_sector;99};100 101struct crypt_config;102 103struct crypt_iv_operations {104	int (*ctr)(struct crypt_config *cc, struct dm_target *ti,105		   const char *opts);106	void (*dtr)(struct crypt_config *cc);107	int (*init)(struct crypt_config *cc);108	int (*wipe)(struct crypt_config *cc);109	int (*generator)(struct crypt_config *cc, u8 *iv,110			 struct dm_crypt_request *dmreq);111	int (*post)(struct crypt_config *cc, u8 *iv,112		    struct dm_crypt_request *dmreq);113};114 115struct iv_benbi_private {116	int shift;117};118 119#define LMK_SEED_SIZE 64 /* hash + 0 */120struct iv_lmk_private {121	struct crypto_shash *hash_tfm;122	u8 *seed;123};124 125#define TCW_WHITENING_SIZE 16126struct iv_tcw_private {127	struct crypto_shash *crc32_tfm;128	u8 *iv_seed;129	u8 *whitening;130};131 132#define ELEPHANT_MAX_KEY_SIZE 32133struct iv_elephant_private {134	struct crypto_skcipher *tfm;135};136 137/*138 * Crypt: maps a linear range of a block device139 * and encrypts / decrypts at the same time.140 */141enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID,142	     DM_CRYPT_SAME_CPU, DM_CRYPT_HIGH_PRIORITY,143	     DM_CRYPT_NO_OFFLOAD, DM_CRYPT_NO_READ_WORKQUEUE,144	     DM_CRYPT_NO_WRITE_WORKQUEUE, DM_CRYPT_WRITE_INLINE };145 146enum cipher_flags {147	CRYPT_MODE_INTEGRITY_AEAD,	/* Use authenticated mode for cipher */148	CRYPT_IV_LARGE_SECTORS,		/* Calculate IV from sector_size, not 512B sectors */149	CRYPT_ENCRYPT_PREPROCESS,	/* Must preprocess data for encryption (elephant) */150	CRYPT_KEY_MAC_SIZE_SET,		/* The integrity_key_size option was used */151};152 153/*154 * The fields in here must be read only after initialization.155 */156struct crypt_config {157	struct dm_dev *dev;158	sector_t start;159 160	struct percpu_counter n_allocated_pages;161 162	struct workqueue_struct *io_queue;163	struct workqueue_struct *crypt_queue;164 165	spinlock_t write_thread_lock;166	struct task_struct *write_thread;167	struct rb_root write_tree;168 169	char *cipher_string;170	char *cipher_auth;171	char *key_string;172 173	const struct crypt_iv_operations *iv_gen_ops;174	union {175		struct iv_benbi_private benbi;176		struct iv_lmk_private lmk;177		struct iv_tcw_private tcw;178		struct iv_elephant_private elephant;179	} iv_gen_private;180	u64 iv_offset;181	unsigned int iv_size;182	unsigned short sector_size;183	unsigned char sector_shift;184 185	union {186		struct crypto_skcipher **tfms;187		struct crypto_aead **tfms_aead;188	} cipher_tfm;189	unsigned int tfms_count;190	int workqueue_id;191	unsigned long cipher_flags;192 193	/*194	 * Layout of each crypto request:195	 *196	 *   struct skcipher_request197	 *      context198	 *      padding199	 *   struct dm_crypt_request200	 *      padding201	 *   IV202	 *203	 * The padding is added so that dm_crypt_request and the IV are204	 * correctly aligned.205	 */206	unsigned int dmreq_start;207 208	unsigned int per_bio_data_size;209 210	unsigned long flags;211	unsigned int key_size;212	unsigned int key_parts;      /* independent parts in key buffer */213	unsigned int key_extra_size; /* additional keys length */214	unsigned int key_mac_size;   /* MAC key size for authenc(...) */215 216	unsigned int integrity_tag_size;217	unsigned int integrity_iv_size;218	unsigned int used_tag_size;219	unsigned int tuple_size;220 221	/*222	 * pool for per bio private data, crypto requests,223	 * encryption requeusts/buffer pages and integrity tags224	 */225	unsigned int tag_pool_max_sectors;226	mempool_t tag_pool;227	mempool_t req_pool;228	mempool_t page_pool;229 230	struct bio_set bs;231	struct mutex bio_alloc_lock;232 233	u8 *authenc_key; /* space for keys in authenc() format (if used) */234	u8 key[] __counted_by(key_size);235};236 237#define MIN_IOS		64238#define MAX_TAG_SIZE	480239#define POOL_ENTRY_SIZE	512240 241static DEFINE_SPINLOCK(dm_crypt_clients_lock);242static unsigned int dm_crypt_clients_n;243static volatile unsigned long dm_crypt_pages_per_client;244#define DM_CRYPT_MEMORY_PERCENT			2245#define DM_CRYPT_MIN_PAGES_PER_CLIENT		(BIO_MAX_VECS * 16)246#define DM_CRYPT_DEFAULT_MAX_READ_SIZE		131072247#define DM_CRYPT_DEFAULT_MAX_WRITE_SIZE		131072248 249static unsigned int max_read_size = 0;250module_param(max_read_size, uint, 0644);251MODULE_PARM_DESC(max_read_size, "Maximum size of a read request");252static unsigned int max_write_size = 0;253module_param(max_write_size, uint, 0644);254MODULE_PARM_DESC(max_write_size, "Maximum size of a write request");255static unsigned get_max_request_size(struct crypt_config *cc, bool wrt)256{257	unsigned val, sector_align;258	val = !wrt ? READ_ONCE(max_read_size) : READ_ONCE(max_write_size);259	if (likely(!val))260		val = !wrt ? DM_CRYPT_DEFAULT_MAX_READ_SIZE : DM_CRYPT_DEFAULT_MAX_WRITE_SIZE;261	if (wrt || cc->used_tag_size) {262		if (unlikely(val > BIO_MAX_VECS << PAGE_SHIFT))263			val = BIO_MAX_VECS << PAGE_SHIFT;264	}265	sector_align = max(bdev_logical_block_size(cc->dev->bdev), (unsigned)cc->sector_size);266	val = round_down(val, sector_align);267	if (unlikely(!val))268		val = sector_align;269	return val >> SECTOR_SHIFT;270}271 272static void crypt_endio(struct bio *clone);273static void kcryptd_queue_crypt(struct dm_crypt_io *io);274static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc,275					     struct scatterlist *sg);276 277static bool crypt_integrity_aead(struct crypt_config *cc);278 279/*280 * Use this to access cipher attributes that are independent of the key.281 */282static struct crypto_skcipher *any_tfm(struct crypt_config *cc)283{284	return cc->cipher_tfm.tfms[0];285}286 287static struct crypto_aead *any_tfm_aead(struct crypt_config *cc)288{289	return cc->cipher_tfm.tfms_aead[0];290}291 292/*293 * Different IV generation algorithms:294 *295 * plain: the initial vector is the 32-bit little-endian version of the sector296 *        number, padded with zeros if necessary.297 *298 * plain64: the initial vector is the 64-bit little-endian version of the sector299 *        number, padded with zeros if necessary.300 *301 * plain64be: the initial vector is the 64-bit big-endian version of the sector302 *        number, padded with zeros if necessary.303 *304 * essiv: "encrypted sector|salt initial vector", the sector number is305 *        encrypted with the bulk cipher using a salt as key. The salt306 *        should be derived from the bulk cipher's key via hashing.307 *308 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1309 *        (needed for LRW-32-AES and possible other narrow block modes)310 *311 * null: the initial vector is always zero.  Provides compatibility with312 *       obsolete loop_fish2 devices.  Do not use for new devices.313 *314 * lmk:  Compatible implementation of the block chaining mode used315 *       by the Loop-AES block device encryption system316 *       designed by Jari Ruusu. See http://loop-aes.sourceforge.net/317 *       It operates on full 512 byte sectors and uses CBC318 *       with an IV derived from the sector number, the data and319 *       optionally extra IV seed.320 *       This means that after decryption the first block321 *       of sector must be tweaked according to decrypted data.322 *       Loop-AES can use three encryption schemes:323 *         version 1: is plain aes-cbc mode324 *         version 2: uses 64 multikey scheme with lmk IV generator325 *         version 3: the same as version 2 with additional IV seed326 *                   (it uses 65 keys, last key is used as IV seed)327 *328 * tcw:  Compatible implementation of the block chaining mode used329 *       by the TrueCrypt device encryption system (prior to version 4.1).330 *       For more info see: https://gitlab.com/cryptsetup/cryptsetup/wikis/TrueCryptOnDiskFormat331 *       It operates on full 512 byte sectors and uses CBC332 *       with an IV derived from initial key and the sector number.333 *       In addition, whitening value is applied on every sector, whitening334 *       is calculated from initial key, sector number and mixed using CRC32.335 *       Note that this encryption scheme is vulnerable to watermarking attacks336 *       and should be used for old compatible containers access only.337 *338 * eboiv: Encrypted byte-offset IV (used in Bitlocker in CBC mode)339 *        The IV is encrypted little-endian byte-offset (with the same key340 *        and cipher as the volume).341 *342 * elephant: The extended version of eboiv with additional Elephant diffuser343 *           used with Bitlocker CBC mode.344 *           This mode was used in older Windows systems345 *           https://download.microsoft.com/download/0/2/3/0238acaf-d3bf-4a6d-b3d6-0a0be4bbb36e/bitlockercipher200608.pdf346 */347 348static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv,349			      struct dm_crypt_request *dmreq)350{351	memset(iv, 0, cc->iv_size);352	*(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff);353 354	return 0;355}356 357static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv,358				struct dm_crypt_request *dmreq)359{360	memset(iv, 0, cc->iv_size);361	*(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);362 363	return 0;364}365 366static int crypt_iv_plain64be_gen(struct crypt_config *cc, u8 *iv,367				  struct dm_crypt_request *dmreq)368{369	memset(iv, 0, cc->iv_size);370	/* iv_size is at least of size u64; usually it is 16 bytes */371	*(__be64 *)&iv[cc->iv_size - sizeof(u64)] = cpu_to_be64(dmreq->iv_sector);372 373	return 0;374}375 376static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv,377			      struct dm_crypt_request *dmreq)378{379	/*380	 * ESSIV encryption of the IV is now handled by the crypto API,381	 * so just pass the plain sector number here.382	 */383	memset(iv, 0, cc->iv_size);384	*(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);385 386	return 0;387}388 389static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti,390			      const char *opts)391{392	unsigned int bs;393	int log;394 395	if (crypt_integrity_aead(cc))396		bs = crypto_aead_blocksize(any_tfm_aead(cc));397	else398		bs = crypto_skcipher_blocksize(any_tfm(cc));399	log = ilog2(bs);400 401	/*402	 * We need to calculate how far we must shift the sector count403	 * to get the cipher block count, we use this shift in _gen.404	 */405	if (1 << log != bs) {406		ti->error = "cypher blocksize is not a power of 2";407		return -EINVAL;408	}409 410	if (log > 9) {411		ti->error = "cypher blocksize is > 512";412		return -EINVAL;413	}414 415	cc->iv_gen_private.benbi.shift = 9 - log;416 417	return 0;418}419 420static void crypt_iv_benbi_dtr(struct crypt_config *cc)421{422}423 424static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv,425			      struct dm_crypt_request *dmreq)426{427	__be64 val;428 429	memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */430 431	val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1);432	put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64)));433 434	return 0;435}436 437static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv,438			     struct dm_crypt_request *dmreq)439{440	memset(iv, 0, cc->iv_size);441 442	return 0;443}444 445static void crypt_iv_lmk_dtr(struct crypt_config *cc)446{447	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;448 449	if (lmk->hash_tfm && !IS_ERR(lmk->hash_tfm))450		crypto_free_shash(lmk->hash_tfm);451	lmk->hash_tfm = NULL;452 453	kfree_sensitive(lmk->seed);454	lmk->seed = NULL;455}456 457static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti,458			    const char *opts)459{460	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;461 462	if (cc->sector_size != (1 << SECTOR_SHIFT)) {463		ti->error = "Unsupported sector size for LMK";464		return -EINVAL;465	}466 467	lmk->hash_tfm = crypto_alloc_shash("md5", 0,468					   CRYPTO_ALG_ALLOCATES_MEMORY);469	if (IS_ERR(lmk->hash_tfm)) {470		ti->error = "Error initializing LMK hash";471		return PTR_ERR(lmk->hash_tfm);472	}473 474	/* No seed in LMK version 2 */475	if (cc->key_parts == cc->tfms_count) {476		lmk->seed = NULL;477		return 0;478	}479 480	lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL);481	if (!lmk->seed) {482		crypt_iv_lmk_dtr(cc);483		ti->error = "Error kmallocing seed storage in LMK";484		return -ENOMEM;485	}486 487	return 0;488}489 490static int crypt_iv_lmk_init(struct crypt_config *cc)491{492	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;493	int subkey_size = cc->key_size / cc->key_parts;494 495	/* LMK seed is on the position of LMK_KEYS + 1 key */496	if (lmk->seed)497		memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size),498		       crypto_shash_digestsize(lmk->hash_tfm));499 500	return 0;501}502 503static int crypt_iv_lmk_wipe(struct crypt_config *cc)504{505	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;506 507	if (lmk->seed)508		memset(lmk->seed, 0, LMK_SEED_SIZE);509 510	return 0;511}512 513static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,514			    struct dm_crypt_request *dmreq,515			    u8 *data)516{517	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;518	SHASH_DESC_ON_STACK(desc, lmk->hash_tfm);519	struct md5_state md5state;520	__le32 buf[4];521	int i, r;522 523	desc->tfm = lmk->hash_tfm;524 525	r = crypto_shash_init(desc);526	if (r)527		return r;528 529	if (lmk->seed) {530		r = crypto_shash_update(desc, lmk->seed, LMK_SEED_SIZE);531		if (r)532			return r;533	}534 535	/* Sector is always 512B, block size 16, add data of blocks 1-31 */536	r = crypto_shash_update(desc, data + 16, 16 * 31);537	if (r)538		return r;539 540	/* Sector is cropped to 56 bits here */541	buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF);542	buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000);543	buf[2] = cpu_to_le32(4024);544	buf[3] = 0;545	r = crypto_shash_update(desc, (u8 *)buf, sizeof(buf));546	if (r)547		return r;548 549	/* No MD5 padding here */550	r = crypto_shash_export(desc, &md5state);551	if (r)552		return r;553 554	for (i = 0; i < MD5_HASH_WORDS; i++)555		__cpu_to_le32s(&md5state.hash[i]);556	memcpy(iv, &md5state.hash, cc->iv_size);557 558	return 0;559}560 561static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv,562			    struct dm_crypt_request *dmreq)563{564	struct scatterlist *sg;565	u8 *src;566	int r = 0;567 568	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {569		sg = crypt_get_sg_data(cc, dmreq->sg_in);570		src = kmap_local_page(sg_page(sg));571		r = crypt_iv_lmk_one(cc, iv, dmreq, src + sg->offset);572		kunmap_local(src);573	} else574		memset(iv, 0, cc->iv_size);575 576	return r;577}578 579static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv,580			     struct dm_crypt_request *dmreq)581{582	struct scatterlist *sg;583	u8 *dst;584	int r;585 586	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE)587		return 0;588 589	sg = crypt_get_sg_data(cc, dmreq->sg_out);590	dst = kmap_local_page(sg_page(sg));591	r = crypt_iv_lmk_one(cc, iv, dmreq, dst + sg->offset);592 593	/* Tweak the first block of plaintext sector */594	if (!r)595		crypto_xor(dst + sg->offset, iv, cc->iv_size);596 597	kunmap_local(dst);598	return r;599}600 601static void crypt_iv_tcw_dtr(struct crypt_config *cc)602{603	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;604 605	kfree_sensitive(tcw->iv_seed);606	tcw->iv_seed = NULL;607	kfree_sensitive(tcw->whitening);608	tcw->whitening = NULL;609 610	if (tcw->crc32_tfm && !IS_ERR(tcw->crc32_tfm))611		crypto_free_shash(tcw->crc32_tfm);612	tcw->crc32_tfm = NULL;613}614 615static int crypt_iv_tcw_ctr(struct crypt_config *cc, struct dm_target *ti,616			    const char *opts)617{618	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;619 620	if (cc->sector_size != (1 << SECTOR_SHIFT)) {621		ti->error = "Unsupported sector size for TCW";622		return -EINVAL;623	}624 625	if (cc->key_size <= (cc->iv_size + TCW_WHITENING_SIZE)) {626		ti->error = "Wrong key size for TCW";627		return -EINVAL;628	}629 630	tcw->crc32_tfm = crypto_alloc_shash("crc32", 0,631					    CRYPTO_ALG_ALLOCATES_MEMORY);632	if (IS_ERR(tcw->crc32_tfm)) {633		ti->error = "Error initializing CRC32 in TCW";634		return PTR_ERR(tcw->crc32_tfm);635	}636 637	tcw->iv_seed = kzalloc(cc->iv_size, GFP_KERNEL);638	tcw->whitening = kzalloc(TCW_WHITENING_SIZE, GFP_KERNEL);639	if (!tcw->iv_seed || !tcw->whitening) {640		crypt_iv_tcw_dtr(cc);641		ti->error = "Error allocating seed storage in TCW";642		return -ENOMEM;643	}644 645	return 0;646}647 648static int crypt_iv_tcw_init(struct crypt_config *cc)649{650	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;651	int key_offset = cc->key_size - cc->iv_size - TCW_WHITENING_SIZE;652 653	memcpy(tcw->iv_seed, &cc->key[key_offset], cc->iv_size);654	memcpy(tcw->whitening, &cc->key[key_offset + cc->iv_size],655	       TCW_WHITENING_SIZE);656 657	return 0;658}659 660static int crypt_iv_tcw_wipe(struct crypt_config *cc)661{662	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;663 664	memset(tcw->iv_seed, 0, cc->iv_size);665	memset(tcw->whitening, 0, TCW_WHITENING_SIZE);666 667	return 0;668}669 670static int crypt_iv_tcw_whitening(struct crypt_config *cc,671				  struct dm_crypt_request *dmreq,672				  u8 *data)673{674	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;675	__le64 sector = cpu_to_le64(dmreq->iv_sector);676	u8 buf[TCW_WHITENING_SIZE];677	SHASH_DESC_ON_STACK(desc, tcw->crc32_tfm);678	int i, r;679 680	/* xor whitening with sector number */681	crypto_xor_cpy(buf, tcw->whitening, (u8 *)&sector, 8);682	crypto_xor_cpy(&buf[8], tcw->whitening + 8, (u8 *)&sector, 8);683 684	/* calculate crc32 for every 32bit part and xor it */685	desc->tfm = tcw->crc32_tfm;686	for (i = 0; i < 4; i++) {687		r = crypto_shash_digest(desc, &buf[i * 4], 4, &buf[i * 4]);688		if (r)689			goto out;690	}691	crypto_xor(&buf[0], &buf[12], 4);692	crypto_xor(&buf[4], &buf[8], 4);693 694	/* apply whitening (8 bytes) to whole sector */695	for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++)696		crypto_xor(data + i * 8, buf, 8);697out:698	memzero_explicit(buf, sizeof(buf));699	return r;700}701 702static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv,703			    struct dm_crypt_request *dmreq)704{705	struct scatterlist *sg;706	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;707	__le64 sector = cpu_to_le64(dmreq->iv_sector);708	u8 *src;709	int r = 0;710 711	/* Remove whitening from ciphertext */712	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {713		sg = crypt_get_sg_data(cc, dmreq->sg_in);714		src = kmap_local_page(sg_page(sg));715		r = crypt_iv_tcw_whitening(cc, dmreq, src + sg->offset);716		kunmap_local(src);717	}718 719	/* Calculate IV */720	crypto_xor_cpy(iv, tcw->iv_seed, (u8 *)&sector, 8);721	if (cc->iv_size > 8)722		crypto_xor_cpy(&iv[8], tcw->iv_seed + 8, (u8 *)&sector,723			       cc->iv_size - 8);724 725	return r;726}727 728static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv,729			     struct dm_crypt_request *dmreq)730{731	struct scatterlist *sg;732	u8 *dst;733	int r;734 735	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)736		return 0;737 738	/* Apply whitening on ciphertext */739	sg = crypt_get_sg_data(cc, dmreq->sg_out);740	dst = kmap_local_page(sg_page(sg));741	r = crypt_iv_tcw_whitening(cc, dmreq, dst + sg->offset);742	kunmap_local(dst);743 744	return r;745}746 747static int crypt_iv_random_gen(struct crypt_config *cc, u8 *iv,748				struct dm_crypt_request *dmreq)749{750	/* Used only for writes, there must be an additional space to store IV */751	get_random_bytes(iv, cc->iv_size);752	return 0;753}754 755static int crypt_iv_eboiv_ctr(struct crypt_config *cc, struct dm_target *ti,756			    const char *opts)757{758	if (crypt_integrity_aead(cc)) {759		ti->error = "AEAD transforms not supported for EBOIV";760		return -EINVAL;761	}762 763	if (crypto_skcipher_blocksize(any_tfm(cc)) != cc->iv_size) {764		ti->error = "Block size of EBOIV cipher does not match IV size of block cipher";765		return -EINVAL;766	}767 768	return 0;769}770 771static int crypt_iv_eboiv_gen(struct crypt_config *cc, u8 *iv,772			    struct dm_crypt_request *dmreq)773{774	struct crypto_skcipher *tfm = any_tfm(cc);775	struct skcipher_request *req;776	struct scatterlist src, dst;777	DECLARE_CRYPTO_WAIT(wait);778	unsigned int reqsize;779	int err;780	u8 *buf;781 782	reqsize = sizeof(*req) + crypto_skcipher_reqsize(tfm);783	reqsize = ALIGN(reqsize, __alignof__(__le64));784 785	req = kmalloc(reqsize + cc->iv_size, GFP_NOIO);786	if (!req)787		return -ENOMEM;788 789	skcipher_request_set_tfm(req, tfm);790 791	buf = (u8 *)req + reqsize;792	memset(buf, 0, cc->iv_size);793	*(__le64 *)buf = cpu_to_le64(dmreq->iv_sector * cc->sector_size);794 795	sg_init_one(&src, page_address(ZERO_PAGE(0)), cc->iv_size);796	sg_init_one(&dst, iv, cc->iv_size);797	skcipher_request_set_crypt(req, &src, &dst, cc->iv_size, buf);798	skcipher_request_set_callback(req, 0, crypto_req_done, &wait);799	err = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);800	kfree_sensitive(req);801 802	return err;803}804 805static void crypt_iv_elephant_dtr(struct crypt_config *cc)806{807	struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;808 809	crypto_free_skcipher(elephant->tfm);810	elephant->tfm = NULL;811}812 813static int crypt_iv_elephant_ctr(struct crypt_config *cc, struct dm_target *ti,814			    const char *opts)815{816	struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;817	int r;818 819	elephant->tfm = crypto_alloc_skcipher("ecb(aes)", 0,820					      CRYPTO_ALG_ALLOCATES_MEMORY);821	if (IS_ERR(elephant->tfm)) {822		r = PTR_ERR(elephant->tfm);823		elephant->tfm = NULL;824		return r;825	}826 827	r = crypt_iv_eboiv_ctr(cc, ti, NULL);828	if (r)829		crypt_iv_elephant_dtr(cc);830	return r;831}832 833static void diffuser_disk_to_cpu(u32 *d, size_t n)834{835#ifndef __LITTLE_ENDIAN836	int i;837 838	for (i = 0; i < n; i++)839		d[i] = le32_to_cpu((__le32)d[i]);840#endif841}842 843static void diffuser_cpu_to_disk(__le32 *d, size_t n)844{845#ifndef __LITTLE_ENDIAN846	int i;847 848	for (i = 0; i < n; i++)849		d[i] = cpu_to_le32((u32)d[i]);850#endif851}852 853static void diffuser_a_decrypt(u32 *d, size_t n)854{855	int i, i1, i2, i3;856 857	for (i = 0; i < 5; i++) {858		i1 = 0;859		i2 = n - 2;860		i3 = n - 5;861 862		while (i1 < (n - 1)) {863			d[i1] += d[i2] ^ (d[i3] << 9 | d[i3] >> 23);864			i1++; i2++; i3++;865 866			if (i3 >= n)867				i3 -= n;868 869			d[i1] += d[i2] ^ d[i3];870			i1++; i2++; i3++;871 872			if (i2 >= n)873				i2 -= n;874 875			d[i1] += d[i2] ^ (d[i3] << 13 | d[i3] >> 19);876			i1++; i2++; i3++;877 878			d[i1] += d[i2] ^ d[i3];879			i1++; i2++; i3++;880		}881	}882}883 884static void diffuser_a_encrypt(u32 *d, size_t n)885{886	int i, i1, i2, i3;887 888	for (i = 0; i < 5; i++) {889		i1 = n - 1;890		i2 = n - 2 - 1;891		i3 = n - 5 - 1;892 893		while (i1 > 0) {894			d[i1] -= d[i2] ^ d[i3];895			i1--; i2--; i3--;896 897			d[i1] -= d[i2] ^ (d[i3] << 13 | d[i3] >> 19);898			i1--; i2--; i3--;899 900			if (i2 < 0)901				i2 += n;902 903			d[i1] -= d[i2] ^ d[i3];904			i1--; i2--; i3--;905 906			if (i3 < 0)907				i3 += n;908 909			d[i1] -= d[i2] ^ (d[i3] << 9 | d[i3] >> 23);910			i1--; i2--; i3--;911		}912	}913}914 915static void diffuser_b_decrypt(u32 *d, size_t n)916{917	int i, i1, i2, i3;918 919	for (i = 0; i < 3; i++) {920		i1 = 0;921		i2 = 2;922		i3 = 5;923 924		while (i1 < (n - 1)) {925			d[i1] += d[i2] ^ d[i3];926			i1++; i2++; i3++;927 928			d[i1] += d[i2] ^ (d[i3] << 10 | d[i3] >> 22);929			i1++; i2++; i3++;930 931			if (i2 >= n)932				i2 -= n;933 934			d[i1] += d[i2] ^ d[i3];935			i1++; i2++; i3++;936 937			if (i3 >= n)938				i3 -= n;939 940			d[i1] += d[i2] ^ (d[i3] << 25 | d[i3] >> 7);941			i1++; i2++; i3++;942		}943	}944}945 946static void diffuser_b_encrypt(u32 *d, size_t n)947{948	int i, i1, i2, i3;949 950	for (i = 0; i < 3; i++) {951		i1 = n - 1;952		i2 = 2 - 1;953		i3 = 5 - 1;954 955		while (i1 > 0) {956			d[i1] -= d[i2] ^ (d[i3] << 25 | d[i3] >> 7);957			i1--; i2--; i3--;958 959			if (i3 < 0)960				i3 += n;961 962			d[i1] -= d[i2] ^ d[i3];963			i1--; i2--; i3--;964 965			if (i2 < 0)966				i2 += n;967 968			d[i1] -= d[i2] ^ (d[i3] << 10 | d[i3] >> 22);969			i1--; i2--; i3--;970 971			d[i1] -= d[i2] ^ d[i3];972			i1--; i2--; i3--;973		}974	}975}976 977static int crypt_iv_elephant(struct crypt_config *cc, struct dm_crypt_request *dmreq)978{979	struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;980	u8 *es, *ks, *data, *data2, *data_offset;981	struct skcipher_request *req;982	struct scatterlist *sg, *sg2, src, dst;983	DECLARE_CRYPTO_WAIT(wait);984	int i, r;985 986	req = skcipher_request_alloc(elephant->tfm, GFP_NOIO);987	es = kzalloc(16, GFP_NOIO); /* Key for AES */988	ks = kzalloc(32, GFP_NOIO); /* Elephant sector key */989 990	if (!req || !es || !ks) {991		r = -ENOMEM;992		goto out;993	}994 995	*(__le64 *)es = cpu_to_le64(dmreq->iv_sector * cc->sector_size);996 997	/* E(Ks, e(s)) */998	sg_init_one(&src, es, 16);999	sg_init_one(&dst, ks, 16);1000	skcipher_request_set_crypt(req, &src, &dst, 16, NULL);1001	skcipher_request_set_callback(req, 0, crypto_req_done, &wait);1002	r = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);1003	if (r)1004		goto out;1005 1006	/* E(Ks, e'(s)) */1007	es[15] = 0x80;1008	sg_init_one(&dst, &ks[16], 16);1009	r = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);1010	if (r)1011		goto out;1012 1013	sg = crypt_get_sg_data(cc, dmreq->sg_out);1014	data = kmap_local_page(sg_page(sg));1015	data_offset = data + sg->offset;1016 1017	/* Cannot modify original bio, copy to sg_out and apply Elephant to it */1018	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {1019		sg2 = crypt_get_sg_data(cc, dmreq->sg_in);1020		data2 = kmap_local_page(sg_page(sg2));1021		memcpy(data_offset, data2 + sg2->offset, cc->sector_size);1022		kunmap_local(data2);1023	}1024 1025	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {1026		diffuser_disk_to_cpu((u32 *)data_offset, cc->sector_size / sizeof(u32));1027		diffuser_b_decrypt((u32 *)data_offset, cc->sector_size / sizeof(u32));1028		diffuser_a_decrypt((u32 *)data_offset, cc->sector_size / sizeof(u32));1029		diffuser_cpu_to_disk((__le32 *)data_offset, cc->sector_size / sizeof(u32));1030	}1031 1032	for (i = 0; i < (cc->sector_size / 32); i++)1033		crypto_xor(data_offset + i * 32, ks, 32);1034 1035	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {1036		diffuser_disk_to_cpu((u32 *)data_offset, cc->sector_size / sizeof(u32));1037		diffuser_a_encrypt((u32 *)data_offset, cc->sector_size / sizeof(u32));1038		diffuser_b_encrypt((u32 *)data_offset, cc->sector_size / sizeof(u32));1039		diffuser_cpu_to_disk((__le32 *)data_offset, cc->sector_size / sizeof(u32));1040	}1041 1042	kunmap_local(data);1043out:1044	kfree_sensitive(ks);1045	kfree_sensitive(es);1046	skcipher_request_free(req);1047	return r;1048}1049 1050static int crypt_iv_elephant_gen(struct crypt_config *cc, u8 *iv,1051			    struct dm_crypt_request *dmreq)1052{1053	int r;1054 1055	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {1056		r = crypt_iv_elephant(cc, dmreq);1057		if (r)1058			return r;1059	}1060 1061	return crypt_iv_eboiv_gen(cc, iv, dmreq);1062}1063 1064static int crypt_iv_elephant_post(struct crypt_config *cc, u8 *iv,1065				  struct dm_crypt_request *dmreq)1066{1067	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)1068		return crypt_iv_elephant(cc, dmreq);1069 1070	return 0;1071}1072 1073static int crypt_iv_elephant_init(struct crypt_config *cc)1074{1075	struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;1076	int key_offset = cc->key_size - cc->key_extra_size;1077 1078	return crypto_skcipher_setkey(elephant->tfm, &cc->key[key_offset], cc->key_extra_size);1079}1080 1081static int crypt_iv_elephant_wipe(struct crypt_config *cc)1082{1083	struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;1084	u8 key[ELEPHANT_MAX_KEY_SIZE];1085 1086	memset(key, 0, cc->key_extra_size);1087	return crypto_skcipher_setkey(elephant->tfm, key, cc->key_extra_size);1088}1089 1090static const struct crypt_iv_operations crypt_iv_plain_ops = {1091	.generator = crypt_iv_plain_gen1092};1093 1094static const struct crypt_iv_operations crypt_iv_plain64_ops = {1095	.generator = crypt_iv_plain64_gen1096};1097 1098static const struct crypt_iv_operations crypt_iv_plain64be_ops = {1099	.generator = crypt_iv_plain64be_gen1100};1101 1102static const struct crypt_iv_operations crypt_iv_essiv_ops = {1103	.generator = crypt_iv_essiv_gen1104};1105 1106static const struct crypt_iv_operations crypt_iv_benbi_ops = {1107	.ctr	   = crypt_iv_benbi_ctr,1108	.dtr	   = crypt_iv_benbi_dtr,1109	.generator = crypt_iv_benbi_gen1110};1111 1112static const struct crypt_iv_operations crypt_iv_null_ops = {1113	.generator = crypt_iv_null_gen1114};1115 1116static const struct crypt_iv_operations crypt_iv_lmk_ops = {1117	.ctr	   = crypt_iv_lmk_ctr,1118	.dtr	   = crypt_iv_lmk_dtr,1119	.init	   = crypt_iv_lmk_init,1120	.wipe	   = crypt_iv_lmk_wipe,1121	.generator = crypt_iv_lmk_gen,1122	.post	   = crypt_iv_lmk_post1123};1124 1125static const struct crypt_iv_operations crypt_iv_tcw_ops = {1126	.ctr	   = crypt_iv_tcw_ctr,1127	.dtr	   = crypt_iv_tcw_dtr,1128	.init	   = crypt_iv_tcw_init,1129	.wipe	   = crypt_iv_tcw_wipe,1130	.generator = crypt_iv_tcw_gen,1131	.post	   = crypt_iv_tcw_post1132};1133 1134static const struct crypt_iv_operations crypt_iv_random_ops = {1135	.generator = crypt_iv_random_gen1136};1137 1138static const struct crypt_iv_operations crypt_iv_eboiv_ops = {1139	.ctr	   = crypt_iv_eboiv_ctr,1140	.generator = crypt_iv_eboiv_gen1141};1142 1143static const struct crypt_iv_operations crypt_iv_elephant_ops = {1144	.ctr	   = crypt_iv_elephant_ctr,1145	.dtr	   = crypt_iv_elephant_dtr,1146	.init	   = crypt_iv_elephant_init,1147	.wipe	   = crypt_iv_elephant_wipe,1148	.generator = crypt_iv_elephant_gen,1149	.post	   = crypt_iv_elephant_post1150};1151 1152/*1153 * Integrity extensions1154 */1155static bool crypt_integrity_aead(struct crypt_config *cc)1156{1157	return test_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags);1158}1159 1160static bool crypt_integrity_hmac(struct crypt_config *cc)1161{1162	return crypt_integrity_aead(cc) && cc->key_mac_size;1163}1164 1165/* Get sg containing data */1166static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc,1167					     struct scatterlist *sg)1168{1169	if (unlikely(crypt_integrity_aead(cc)))1170		return &sg[2];1171 1172	return sg;1173}1174 1175static int dm_crypt_integrity_io_alloc(struct dm_crypt_io *io, struct bio *bio)1176{1177	struct bio_integrity_payload *bip;1178	unsigned int tag_len;1179	int ret;1180 1181	if (!bio_sectors(bio) || !io->cc->tuple_size)1182		return 0;1183 1184	bip = bio_integrity_alloc(bio, GFP_NOIO, 1);1185	if (IS_ERR(bip))1186		return PTR_ERR(bip);1187 1188	tag_len = io->cc->tuple_size * (bio_sectors(bio) >> io->cc->sector_shift);1189 1190	bip->bip_iter.bi_sector = io->cc->start + io->sector;1191 1192	ret = bio_integrity_add_page(bio, virt_to_page(io->integrity_metadata),1193				     tag_len, offset_in_page(io->integrity_metadata));1194	if (unlikely(ret != tag_len))1195		return -ENOMEM;1196 1197	return 0;1198}1199 1200static int crypt_integrity_ctr(struct crypt_config *cc, struct dm_target *ti)1201{1202#ifdef CONFIG_BLK_DEV_INTEGRITY1203	struct blk_integrity *bi = blk_get_integrity(cc->dev->bdev->bd_disk);1204	struct mapped_device *md = dm_table_get_md(ti->table);1205 1206	/* We require an underlying device with non-PI metadata */1207	if (!bi || bi->csum_type != BLK_INTEGRITY_CSUM_NONE) {1208		ti->error = "Integrity profile not supported.";1209		return -EINVAL;1210	}1211 1212	if (bi->tuple_size < cc->used_tag_size) {1213		ti->error = "Integrity profile tag size mismatch.";1214		return -EINVAL;1215	}1216	cc->tuple_size = bi->tuple_size;1217	if (1 << bi->interval_exp != cc->sector_size) {1218		ti->error = "Integrity profile sector size mismatch.";1219		return -EINVAL;1220	}1221 1222	if (crypt_integrity_aead(cc)) {1223		cc->integrity_tag_size = cc->used_tag_size - cc->integrity_iv_size;1224		DMDEBUG("%s: Integrity AEAD, tag size %u, IV size %u.", dm_device_name(md),1225		       cc->integrity_tag_size, cc->integrity_iv_size);1226 1227		if (crypto_aead_setauthsize(any_tfm_aead(cc), cc->integrity_tag_size)) {1228			ti->error = "Integrity AEAD auth tag size is not supported.";1229			return -EINVAL;1230		}1231	} else if (cc->integrity_iv_size)1232		DMDEBUG("%s: Additional per-sector space %u bytes for IV.", dm_device_name(md),1233		       cc->integrity_iv_size);1234 1235	if ((cc->integrity_tag_size + cc->integrity_iv_size) > cc->tuple_size) {1236		ti->error = "Not enough space for integrity tag in the profile.";1237		return -EINVAL;1238	}1239 1240	return 0;1241#else1242	ti->error = "Integrity profile not supported.";1243	return -EINVAL;1244#endif1245}1246 1247static void crypt_convert_init(struct crypt_config *cc,1248			       struct convert_context *ctx,1249			       struct bio *bio_out, struct bio *bio_in,1250			       sector_t sector)1251{1252	ctx->bio_in = bio_in;1253	ctx->bio_out = bio_out;1254	if (bio_in)1255		ctx->iter_in = bio_in->bi_iter;1256	if (bio_out)1257		ctx->iter_out = bio_out->bi_iter;1258	ctx->cc_sector = sector + cc->iv_offset;1259	init_completion(&ctx->restart);1260}1261 1262static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc,1263					     void *req)1264{1265	return (struct dm_crypt_request *)((char *)req + cc->dmreq_start);1266}1267 1268static void *req_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq)1269{1270	return (void *)((char *)dmreq - cc->dmreq_start);1271}1272 1273static u8 *iv_of_dmreq(struct crypt_config *cc,1274		       struct dm_crypt_request *dmreq)1275{1276	if (crypt_integrity_aead(cc))1277		return (u8 *)ALIGN((unsigned long)(dmreq + 1),1278			crypto_aead_alignmask(any_tfm_aead(cc)) + 1);1279	else1280		return (u8 *)ALIGN((unsigned long)(dmreq + 1),1281			crypto_skcipher_alignmask(any_tfm(cc)) + 1);1282}1283 1284static u8 *org_iv_of_dmreq(struct crypt_config *cc,1285		       struct dm_crypt_request *dmreq)1286{1287	return iv_of_dmreq(cc, dmreq) + cc->iv_size;1288}1289 1290static __le64 *org_sector_of_dmreq(struct crypt_config *cc,1291		       struct dm_crypt_request *dmreq)1292{1293	u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size + cc->iv_size;1294 1295	return (__le64 *) ptr;1296}1297 1298static unsigned int *org_tag_of_dmreq(struct crypt_config *cc,1299		       struct dm_crypt_request *dmreq)1300{1301	u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size +1302		  cc->iv_size + sizeof(uint64_t);1303 1304	return (unsigned int *)ptr;1305}1306 1307static void *tag_from_dmreq(struct crypt_config *cc,1308				struct dm_crypt_request *dmreq)1309{1310	struct convert_context *ctx = dmreq->ctx;1311	struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);1312 1313	return &io->integrity_metadata[*org_tag_of_dmreq(cc, dmreq) *1314		cc->tuple_size];1315}1316 1317static void *iv_tag_from_dmreq(struct crypt_config *cc,1318			       struct dm_crypt_request *dmreq)1319{1320	return tag_from_dmreq(cc, dmreq) + cc->integrity_tag_size;1321}1322 1323static int crypt_convert_block_aead(struct crypt_config *cc,1324				     struct convert_context *ctx,1325				     struct aead_request *req,1326				     unsigned int tag_offset)1327{1328	struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);1329	struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);1330	struct dm_crypt_request *dmreq;1331	u8 *iv, *org_iv, *tag_iv, *tag;1332	__le64 *sector;1333	int r = 0;1334 1335	BUG_ON(cc->integrity_iv_size && cc->integrity_iv_size != cc->iv_size);1336 1337	/* Reject unexpected unaligned bio. */1338	if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))1339		return -EIO;1340 1341	dmreq = dmreq_of_req(cc, req);1342	dmreq->iv_sector = ctx->cc_sector;1343	if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))1344		dmreq->iv_sector >>= cc->sector_shift;1345	dmreq->ctx = ctx;1346 1347	*org_tag_of_dmreq(cc, dmreq) = tag_offset;1348 1349	sector = org_sector_of_dmreq(cc, dmreq);1350	*sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);1351 1352	iv = iv_of_dmreq(cc, dmreq);1353	org_iv = org_iv_of_dmreq(cc, dmreq);1354	tag = tag_from_dmreq(cc, dmreq);1355	tag_iv = iv_tag_from_dmreq(cc, dmreq);1356 1357	/* AEAD request:1358	 *  |----- AAD -------|------ DATA -------|-- AUTH TAG --|1359	 *  | (authenticated) | (auth+encryption) |              |1360	 *  | sector_LE |  IV |  sector in/out    |  tag in/out  |1361	 */1362	sg_init_table(dmreq->sg_in, 4);1363	sg_set_buf(&dmreq->sg_in[0], sector, sizeof(uint64_t));1364	sg_set_buf(&dmreq->sg_in[1], org_iv, cc->iv_size);1365	sg_set_page(&dmreq->sg_in[2], bv_in.bv_page, cc->sector_size, bv_in.bv_offset);1366	sg_set_buf(&dmreq->sg_in[3], tag, cc->integrity_tag_size);1367 1368	sg_init_table(dmreq->sg_out, 4);1369	sg_set_buf(&dmreq->sg_out[0], sector, sizeof(uint64_t));1370	sg_set_buf(&dmreq->sg_out[1], org_iv, cc->iv_size);1371	sg_set_page(&dmreq->sg_out[2], bv_out.bv_page, cc->sector_size, bv_out.bv_offset);1372	sg_set_buf(&dmreq->sg_out[3], tag, cc->integrity_tag_size);1373 1374	if (cc->iv_gen_ops) {1375		/* For READs use IV stored in integrity metadata */1376		if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) {1377			memcpy(org_iv, tag_iv, cc->iv_size);1378		} else {1379			r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);1380			if (r < 0)1381				return r;1382			/* Store generated IV in integrity metadata */1383			if (cc->integrity_iv_size)1384				memcpy(tag_iv, org_iv, cc->iv_size);1385		}1386		/* Working copy of IV, to be modified in crypto API */1387		memcpy(iv, org_iv, cc->iv_size);1388	}1389 1390	aead_request_set_ad(req, sizeof(uint64_t) + cc->iv_size);1391	if (bio_data_dir(ctx->bio_in) == WRITE) {1392		aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,1393				       cc->sector_size, iv);1394		r = crypto_aead_encrypt(req);1395		if (cc->integrity_tag_size + cc->integrity_iv_size != cc->tuple_size)1396			memset(tag + cc->integrity_tag_size + cc->integrity_iv_size, 0,1397			       cc->tuple_size - (cc->integrity_tag_size + cc->integrity_iv_size));1398	} else {1399		aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,1400				       cc->sector_size + cc->integrity_tag_size, iv);1401		r = crypto_aead_decrypt(req);1402	}1403 1404	if (r == -EBADMSG) {1405		sector_t s = le64_to_cpu(*sector);1406 1407		ctx->aead_failed = true;1408		if (ctx->aead_recheck) {1409			DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu",1410				    ctx->bio_in->bi_bdev, s);1411			dm_audit_log_bio(DM_MSG_PREFIX, "integrity-aead",1412					 ctx->bio_in, s, 0);1413		}1414	}1415 1416	if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)1417		r = cc->iv_gen_ops->post(cc, org_iv, dmreq);1418 1419	bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size);1420	bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size);1421 1422	return r;1423}1424 1425static int crypt_convert_block_skcipher(struct crypt_config *cc,1426					struct convert_context *ctx,1427					struct skcipher_request *req,1428					unsigned int tag_offset)1429{1430	struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);1431	struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);1432	struct scatterlist *sg_in, *sg_out;1433	struct dm_crypt_request *dmreq;1434	u8 *iv, *org_iv, *tag_iv;1435	__le64 *sector;1436	int r = 0;1437 1438	/* Reject unexpected unaligned bio. */1439	if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))1440		return -EIO;1441 1442	dmreq = dmreq_of_req(cc, req);1443	dmreq->iv_sector = ctx->cc_sector;1444	if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))1445		dmreq->iv_sector >>= cc->sector_shift;1446	dmreq->ctx = ctx;1447 1448	*org_tag_of_dmreq(cc, dmreq) = tag_offset;1449 1450	iv = iv_of_dmreq(cc, dmreq);1451	org_iv = org_iv_of_dmreq(cc, dmreq);1452	tag_iv = iv_tag_from_dmreq(cc, dmreq);1453 1454	sector = org_sector_of_dmreq(cc, dmreq);1455	*sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);1456 1457	/* For skcipher we use only the first sg item */1458	sg_in  = &dmreq->sg_in[0];1459	sg_out = &dmreq->sg_out[0];1460 1461	sg_init_table(sg_in, 1);1462	sg_set_page(sg_in, bv_in.bv_page, cc->sector_size, bv_in.bv_offset);1463 1464	sg_init_table(sg_out, 1);1465	sg_set_page(sg_out, bv_out.bv_page, cc->sector_size, bv_out.bv_offset);1466 1467	if (cc->iv_gen_ops) {1468		/* For READs use IV stored in integrity metadata */1469		if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) {1470			memcpy(org_iv, tag_iv, cc->integrity_iv_size);1471		} else {1472			r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);1473			if (r < 0)1474				return r;1475			/* Data can be already preprocessed in generator */1476			if (test_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags))1477				sg_in = sg_out;1478			/* Store generated IV in integrity metadata */1479			if (cc->integrity_iv_size)1480				memcpy(tag_iv, org_iv, cc->integrity_iv_size);1481		}1482		/* Working copy of IV, to be modified in crypto API */1483		memcpy(iv, org_iv, cc->iv_size);1484	}1485 1486	skcipher_request_set_crypt(req, sg_in, sg_out, cc->sector_size, iv);1487 1488	if (bio_data_dir(ctx->bio_in) == WRITE)1489		r = crypto_skcipher_encrypt(req);1490	else1491		r = crypto_skcipher_decrypt(req);1492 1493	if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)1494		r = cc->iv_gen_ops->post(cc, org_iv, dmreq);1495 1496	bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size);1497	bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size);1498 1499	return r;1500}1501 1502static void kcryptd_async_done(void *async_req, int error);1503 1504static int crypt_alloc_req_skcipher(struct crypt_config *cc,1505				     struct convert_context *ctx)1506{1507	unsigned int key_index = ctx->cc_sector & (cc->tfms_count - 1);1508 1509	if (!ctx->r.req) {1510		ctx->r.req = mempool_alloc(&cc->req_pool, in_interrupt() ? GFP_ATOMIC : GFP_NOIO);1511		if (!ctx->r.req)1512			return -ENOMEM;1513	}1514 1515	skcipher_request_set_tfm(ctx->r.req, cc->cipher_tfm.tfms[key_index]);1516 1517	/*1518	 * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs1519	 * requests if driver request queue is full.1520	 */1521	skcipher_request_set_callback(ctx->r.req,1522	    CRYPTO_TFM_REQ_MAY_BACKLOG,1523	    kcryptd_async_done, dmreq_of_req(cc, ctx->r.req));1524 1525	return 0;1526}1527 1528static int crypt_alloc_req_aead(struct crypt_config *cc,1529				 struct convert_context *ctx)1530{1531	if (!ctx->r.req_aead) {1532		ctx->r.req_aead = mempool_alloc(&cc->req_pool, in_interrupt() ? GFP_ATOMIC : GFP_NOIO);1533		if (!ctx->r.req_aead)1534			return -ENOMEM;1535	}1536 1537	aead_request_set_tfm(ctx->r.req_aead, cc->cipher_tfm.tfms_aead[0]);1538 1539	/*1540	 * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs1541	 * requests if driver request queue is full.1542	 */1543	aead_request_set_callback(ctx->r.req_aead,1544	    CRYPTO_TFM_REQ_MAY_BACKLOG,1545	    kcryptd_async_done, dmreq_of_req(cc, ctx->r.req_aead));1546 1547	return 0;1548}1549 1550static int crypt_alloc_req(struct crypt_config *cc,1551			    struct convert_context *ctx)1552{1553	if (crypt_integrity_aead(cc))1554		return crypt_alloc_req_aead(cc, ctx);1555	else1556		return crypt_alloc_req_skcipher(cc, ctx);1557}1558 1559static void crypt_free_req_skcipher(struct crypt_config *cc,1560				    struct skcipher_request *req, struct bio *base_bio)1561{1562	struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);1563 1564	if ((struct skcipher_request *)(io + 1) != req)1565		mempool_free(req, &cc->req_pool);1566}1567 1568static void crypt_free_req_aead(struct crypt_config *cc,1569				struct aead_request *req, struct bio *base_bio)1570{1571	struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);1572 1573	if ((struct aead_request *)(io + 1) != req)1574		mempool_free(req, &cc->req_pool);1575}1576 1577static void crypt_free_req(struct crypt_config *cc, void *req, struct bio *base_bio)1578{1579	if (crypt_integrity_aead(cc))1580		crypt_free_req_aead(cc, req, base_bio);1581	else1582		crypt_free_req_skcipher(cc, req, base_bio);1583}1584 1585/*1586 * Encrypt / decrypt data from one bio to another one (can be the same one)1587 */1588static blk_status_t crypt_convert(struct crypt_config *cc,1589			 struct convert_context *ctx, bool atomic, bool reset_pending)1590{1591	unsigned int tag_offset = 0;1592	unsigned int sector_step = cc->sector_size >> SECTOR_SHIFT;1593	int r;1594 1595	/*1596	 * if reset_pending is set we are dealing with the bio for the first time,1597	 * else we're continuing to work on the previous bio, so don't mess with1598	 * the cc_pending counter1599	 */1600	if (reset_pending)1601		atomic_set(&ctx->cc_pending, 1);1602 1603	while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) {1604 1605		r = crypt_alloc_req(cc, ctx);1606		if (r) {1607			complete(&ctx->restart);1608			return BLK_STS_DEV_RESOURCE;1609		}1610 1611		atomic_inc(&ctx->cc_pending);1612 1613		if (crypt_integrity_aead(cc))1614			r = crypt_convert_block_aead(cc, ctx, ctx->r.req_aead, tag_offset);1615		else1616			r = crypt_convert_block_skcipher(cc, ctx, ctx->r.req, tag_offset);1617 1618		switch (r) {1619		/*1620		 * The request was queued by a crypto driver1621		 * but the driver request queue is full, let's wait.1622		 */1623		case -EBUSY:1624			if (in_interrupt()) {1625				if (try_wait_for_completion(&ctx->restart)) {1626					/*1627					 * we don't have to block to wait for completion,1628					 * so proceed1629					 */1630				} else {1631					/*1632					 * we can't wait for completion without blocking1633					 * exit and continue processing in a workqueue1634					 */1635					ctx->r.req = NULL;1636					ctx->cc_sector += sector_step;1637					tag_offset++;1638					return BLK_STS_DEV_RESOURCE;1639				}1640			} else {1641				wait_for_completion(&ctx->restart);1642			}1643			reinit_completion(&ctx->restart);1644			fallthrough;1645		/*1646		 * The request is queued and processed asynchronously,1647		 * completion function kcryptd_async_done() will be called.1648		 */1649		case -EINPROGRESS:1650			ctx->r.req = NULL;1651			ctx->cc_sector += sector_step;1652			tag_offset++;1653			continue;1654		/*1655		 * The request was already processed (synchronously).1656		 */1657		case 0:1658			atomic_dec(&ctx->cc_pending);1659			ctx->cc_sector += sector_step;1660			tag_offset++;1661			if (!atomic)1662				cond_resched();1663			continue;1664		/*1665		 * There was a data integrity error.1666		 */1667		case -EBADMSG:1668			atomic_dec(&ctx->cc_pending);1669			return BLK_STS_PROTECTION;1670		/*1671		 * There was an error while processing the request.1672		 */1673		default:1674			atomic_dec(&ctx->cc_pending);1675			return BLK_STS_IOERR;1676		}1677	}1678 1679	return 0;1680}1681 1682static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone);1683 1684/*1685 * Generate a new unfragmented bio with the given size1686 * This should never violate the device limitations (but if it did then block1687 * core should split the bio as needed).1688 *1689 * This function may be called concurrently. If we allocate from the mempool1690 * concurrently, there is a possibility of deadlock. For example, if we have1691 * mempool of 256 pages, two processes, each wanting 256, pages allocate from1692 * the mempool concurrently, it may deadlock in a situation where both processes1693 * have allocated 128 pages and the mempool is exhausted.1694 *1695 * In order to avoid this scenario we allocate the pages under a mutex.1696 *1697 * In order to not degrade performance with excessive locking, we try1698 * non-blocking allocations without a mutex first but on failure we fallback1699 * to blocking allocations with a mutex.1700 *1701 * In order to reduce allocation overhead, we try to allocate compound pages in1702 * the first pass. If they are not available, we fall back to the mempool.1703 */1704static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned int size)1705{1706	struct crypt_config *cc = io->cc;1707	struct bio *clone;1708	unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;1709	gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM;1710	unsigned int remaining_size;1711	unsigned int order = MAX_PAGE_ORDER;1712 1713retry:1714	if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))1715		mutex_lock(&cc->bio_alloc_lock);1716 1717	clone = bio_alloc_bioset(cc->dev->bdev, nr_iovecs, io->base_bio->bi_opf,1718				 GFP_NOIO, &cc->bs);1719	clone->bi_private = io;1720	clone->bi_end_io = crypt_endio;1721	clone->bi_ioprio = io->base_bio->bi_ioprio;1722 1723	remaining_size = size;1724 1725	while (remaining_size) {1726		struct page *pages;1727		unsigned size_to_add;1728		unsigned remaining_order = __fls((remaining_size + PAGE_SIZE - 1) >> PAGE_SHIFT);1729		order = min(order, remaining_order);1730 1731		while (order > 0) {1732			if (unlikely(percpu_counter_read_positive(&cc->n_allocated_pages) +1733					(1 << order) > dm_crypt_pages_per_client))1734				goto decrease_order;1735			pages = alloc_pages(gfp_mask1736				| __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN | __GFP_COMP,1737				order);1738			if (likely(pages != NULL)) {1739				percpu_counter_add(&cc->n_allocated_pages, 1 << order);1740				goto have_pages;1741			}1742decrease_order:1743			order--;1744		}1745 1746		pages = mempool_alloc(&cc->page_pool, gfp_mask);1747		if (!pages) {1748			crypt_free_buffer_pages(cc, clone);1749			bio_put(clone);1750			gfp_mask |= __GFP_DIRECT_RECLAIM;1751			order = 0;1752			goto retry;1753		}1754 1755have_pages:1756		size_to_add = min((unsigned)PAGE_SIZE << order, remaining_size);1757		__bio_add_page(clone, pages, size_to_add, 0);1758		remaining_size -= size_to_add;1759	}1760 1761	/* Allocate space for integrity tags */1762	if (dm_crypt_integrity_io_alloc(io, clone)) {1763		crypt_free_buffer_pages(cc, clone);1764		bio_put(clone);1765		clone = NULL;1766	}1767 1768	if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))1769		mutex_unlock(&cc->bio_alloc_lock);1770 1771	return clone;1772}1773 1774static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)1775{1776	struct folio_iter fi;1777 1778	if (clone->bi_vcnt > 0) { /* bio_for_each_folio_all crashes with an empty bio */1779		bio_for_each_folio_all(fi, clone) {1780			if (folio_test_large(fi.folio)) {1781				percpu_counter_sub(&cc->n_allocated_pages,1782						1 << folio_order(fi.folio));1783				folio_put(fi.folio);1784			} else {1785				mempool_free(&fi.folio->page, &cc->page_pool);1786			}1787		}1788	}1789}1790 1791static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc,1792			  struct bio *bio, sector_t sector)1793{1794	io->cc = cc;1795	io->base_bio = bio;1796	io->sector = sector;1797	io->error = 0;1798	io->ctx.aead_recheck = false;1799	io->ctx.aead_failed = false;1800	io->ctx.r.req = NULL;1801	io->integrity_metadata = NULL;1802	io->integrity_metadata_from_pool = false;1803	atomic_set(&io->io_pending, 0);1804}1805 1806static void crypt_inc_pending(struct dm_crypt_io *io)1807{1808	atomic_inc(&io->io_pending);1809}1810 1811static void kcryptd_queue_read(struct dm_crypt_io *io);1812 1813/*1814 * One of the bios was finished. Check for completion of1815 * the whole request and correctly clean up the buffer.1816 */1817static void crypt_dec_pending(struct dm_crypt_io *io)1818{1819	struct crypt_config *cc = io->cc;1820	struct bio *base_bio = io->base_bio;1821	blk_status_t error = io->error;1822 1823	if (!atomic_dec_and_test(&io->io_pending))1824		return;1825 1826	if (likely(!io->ctx.aead_recheck) && unlikely(io->ctx.aead_failed) &&1827	    cc->used_tag_size && bio_data_dir(base_bio) == READ) {1828		io->ctx.aead_recheck = true;1829		io->ctx.aead_failed = false;1830		io->error = 0;1831		kcryptd_queue_read(io);1832		return;1833	}1834 1835	if (io->ctx.r.req)1836		crypt_free_req(cc, io->ctx.r.req, base_bio);1837 1838	if (unlikely(io->integrity_metadata_from_pool))1839		mempool_free(io->integrity_metadata, &io->cc->tag_pool);1840	else1841		kfree(io->integrity_metadata);1842 1843	base_bio->bi_status = error;1844 1845	bio_endio(base_bio);1846}1847 1848/*1849 * kcryptd/kcryptd_io:1850 *1851 * Needed because it would be very unwise to do decryption in an1852 * interrupt context.1853 *1854 * kcryptd performs the actual encryption or decryption.1855 *1856 * kcryptd_io performs the IO submission.1857 *1858 * They must be separated as otherwise the final stages could be1859 * starved by new requests which can block in the first stages due1860 * to memory allocation.1861 *1862 * The work is done per CPU global for all dm-crypt instances.1863 * They should not depend on each other and do not block.1864 */1865static void crypt_endio(struct bio *clone)1866{1867	struct dm_crypt_io *io = clone->bi_private;1868	struct crypt_config *cc = io->cc;1869	unsigned int rw = bio_data_dir(clone);1870	blk_status_t error = clone->bi_status;1871 1872	if (io->ctx.aead_recheck && !error) {1873		kcryptd_queue_crypt(io);1874		return;1875	}1876 1877	/*1878	 * free the processed pages1879	 */1880	if (rw == WRITE || io->ctx.aead_recheck)1881		crypt_free_buffer_pages(cc, clone);1882 1883	bio_put(clone);1884 1885	if (rw == READ && !error) {1886		kcryptd_queue_crypt(io);1887		return;1888	}1889 1890	if (unlikely(error))1891		io->error = error;1892 1893	crypt_dec_pending(io);1894}1895 1896#define CRYPT_MAP_READ_GFP GFP_NOWAIT1897 1898static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp)1899{1900	struct crypt_config *cc = io->cc;1901	struct bio *clone;1902 1903	if (io->ctx.aead_recheck) {1904		if (!(gfp & __GFP_DIRECT_RECLAIM))1905			return 1;1906		crypt_inc_pending(io);1907		clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size);1908		if (unlikely(!clone)) {1909			crypt_dec_pending(io);1910			return 1;1911		}1912		clone->bi_iter.bi_sector = cc->start + io->sector;1913		crypt_convert_init(cc, &io->ctx, clone, clone, io->sector);1914		io->saved_bi_iter = clone->bi_iter;1915		dm_submit_bio_remap(io->base_bio, clone);1916		return 0;1917	}1918 1919	/*1920	 * We need the original biovec array in order to decrypt the whole bio1921	 * data *afterwards* -- thanks to immutable biovecs we don't need to1922	 * worry about the block layer modifying the biovec array; so leverage1923	 * bio_alloc_clone().1924	 */1925	clone = bio_alloc_clone(cc->dev->bdev, io->base_bio, gfp, &cc->bs);1926	if (!clone)1927		return 1;1928	clone->bi_private = io;1929	clone->bi_end_io = crypt_endio;1930 1931	crypt_inc_pending(io);1932 1933	clone->bi_iter.bi_sector = cc->start + io->sector;1934 1935	if (dm_crypt_integrity_io_alloc(io, clone)) {1936		crypt_dec_pending(io);1937		bio_put(clone);1938		return 1;1939	}1940 1941	dm_submit_bio_remap(io->base_bio, clone);1942	return 0;1943}1944 1945static void kcryptd_io_read_work(struct work_struct *work)1946{1947	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);1948 1949	crypt_inc_pending(io);1950	if (kcryptd_io_read(io, GFP_NOIO))1951		io->error = BLK_STS_RESOURCE;1952	crypt_dec_pending(io);1953}1954 1955static void kcryptd_queue_read(struct dm_crypt_io *io)1956{1957	struct crypt_config *cc = io->cc;1958 1959	INIT_WORK(&io->work, kcryptd_io_read_work);1960	queue_work(cc->io_queue, &io->work);1961}1962 1963static void kcryptd_io_write(struct dm_crypt_io *io)1964{1965	struct bio *clone = io->ctx.bio_out;1966 1967	dm_submit_bio_remap(io->base_bio, clone);1968}1969 1970#define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node)1971 1972static int dmcrypt_write(void *data)1973{1974	struct crypt_config *cc = data;1975	struct dm_crypt_io *io;1976 1977	while (1) {1978		struct rb_root write_tree;1979		struct blk_plug plug;1980 1981		spin_lock_irq(&cc->write_thread_lock);1982continue_locked:1983 1984		if (!RB_EMPTY_ROOT(&cc->write_tree))1985			goto pop_from_list;1986 1987		set_current_state(TASK_INTERRUPTIBLE);1988 1989		spin_unlock_irq(&cc->write_thread_lock);1990 1991		if (unlikely(kthread_should_stop())) {1992			set_current_state(TASK_RUNNING);1993			break;1994		}1995 1996		schedule();1997 1998		spin_lock_irq(&cc->write_thread_lock);1999		goto continue_locked;2000 2001pop_from_list:2002		write_tree = cc->write_tree;2003		cc->write_tree = RB_ROOT;2004		spin_unlock_irq(&cc->write_thread_lock);2005 2006		BUG_ON(rb_parent(write_tree.rb_node));2007 2008		/*2009		 * Note: we cannot walk the tree here with rb_next because2010		 * the structures may be freed when kcryptd_io_write is called.2011		 */2012		blk_start_plug(&plug);2013		do {2014			io = crypt_io_from_node(rb_first(&write_tree));2015			rb_erase(&io->rb_node, &write_tree);2016			kcryptd_io_write(io);2017			cond_resched();2018		} while (!RB_EMPTY_ROOT(&write_tree));2019		blk_finish_plug(&plug);2020	}2021	return 0;2022}2023 2024static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async)2025{2026	struct bio *clone = io->ctx.bio_out;2027	struct crypt_config *cc = io->cc;2028	unsigned long flags;2029	sector_t sector;2030	struct rb_node **rbp, *parent;2031 2032	if (unlikely(io->error)) {2033		crypt_free_buffer_pages(cc, clone);2034		bio_put(clone);2035		crypt_dec_pending(io);2036		return;2037	}2038 2039	/* crypt_convert should have filled the clone bio */2040	BUG_ON(io->ctx.iter_out.bi_size);2041 2042	clone->bi_iter.bi_sector = cc->start + io->sector;2043 2044	if ((likely(!async) && test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) ||2045	    test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags)) {2046		dm_submit_bio_remap(io->base_bio, clone);2047		return;2048	}2049 2050	spin_lock_irqsave(&cc->write_thread_lock, flags);2051	if (RB_EMPTY_ROOT(&cc->write_tree))2052		wake_up_process(cc->write_thread);2053	rbp = &cc->write_tree.rb_node;2054	parent = NULL;2055	sector = io->sector;2056	while (*rbp) {2057		parent = *rbp;2058		if (sector < crypt_io_from_node(parent)->sector)2059			rbp = &(*rbp)->rb_left;2060		else2061			rbp = &(*rbp)->rb_right;2062	}2063	rb_link_node(&io->rb_node, parent, rbp);2064	rb_insert_color(&io->rb_node, &cc->write_tree);2065	spin_unlock_irqrestore(&cc->write_thread_lock, flags);2066}2067 2068static bool kcryptd_crypt_write_inline(struct crypt_config *cc,2069				       struct convert_context *ctx)2070 2071{2072	if (!test_bit(DM_CRYPT_WRITE_INLINE, &cc->flags))2073		return false;2074 2075	/*2076	 * Note: zone append writes (REQ_OP_ZONE_APPEND) do not have ordering2077	 * constraints so they do not need to be issued inline by2078	 * kcryptd_crypt_write_convert().2079	 */2080	switch (bio_op(ctx->bio_in)) {2081	case REQ_OP_WRITE:2082	case REQ_OP_WRITE_ZEROES:2083		return true;2084	default:2085		return false;2086	}2087}2088 2089static void kcryptd_crypt_write_continue(struct work_struct *work)2090{2091	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);2092	struct crypt_config *cc = io->cc;2093	struct convert_context *ctx = &io->ctx;2094	int crypt_finished;2095	sector_t sector = io->sector;2096	blk_status_t r;2097 2098	wait_for_completion(&ctx->restart);2099	reinit_completion(&ctx->restart);2100 2101	r = crypt_convert(cc, &io->ctx, true, false);2102	if (r)2103		io->error = r;2104	crypt_finished = atomic_dec_and_test(&ctx->cc_pending);2105	if (!crypt_finished && kcryptd_crypt_write_inline(cc, ctx)) {2106		/* Wait for completion signaled by kcryptd_async_done() */2107		wait_for_completion(&ctx->restart);2108		crypt_finished = 1;2109	}2110 2111	/* Encryption was already finished, submit io now */2112	if (crypt_finished) {2113		kcryptd_crypt_write_io_submit(io, 0);2114		io->sector = sector;2115	}2116 2117	crypt_dec_pending(io);2118}2119 2120static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)2121{2122	struct crypt_config *cc = io->cc;2123	struct convert_context *ctx = &io->ctx;2124	struct bio *clone;2125	int crypt_finished;2126	sector_t sector = io->sector;2127	blk_status_t r;2128 2129	/*2130	 * Prevent io from disappearing until this function completes.2131	 */2132	crypt_inc_pending(io);2133	crypt_convert_init(cc, ctx, NULL, io->base_bio, sector);2134 2135	clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size);2136	if (unlikely(!clone)) {2137		io->error = BLK_STS_IOERR;2138		goto dec;2139	}2140 2141	io->ctx.bio_out = clone;2142	io->ctx.iter_out = clone->bi_iter;2143 2144	if (crypt_integrity_aead(cc)) {2145		bio_copy_data(clone, io->base_bio);2146		io->ctx.bio_in = clone;2147		io->ctx.iter_in = clone->bi_iter;2148	}2149 2150	sector += bio_sectors(clone);2151 2152	crypt_inc_pending(io);2153	r = crypt_convert(cc, ctx,2154			  test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags), true);2155	/*2156	 * Crypto API backlogged the request, because its queue was full2157	 * and we're in softirq context, so continue from a workqueue2158	 * (TODO: is it actually possible to be in softirq in the write path?)2159	 */2160	if (r == BLK_STS_DEV_RESOURCE) {2161		INIT_WORK(&io->work, kcryptd_crypt_write_continue);2162		queue_work(cc->crypt_queue, &io->work);2163		return;2164	}2165	if (r)2166		io->error = r;2167	crypt_finished = atomic_dec_and_test(&ctx->cc_pending);2168	if (!crypt_finished && kcryptd_crypt_write_inline(cc, ctx)) {2169		/* Wait for completion signaled by kcryptd_async_done() */2170		wait_for_completion(&ctx->restart);2171		crypt_finished = 1;2172	}2173 2174	/* Encryption was already finished, submit io now */2175	if (crypt_finished) {2176		kcryptd_crypt_write_io_submit(io, 0);2177		io->sector = sector;2178	}2179 2180dec:2181	crypt_dec_pending(io);2182}2183 2184static void kcryptd_crypt_read_done(struct dm_crypt_io *io)2185{2186	if (io->ctx.aead_recheck) {2187		if (!io->error) {2188			io->ctx.bio_in->bi_iter = io->saved_bi_iter;2189			bio_copy_data(io->base_bio, io->ctx.bio_in);2190		}2191		crypt_free_buffer_pages(io->cc, io->ctx.bio_in);2192		bio_put(io->ctx.bio_in);2193	}2194	crypt_dec_pending(io);2195}2196 2197static void kcryptd_crypt_read_continue(struct work_struct *work)2198{2199	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);2200	struct crypt_config *cc = io->cc;2201	blk_status_t r;2202 2203	wait_for_completion(&io->ctx.restart);2204	reinit_completion(&io->ctx.restart);2205 2206	r = crypt_convert(cc, &io->ctx, true, false);2207	if (r)2208		io->error = r;2209 2210	if (atomic_dec_and_test(&io->ctx.cc_pending))2211		kcryptd_crypt_read_done(io);2212 2213	crypt_dec_pending(io);2214}2215 2216static void kcryptd_crypt_read_convert(struct dm_crypt_io *io)2217{2218	struct crypt_config *cc = io->cc;2219	blk_status_t r;2220 2221	crypt_inc_pending(io);2222 2223	if (io->ctx.aead_recheck) {2224		io->ctx.cc_sector = io->sector + cc->iv_offset;2225		r = crypt_convert(cc, &io->ctx,2226				  test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags), true);2227	} else {2228		crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio,2229				   io->sector);2230 2231		r = crypt_convert(cc, &io->ctx,2232				  test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags), true);2233	}2234	/*2235	 * Crypto API backlogged the request, because its queue was full2236	 * and we're in softirq context, so continue from a workqueue2237	 */2238	if (r == BLK_STS_DEV_RESOURCE) {2239		INIT_WORK(&io->work, kcryptd_crypt_read_continue);2240		queue_work(cc->crypt_queue, &io->work);2241		return;2242	}2243	if (r)2244		io->error = r;2245 2246	if (atomic_dec_and_test(&io->ctx.cc_pending))2247		kcryptd_crypt_read_done(io);2248 2249	crypt_dec_pending(io);2250}2251 2252static void kcryptd_async_done(void *data, int error)2253{2254	struct dm_crypt_request *dmreq = data;2255	struct convert_context *ctx = dmreq->ctx;2256	struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);2257	struct crypt_config *cc = io->cc;2258 2259	/*2260	 * A request from crypto driver backlog is going to be processed now,2261	 * finish the completion and continue in crypt_convert().2262	 * (Callback will be called for the second time for this request.)2263	 */2264	if (error == -EINPROGRESS) {2265		complete(&ctx->restart);2266		return;2267	}2268 2269	if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post)2270		error = cc->iv_gen_ops->post(cc, org_iv_of_dmreq(cc, dmreq), dmreq);2271 2272	if (error == -EBADMSG) {2273		sector_t s = le64_to_cpu(*org_sector_of_dmreq(cc, dmreq));2274 2275		ctx->aead_failed = true;2276		if (ctx->aead_recheck) {2277			DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu",2278				    ctx->bio_in->bi_bdev, s);2279			dm_audit_log_bio(DM_MSG_PREFIX, "integrity-aead",2280					 ctx->bio_in, s, 0);2281		}2282		io->error = BLK_STS_PROTECTION;2283	} else if (error < 0)2284		io->error = BLK_STS_IOERR;2285 2286	crypt_free_req(cc, req_of_dmreq(cc, dmreq), io->base_bio);2287 2288	if (!atomic_dec_and_test(&ctx->cc_pending))2289		return;2290 2291	/*2292	 * The request is fully completed: for inline writes, let2293	 * kcryptd_crypt_write_convert() do the IO submission.2294	 */2295	if (bio_data_dir(io->base_bio) == READ) {2296		kcryptd_crypt_read_done(io);2297		return;2298	}2299 2300	if (kcryptd_crypt_write_inline(cc, ctx)) {2301		complete(&ctx->restart);2302		return;2303	}2304 2305	kcryptd_crypt_write_io_submit(io, 1);2306}2307 2308static void kcryptd_crypt(struct work_struct *work)2309{2310	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);2311 2312	if (bio_data_dir(io->base_bio) == READ)2313		kcryptd_crypt_read_convert(io);2314	else2315		kcryptd_crypt_write_convert(io);2316}2317 2318static void kcryptd_queue_crypt(struct dm_crypt_io *io)2319{2320	struct crypt_config *cc = io->cc;2321 2322	if ((bio_data_dir(io->base_bio) == READ && test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags)) ||2323	    (bio_data_dir(io->base_bio) == WRITE && test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))) {2324		/*2325		 * in_hardirq(): Crypto API's skcipher_walk_first() refuses to work in hard IRQ context.2326		 * irqs_disabled(): the kernel may run some IO completion from the idle thread, but2327		 * it is being executed with irqs disabled.2328		 */2329		if (in_hardirq() || irqs_disabled()) {2330			INIT_WORK(&io->work, kcryptd_crypt);2331			queue_work(system_bh_wq, &io->work);2332			return;2333		} else {2334			kcryptd_crypt(&io->work);2335			return;2336		}2337	}2338 2339	INIT_WORK(&io->work, kcryptd_crypt);2340	queue_work(cc->crypt_queue, &io->work);2341}2342 2343static void crypt_free_tfms_aead(struct crypt_config *cc)2344{2345	if (!cc->cipher_tfm.tfms_aead)2346		return;2347 2348	if (cc->cipher_tfm.tfms_aead[0] && !IS_ERR(cc->cipher_tfm.tfms_aead[0])) {2349		crypto_free_aead(cc->cipher_tfm.tfms_aead[0]);2350		cc->cipher_tfm.tfms_aead[0] = NULL;2351	}2352 2353	kfree(cc->cipher_tfm.tfms_aead);2354	cc->cipher_tfm.tfms_aead = NULL;2355}2356 2357static void crypt_free_tfms_skcipher(struct crypt_config *cc)2358{2359	unsigned int i;2360 2361	if (!cc->cipher_tfm.tfms)2362		return;2363 2364	for (i = 0; i < cc->tfms_count; i++)2365		if (cc->cipher_tfm.tfms[i] && !IS_ERR(cc->cipher_tfm.tfms[i])) {2366			crypto_free_skcipher(cc->cipher_tfm.tfms[i]);2367			cc->cipher_tfm.tfms[i] = NULL;2368		}2369 2370	kfree(cc->cipher_tfm.tfms);2371	cc->cipher_tfm.tfms = NULL;2372}2373 2374static void crypt_free_tfms(struct crypt_config *cc)2375{2376	if (crypt_integrity_aead(cc))2377		crypt_free_tfms_aead(cc);2378	else2379		crypt_free_tfms_skcipher(cc);2380}2381 2382static int crypt_alloc_tfms_skcipher(struct crypt_config *cc, char *ciphermode)2383{2384	unsigned int i;2385	int err;2386 2387	cc->cipher_tfm.tfms = kcalloc(cc->tfms_count,2388				      sizeof(struct crypto_skcipher *),2389				      GFP_KERNEL);2390	if (!cc->cipher_tfm.tfms)2391		return -ENOMEM;2392 2393	for (i = 0; i < cc->tfms_count; i++) {2394		cc->cipher_tfm.tfms[i] = crypto_alloc_skcipher(ciphermode, 0,2395						CRYPTO_ALG_ALLOCATES_MEMORY);2396		if (IS_ERR(cc->cipher_tfm.tfms[i])) {2397			err = PTR_ERR(cc->cipher_tfm.tfms[i]);2398			crypt_free_tfms(cc);2399			return err;2400		}2401	}2402 2403	/*2404	 * dm-crypt performance can vary greatly depending on which crypto2405	 * algorithm implementation is used.  Help people debug performance2406	 * problems by logging the ->cra_driver_name.2407	 */2408	DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode,2409	       crypto_skcipher_alg(any_tfm(cc))->base.cra_driver_name);2410	return 0;2411}2412 2413static int crypt_alloc_tfms_aead(struct crypt_config *cc, char *ciphermode)2414{2415	int err;2416 2417	cc->cipher_tfm.tfms = kmalloc(sizeof(struct crypto_aead *), GFP_KERNEL);2418	if (!cc->cipher_tfm.tfms)2419		return -ENOMEM;2420 2421	cc->cipher_tfm.tfms_aead[0] = crypto_alloc_aead(ciphermode, 0,2422						CRYPTO_ALG_ALLOCATES_MEMORY);2423	if (IS_ERR(cc->cipher_tfm.tfms_aead[0])) {2424		err = PTR_ERR(cc->cipher_tfm.tfms_aead[0]);2425		crypt_free_tfms(cc);2426		return err;2427	}2428 2429	DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode,2430	       crypto_aead_alg(any_tfm_aead(cc))->base.cra_driver_name);2431	return 0;2432}2433 2434static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode)2435{2436	if (crypt_integrity_aead(cc))2437		return crypt_alloc_tfms_aead(cc, ciphermode);2438	else2439		return crypt_alloc_tfms_skcipher(cc, ciphermode);2440}2441 2442static unsigned int crypt_subkey_size(struct crypt_config *cc)2443{2444	return (cc->key_size - cc->key_extra_size) >> ilog2(cc->tfms_count);2445}2446 2447static unsigned int crypt_authenckey_size(struct crypt_config *cc)2448{2449	return crypt_subkey_size(cc) + RTA_SPACE(sizeof(struct crypto_authenc_key_param));2450}2451 2452/*2453 * If AEAD is composed like authenc(hmac(sha256),xts(aes)),2454 * the key must be for some reason in special format.2455 * This funcion converts cc->key to this special format.2456 */2457static void crypt_copy_authenckey(char *p, const void *key,2458				  unsigned int enckeylen, unsigned int authkeylen)2459{2460	struct crypto_authenc_key_param *param;2461	struct rtattr *rta;2462 2463	rta = (struct rtattr *)p;2464	param = RTA_DATA(rta);2465	param->enckeylen = cpu_to_be32(enckeylen);2466	rta->rta_len = RTA_LENGTH(sizeof(*param));2467	rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM;2468	p += RTA_SPACE(sizeof(*param));2469	memcpy(p, key + enckeylen, authkeylen);2470	p += authkeylen;2471	memcpy(p, key, enckeylen);2472}2473 2474static int crypt_setkey(struct crypt_config *cc)2475{2476	unsigned int subkey_size;2477	int err = 0, i, r;2478 2479	/* Ignore extra keys (which are used for IV etc) */2480	subkey_size = crypt_subkey_size(cc);2481 2482	if (crypt_integrity_hmac(cc)) {2483		if (subkey_size < cc->key_mac_size)2484			return -EINVAL;2485 2486		crypt_copy_authenckey(cc->authenc_key, cc->key,2487				      subkey_size - cc->key_mac_size,2488				      cc->key_mac_size);2489	}2490 2491	for (i = 0; i < cc->tfms_count; i++) {2492		if (crypt_integrity_hmac(cc))2493			r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i],2494				cc->authenc_key, crypt_authenckey_size(cc));2495		else if (crypt_integrity_aead(cc))2496			r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i],2497					       cc->key + (i * subkey_size),2498					       subkey_size);2499		else2500			r = crypto_skcipher_setkey(cc->cipher_tfm.tfms[i],2501						   cc->key + (i * subkey_size),2502						   subkey_size);2503		if (r)2504			err = r;2505	}2506 2507	if (crypt_integrity_hmac(cc))2508		memzero_explicit(cc->authenc_key, crypt_authenckey_size(cc));2509 2510	return err;2511}2512 2513#ifdef CONFIG_KEYS2514 2515static bool contains_whitespace(const char *str)2516{2517	while (*str)2518		if (isspace(*str++))2519			return true;2520	return false;2521}2522 2523static int set_key_user(struct crypt_config *cc, struct key *key)2524{2525	const struct user_key_payload *ukp;2526 2527	ukp = user_key_payload_locked(key);2528	if (!ukp)2529		return -EKEYREVOKED;2530 2531	if (cc->key_size != ukp->datalen)2532		return -EINVAL;2533 2534	memcpy(cc->key, ukp->data, cc->key_size);2535 2536	return 0;2537}2538 2539static int set_key_encrypted(struct crypt_config *cc, struct key *key)2540{2541	const struct encrypted_key_payload *ekp;2542 2543	ekp = key->payload.data[0];2544	if (!ekp)2545		return -EKEYREVOKED;2546 2547	if (cc->key_size != ekp->decrypted_datalen)2548		return -EINVAL;2549 2550	memcpy(cc->key, ekp->decrypted_data, cc->key_size);2551 2552	return 0;2553}2554 2555static int set_key_trusted(struct crypt_config *cc, struct key *key)2556{2557	const struct trusted_key_payload *tkp;2558 2559	tkp = key->payload.data[0];2560	if (!tkp)2561		return -EKEYREVOKED;2562 2563	if (cc->key_size != tkp->key_len)2564		return -EINVAL;2565 2566	memcpy(cc->key, tkp->key, cc->key_size);2567 2568	return 0;2569}2570 2571static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)2572{2573	char *new_key_string, *key_desc;2574	int ret;2575	struct key_type *type;2576	struct key *key;2577	int (*set_key)(struct crypt_config *cc, struct key *key);2578 2579	/*2580	 * Reject key_string with whitespace. dm core currently lacks code for2581	 * proper whitespace escaping in arguments on DM_TABLE_STATUS path.2582	 */2583	if (contains_whitespace(key_string)) {2584		DMERR("whitespace chars not allowed in key string");2585		return -EINVAL;2586	}2587 2588	/* look for next ':' separating key_type from key_description */2589	key_desc = strchr(key_string, ':');2590	if (!key_desc || key_desc == key_string || !strlen(key_desc + 1))2591		return -EINVAL;2592 2593	if (!strncmp(key_string, "logon:", key_desc - key_string + 1)) {2594		type = &key_type_logon;2595		set_key = set_key_user;2596	} else if (!strncmp(key_string, "user:", key_desc - key_string + 1)) {2597		type = &key_type_user;2598		set_key = set_key_user;2599	} else if (IS_ENABLED(CONFIG_ENCRYPTED_KEYS) &&2600		   !strncmp(key_string, "encrypted:", key_desc - key_string + 1)) {2601		type = &key_type_encrypted;2602		set_key = set_key_encrypted;2603	} else if (IS_ENABLED(CONFIG_TRUSTED_KEYS) &&2604		   !strncmp(key_string, "trusted:", key_desc - key_string + 1)) {2605		type = &key_type_trusted;2606		set_key = set_key_trusted;2607	} else {2608		return -EINVAL;2609	}2610 2611	new_key_string = kstrdup(key_string, GFP_KERNEL);2612	if (!new_key_string)2613		return -ENOMEM;2614 2615	key = request_key(type, key_desc + 1, NULL);2616	if (IS_ERR(key)) {2617		ret = PTR_ERR(key);2618		goto free_new_key_string;2619	}2620 2621	down_read(&key->sem);2622	ret = set_key(cc, key);2623	up_read(&key->sem);2624	key_put(key);2625	if (ret < 0)2626		goto free_new_key_string;2627 2628	/* clear the flag since following operations may invalidate previously valid key */2629	clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);2630 2631	ret = crypt_setkey(cc);2632	if (ret)2633		goto free_new_key_string;2634 2635	set_bit(DM_CRYPT_KEY_VALID, &cc->flags);2636	kfree_sensitive(cc->key_string);2637	cc->key_string = new_key_string;2638	return 0;2639 2640free_new_key_string:2641	kfree_sensitive(new_key_string);2642	return ret;2643}2644 2645static int get_key_size(char **key_string)2646{2647	char *colon, dummy;2648	int ret;2649 2650	if (*key_string[0] != ':')2651		return strlen(*key_string) >> 1;2652 2653	/* look for next ':' in key string */2654	colon = strpbrk(*key_string + 1, ":");2655	if (!colon)2656		return -EINVAL;2657 2658	if (sscanf(*key_string + 1, "%u%c", &ret, &dummy) != 2 || dummy != ':')2659		return -EINVAL;2660 2661	*key_string = colon;2662 2663	/* remaining key string should be :<logon|user>:<key_desc> */2664 2665	return ret;2666}2667 2668#else2669 2670static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)2671{2672	return -EINVAL;2673}2674 2675static int get_key_size(char **key_string)2676{2677	return (*key_string[0] == ':') ? -EINVAL : (int)(strlen(*key_string) >> 1);2678}2679 2680#endif /* CONFIG_KEYS */2681 2682static int crypt_set_key(struct crypt_config *cc, char *key)2683{2684	int r = -EINVAL;2685	int key_string_len = strlen(key);2686 2687	/* Hyphen (which gives a key_size of zero) means there is no key. */2688	if (!cc->key_size && strcmp(key, "-"))2689		goto out;2690 2691	/* ':' means the key is in kernel keyring, short-circuit normal key processing */2692	if (key[0] == ':') {2693		r = crypt_set_keyring_key(cc, key + 1);2694		goto out;2695	}2696 2697	/* clear the flag since following operations may invalidate previously valid key */2698	clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);2699 2700	/* wipe references to any kernel keyring key */2701	kfree_sensitive(cc->key_string);2702	cc->key_string = NULL;2703 2704	/* Decode key from its hex representation. */2705	if (cc->key_size && hex2bin(cc->key, key, cc->key_size) < 0)2706		goto out;2707 2708	r = crypt_setkey(cc);2709	if (!r)2710		set_bit(DM_CRYPT_KEY_VALID, &cc->flags);2711 2712out:2713	/* Hex key string not needed after here, so wipe it. */2714	memset(key, '0', key_string_len);2715 2716	return r;2717}2718 2719static int crypt_wipe_key(struct crypt_config *cc)2720{2721	int r;2722 2723	clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);2724	get_random_bytes(&cc->key, cc->key_size);2725 2726	/* Wipe IV private keys */2727	if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) {2728		r = cc->iv_gen_ops->wipe(cc);2729		if (r)2730			return r;2731	}2732 2733	kfree_sensitive(cc->key_string);2734	cc->key_string = NULL;2735	r = crypt_setkey(cc);2736	memset(&cc->key, 0, cc->key_size * sizeof(u8));2737 2738	return r;2739}2740 2741static void crypt_calculate_pages_per_client(void)2742{2743	unsigned long pages = (totalram_pages() - totalhigh_pages()) * DM_CRYPT_MEMORY_PERCENT / 100;2744 2745	if (!dm_crypt_clients_n)2746		return;2747 2748	pages /= dm_crypt_clients_n;2749	if (pages < DM_CRYPT_MIN_PAGES_PER_CLIENT)2750		pages = DM_CRYPT_MIN_PAGES_PER_CLIENT;2751	dm_crypt_pages_per_client = pages;2752}2753 2754static void *crypt_page_alloc(gfp_t gfp_mask, void *pool_data)2755{2756	struct crypt_config *cc = pool_data;2757	struct page *page;2758 2759	/*2760	 * Note, percpu_counter_read_positive() may over (and under) estimate2761	 * the current usage by at most (batch - 1) * num_online_cpus() pages,2762	 * but avoids potential spinlock contention of an exact result.2763	 */2764	if (unlikely(percpu_counter_read_positive(&cc->n_allocated_pages) >= dm_crypt_pages_per_client) &&2765	    likely(gfp_mask & __GFP_NORETRY))2766		return NULL;2767 2768	page = alloc_page(gfp_mask);2769	if (likely(page != NULL))2770		percpu_counter_add(&cc->n_allocated_pages, 1);2771 2772	return page;2773}2774 2775static void crypt_page_free(void *page, void *pool_data)2776{2777	struct crypt_config *cc = pool_data;2778 2779	__free_page(page);2780	percpu_counter_sub(&cc->n_allocated_pages, 1);2781}2782 2783static void crypt_dtr(struct dm_target *ti)2784{2785	struct crypt_config *cc = ti->private;2786 2787	ti->private = NULL;2788 2789	if (!cc)2790		return;2791 2792	if (cc->write_thread)2793		kthread_stop(cc->write_thread);2794 2795	if (cc->io_queue)2796		destroy_workqueue(cc->io_queue);2797	if (cc->crypt_queue)2798		destroy_workqueue(cc->crypt_queue);2799 2800	if (cc->workqueue_id)2801		ida_free(&workqueue_ida, cc->workqueue_id);2802 2803	crypt_free_tfms(cc);2804 2805	bioset_exit(&cc->bs);2806 2807	mempool_exit(&cc->page_pool);2808	mempool_exit(&cc->req_pool);2809	mempool_exit(&cc->tag_pool);2810 2811	WARN_ON(percpu_counter_sum(&cc->n_allocated_pages) != 0);2812	percpu_counter_destroy(&cc->n_allocated_pages);2813 2814	if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)2815		cc->iv_gen_ops->dtr(cc);2816 2817	if (cc->dev)2818		dm_put_device(ti, cc->dev);2819 2820	kfree_sensitive(cc->cipher_string);2821	kfree_sensitive(cc->key_string);2822	kfree_sensitive(cc->cipher_auth);2823	kfree_sensitive(cc->authenc_key);2824 2825	mutex_destroy(&cc->bio_alloc_lock);2826 2827	/* Must zero key material before freeing */2828	kfree_sensitive(cc);2829 2830	spin_lock(&dm_crypt_clients_lock);2831	WARN_ON(!dm_crypt_clients_n);2832	dm_crypt_clients_n--;2833	crypt_calculate_pages_per_client();2834	spin_unlock(&dm_crypt_clients_lock);2835 2836	dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1);2837}2838 2839static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)2840{2841	struct crypt_config *cc = ti->private;2842 2843	if (crypt_integrity_aead(cc))2844		cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc));2845	else2846		cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));2847 2848	if (cc->iv_size)2849		/* at least a 64 bit sector number should fit in our buffer */2850		cc->iv_size = max(cc->iv_size,2851				  (unsigned int)(sizeof(u64) / sizeof(u8)));2852	else if (ivmode) {2853		DMWARN("Selected cipher does not support IVs");2854		ivmode = NULL;2855	}2856 2857	/* Choose ivmode, see comments at iv code. */2858	if (ivmode == NULL)2859		cc->iv_gen_ops = NULL;2860	else if (strcmp(ivmode, "plain") == 0)2861		cc->iv_gen_ops = &crypt_iv_plain_ops;2862	else if (strcmp(ivmode, "plain64") == 0)2863		cc->iv_gen_ops = &crypt_iv_plain64_ops;2864	else if (strcmp(ivmode, "plain64be") == 0)2865		cc->iv_gen_ops = &crypt_iv_plain64be_ops;2866	else if (strcmp(ivmode, "essiv") == 0)2867		cc->iv_gen_ops = &crypt_iv_essiv_ops;2868	else if (strcmp(ivmode, "benbi") == 0)2869		cc->iv_gen_ops = &crypt_iv_benbi_ops;2870	else if (strcmp(ivmode, "null") == 0)2871		cc->iv_gen_ops = &crypt_iv_null_ops;2872	else if (strcmp(ivmode, "eboiv") == 0)2873		cc->iv_gen_ops = &crypt_iv_eboiv_ops;2874	else if (strcmp(ivmode, "elephant") == 0) {2875		cc->iv_gen_ops = &crypt_iv_elephant_ops;2876		cc->key_parts = 2;2877		cc->key_extra_size = cc->key_size / 2;2878		if (cc->key_extra_size > ELEPHANT_MAX_KEY_SIZE)2879			return -EINVAL;2880		set_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags);2881	} else if (strcmp(ivmode, "lmk") == 0) {2882		cc->iv_gen_ops = &crypt_iv_lmk_ops;2883		/*2884		 * Version 2 and 3 is recognised according2885		 * to length of provided multi-key string.2886		 * If present (version 3), last key is used as IV seed.2887		 * All keys (including IV seed) are always the same size.2888		 */2889		if (cc->key_size % cc->key_parts) {2890			cc->key_parts++;2891			cc->key_extra_size = cc->key_size / cc->key_parts;2892		}2893	} else if (strcmp(ivmode, "tcw") == 0) {2894		cc->iv_gen_ops = &crypt_iv_tcw_ops;2895		cc->key_parts += 2; /* IV + whitening */2896		cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE;2897	} else if (strcmp(ivmode, "random") == 0) {2898		cc->iv_gen_ops = &crypt_iv_random_ops;2899		/* Need storage space in integrity fields. */2900		cc->integrity_iv_size = cc->iv_size;2901	} else {2902		ti->error = "Invalid IV mode";2903		return -EINVAL;2904	}2905 2906	return 0;2907}2908 2909/*2910 * Workaround to parse HMAC algorithm from AEAD crypto API spec.2911 * The HMAC is needed to calculate tag size (HMAC digest size).2912 * This should be probably done by crypto-api calls (once available...)2913 */2914static int crypt_ctr_auth_cipher(struct crypt_config *cc, char *cipher_api)2915{2916	char *start, *end, *mac_alg = NULL;2917	struct crypto_ahash *mac;2918 2919	if (!strstarts(cipher_api, "authenc("))2920		return 0;2921 2922	start = strchr(cipher_api, '(');2923	end = strchr(cipher_api, ',');2924	if (!start || !end || ++start > end)2925		return -EINVAL;2926 2927	mac_alg = kmemdup_nul(start, end - start, GFP_KERNEL);2928	if (!mac_alg)2929		return -ENOMEM;2930 2931	mac = crypto_alloc_ahash(mac_alg, 0, CRYPTO_ALG_ALLOCATES_MEMORY);2932	kfree(mac_alg);2933 2934	if (IS_ERR(mac))2935		return PTR_ERR(mac);2936 2937	if (!test_bit(CRYPT_KEY_MAC_SIZE_SET, &cc->cipher_flags))2938		cc->key_mac_size = crypto_ahash_digestsize(mac);2939	crypto_free_ahash(mac);2940 2941	cc->authenc_key = kmalloc(crypt_authenckey_size(cc), GFP_KERNEL);2942	if (!cc->authenc_key)2943		return -ENOMEM;2944 2945	return 0;2946}2947 2948static int crypt_ctr_cipher_new(struct dm_target *ti, char *cipher_in, char *key,2949				char **ivmode, char **ivopts)2950{2951	struct crypt_config *cc = ti->private;2952	char *tmp, *cipher_api, buf[CRYPTO_MAX_ALG_NAME];2953	int ret = -EINVAL;2954 2955	cc->tfms_count = 1;2956 2957	/*2958	 * New format (capi: prefix)2959	 * capi:cipher_api_spec-iv:ivopts2960	 */2961	tmp = &cipher_in[strlen("capi:")];2962 2963	/* Separate IV options if present, it can contain another '-' in hash name */2964	*ivopts = strrchr(tmp, ':');2965	if (*ivopts) {2966		**ivopts = '\0';2967		(*ivopts)++;2968	}2969	/* Parse IV mode */2970	*ivmode = strrchr(tmp, '-');2971	if (*ivmode) {2972		**ivmode = '\0';2973		(*ivmode)++;2974	}2975	/* The rest is crypto API spec */2976	cipher_api = tmp;2977 2978	/* Alloc AEAD, can be used only in new format. */2979	if (crypt_integrity_aead(cc)) {2980		ret = crypt_ctr_auth_cipher(cc, cipher_api);2981		if (ret < 0) {2982			ti->error = "Invalid AEAD cipher spec";2983			return ret;2984		}2985	}2986 2987	if (*ivmode && !strcmp(*ivmode, "lmk"))2988		cc->tfms_count = 64;2989 2990	if (*ivmode && !strcmp(*ivmode, "essiv")) {2991		if (!*ivopts) {2992			ti->error = "Digest algorithm missing for ESSIV mode";2993			return -EINVAL;2994		}2995		ret = snprintf(buf, CRYPTO_MAX_ALG_NAME, "essiv(%s,%s)",2996			       cipher_api, *ivopts);2997		if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) {2998			ti->error = "Cannot allocate cipher string";2999			return -ENOMEM;3000		}3001		cipher_api = buf;3002	}3003 3004	cc->key_parts = cc->tfms_count;3005 3006	/* Allocate cipher */3007	ret = crypt_alloc_tfms(cc, cipher_api);3008	if (ret < 0) {3009		ti->error = "Error allocating crypto tfm";3010		return ret;3011	}3012 3013	if (crypt_integrity_aead(cc))3014		cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc));3015	else3016		cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));3017 3018	return 0;3019}3020 3021static int crypt_ctr_cipher_old(struct dm_target *ti, char *cipher_in, char *key,3022				char **ivmode, char **ivopts)3023{3024	struct crypt_config *cc = ti->private;3025	char *tmp, *cipher, *chainmode, *keycount;3026	char *cipher_api = NULL;3027	int ret = -EINVAL;3028	char dummy;3029 3030	if (strchr(cipher_in, '(') || crypt_integrity_aead(cc)) {3031		ti->error = "Bad cipher specification";3032		return -EINVAL;3033	}3034 3035	/*3036	 * Legacy dm-crypt cipher specification3037	 * cipher[:keycount]-mode-iv:ivopts3038	 */3039	tmp = cipher_in;3040	keycount = strsep(&tmp, "-");3041	cipher = strsep(&keycount, ":");3042 3043	if (!keycount)3044		cc->tfms_count = 1;3045	else if (sscanf(keycount, "%u%c", &cc->tfms_count, &dummy) != 1 ||3046		 !is_power_of_2(cc->tfms_count)) {3047		ti->error = "Bad cipher key count specification";3048		return -EINVAL;3049	}3050	cc->key_parts = cc->tfms_count;3051 3052	chainmode = strsep(&tmp, "-");3053	*ivmode = strsep(&tmp, ":");3054	*ivopts = tmp;3055 3056	/*3057	 * For compatibility with the original dm-crypt mapping format, if3058	 * only the cipher name is supplied, use cbc-plain.3059	 */3060	if (!chainmode || (!strcmp(chainmode, "plain") && !*ivmode)) {3061		chainmode = "cbc";3062		*ivmode = "plain";3063	}3064 3065	if (strcmp(chainmode, "ecb") && !*ivmode) {3066		ti->error = "IV mechanism required";3067		return -EINVAL;3068	}3069 3070	cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL);3071	if (!cipher_api)3072		goto bad_mem;3073 3074	if (*ivmode && !strcmp(*ivmode, "essiv")) {3075		if (!*ivopts) {3076			ti->error = "Digest algorithm missing for ESSIV mode";3077			kfree(cipher_api);3078			return -EINVAL;3079		}3080		ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME,3081			       "essiv(%s(%s),%s)", chainmode, cipher, *ivopts);3082	} else {3083		ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME,3084			       "%s(%s)", chainmode, cipher);3085	}3086	if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) {3087		kfree(cipher_api);3088		goto bad_mem;3089	}3090 3091	/* Allocate cipher */3092	ret = crypt_alloc_tfms(cc, cipher_api);3093	if (ret < 0) {3094		ti->error = "Error allocating crypto tfm";3095		kfree(cipher_api);3096		return ret;3097	}3098	kfree(cipher_api);3099 3100	return 0;3101bad_mem:3102	ti->error = "Cannot allocate cipher strings";3103	return -ENOMEM;3104}3105 3106static int crypt_ctr_cipher(struct dm_target *ti, char *cipher_in, char *key)3107{3108	struct crypt_config *cc = ti->private;3109	char *ivmode = NULL, *ivopts = NULL;3110	int ret;3111 3112	cc->cipher_string = kstrdup(cipher_in, GFP_KERNEL);3113	if (!cc->cipher_string) {3114		ti->error = "Cannot allocate cipher strings";3115		return -ENOMEM;3116	}3117 3118	if (strstarts(cipher_in, "capi:"))3119		ret = crypt_ctr_cipher_new(ti, cipher_in, key, &ivmode, &ivopts);3120	else3121		ret = crypt_ctr_cipher_old(ti, cipher_in, key, &ivmode, &ivopts);3122	if (ret)3123		return ret;3124 3125	/* Initialize IV */3126	ret = crypt_ctr_ivmode(ti, ivmode);3127	if (ret < 0)3128		return ret;3129 3130	/* Initialize and set key */3131	ret = crypt_set_key(cc, key);3132	if (ret < 0) {3133		ti->error = "Error decoding and setting key";3134		return ret;3135	}3136 3137	/* Allocate IV */3138	if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) {3139		ret = cc->iv_gen_ops->ctr(cc, ti, ivopts);3140		if (ret < 0) {3141			ti->error = "Error creating IV";3142			return ret;3143		}3144	}3145 3146	/* Initialize IV (set keys for ESSIV etc) */3147	if (cc->iv_gen_ops && cc->iv_gen_ops->init) {3148		ret = cc->iv_gen_ops->init(cc);3149		if (ret < 0) {3150			ti->error = "Error initialising IV";3151			return ret;3152		}3153	}3154 3155	/* wipe the kernel key payload copy */3156	if (cc->key_string)3157		memset(cc->key, 0, cc->key_size * sizeof(u8));3158 3159	return ret;3160}3161 3162static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **argv)3163{3164	struct crypt_config *cc = ti->private;3165	struct dm_arg_set as;3166	static const struct dm_arg _args[] = {3167		{0, 9, "Invalid number of feature args"},3168	};3169	unsigned int opt_params, val;3170	const char *opt_string, *sval;3171	char dummy;3172	int ret;3173 3174	/* Optional parameters */3175	as.argc = argc;3176	as.argv = argv;3177 3178	ret = dm_read_arg_group(_args, &as, &opt_params, &ti->error);3179	if (ret)3180		return ret;3181 3182	while (opt_params--) {3183		opt_string = dm_shift_arg(&as);3184		if (!opt_string) {3185			ti->error = "Not enough feature arguments";3186			return -EINVAL;3187		}3188 3189		if (!strcasecmp(opt_string, "allow_discards"))3190			ti->num_discard_bios = 1;3191 3192		else if (!strcasecmp(opt_string, "same_cpu_crypt"))3193			set_bit(DM_CRYPT_SAME_CPU, &cc->flags);3194		else if (!strcasecmp(opt_string, "high_priority"))3195			set_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags);3196 3197		else if (!strcasecmp(opt_string, "submit_from_crypt_cpus"))3198			set_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags);3199		else if (!strcasecmp(opt_string, "no_read_workqueue"))3200			set_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags);3201		else if (!strcasecmp(opt_string, "no_write_workqueue"))3202			set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags);3203		else if (sscanf(opt_string, "integrity:%u:", &val) == 1) {3204			if (val == 0 || val > MAX_TAG_SIZE) {3205				ti->error = "Invalid integrity arguments";3206				return -EINVAL;3207			}3208			cc->used_tag_size = val;3209			sval = strchr(opt_string + strlen("integrity:"), ':') + 1;3210			if (!strcasecmp(sval, "aead")) {3211				set_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags);3212			} else if (strcasecmp(sval, "none")) {3213				ti->error = "Unknown integrity profile";3214				return -EINVAL;3215			}3216 3217			cc->cipher_auth = kstrdup(sval, GFP_KERNEL);3218			if (!cc->cipher_auth)3219				return -ENOMEM;3220		} else if (sscanf(opt_string, "integrity_key_size:%u%c", &val, &dummy) == 1) {3221			if (!val) {3222				ti->error = "Invalid integrity_key_size argument";3223				return -EINVAL;3224			}3225			cc->key_mac_size = val;3226			set_bit(CRYPT_KEY_MAC_SIZE_SET, &cc->cipher_flags);3227		} else if (sscanf(opt_string, "sector_size:%hu%c", &cc->sector_size, &dummy) == 1) {3228			if (cc->sector_size < (1 << SECTOR_SHIFT) ||3229			    cc->sector_size > 4096 ||3230			    (cc->sector_size & (cc->sector_size - 1))) {3231				ti->error = "Invalid feature value for sector_size";3232				return -EINVAL;3233			}3234			if (ti->len & ((cc->sector_size >> SECTOR_SHIFT) - 1)) {3235				ti->error = "Device size is not multiple of sector_size feature";3236				return -EINVAL;3237			}3238			cc->sector_shift = __ffs(cc->sector_size) - SECTOR_SHIFT;3239		} else if (!strcasecmp(opt_string, "iv_large_sectors"))3240			set_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags);3241		else {3242			ti->error = "Invalid feature arguments";3243			return -EINVAL;3244		}3245	}3246 3247	return 0;3248}3249 3250#ifdef CONFIG_BLK_DEV_ZONED3251static int crypt_report_zones(struct dm_target *ti,3252		struct dm_report_zones_args *args, unsigned int nr_zones)3253{3254	struct crypt_config *cc = ti->private;3255 3256	return dm_report_zones(cc->dev->bdev, cc->start,3257			cc->start + dm_target_offset(ti, args->next_sector),3258			args, nr_zones);3259}3260#else3261#define crypt_report_zones NULL3262#endif3263 3264/*3265 * Construct an encryption mapping:3266 * <cipher> [<key>|:<key_size>:<user|logon>:<key_description>] <iv_offset> <dev_path> <start>3267 */3268static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)3269{3270	struct crypt_config *cc;3271	const char *devname = dm_table_device_name(ti->table);3272	int key_size, wq_id;3273	unsigned int align_mask;3274	unsigned int common_wq_flags;3275	unsigned long long tmpll;3276	int ret;3277	size_t iv_size_padding, additional_req_size;3278	char dummy;3279 3280	if (argc < 5) {3281		ti->error = "Not enough arguments";3282		return -EINVAL;3283	}3284 3285	key_size = get_key_size(&argv[1]);3286	if (key_size < 0) {3287		ti->error = "Cannot parse key size";3288		return -EINVAL;3289	}3290 3291	cc = kzalloc(struct_size(cc, key, key_size), GFP_KERNEL);3292	if (!cc) {3293		ti->error = "Cannot allocate encryption context";3294		return -ENOMEM;3295	}3296	cc->key_size = key_size;3297	cc->sector_size = (1 << SECTOR_SHIFT);3298	cc->sector_shift = 0;3299 3300	ti->private = cc;3301 3302	spin_lock(&dm_crypt_clients_lock);3303	dm_crypt_clients_n++;3304	crypt_calculate_pages_per_client();3305	spin_unlock(&dm_crypt_clients_lock);3306 3307	ret = percpu_counter_init(&cc->n_allocated_pages, 0, GFP_KERNEL);3308	if (ret < 0)3309		goto bad;3310 3311	/* Optional parameters need to be read before cipher constructor */3312	if (argc > 5) {3313		ret = crypt_ctr_optional(ti, argc - 5, &argv[5]);3314		if (ret)3315			goto bad;3316	}3317 3318	ret = crypt_ctr_cipher(ti, argv[0], argv[1]);3319	if (ret < 0)3320		goto bad;3321 3322	if (crypt_integrity_aead(cc)) {3323		cc->dmreq_start = sizeof(struct aead_request);3324		cc->dmreq_start += crypto_aead_reqsize(any_tfm_aead(cc));3325		align_mask = crypto_aead_alignmask(any_tfm_aead(cc));3326	} else {3327		cc->dmreq_start = sizeof(struct skcipher_request);3328		cc->dmreq_start += crypto_skcipher_reqsize(any_tfm(cc));3329		align_mask = crypto_skcipher_alignmask(any_tfm(cc));3330	}3331	cc->dmreq_start = ALIGN(cc->dmreq_start, __alignof__(struct dm_crypt_request));3332 3333	if (align_mask < CRYPTO_MINALIGN) {3334		/* Allocate the padding exactly */3335		iv_size_padding = -(cc->dmreq_start + sizeof(struct dm_crypt_request))3336				& align_mask;3337	} else {3338		/*3339		 * If the cipher requires greater alignment than kmalloc3340		 * alignment, we don't know the exact position of the3341		 * initialization vector. We must assume worst case.3342		 */3343		iv_size_padding = align_mask;3344	}3345 3346	/*  ...| IV + padding | original IV | original sec. number | bio tag offset | */3347	additional_req_size = sizeof(struct dm_crypt_request) +3348		iv_size_padding + cc->iv_size +3349		cc->iv_size +3350		sizeof(uint64_t) +3351		sizeof(unsigned int);3352 3353	ret = mempool_init_kmalloc_pool(&cc->req_pool, MIN_IOS, cc->dmreq_start + additional_req_size);3354	if (ret) {3355		ti->error = "Cannot allocate crypt request mempool";3356		goto bad;3357	}3358 3359	cc->per_bio_data_size = ti->per_io_data_size =3360		ALIGN(sizeof(struct dm_crypt_io) + cc->dmreq_start + additional_req_size,3361		      ARCH_DMA_MINALIGN);3362 3363	ret = mempool_init(&cc->page_pool, BIO_MAX_VECS, crypt_page_alloc, crypt_page_free, cc);3364	if (ret) {3365		ti->error = "Cannot allocate page mempool";3366		goto bad;3367	}3368 3369	ret = bioset_init(&cc->bs, MIN_IOS, 0, BIOSET_NEED_BVECS);3370	if (ret) {3371		ti->error = "Cannot allocate crypt bioset";3372		goto bad;3373	}3374 3375	mutex_init(&cc->bio_alloc_lock);3376 3377	ret = -EINVAL;3378	if ((sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) ||3379	    (tmpll & ((cc->sector_size >> SECTOR_SHIFT) - 1))) {3380		ti->error = "Invalid iv_offset sector";3381		goto bad;3382	}3383	cc->iv_offset = tmpll;3384 3385	ret = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev);3386	if (ret) {3387		ti->error = "Device lookup failed";3388		goto bad;3389	}3390 3391	ret = -EINVAL;3392	if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1 || tmpll != (sector_t)tmpll) {3393		ti->error = "Invalid device sector";3394		goto bad;3395	}3396	cc->start = tmpll;3397 3398	if (bdev_is_zoned(cc->dev->bdev)) {3399		/*3400		 * For zoned block devices, we need to preserve the issuer write3401		 * ordering. To do so, disable write workqueues and force inline3402		 * encryption completion.3403		 */3404		set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags);3405		set_bit(DM_CRYPT_WRITE_INLINE, &cc->flags);3406 3407		/*3408		 * All zone append writes to a zone of a zoned block device will3409		 * have the same BIO sector, the start of the zone. When the3410		 * cypher IV mode uses sector values, all data targeting a3411		 * zone will be encrypted using the first sector numbers of the3412		 * zone. This will not result in write errors but will3413		 * cause most reads to fail as reads will use the sector values3414		 * for the actual data locations, resulting in IV mismatch.3415		 * To avoid this problem, ask DM core to emulate zone append3416		 * operations with regular writes.3417		 */3418		DMDEBUG("Zone append operations will be emulated");3419		ti->emulate_zone_append = true;3420	}3421 3422	if (crypt_integrity_aead(cc) || cc->integrity_iv_size) {3423		ret = crypt_integrity_ctr(cc, ti);3424		if (ret)3425			goto bad;3426 3427		cc->tag_pool_max_sectors = POOL_ENTRY_SIZE / cc->tuple_size;3428		if (!cc->tag_pool_max_sectors)3429			cc->tag_pool_max_sectors = 1;3430 3431		ret = mempool_init_kmalloc_pool(&cc->tag_pool, MIN_IOS,3432			cc->tag_pool_max_sectors * cc->tuple_size);3433		if (ret) {3434			ti->error = "Cannot allocate integrity tags mempool";3435			goto bad;3436		}3437 3438		cc->tag_pool_max_sectors <<= cc->sector_shift;3439	}3440 3441	wq_id = ida_alloc_min(&workqueue_ida, 1, GFP_KERNEL);3442	if (wq_id < 0) {3443		ti->error = "Couldn't get workqueue id";3444		ret = wq_id;3445		goto bad;3446	}3447	cc->workqueue_id = wq_id;3448 3449	ret = -ENOMEM;3450	common_wq_flags = WQ_MEM_RECLAIM | WQ_SYSFS;3451	if (test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags))3452		common_wq_flags |= WQ_HIGHPRI;3453 3454	cc->io_queue = alloc_workqueue("kcryptd_io-%s-%d", common_wq_flags, 1, devname, wq_id);3455	if (!cc->io_queue) {3456		ti->error = "Couldn't create kcryptd io queue";3457		goto bad;3458	}3459 3460	if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) {3461		cc->crypt_queue = alloc_workqueue("kcryptd-%s-%d",3462						  common_wq_flags | WQ_CPU_INTENSIVE,3463						  1, devname, wq_id);3464	} else {3465		/*3466		 * While crypt_queue is certainly CPU intensive, the use of3467		 * WQ_CPU_INTENSIVE is meaningless with WQ_UNBOUND.3468		 */3469		cc->crypt_queue = alloc_workqueue("kcryptd-%s-%d",3470						  common_wq_flags | WQ_UNBOUND,3471						  num_online_cpus(), devname, wq_id);3472	}3473	if (!cc->crypt_queue) {3474		ti->error = "Couldn't create kcryptd queue";3475		goto bad;3476	}3477 3478	spin_lock_init(&cc->write_thread_lock);3479	cc->write_tree = RB_ROOT;3480 3481	cc->write_thread = kthread_run(dmcrypt_write, cc, "dmcrypt_write/%s", devname);3482	if (IS_ERR(cc->write_thread)) {3483		ret = PTR_ERR(cc->write_thread);3484		cc->write_thread = NULL;3485		ti->error = "Couldn't spawn write thread";3486		goto bad;3487	}3488	if (test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags))3489		set_user_nice(cc->write_thread, MIN_NICE);3490 3491	ti->num_flush_bios = 1;3492	ti->limit_swap_bios = true;3493	ti->accounts_remapped_io = true;3494 3495	dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1);3496	return 0;3497 3498bad:3499	dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0);3500	crypt_dtr(ti);3501	return ret;3502}3503 3504static int crypt_map(struct dm_target *ti, struct bio *bio)3505{3506	struct dm_crypt_io *io;3507	struct crypt_config *cc = ti->private;3508	unsigned max_sectors;3509 3510	/*3511	 * If bio is REQ_PREFLUSH or REQ_OP_DISCARD, just bypass crypt queues.3512	 * - for REQ_PREFLUSH device-mapper core ensures that no IO is in-flight3513	 * - for REQ_OP_DISCARD caller must use flush if IO ordering matters3514	 */3515	if (unlikely(bio->bi_opf & REQ_PREFLUSH ||3516	    bio_op(bio) == REQ_OP_DISCARD)) {3517		bio_set_dev(bio, cc->dev->bdev);3518		if (bio_sectors(bio))3519			bio->bi_iter.bi_sector = cc->start +3520				dm_target_offset(ti, bio->bi_iter.bi_sector);3521		return DM_MAPIO_REMAPPED;3522	}3523 3524	/*3525	 * Check if bio is too large, split as needed.3526	 */3527	max_sectors = get_max_request_size(cc, bio_data_dir(bio) == WRITE);3528	if (unlikely(bio_sectors(bio) > max_sectors))3529		dm_accept_partial_bio(bio, max_sectors);3530 3531	/*3532	 * Ensure that bio is a multiple of internal sector encryption size3533	 * and is aligned to this size as defined in IO hints.3534	 */3535	if (unlikely((bio->bi_iter.bi_sector & ((cc->sector_size >> SECTOR_SHIFT) - 1)) != 0))3536		return DM_MAPIO_KILL;3537 3538	if (unlikely(bio->bi_iter.bi_size & (cc->sector_size - 1)))3539		return DM_MAPIO_KILL;3540 3541	io = dm_per_bio_data(bio, cc->per_bio_data_size);3542	crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector));3543 3544	if (cc->tuple_size) {3545		unsigned int tag_len = cc->tuple_size * (bio_sectors(bio) >> cc->sector_shift);3546 3547		if (unlikely(tag_len > KMALLOC_MAX_SIZE))3548			io->integrity_metadata = NULL;3549		else3550			io->integrity_metadata = kmalloc(tag_len, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);3551 3552		if (unlikely(!io->integrity_metadata)) {3553			if (bio_sectors(bio) > cc->tag_pool_max_sectors)3554				dm_accept_partial_bio(bio, cc->tag_pool_max_sectors);3555			io->integrity_metadata = mempool_alloc(&cc->tag_pool, GFP_NOIO);3556			io->integrity_metadata_from_pool = true;3557		}3558	}3559 3560	if (crypt_integrity_aead(cc))3561		io->ctx.r.req_aead = (struct aead_request *)(io + 1);3562	else3563		io->ctx.r.req = (struct skcipher_request *)(io + 1);3564 3565	if (bio_data_dir(io->base_bio) == READ) {3566		if (kcryptd_io_read(io, CRYPT_MAP_READ_GFP))3567			kcryptd_queue_read(io);3568	} else3569		kcryptd_queue_crypt(io);3570 3571	return DM_MAPIO_SUBMITTED;3572}3573 3574static char hex2asc(unsigned char c)3575{3576	return c + '0' + ((unsigned int)(9 - c) >> 4 & 0x27);3577}3578 3579static void crypt_status(struct dm_target *ti, status_type_t type,3580			 unsigned int status_flags, char *result, unsigned int maxlen)3581{3582	struct crypt_config *cc = ti->private;3583	unsigned int i, sz = 0;3584	int num_feature_args = 0;3585 3586	switch (type) {3587	case STATUSTYPE_INFO:3588		result[0] = '\0';3589		break;3590 3591	case STATUSTYPE_TABLE:3592		DMEMIT("%s ", cc->cipher_string);3593 3594		if (cc->key_size > 0) {3595			if (cc->key_string)3596				DMEMIT(":%u:%s", cc->key_size, cc->key_string);3597			else {3598				for (i = 0; i < cc->key_size; i++) {3599					DMEMIT("%c%c", hex2asc(cc->key[i] >> 4),3600					       hex2asc(cc->key[i] & 0xf));3601				}3602			}3603		} else3604			DMEMIT("-");3605 3606		DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset,3607				cc->dev->name, (unsigned long long)cc->start);3608 3609		num_feature_args += !!ti->num_discard_bios;3610		num_feature_args += test_bit(DM_CRYPT_SAME_CPU, &cc->flags);3611		num_feature_args += test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags);3612		num_feature_args += test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags);3613		num_feature_args += test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags);3614		num_feature_args += test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags);3615		num_feature_args += !!cc->used_tag_size;3616		num_feature_args += cc->sector_size != (1 << SECTOR_SHIFT);3617		num_feature_args += test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags);3618		num_feature_args += test_bit(CRYPT_KEY_MAC_SIZE_SET, &cc->cipher_flags);3619		if (num_feature_args) {3620			DMEMIT(" %d", num_feature_args);3621			if (ti->num_discard_bios)3622				DMEMIT(" allow_discards");3623			if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags))3624				DMEMIT(" same_cpu_crypt");3625			if (test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags))3626				DMEMIT(" high_priority");3627			if (test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags))3628				DMEMIT(" submit_from_crypt_cpus");3629			if (test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags))3630				DMEMIT(" no_read_workqueue");3631			if (test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))3632				DMEMIT(" no_write_workqueue");3633			if (cc->used_tag_size)3634				DMEMIT(" integrity:%u:%s", cc->used_tag_size, cc->cipher_auth);3635			if (cc->sector_size != (1 << SECTOR_SHIFT))3636				DMEMIT(" sector_size:%d", cc->sector_size);3637			if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))3638				DMEMIT(" iv_large_sectors");3639			if (test_bit(CRYPT_KEY_MAC_SIZE_SET, &cc->cipher_flags))3640				DMEMIT(" integrity_key_size:%u", cc->key_mac_size);3641		}3642		break;3643 3644	case STATUSTYPE_IMA:3645		DMEMIT_TARGET_NAME_VERSION(ti->type);3646		DMEMIT(",allow_discards=%c", ti->num_discard_bios ? 'y' : 'n');3647		DMEMIT(",same_cpu_crypt=%c", test_bit(DM_CRYPT_SAME_CPU, &cc->flags) ? 'y' : 'n');3648		DMEMIT(",high_priority=%c", test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags) ? 'y' : 'n');3649		DMEMIT(",submit_from_crypt_cpus=%c", test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags) ?3650		       'y' : 'n');3651		DMEMIT(",no_read_workqueue=%c", test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags) ?3652		       'y' : 'n');3653		DMEMIT(",no_write_workqueue=%c", test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags) ?3654		       'y' : 'n');3655		DMEMIT(",iv_large_sectors=%c", test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags) ?3656		       'y' : 'n');3657 3658		if (cc->used_tag_size)3659			DMEMIT(",integrity_tag_size=%u,cipher_auth=%s",3660			       cc->used_tag_size, cc->cipher_auth);3661		if (cc->sector_size != (1 << SECTOR_SHIFT))3662			DMEMIT(",sector_size=%d", cc->sector_size);3663		if (cc->cipher_string)3664			DMEMIT(",cipher_string=%s", cc->cipher_string);3665 3666		DMEMIT(",key_size=%u", cc->key_size);3667		DMEMIT(",key_parts=%u", cc->key_parts);3668		DMEMIT(",key_extra_size=%u", cc->key_extra_size);3669		DMEMIT(",key_mac_size=%u", cc->key_mac_size);3670		DMEMIT(";");3671		break;3672	}3673}3674 3675static void crypt_postsuspend(struct dm_target *ti)3676{3677	struct crypt_config *cc = ti->private;3678 3679	set_bit(DM_CRYPT_SUSPENDED, &cc->flags);3680}3681 3682static int crypt_preresume(struct dm_target *ti)3683{3684	struct crypt_config *cc = ti->private;3685 3686	if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) {3687		DMERR("aborting resume - crypt key is not set.");3688		return -EAGAIN;3689	}3690 3691	return 0;3692}3693 3694static void crypt_resume(struct dm_target *ti)3695{3696	struct crypt_config *cc = ti->private;3697 3698	clear_bit(DM_CRYPT_SUSPENDED, &cc->flags);3699}3700 3701/* Message interface3702 *	key set <key>3703 *	key wipe3704 */3705static int crypt_message(struct dm_target *ti, unsigned int argc, char **argv,3706			 char *result, unsigned int maxlen)3707{3708	struct crypt_config *cc = ti->private;3709	int key_size, ret = -EINVAL;3710 3711	if (argc < 2)3712		goto error;3713 3714	if (!strcasecmp(argv[0], "key")) {3715		if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) {3716			DMWARN("not suspended during key manipulation.");3717			return -EINVAL;3718		}3719		if (argc == 3 && !strcasecmp(argv[1], "set")) {3720			/* The key size may not be changed. */3721			key_size = get_key_size(&argv[2]);3722			if (key_size < 0 || cc->key_size != key_size) {3723				memset(argv[2], '0', strlen(argv[2]));3724				return -EINVAL;3725			}3726 3727			ret = crypt_set_key(cc, argv[2]);3728			if (ret)3729				return ret;3730			if (cc->iv_gen_ops && cc->iv_gen_ops->init)3731				ret = cc->iv_gen_ops->init(cc);3732			/* wipe the kernel key payload copy */3733			if (cc->key_string)3734				memset(cc->key, 0, cc->key_size * sizeof(u8));3735			return ret;3736		}3737		if (argc == 2 && !strcasecmp(argv[1], "wipe"))3738			return crypt_wipe_key(cc);3739	}3740 3741error:3742	DMWARN("unrecognised message received.");3743	return -EINVAL;3744}3745 3746static int crypt_iterate_devices(struct dm_target *ti,3747				 iterate_devices_callout_fn fn, void *data)3748{3749	struct crypt_config *cc = ti->private;3750 3751	return fn(ti, cc->dev, cc->start, ti->len, data);3752}3753 3754static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)3755{3756	struct crypt_config *cc = ti->private;3757 3758	limits->logical_block_size =3759		max_t(unsigned int, limits->logical_block_size, cc->sector_size);3760	limits->physical_block_size =3761		max_t(unsigned int, limits->physical_block_size, cc->sector_size);3762	limits->io_min = max_t(unsigned int, limits->io_min, cc->sector_size);3763	limits->dma_alignment = limits->logical_block_size - 1;3764}3765 3766static struct target_type crypt_target = {3767	.name   = "crypt",3768	.version = {1, 28, 0},3769	.module = THIS_MODULE,3770	.ctr    = crypt_ctr,3771	.dtr    = crypt_dtr,3772	.features = DM_TARGET_ZONED_HM,3773	.report_zones = crypt_report_zones,3774	.map    = crypt_map,3775	.status = crypt_status,3776	.postsuspend = crypt_postsuspend,3777	.preresume = crypt_preresume,3778	.resume = crypt_resume,3779	.message = crypt_message,3780	.iterate_devices = crypt_iterate_devices,3781	.io_hints = crypt_io_hints,3782};3783module_dm(crypt);3784 3785MODULE_AUTHOR("Jana Saout <jana@saout.de>");3786MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");3787MODULE_LICENSE("GPL");3788