477 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/* XTS: as defined in IEEE1619/D163 * http://grouper.ieee.org/groups/1619/email/pdf00086.pdf4 *5 * Copyright (c) 2007 Rik Snel <rsnel@cube.dyndns.org>6 *7 * Based on ecb.c8 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>9 */10#include <crypto/internal/cipher.h>11#include <crypto/internal/skcipher.h>12#include <crypto/scatterwalk.h>13#include <linux/err.h>14#include <linux/init.h>15#include <linux/kernel.h>16#include <linux/module.h>17#include <linux/scatterlist.h>18#include <linux/slab.h>19 20#include <crypto/xts.h>21#include <crypto/b128ops.h>22#include <crypto/gf128mul.h>23 24struct xts_tfm_ctx {25 struct crypto_skcipher *child;26 struct crypto_cipher *tweak;27};28 29struct xts_instance_ctx {30 struct crypto_skcipher_spawn spawn;31 struct crypto_cipher_spawn tweak_spawn;32};33 34struct xts_request_ctx {35 le128 t;36 struct scatterlist *tail;37 struct scatterlist sg[2];38 struct skcipher_request subreq;39};40 41static int xts_setkey(struct crypto_skcipher *parent, const u8 *key,42 unsigned int keylen)43{44 struct xts_tfm_ctx *ctx = crypto_skcipher_ctx(parent);45 struct crypto_skcipher *child;46 struct crypto_cipher *tweak;47 int err;48 49 err = xts_verify_key(parent, key, keylen);50 if (err)51 return err;52 53 keylen /= 2;54 55 /* we need two cipher instances: one to compute the initial 'tweak'56 * by encrypting the IV (usually the 'plain' iv) and the other57 * one to encrypt and decrypt the data */58 59 /* tweak cipher, uses Key2 i.e. the second half of *key */60 tweak = ctx->tweak;61 crypto_cipher_clear_flags(tweak, CRYPTO_TFM_REQ_MASK);62 crypto_cipher_set_flags(tweak, crypto_skcipher_get_flags(parent) &63 CRYPTO_TFM_REQ_MASK);64 err = crypto_cipher_setkey(tweak, key + keylen, keylen);65 if (err)66 return err;67 68 /* data cipher, uses Key1 i.e. the first half of *key */69 child = ctx->child;70 crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);71 crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) &72 CRYPTO_TFM_REQ_MASK);73 return crypto_skcipher_setkey(child, key, keylen);74}75 76/*77 * We compute the tweak masks twice (both before and after the ECB encryption or78 * decryption) to avoid having to allocate a temporary buffer and/or make79 * mutliple calls to the 'ecb(..)' instance, which usually would be slower than80 * just doing the gf128mul_x_ble() calls again.81 */82static int xts_xor_tweak(struct skcipher_request *req, bool second_pass,83 bool enc)84{85 struct xts_request_ctx *rctx = skcipher_request_ctx(req);86 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);87 const bool cts = (req->cryptlen % XTS_BLOCK_SIZE);88 const int bs = XTS_BLOCK_SIZE;89 struct skcipher_walk w;90 le128 t = rctx->t;91 int err;92 93 if (second_pass) {94 req = &rctx->subreq;95 /* set to our TFM to enforce correct alignment: */96 skcipher_request_set_tfm(req, tfm);97 }98 err = skcipher_walk_virt(&w, req, false);99 100 while (w.nbytes) {101 unsigned int avail = w.nbytes;102 le128 *wsrc;103 le128 *wdst;104 105 wsrc = w.src.virt.addr;106 wdst = w.dst.virt.addr;107 108 do {109 if (unlikely(cts) &&110 w.total - w.nbytes + avail < 2 * XTS_BLOCK_SIZE) {111 if (!enc) {112 if (second_pass)113 rctx->t = t;114 gf128mul_x_ble(&t, &t);115 }116 le128_xor(wdst, &t, wsrc);117 if (enc && second_pass)118 gf128mul_x_ble(&rctx->t, &t);119 skcipher_walk_done(&w, avail - bs);120 return 0;121 }122 123 le128_xor(wdst++, &t, wsrc++);124 gf128mul_x_ble(&t, &t);125 } while ((avail -= bs) >= bs);126 127 err = skcipher_walk_done(&w, avail);128 }129 130 return err;131}132 133static int xts_xor_tweak_pre(struct skcipher_request *req, bool enc)134{135 return xts_xor_tweak(req, false, enc);136}137 138static int xts_xor_tweak_post(struct skcipher_request *req, bool enc)139{140 return xts_xor_tweak(req, true, enc);141}142 143static void xts_cts_done(void *data, int err)144{145 struct skcipher_request *req = data;146 le128 b;147 148 if (!err) {149 struct xts_request_ctx *rctx = skcipher_request_ctx(req);150 151 scatterwalk_map_and_copy(&b, rctx->tail, 0, XTS_BLOCK_SIZE, 0);152 le128_xor(&b, &rctx->t, &b);153 scatterwalk_map_and_copy(&b, rctx->tail, 0, XTS_BLOCK_SIZE, 1);154 }155 156 skcipher_request_complete(req, err);157}158 159static int xts_cts_final(struct skcipher_request *req,160 int (*crypt)(struct skcipher_request *req))161{162 const struct xts_tfm_ctx *ctx =163 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));164 int offset = req->cryptlen & ~(XTS_BLOCK_SIZE - 1);165 struct xts_request_ctx *rctx = skcipher_request_ctx(req);166 struct skcipher_request *subreq = &rctx->subreq;167 int tail = req->cryptlen % XTS_BLOCK_SIZE;168 le128 b[2];169 int err;170 171 rctx->tail = scatterwalk_ffwd(rctx->sg, req->dst,172 offset - XTS_BLOCK_SIZE);173 174 scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE, 0);175 b[1] = b[0];176 scatterwalk_map_and_copy(b, req->src, offset, tail, 0);177 178 le128_xor(b, &rctx->t, b);179 180 scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE + tail, 1);181 182 skcipher_request_set_tfm(subreq, ctx->child);183 skcipher_request_set_callback(subreq, req->base.flags, xts_cts_done,184 req);185 skcipher_request_set_crypt(subreq, rctx->tail, rctx->tail,186 XTS_BLOCK_SIZE, NULL);187 188 err = crypt(subreq);189 if (err)190 return err;191 192 scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE, 0);193 le128_xor(b, &rctx->t, b);194 scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE, 1);195 196 return 0;197}198 199static void xts_encrypt_done(void *data, int err)200{201 struct skcipher_request *req = data;202 203 if (!err) {204 struct xts_request_ctx *rctx = skcipher_request_ctx(req);205 206 rctx->subreq.base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG;207 err = xts_xor_tweak_post(req, true);208 209 if (!err && unlikely(req->cryptlen % XTS_BLOCK_SIZE)) {210 err = xts_cts_final(req, crypto_skcipher_encrypt);211 if (err == -EINPROGRESS || err == -EBUSY)212 return;213 }214 }215 216 skcipher_request_complete(req, err);217}218 219static void xts_decrypt_done(void *data, int err)220{221 struct skcipher_request *req = data;222 223 if (!err) {224 struct xts_request_ctx *rctx = skcipher_request_ctx(req);225 226 rctx->subreq.base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG;227 err = xts_xor_tweak_post(req, false);228 229 if (!err && unlikely(req->cryptlen % XTS_BLOCK_SIZE)) {230 err = xts_cts_final(req, crypto_skcipher_decrypt);231 if (err == -EINPROGRESS || err == -EBUSY)232 return;233 }234 }235 236 skcipher_request_complete(req, err);237}238 239static int xts_init_crypt(struct skcipher_request *req,240 crypto_completion_t compl)241{242 const struct xts_tfm_ctx *ctx =243 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));244 struct xts_request_ctx *rctx = skcipher_request_ctx(req);245 struct skcipher_request *subreq = &rctx->subreq;246 247 if (req->cryptlen < XTS_BLOCK_SIZE)248 return -EINVAL;249 250 skcipher_request_set_tfm(subreq, ctx->child);251 skcipher_request_set_callback(subreq, req->base.flags, compl, req);252 skcipher_request_set_crypt(subreq, req->dst, req->dst,253 req->cryptlen & ~(XTS_BLOCK_SIZE - 1), NULL);254 255 /* calculate first value of T */256 crypto_cipher_encrypt_one(ctx->tweak, (u8 *)&rctx->t, req->iv);257 258 return 0;259}260 261static int xts_encrypt(struct skcipher_request *req)262{263 struct xts_request_ctx *rctx = skcipher_request_ctx(req);264 struct skcipher_request *subreq = &rctx->subreq;265 int err;266 267 err = xts_init_crypt(req, xts_encrypt_done) ?:268 xts_xor_tweak_pre(req, true) ?:269 crypto_skcipher_encrypt(subreq) ?:270 xts_xor_tweak_post(req, true);271 272 if (err || likely((req->cryptlen % XTS_BLOCK_SIZE) == 0))273 return err;274 275 return xts_cts_final(req, crypto_skcipher_encrypt);276}277 278static int xts_decrypt(struct skcipher_request *req)279{280 struct xts_request_ctx *rctx = skcipher_request_ctx(req);281 struct skcipher_request *subreq = &rctx->subreq;282 int err;283 284 err = xts_init_crypt(req, xts_decrypt_done) ?:285 xts_xor_tweak_pre(req, false) ?:286 crypto_skcipher_decrypt(subreq) ?:287 xts_xor_tweak_post(req, false);288 289 if (err || likely((req->cryptlen % XTS_BLOCK_SIZE) == 0))290 return err;291 292 return xts_cts_final(req, crypto_skcipher_decrypt);293}294 295static int xts_init_tfm(struct crypto_skcipher *tfm)296{297 struct skcipher_instance *inst = skcipher_alg_instance(tfm);298 struct xts_instance_ctx *ictx = skcipher_instance_ctx(inst);299 struct xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);300 struct crypto_skcipher *child;301 struct crypto_cipher *tweak;302 303 child = crypto_spawn_skcipher(&ictx->spawn);304 if (IS_ERR(child))305 return PTR_ERR(child);306 307 ctx->child = child;308 309 tweak = crypto_spawn_cipher(&ictx->tweak_spawn);310 if (IS_ERR(tweak)) {311 crypto_free_skcipher(ctx->child);312 return PTR_ERR(tweak);313 }314 315 ctx->tweak = tweak;316 317 crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) +318 sizeof(struct xts_request_ctx));319 320 return 0;321}322 323static void xts_exit_tfm(struct crypto_skcipher *tfm)324{325 struct xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);326 327 crypto_free_skcipher(ctx->child);328 crypto_free_cipher(ctx->tweak);329}330 331static void xts_free_instance(struct skcipher_instance *inst)332{333 struct xts_instance_ctx *ictx = skcipher_instance_ctx(inst);334 335 crypto_drop_skcipher(&ictx->spawn);336 crypto_drop_cipher(&ictx->tweak_spawn);337 kfree(inst);338}339 340static int xts_create(struct crypto_template *tmpl, struct rtattr **tb)341{342 struct skcipher_alg_common *alg;343 char name[CRYPTO_MAX_ALG_NAME];344 struct skcipher_instance *inst;345 struct xts_instance_ctx *ctx;346 const char *cipher_name;347 u32 mask;348 int err;349 350 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SKCIPHER, &mask);351 if (err)352 return err;353 354 cipher_name = crypto_attr_alg_name(tb[1]);355 if (IS_ERR(cipher_name))356 return PTR_ERR(cipher_name);357 358 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);359 if (!inst)360 return -ENOMEM;361 362 ctx = skcipher_instance_ctx(inst);363 364 err = crypto_grab_skcipher(&ctx->spawn, skcipher_crypto_instance(inst),365 cipher_name, 0, mask);366 if (err == -ENOENT) {367 err = -ENAMETOOLONG;368 if (snprintf(name, CRYPTO_MAX_ALG_NAME, "ecb(%s)",369 cipher_name) >= CRYPTO_MAX_ALG_NAME)370 goto err_free_inst;371 372 err = crypto_grab_skcipher(&ctx->spawn,373 skcipher_crypto_instance(inst),374 name, 0, mask);375 }376 377 if (err)378 goto err_free_inst;379 380 alg = crypto_spawn_skcipher_alg_common(&ctx->spawn);381 382 err = -EINVAL;383 if (alg->base.cra_blocksize != XTS_BLOCK_SIZE)384 goto err_free_inst;385 386 if (alg->ivsize)387 goto err_free_inst;388 389 err = crypto_inst_setname(skcipher_crypto_instance(inst), "xts",390 &alg->base);391 if (err)392 goto err_free_inst;393 394 err = -EINVAL;395 cipher_name = alg->base.cra_name;396 397 /* Alas we screwed up the naming so we have to mangle the398 * cipher name.399 */400 if (!strncmp(cipher_name, "ecb(", 4)) {401 int len;402 403 len = strscpy(name, cipher_name + 4, sizeof(name));404 if (len < 2)405 goto err_free_inst;406 407 if (name[len - 1] != ')')408 goto err_free_inst;409 410 name[len - 1] = 0;411 412 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,413 "xts(%s)", name) >= CRYPTO_MAX_ALG_NAME) {414 err = -ENAMETOOLONG;415 goto err_free_inst;416 }417 } else418 goto err_free_inst;419 420 err = crypto_grab_cipher(&ctx->tweak_spawn,421 skcipher_crypto_instance(inst), name, 0, mask);422 if (err)423 goto err_free_inst;424 425 inst->alg.base.cra_priority = alg->base.cra_priority;426 inst->alg.base.cra_blocksize = XTS_BLOCK_SIZE;427 inst->alg.base.cra_alignmask = alg->base.cra_alignmask |428 (__alignof__(u64) - 1);429 430 inst->alg.ivsize = XTS_BLOCK_SIZE;431 inst->alg.min_keysize = alg->min_keysize * 2;432 inst->alg.max_keysize = alg->max_keysize * 2;433 434 inst->alg.base.cra_ctxsize = sizeof(struct xts_tfm_ctx);435 436 inst->alg.init = xts_init_tfm;437 inst->alg.exit = xts_exit_tfm;438 439 inst->alg.setkey = xts_setkey;440 inst->alg.encrypt = xts_encrypt;441 inst->alg.decrypt = xts_decrypt;442 443 inst->free = xts_free_instance;444 445 err = skcipher_register_instance(tmpl, inst);446 if (err) {447err_free_inst:448 xts_free_instance(inst);449 }450 return err;451}452 453static struct crypto_template xts_tmpl = {454 .name = "xts",455 .create = xts_create,456 .module = THIS_MODULE,457};458 459static int __init xts_module_init(void)460{461 return crypto_register_template(&xts_tmpl);462}463 464static void __exit xts_module_exit(void)465{466 crypto_unregister_template(&xts_tmpl);467}468 469subsys_initcall(xts_module_init);470module_exit(xts_module_exit);471 472MODULE_LICENSE("GPL");473MODULE_DESCRIPTION("XTS block cipher mode");474MODULE_ALIAS_CRYPTO("xts");475MODULE_IMPORT_NS(CRYPTO_INTERNAL);476MODULE_SOFTDEP("pre: ecb");477