678 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * ChaCha20-Poly1305 AEAD, RFC75394 *5 * Copyright (C) 2015 Martin Willi6 */7 8#include <crypto/internal/aead.h>9#include <crypto/internal/hash.h>10#include <crypto/internal/skcipher.h>11#include <crypto/scatterwalk.h>12#include <crypto/chacha.h>13#include <crypto/poly1305.h>14#include <linux/err.h>15#include <linux/init.h>16#include <linux/kernel.h>17#include <linux/module.h>18 19struct chachapoly_instance_ctx {20 struct crypto_skcipher_spawn chacha;21 struct crypto_ahash_spawn poly;22 unsigned int saltlen;23};24 25struct chachapoly_ctx {26 struct crypto_skcipher *chacha;27 struct crypto_ahash *poly;28 /* key bytes we use for the ChaCha20 IV */29 unsigned int saltlen;30 u8 salt[] __counted_by(saltlen);31};32 33struct poly_req {34 /* zero byte padding for AD/ciphertext, as needed */35 u8 pad[POLY1305_BLOCK_SIZE];36 /* tail data with AD/ciphertext lengths */37 struct {38 __le64 assoclen;39 __le64 cryptlen;40 } tail;41 struct scatterlist src[1];42 struct ahash_request req; /* must be last member */43};44 45struct chacha_req {46 u8 iv[CHACHA_IV_SIZE];47 struct scatterlist src[1];48 struct skcipher_request req; /* must be last member */49};50 51struct chachapoly_req_ctx {52 struct scatterlist src[2];53 struct scatterlist dst[2];54 /* the key we generate for Poly1305 using Chacha20 */55 u8 key[POLY1305_KEY_SIZE];56 /* calculated Poly1305 tag */57 u8 tag[POLY1305_DIGEST_SIZE];58 /* length of data to en/decrypt, without ICV */59 unsigned int cryptlen;60 /* Actual AD, excluding IV */61 unsigned int assoclen;62 /* request flags, with MAY_SLEEP cleared if needed */63 u32 flags;64 union {65 struct poly_req poly;66 struct chacha_req chacha;67 } u;68};69 70static inline void async_done_continue(struct aead_request *req, int err,71 int (*cont)(struct aead_request *))72{73 if (!err) {74 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);75 76 rctx->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;77 err = cont(req);78 }79 80 if (err != -EINPROGRESS && err != -EBUSY)81 aead_request_complete(req, err);82}83 84static void chacha_iv(u8 *iv, struct aead_request *req, u32 icb)85{86 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));87 __le32 leicb = cpu_to_le32(icb);88 89 memcpy(iv, &leicb, sizeof(leicb));90 memcpy(iv + sizeof(leicb), ctx->salt, ctx->saltlen);91 memcpy(iv + sizeof(leicb) + ctx->saltlen, req->iv,92 CHACHA_IV_SIZE - sizeof(leicb) - ctx->saltlen);93}94 95static int poly_verify_tag(struct aead_request *req)96{97 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);98 u8 tag[sizeof(rctx->tag)];99 100 scatterwalk_map_and_copy(tag, req->src,101 req->assoclen + rctx->cryptlen,102 sizeof(tag), 0);103 if (crypto_memneq(tag, rctx->tag, sizeof(tag)))104 return -EBADMSG;105 return 0;106}107 108static int poly_copy_tag(struct aead_request *req)109{110 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);111 112 scatterwalk_map_and_copy(rctx->tag, req->dst,113 req->assoclen + rctx->cryptlen,114 sizeof(rctx->tag), 1);115 return 0;116}117 118static void chacha_decrypt_done(void *data, int err)119{120 async_done_continue(data, err, poly_verify_tag);121}122 123static int chacha_decrypt(struct aead_request *req)124{125 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));126 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);127 struct chacha_req *creq = &rctx->u.chacha;128 struct scatterlist *src, *dst;129 int err;130 131 if (rctx->cryptlen == 0)132 goto skip;133 134 chacha_iv(creq->iv, req, 1);135 136 src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);137 dst = src;138 if (req->src != req->dst)139 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);140 141 skcipher_request_set_callback(&creq->req, rctx->flags,142 chacha_decrypt_done, req);143 skcipher_request_set_tfm(&creq->req, ctx->chacha);144 skcipher_request_set_crypt(&creq->req, src, dst,145 rctx->cryptlen, creq->iv);146 err = crypto_skcipher_decrypt(&creq->req);147 if (err)148 return err;149 150skip:151 return poly_verify_tag(req);152}153 154static int poly_tail_continue(struct aead_request *req)155{156 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);157 158 if (rctx->cryptlen == req->cryptlen) /* encrypting */159 return poly_copy_tag(req);160 161 return chacha_decrypt(req);162}163 164static void poly_tail_done(void *data, int err)165{166 async_done_continue(data, err, poly_tail_continue);167}168 169static int poly_tail(struct aead_request *req)170{171 struct crypto_aead *tfm = crypto_aead_reqtfm(req);172 struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);173 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);174 struct poly_req *preq = &rctx->u.poly;175 int err;176 177 preq->tail.assoclen = cpu_to_le64(rctx->assoclen);178 preq->tail.cryptlen = cpu_to_le64(rctx->cryptlen);179 sg_init_one(preq->src, &preq->tail, sizeof(preq->tail));180 181 ahash_request_set_callback(&preq->req, rctx->flags,182 poly_tail_done, req);183 ahash_request_set_tfm(&preq->req, ctx->poly);184 ahash_request_set_crypt(&preq->req, preq->src,185 rctx->tag, sizeof(preq->tail));186 187 err = crypto_ahash_finup(&preq->req);188 if (err)189 return err;190 191 return poly_tail_continue(req);192}193 194static void poly_cipherpad_done(void *data, int err)195{196 async_done_continue(data, err, poly_tail);197}198 199static int poly_cipherpad(struct aead_request *req)200{201 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));202 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);203 struct poly_req *preq = &rctx->u.poly;204 unsigned int padlen;205 int err;206 207 padlen = -rctx->cryptlen % POLY1305_BLOCK_SIZE;208 memset(preq->pad, 0, sizeof(preq->pad));209 sg_init_one(preq->src, preq->pad, padlen);210 211 ahash_request_set_callback(&preq->req, rctx->flags,212 poly_cipherpad_done, req);213 ahash_request_set_tfm(&preq->req, ctx->poly);214 ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);215 216 err = crypto_ahash_update(&preq->req);217 if (err)218 return err;219 220 return poly_tail(req);221}222 223static void poly_cipher_done(void *data, int err)224{225 async_done_continue(data, err, poly_cipherpad);226}227 228static int poly_cipher(struct aead_request *req)229{230 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));231 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);232 struct poly_req *preq = &rctx->u.poly;233 struct scatterlist *crypt = req->src;234 int err;235 236 if (rctx->cryptlen == req->cryptlen) /* encrypting */237 crypt = req->dst;238 239 crypt = scatterwalk_ffwd(rctx->src, crypt, req->assoclen);240 241 ahash_request_set_callback(&preq->req, rctx->flags,242 poly_cipher_done, req);243 ahash_request_set_tfm(&preq->req, ctx->poly);244 ahash_request_set_crypt(&preq->req, crypt, NULL, rctx->cryptlen);245 246 err = crypto_ahash_update(&preq->req);247 if (err)248 return err;249 250 return poly_cipherpad(req);251}252 253static void poly_adpad_done(void *data, int err)254{255 async_done_continue(data, err, poly_cipher);256}257 258static int poly_adpad(struct aead_request *req)259{260 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));261 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);262 struct poly_req *preq = &rctx->u.poly;263 unsigned int padlen;264 int err;265 266 padlen = -rctx->assoclen % POLY1305_BLOCK_SIZE;267 memset(preq->pad, 0, sizeof(preq->pad));268 sg_init_one(preq->src, preq->pad, padlen);269 270 ahash_request_set_callback(&preq->req, rctx->flags,271 poly_adpad_done, req);272 ahash_request_set_tfm(&preq->req, ctx->poly);273 ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);274 275 err = crypto_ahash_update(&preq->req);276 if (err)277 return err;278 279 return poly_cipher(req);280}281 282static void poly_ad_done(void *data, int err)283{284 async_done_continue(data, err, poly_adpad);285}286 287static int poly_ad(struct aead_request *req)288{289 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));290 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);291 struct poly_req *preq = &rctx->u.poly;292 int err;293 294 ahash_request_set_callback(&preq->req, rctx->flags,295 poly_ad_done, req);296 ahash_request_set_tfm(&preq->req, ctx->poly);297 ahash_request_set_crypt(&preq->req, req->src, NULL, rctx->assoclen);298 299 err = crypto_ahash_update(&preq->req);300 if (err)301 return err;302 303 return poly_adpad(req);304}305 306static void poly_setkey_done(void *data, int err)307{308 async_done_continue(data, err, poly_ad);309}310 311static int poly_setkey(struct aead_request *req)312{313 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));314 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);315 struct poly_req *preq = &rctx->u.poly;316 int err;317 318 sg_init_one(preq->src, rctx->key, sizeof(rctx->key));319 320 ahash_request_set_callback(&preq->req, rctx->flags,321 poly_setkey_done, req);322 ahash_request_set_tfm(&preq->req, ctx->poly);323 ahash_request_set_crypt(&preq->req, preq->src, NULL, sizeof(rctx->key));324 325 err = crypto_ahash_update(&preq->req);326 if (err)327 return err;328 329 return poly_ad(req);330}331 332static void poly_init_done(void *data, int err)333{334 async_done_continue(data, err, poly_setkey);335}336 337static int poly_init(struct aead_request *req)338{339 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));340 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);341 struct poly_req *preq = &rctx->u.poly;342 int err;343 344 ahash_request_set_callback(&preq->req, rctx->flags,345 poly_init_done, req);346 ahash_request_set_tfm(&preq->req, ctx->poly);347 348 err = crypto_ahash_init(&preq->req);349 if (err)350 return err;351 352 return poly_setkey(req);353}354 355static void poly_genkey_done(void *data, int err)356{357 async_done_continue(data, err, poly_init);358}359 360static int poly_genkey(struct aead_request *req)361{362 struct crypto_aead *tfm = crypto_aead_reqtfm(req);363 struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);364 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);365 struct chacha_req *creq = &rctx->u.chacha;366 int err;367 368 rctx->assoclen = req->assoclen;369 370 if (crypto_aead_ivsize(tfm) == 8) {371 if (rctx->assoclen < 8)372 return -EINVAL;373 rctx->assoclen -= 8;374 }375 376 memset(rctx->key, 0, sizeof(rctx->key));377 sg_init_one(creq->src, rctx->key, sizeof(rctx->key));378 379 chacha_iv(creq->iv, req, 0);380 381 skcipher_request_set_callback(&creq->req, rctx->flags,382 poly_genkey_done, req);383 skcipher_request_set_tfm(&creq->req, ctx->chacha);384 skcipher_request_set_crypt(&creq->req, creq->src, creq->src,385 POLY1305_KEY_SIZE, creq->iv);386 387 err = crypto_skcipher_decrypt(&creq->req);388 if (err)389 return err;390 391 return poly_init(req);392}393 394static void chacha_encrypt_done(void *data, int err)395{396 async_done_continue(data, err, poly_genkey);397}398 399static int chacha_encrypt(struct aead_request *req)400{401 struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));402 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);403 struct chacha_req *creq = &rctx->u.chacha;404 struct scatterlist *src, *dst;405 int err;406 407 if (req->cryptlen == 0)408 goto skip;409 410 chacha_iv(creq->iv, req, 1);411 412 src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);413 dst = src;414 if (req->src != req->dst)415 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);416 417 skcipher_request_set_callback(&creq->req, rctx->flags,418 chacha_encrypt_done, req);419 skcipher_request_set_tfm(&creq->req, ctx->chacha);420 skcipher_request_set_crypt(&creq->req, src, dst,421 req->cryptlen, creq->iv);422 err = crypto_skcipher_encrypt(&creq->req);423 if (err)424 return err;425 426skip:427 return poly_genkey(req);428}429 430static int chachapoly_encrypt(struct aead_request *req)431{432 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);433 434 rctx->cryptlen = req->cryptlen;435 rctx->flags = aead_request_flags(req);436 437 /* encrypt call chain:438 * - chacha_encrypt/done()439 * - poly_genkey/done()440 * - poly_init/done()441 * - poly_setkey/done()442 * - poly_ad/done()443 * - poly_adpad/done()444 * - poly_cipher/done()445 * - poly_cipherpad/done()446 * - poly_tail/done/continue()447 * - poly_copy_tag()448 */449 return chacha_encrypt(req);450}451 452static int chachapoly_decrypt(struct aead_request *req)453{454 struct chachapoly_req_ctx *rctx = aead_request_ctx(req);455 456 rctx->cryptlen = req->cryptlen - POLY1305_DIGEST_SIZE;457 rctx->flags = aead_request_flags(req);458 459 /* decrypt call chain:460 * - poly_genkey/done()461 * - poly_init/done()462 * - poly_setkey/done()463 * - poly_ad/done()464 * - poly_adpad/done()465 * - poly_cipher/done()466 * - poly_cipherpad/done()467 * - poly_tail/done/continue()468 * - chacha_decrypt/done()469 * - poly_verify_tag()470 */471 return poly_genkey(req);472}473 474static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,475 unsigned int keylen)476{477 struct chachapoly_ctx *ctx = crypto_aead_ctx(aead);478 479 if (keylen != ctx->saltlen + CHACHA_KEY_SIZE)480 return -EINVAL;481 482 keylen -= ctx->saltlen;483 memcpy(ctx->salt, key + keylen, ctx->saltlen);484 485 crypto_skcipher_clear_flags(ctx->chacha, CRYPTO_TFM_REQ_MASK);486 crypto_skcipher_set_flags(ctx->chacha, crypto_aead_get_flags(aead) &487 CRYPTO_TFM_REQ_MASK);488 return crypto_skcipher_setkey(ctx->chacha, key, keylen);489}490 491static int chachapoly_setauthsize(struct crypto_aead *tfm,492 unsigned int authsize)493{494 if (authsize != POLY1305_DIGEST_SIZE)495 return -EINVAL;496 497 return 0;498}499 500static int chachapoly_init(struct crypto_aead *tfm)501{502 struct aead_instance *inst = aead_alg_instance(tfm);503 struct chachapoly_instance_ctx *ictx = aead_instance_ctx(inst);504 struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);505 struct crypto_skcipher *chacha;506 struct crypto_ahash *poly;507 unsigned long align;508 509 poly = crypto_spawn_ahash(&ictx->poly);510 if (IS_ERR(poly))511 return PTR_ERR(poly);512 513 chacha = crypto_spawn_skcipher(&ictx->chacha);514 if (IS_ERR(chacha)) {515 crypto_free_ahash(poly);516 return PTR_ERR(chacha);517 }518 519 ctx->chacha = chacha;520 ctx->poly = poly;521 ctx->saltlen = ictx->saltlen;522 523 align = crypto_aead_alignmask(tfm);524 align &= ~(crypto_tfm_ctx_alignment() - 1);525 crypto_aead_set_reqsize(526 tfm,527 align + offsetof(struct chachapoly_req_ctx, u) +528 max(offsetof(struct chacha_req, req) +529 sizeof(struct skcipher_request) +530 crypto_skcipher_reqsize(chacha),531 offsetof(struct poly_req, req) +532 sizeof(struct ahash_request) +533 crypto_ahash_reqsize(poly)));534 535 return 0;536}537 538static void chachapoly_exit(struct crypto_aead *tfm)539{540 struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);541 542 crypto_free_ahash(ctx->poly);543 crypto_free_skcipher(ctx->chacha);544}545 546static void chachapoly_free(struct aead_instance *inst)547{548 struct chachapoly_instance_ctx *ctx = aead_instance_ctx(inst);549 550 crypto_drop_skcipher(&ctx->chacha);551 crypto_drop_ahash(&ctx->poly);552 kfree(inst);553}554 555static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb,556 const char *name, unsigned int ivsize)557{558 u32 mask;559 struct aead_instance *inst;560 struct chachapoly_instance_ctx *ctx;561 struct skcipher_alg_common *chacha;562 struct hash_alg_common *poly;563 int err;564 565 if (ivsize > CHACHAPOLY_IV_SIZE)566 return -EINVAL;567 568 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask);569 if (err)570 return err;571 572 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);573 if (!inst)574 return -ENOMEM;575 ctx = aead_instance_ctx(inst);576 ctx->saltlen = CHACHAPOLY_IV_SIZE - ivsize;577 578 err = crypto_grab_skcipher(&ctx->chacha, aead_crypto_instance(inst),579 crypto_attr_alg_name(tb[1]), 0, mask);580 if (err)581 goto err_free_inst;582 chacha = crypto_spawn_skcipher_alg_common(&ctx->chacha);583 584 err = crypto_grab_ahash(&ctx->poly, aead_crypto_instance(inst),585 crypto_attr_alg_name(tb[2]), 0, mask);586 if (err)587 goto err_free_inst;588 poly = crypto_spawn_ahash_alg(&ctx->poly);589 590 err = -EINVAL;591 if (poly->digestsize != POLY1305_DIGEST_SIZE)592 goto err_free_inst;593 /* Need 16-byte IV size, including Initial Block Counter value */594 if (chacha->ivsize != CHACHA_IV_SIZE)595 goto err_free_inst;596 /* Not a stream cipher? */597 if (chacha->base.cra_blocksize != 1)598 goto err_free_inst;599 600 err = -ENAMETOOLONG;601 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,602 "%s(%s,%s)", name, chacha->base.cra_name,603 poly->base.cra_name) >= CRYPTO_MAX_ALG_NAME)604 goto err_free_inst;605 if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,606 "%s(%s,%s)", name, chacha->base.cra_driver_name,607 poly->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)608 goto err_free_inst;609 610 inst->alg.base.cra_priority = (chacha->base.cra_priority +611 poly->base.cra_priority) / 2;612 inst->alg.base.cra_blocksize = 1;613 inst->alg.base.cra_alignmask = chacha->base.cra_alignmask;614 inst->alg.base.cra_ctxsize = sizeof(struct chachapoly_ctx) +615 ctx->saltlen;616 inst->alg.ivsize = ivsize;617 inst->alg.chunksize = chacha->chunksize;618 inst->alg.maxauthsize = POLY1305_DIGEST_SIZE;619 inst->alg.init = chachapoly_init;620 inst->alg.exit = chachapoly_exit;621 inst->alg.encrypt = chachapoly_encrypt;622 inst->alg.decrypt = chachapoly_decrypt;623 inst->alg.setkey = chachapoly_setkey;624 inst->alg.setauthsize = chachapoly_setauthsize;625 626 inst->free = chachapoly_free;627 628 err = aead_register_instance(tmpl, inst);629 if (err) {630err_free_inst:631 chachapoly_free(inst);632 }633 return err;634}635 636static int rfc7539_create(struct crypto_template *tmpl, struct rtattr **tb)637{638 return chachapoly_create(tmpl, tb, "rfc7539", 12);639}640 641static int rfc7539esp_create(struct crypto_template *tmpl, struct rtattr **tb)642{643 return chachapoly_create(tmpl, tb, "rfc7539esp", 8);644}645 646static struct crypto_template rfc7539_tmpls[] = {647 {648 .name = "rfc7539",649 .create = rfc7539_create,650 .module = THIS_MODULE,651 }, {652 .name = "rfc7539esp",653 .create = rfc7539esp_create,654 .module = THIS_MODULE,655 },656};657 658static int __init chacha20poly1305_module_init(void)659{660 return crypto_register_templates(rfc7539_tmpls,661 ARRAY_SIZE(rfc7539_tmpls));662}663 664static void __exit chacha20poly1305_module_exit(void)665{666 crypto_unregister_templates(rfc7539_tmpls,667 ARRAY_SIZE(rfc7539_tmpls));668}669 670subsys_initcall(chacha20poly1305_module_init);671module_exit(chacha20poly1305_module_exit);672 673MODULE_LICENSE("GPL");674MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");675MODULE_DESCRIPTION("ChaCha20-Poly1305 AEAD");676MODULE_ALIAS_CRYPTO("rfc7539");677MODULE_ALIAS_CRYPTO("rfc7539esp");678