1165 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * GCM: Galois/Counter Mode.4 *5 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>6 */7 8#include <crypto/gf128mul.h>9#include <crypto/internal/aead.h>10#include <crypto/internal/skcipher.h>11#include <crypto/internal/hash.h>12#include <crypto/null.h>13#include <crypto/scatterwalk.h>14#include <crypto/gcm.h>15#include <crypto/hash.h>16#include <linux/err.h>17#include <linux/init.h>18#include <linux/kernel.h>19#include <linux/module.h>20#include <linux/slab.h>21 22struct gcm_instance_ctx {23 struct crypto_skcipher_spawn ctr;24 struct crypto_ahash_spawn ghash;25};26 27struct crypto_gcm_ctx {28 struct crypto_skcipher *ctr;29 struct crypto_ahash *ghash;30};31 32struct crypto_rfc4106_ctx {33 struct crypto_aead *child;34 u8 nonce[4];35};36 37struct crypto_rfc4106_req_ctx {38 struct scatterlist src[3];39 struct scatterlist dst[3];40 struct aead_request subreq;41};42 43struct crypto_rfc4543_instance_ctx {44 struct crypto_aead_spawn aead;45};46 47struct crypto_rfc4543_ctx {48 struct crypto_aead *child;49 struct crypto_sync_skcipher *null;50 u8 nonce[4];51};52 53struct crypto_rfc4543_req_ctx {54 struct aead_request subreq;55};56 57struct crypto_gcm_ghash_ctx {58 unsigned int cryptlen;59 struct scatterlist *src;60 int (*complete)(struct aead_request *req, u32 flags);61};62 63struct crypto_gcm_req_priv_ctx {64 u8 iv[16];65 u8 auth_tag[16];66 u8 iauth_tag[16];67 struct scatterlist src[3];68 struct scatterlist dst[3];69 struct scatterlist sg;70 struct crypto_gcm_ghash_ctx ghash_ctx;71 union {72 struct ahash_request ahreq;73 struct skcipher_request skreq;74 } u;75};76 77static struct {78 u8 buf[16];79 struct scatterlist sg;80} *gcm_zeroes;81 82static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc);83 84static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(85 struct aead_request *req)86{87 unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));88 89 return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);90}91 92static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,93 unsigned int keylen)94{95 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);96 struct crypto_ahash *ghash = ctx->ghash;97 struct crypto_skcipher *ctr = ctx->ctr;98 struct {99 be128 hash;100 u8 iv[16];101 102 struct crypto_wait wait;103 104 struct scatterlist sg[1];105 struct skcipher_request req;106 } *data;107 int err;108 109 crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);110 crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) &111 CRYPTO_TFM_REQ_MASK);112 err = crypto_skcipher_setkey(ctr, key, keylen);113 if (err)114 return err;115 116 data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr),117 GFP_KERNEL);118 if (!data)119 return -ENOMEM;120 121 crypto_init_wait(&data->wait);122 sg_init_one(data->sg, &data->hash, sizeof(data->hash));123 skcipher_request_set_tfm(&data->req, ctr);124 skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |125 CRYPTO_TFM_REQ_MAY_BACKLOG,126 crypto_req_done,127 &data->wait);128 skcipher_request_set_crypt(&data->req, data->sg, data->sg,129 sizeof(data->hash), data->iv);130 131 err = crypto_wait_req(crypto_skcipher_encrypt(&data->req),132 &data->wait);133 134 if (err)135 goto out;136 137 crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK);138 crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) &139 CRYPTO_TFM_REQ_MASK);140 err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128));141out:142 kfree_sensitive(data);143 return err;144}145 146static int crypto_gcm_setauthsize(struct crypto_aead *tfm,147 unsigned int authsize)148{149 return crypto_gcm_check_authsize(authsize);150}151 152static void crypto_gcm_init_common(struct aead_request *req)153{154 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);155 __be32 counter = cpu_to_be32(1);156 struct scatterlist *sg;157 158 memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));159 memcpy(pctx->iv, req->iv, GCM_AES_IV_SIZE);160 memcpy(pctx->iv + GCM_AES_IV_SIZE, &counter, 4);161 162 sg_init_table(pctx->src, 3);163 sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));164 sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen);165 if (sg != pctx->src + 1)166 sg_chain(pctx->src, 2, sg);167 168 if (req->src != req->dst) {169 sg_init_table(pctx->dst, 3);170 sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));171 sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen);172 if (sg != pctx->dst + 1)173 sg_chain(pctx->dst, 2, sg);174 }175}176 177static void crypto_gcm_init_crypt(struct aead_request *req,178 unsigned int cryptlen)179{180 struct crypto_aead *aead = crypto_aead_reqtfm(req);181 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);182 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);183 struct skcipher_request *skreq = &pctx->u.skreq;184 struct scatterlist *dst;185 186 dst = req->src == req->dst ? pctx->src : pctx->dst;187 188 skcipher_request_set_tfm(skreq, ctx->ctr);189 skcipher_request_set_crypt(skreq, pctx->src, dst,190 cryptlen + sizeof(pctx->auth_tag),191 pctx->iv);192}193 194static inline unsigned int gcm_remain(unsigned int len)195{196 len &= 0xfU;197 return len ? 16 - len : 0;198}199 200static void gcm_hash_len_done(void *data, int err);201 202static int gcm_hash_update(struct aead_request *req,203 crypto_completion_t compl,204 struct scatterlist *src,205 unsigned int len, u32 flags)206{207 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);208 struct ahash_request *ahreq = &pctx->u.ahreq;209 210 ahash_request_set_callback(ahreq, flags, compl, req);211 ahash_request_set_crypt(ahreq, src, NULL, len);212 213 return crypto_ahash_update(ahreq);214}215 216static int gcm_hash_remain(struct aead_request *req,217 unsigned int remain,218 crypto_completion_t compl, u32 flags)219{220 return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags);221}222 223static int gcm_hash_len(struct aead_request *req, u32 flags)224{225 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);226 struct ahash_request *ahreq = &pctx->u.ahreq;227 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;228 be128 lengths;229 230 lengths.a = cpu_to_be64(req->assoclen * 8);231 lengths.b = cpu_to_be64(gctx->cryptlen * 8);232 memcpy(pctx->iauth_tag, &lengths, 16);233 sg_init_one(&pctx->sg, pctx->iauth_tag, 16);234 ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req);235 ahash_request_set_crypt(ahreq, &pctx->sg,236 pctx->iauth_tag, sizeof(lengths));237 238 return crypto_ahash_finup(ahreq);239}240 241static int gcm_hash_len_continue(struct aead_request *req, u32 flags)242{243 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);244 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;245 246 return gctx->complete(req, flags);247}248 249static void gcm_hash_len_done(void *data, int err)250{251 struct aead_request *req = data;252 253 if (err)254 goto out;255 256 err = gcm_hash_len_continue(req, 0);257 if (err == -EINPROGRESS)258 return;259 260out:261 aead_request_complete(req, err);262}263 264static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags)265{266 return gcm_hash_len(req, flags) ?:267 gcm_hash_len_continue(req, flags);268}269 270static void gcm_hash_crypt_remain_done(void *data, int err)271{272 struct aead_request *req = data;273 274 if (err)275 goto out;276 277 err = gcm_hash_crypt_remain_continue(req, 0);278 if (err == -EINPROGRESS)279 return;280 281out:282 aead_request_complete(req, err);283}284 285static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags)286{287 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);288 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;289 unsigned int remain;290 291 remain = gcm_remain(gctx->cryptlen);292 if (remain)293 return gcm_hash_remain(req, remain,294 gcm_hash_crypt_remain_done, flags) ?:295 gcm_hash_crypt_remain_continue(req, flags);296 297 return gcm_hash_crypt_remain_continue(req, flags);298}299 300static void gcm_hash_crypt_done(void *data, int err)301{302 struct aead_request *req = data;303 304 if (err)305 goto out;306 307 err = gcm_hash_crypt_continue(req, 0);308 if (err == -EINPROGRESS)309 return;310 311out:312 aead_request_complete(req, err);313}314 315static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags)316{317 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);318 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;319 320 if (gctx->cryptlen)321 return gcm_hash_update(req, gcm_hash_crypt_done,322 gctx->src, gctx->cryptlen, flags) ?:323 gcm_hash_crypt_continue(req, flags);324 325 return gcm_hash_crypt_remain_continue(req, flags);326}327 328static void gcm_hash_assoc_remain_done(void *data, int err)329{330 struct aead_request *req = data;331 332 if (err)333 goto out;334 335 err = gcm_hash_assoc_remain_continue(req, 0);336 if (err == -EINPROGRESS)337 return;338 339out:340 aead_request_complete(req, err);341}342 343static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags)344{345 unsigned int remain;346 347 remain = gcm_remain(req->assoclen);348 if (remain)349 return gcm_hash_remain(req, remain,350 gcm_hash_assoc_remain_done, flags) ?:351 gcm_hash_assoc_remain_continue(req, flags);352 353 return gcm_hash_assoc_remain_continue(req, flags);354}355 356static void gcm_hash_assoc_done(void *data, int err)357{358 struct aead_request *req = data;359 360 if (err)361 goto out;362 363 err = gcm_hash_assoc_continue(req, 0);364 if (err == -EINPROGRESS)365 return;366 367out:368 aead_request_complete(req, err);369}370 371static int gcm_hash_init_continue(struct aead_request *req, u32 flags)372{373 if (req->assoclen)374 return gcm_hash_update(req, gcm_hash_assoc_done,375 req->src, req->assoclen, flags) ?:376 gcm_hash_assoc_continue(req, flags);377 378 return gcm_hash_assoc_remain_continue(req, flags);379}380 381static void gcm_hash_init_done(void *data, int err)382{383 struct aead_request *req = data;384 385 if (err)386 goto out;387 388 err = gcm_hash_init_continue(req, 0);389 if (err == -EINPROGRESS)390 return;391 392out:393 aead_request_complete(req, err);394}395 396static int gcm_hash(struct aead_request *req, u32 flags)397{398 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);399 struct ahash_request *ahreq = &pctx->u.ahreq;400 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));401 402 ahash_request_set_tfm(ahreq, ctx->ghash);403 404 ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req);405 return crypto_ahash_init(ahreq) ?:406 gcm_hash_init_continue(req, flags);407}408 409static int gcm_enc_copy_hash(struct aead_request *req, u32 flags)410{411 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);412 struct crypto_aead *aead = crypto_aead_reqtfm(req);413 u8 *auth_tag = pctx->auth_tag;414 415 crypto_xor(auth_tag, pctx->iauth_tag, 16);416 scatterwalk_map_and_copy(auth_tag, req->dst,417 req->assoclen + req->cryptlen,418 crypto_aead_authsize(aead), 1);419 return 0;420}421 422static int gcm_encrypt_continue(struct aead_request *req, u32 flags)423{424 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);425 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;426 427 gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst);428 gctx->cryptlen = req->cryptlen;429 gctx->complete = gcm_enc_copy_hash;430 431 return gcm_hash(req, flags);432}433 434static void gcm_encrypt_done(void *data, int err)435{436 struct aead_request *req = data;437 438 if (err)439 goto out;440 441 err = gcm_encrypt_continue(req, 0);442 if (err == -EINPROGRESS)443 return;444 445out:446 aead_request_complete(req, err);447}448 449static int crypto_gcm_encrypt(struct aead_request *req)450{451 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);452 struct skcipher_request *skreq = &pctx->u.skreq;453 u32 flags = aead_request_flags(req);454 455 crypto_gcm_init_common(req);456 crypto_gcm_init_crypt(req, req->cryptlen);457 skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req);458 459 return crypto_skcipher_encrypt(skreq) ?:460 gcm_encrypt_continue(req, flags);461}462 463static int crypto_gcm_verify(struct aead_request *req)464{465 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);466 struct crypto_aead *aead = crypto_aead_reqtfm(req);467 u8 *auth_tag = pctx->auth_tag;468 u8 *iauth_tag = pctx->iauth_tag;469 unsigned int authsize = crypto_aead_authsize(aead);470 unsigned int cryptlen = req->cryptlen - authsize;471 472 crypto_xor(auth_tag, iauth_tag, 16);473 scatterwalk_map_and_copy(iauth_tag, req->src,474 req->assoclen + cryptlen, authsize, 0);475 return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;476}477 478static void gcm_decrypt_done(void *data, int err)479{480 struct aead_request *req = data;481 482 if (!err)483 err = crypto_gcm_verify(req);484 485 aead_request_complete(req, err);486}487 488static int gcm_dec_hash_continue(struct aead_request *req, u32 flags)489{490 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);491 struct skcipher_request *skreq = &pctx->u.skreq;492 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;493 494 crypto_gcm_init_crypt(req, gctx->cryptlen);495 skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req);496 return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req);497}498 499static int crypto_gcm_decrypt(struct aead_request *req)500{501 struct crypto_aead *aead = crypto_aead_reqtfm(req);502 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);503 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;504 unsigned int authsize = crypto_aead_authsize(aead);505 unsigned int cryptlen = req->cryptlen;506 u32 flags = aead_request_flags(req);507 508 cryptlen -= authsize;509 510 crypto_gcm_init_common(req);511 512 gctx->src = sg_next(pctx->src);513 gctx->cryptlen = cryptlen;514 gctx->complete = gcm_dec_hash_continue;515 516 return gcm_hash(req, flags);517}518 519static int crypto_gcm_init_tfm(struct crypto_aead *tfm)520{521 struct aead_instance *inst = aead_alg_instance(tfm);522 struct gcm_instance_ctx *ictx = aead_instance_ctx(inst);523 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);524 struct crypto_skcipher *ctr;525 struct crypto_ahash *ghash;526 unsigned long align;527 int err;528 529 ghash = crypto_spawn_ahash(&ictx->ghash);530 if (IS_ERR(ghash))531 return PTR_ERR(ghash);532 533 ctr = crypto_spawn_skcipher(&ictx->ctr);534 err = PTR_ERR(ctr);535 if (IS_ERR(ctr))536 goto err_free_hash;537 538 ctx->ctr = ctr;539 ctx->ghash = ghash;540 541 align = crypto_aead_alignmask(tfm);542 align &= ~(crypto_tfm_ctx_alignment() - 1);543 crypto_aead_set_reqsize(tfm,544 align + offsetof(struct crypto_gcm_req_priv_ctx, u) +545 max(sizeof(struct skcipher_request) +546 crypto_skcipher_reqsize(ctr),547 sizeof(struct ahash_request) +548 crypto_ahash_reqsize(ghash)));549 550 return 0;551 552err_free_hash:553 crypto_free_ahash(ghash);554 return err;555}556 557static void crypto_gcm_exit_tfm(struct crypto_aead *tfm)558{559 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);560 561 crypto_free_ahash(ctx->ghash);562 crypto_free_skcipher(ctx->ctr);563}564 565static void crypto_gcm_free(struct aead_instance *inst)566{567 struct gcm_instance_ctx *ctx = aead_instance_ctx(inst);568 569 crypto_drop_skcipher(&ctx->ctr);570 crypto_drop_ahash(&ctx->ghash);571 kfree(inst);572}573 574static int crypto_gcm_create_common(struct crypto_template *tmpl,575 struct rtattr **tb,576 const char *ctr_name,577 const char *ghash_name)578{579 struct skcipher_alg_common *ctr;580 u32 mask;581 struct aead_instance *inst;582 struct gcm_instance_ctx *ctx;583 struct hash_alg_common *ghash;584 int err;585 586 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask);587 if (err)588 return err;589 590 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);591 if (!inst)592 return -ENOMEM;593 ctx = aead_instance_ctx(inst);594 595 err = crypto_grab_ahash(&ctx->ghash, aead_crypto_instance(inst),596 ghash_name, 0, mask);597 if (err)598 goto err_free_inst;599 ghash = crypto_spawn_ahash_alg(&ctx->ghash);600 601 err = -EINVAL;602 if (strcmp(ghash->base.cra_name, "ghash") != 0 ||603 ghash->digestsize != 16)604 goto err_free_inst;605 606 err = crypto_grab_skcipher(&ctx->ctr, aead_crypto_instance(inst),607 ctr_name, 0, mask);608 if (err)609 goto err_free_inst;610 ctr = crypto_spawn_skcipher_alg_common(&ctx->ctr);611 612 /* The skcipher algorithm must be CTR mode, using 16-byte blocks. */613 err = -EINVAL;614 if (strncmp(ctr->base.cra_name, "ctr(", 4) != 0 ||615 ctr->ivsize != 16 || ctr->base.cra_blocksize != 1)616 goto err_free_inst;617 618 err = -ENAMETOOLONG;619 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,620 "gcm(%s", ctr->base.cra_name + 4) >= CRYPTO_MAX_ALG_NAME)621 goto err_free_inst;622 623 if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,624 "gcm_base(%s,%s)", ctr->base.cra_driver_name,625 ghash->base.cra_driver_name) >=626 CRYPTO_MAX_ALG_NAME)627 goto err_free_inst;628 629 inst->alg.base.cra_priority = (ghash->base.cra_priority +630 ctr->base.cra_priority) / 2;631 inst->alg.base.cra_blocksize = 1;632 inst->alg.base.cra_alignmask = ctr->base.cra_alignmask;633 inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx);634 inst->alg.ivsize = GCM_AES_IV_SIZE;635 inst->alg.chunksize = ctr->chunksize;636 inst->alg.maxauthsize = 16;637 inst->alg.init = crypto_gcm_init_tfm;638 inst->alg.exit = crypto_gcm_exit_tfm;639 inst->alg.setkey = crypto_gcm_setkey;640 inst->alg.setauthsize = crypto_gcm_setauthsize;641 inst->alg.encrypt = crypto_gcm_encrypt;642 inst->alg.decrypt = crypto_gcm_decrypt;643 644 inst->free = crypto_gcm_free;645 646 err = aead_register_instance(tmpl, inst);647 if (err) {648err_free_inst:649 crypto_gcm_free(inst);650 }651 return err;652}653 654static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb)655{656 const char *cipher_name;657 char ctr_name[CRYPTO_MAX_ALG_NAME];658 659 cipher_name = crypto_attr_alg_name(tb[1]);660 if (IS_ERR(cipher_name))661 return PTR_ERR(cipher_name);662 663 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >=664 CRYPTO_MAX_ALG_NAME)665 return -ENAMETOOLONG;666 667 return crypto_gcm_create_common(tmpl, tb, ctr_name, "ghash");668}669 670static int crypto_gcm_base_create(struct crypto_template *tmpl,671 struct rtattr **tb)672{673 const char *ctr_name;674 const char *ghash_name;675 676 ctr_name = crypto_attr_alg_name(tb[1]);677 if (IS_ERR(ctr_name))678 return PTR_ERR(ctr_name);679 680 ghash_name = crypto_attr_alg_name(tb[2]);681 if (IS_ERR(ghash_name))682 return PTR_ERR(ghash_name);683 684 return crypto_gcm_create_common(tmpl, tb, ctr_name, ghash_name);685}686 687static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key,688 unsigned int keylen)689{690 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);691 struct crypto_aead *child = ctx->child;692 693 if (keylen < 4)694 return -EINVAL;695 696 keylen -= 4;697 memcpy(ctx->nonce, key + keylen, 4);698 699 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);700 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &701 CRYPTO_TFM_REQ_MASK);702 return crypto_aead_setkey(child, key, keylen);703}704 705static int crypto_rfc4106_setauthsize(struct crypto_aead *parent,706 unsigned int authsize)707{708 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);709 int err;710 711 err = crypto_rfc4106_check_authsize(authsize);712 if (err)713 return err;714 715 return crypto_aead_setauthsize(ctx->child, authsize);716}717 718static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)719{720 struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req);721 struct crypto_aead *aead = crypto_aead_reqtfm(req);722 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead);723 struct aead_request *subreq = &rctx->subreq;724 struct crypto_aead *child = ctx->child;725 struct scatterlist *sg;726 u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child),727 crypto_aead_alignmask(child) + 1);728 729 scatterwalk_map_and_copy(iv + GCM_AES_IV_SIZE, req->src, 0, req->assoclen - 8, 0);730 731 memcpy(iv, ctx->nonce, 4);732 memcpy(iv + 4, req->iv, 8);733 734 sg_init_table(rctx->src, 3);735 sg_set_buf(rctx->src, iv + GCM_AES_IV_SIZE, req->assoclen - 8);736 sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen);737 if (sg != rctx->src + 1)738 sg_chain(rctx->src, 2, sg);739 740 if (req->src != req->dst) {741 sg_init_table(rctx->dst, 3);742 sg_set_buf(rctx->dst, iv + GCM_AES_IV_SIZE, req->assoclen - 8);743 sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen);744 if (sg != rctx->dst + 1)745 sg_chain(rctx->dst, 2, sg);746 }747 748 aead_request_set_tfm(subreq, child);749 aead_request_set_callback(subreq, req->base.flags, req->base.complete,750 req->base.data);751 aead_request_set_crypt(subreq, rctx->src,752 req->src == req->dst ? rctx->src : rctx->dst,753 req->cryptlen, iv);754 aead_request_set_ad(subreq, req->assoclen - 8);755 756 return subreq;757}758 759static int crypto_rfc4106_encrypt(struct aead_request *req)760{761 int err;762 763 err = crypto_ipsec_check_assoclen(req->assoclen);764 if (err)765 return err;766 767 req = crypto_rfc4106_crypt(req);768 769 return crypto_aead_encrypt(req);770}771 772static int crypto_rfc4106_decrypt(struct aead_request *req)773{774 int err;775 776 err = crypto_ipsec_check_assoclen(req->assoclen);777 if (err)778 return err;779 780 req = crypto_rfc4106_crypt(req);781 782 return crypto_aead_decrypt(req);783}784 785static int crypto_rfc4106_init_tfm(struct crypto_aead *tfm)786{787 struct aead_instance *inst = aead_alg_instance(tfm);788 struct crypto_aead_spawn *spawn = aead_instance_ctx(inst);789 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm);790 struct crypto_aead *aead;791 unsigned long align;792 793 aead = crypto_spawn_aead(spawn);794 if (IS_ERR(aead))795 return PTR_ERR(aead);796 797 ctx->child = aead;798 799 align = crypto_aead_alignmask(aead);800 align &= ~(crypto_tfm_ctx_alignment() - 1);801 crypto_aead_set_reqsize(802 tfm,803 sizeof(struct crypto_rfc4106_req_ctx) +804 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) +805 align + 24);806 807 return 0;808}809 810static void crypto_rfc4106_exit_tfm(struct crypto_aead *tfm)811{812 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm);813 814 crypto_free_aead(ctx->child);815}816 817static void crypto_rfc4106_free(struct aead_instance *inst)818{819 crypto_drop_aead(aead_instance_ctx(inst));820 kfree(inst);821}822 823static int crypto_rfc4106_create(struct crypto_template *tmpl,824 struct rtattr **tb)825{826 u32 mask;827 struct aead_instance *inst;828 struct crypto_aead_spawn *spawn;829 struct aead_alg *alg;830 int err;831 832 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask);833 if (err)834 return err;835 836 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);837 if (!inst)838 return -ENOMEM;839 840 spawn = aead_instance_ctx(inst);841 err = crypto_grab_aead(spawn, aead_crypto_instance(inst),842 crypto_attr_alg_name(tb[1]), 0, mask);843 if (err)844 goto err_free_inst;845 846 alg = crypto_spawn_aead_alg(spawn);847 848 err = -EINVAL;849 850 /* Underlying IV size must be 12. */851 if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE)852 goto err_free_inst;853 854 /* Not a stream cipher? */855 if (alg->base.cra_blocksize != 1)856 goto err_free_inst;857 858 err = -ENAMETOOLONG;859 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,860 "rfc4106(%s)", alg->base.cra_name) >=861 CRYPTO_MAX_ALG_NAME ||862 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,863 "rfc4106(%s)", alg->base.cra_driver_name) >=864 CRYPTO_MAX_ALG_NAME)865 goto err_free_inst;866 867 inst->alg.base.cra_priority = alg->base.cra_priority;868 inst->alg.base.cra_blocksize = 1;869 inst->alg.base.cra_alignmask = alg->base.cra_alignmask;870 871 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx);872 873 inst->alg.ivsize = GCM_RFC4106_IV_SIZE;874 inst->alg.chunksize = crypto_aead_alg_chunksize(alg);875 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);876 877 inst->alg.init = crypto_rfc4106_init_tfm;878 inst->alg.exit = crypto_rfc4106_exit_tfm;879 880 inst->alg.setkey = crypto_rfc4106_setkey;881 inst->alg.setauthsize = crypto_rfc4106_setauthsize;882 inst->alg.encrypt = crypto_rfc4106_encrypt;883 inst->alg.decrypt = crypto_rfc4106_decrypt;884 885 inst->free = crypto_rfc4106_free;886 887 err = aead_register_instance(tmpl, inst);888 if (err) {889err_free_inst:890 crypto_rfc4106_free(inst);891 }892 return err;893}894 895static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key,896 unsigned int keylen)897{898 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent);899 struct crypto_aead *child = ctx->child;900 901 if (keylen < 4)902 return -EINVAL;903 904 keylen -= 4;905 memcpy(ctx->nonce, key + keylen, 4);906 907 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);908 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &909 CRYPTO_TFM_REQ_MASK);910 return crypto_aead_setkey(child, key, keylen);911}912 913static int crypto_rfc4543_setauthsize(struct crypto_aead *parent,914 unsigned int authsize)915{916 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent);917 918 if (authsize != 16)919 return -EINVAL;920 921 return crypto_aead_setauthsize(ctx->child, authsize);922}923 924static int crypto_rfc4543_crypt(struct aead_request *req, bool enc)925{926 struct crypto_aead *aead = crypto_aead_reqtfm(req);927 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);928 struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req);929 struct aead_request *subreq = &rctx->subreq;930 unsigned int authsize = crypto_aead_authsize(aead);931 u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child),932 crypto_aead_alignmask(ctx->child) + 1);933 int err;934 935 if (req->src != req->dst) {936 err = crypto_rfc4543_copy_src_to_dst(req, enc);937 if (err)938 return err;939 }940 941 memcpy(iv, ctx->nonce, 4);942 memcpy(iv + 4, req->iv, 8);943 944 aead_request_set_tfm(subreq, ctx->child);945 aead_request_set_callback(subreq, req->base.flags,946 req->base.complete, req->base.data);947 aead_request_set_crypt(subreq, req->src, req->dst,948 enc ? 0 : authsize, iv);949 aead_request_set_ad(subreq, req->assoclen + req->cryptlen -950 subreq->cryptlen);951 952 return enc ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq);953}954 955static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)956{957 struct crypto_aead *aead = crypto_aead_reqtfm(req);958 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);959 unsigned int authsize = crypto_aead_authsize(aead);960 unsigned int nbytes = req->assoclen + req->cryptlen -961 (enc ? 0 : authsize);962 SYNC_SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null);963 964 skcipher_request_set_sync_tfm(nreq, ctx->null);965 skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL);966 skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL);967 968 return crypto_skcipher_encrypt(nreq);969}970 971static int crypto_rfc4543_encrypt(struct aead_request *req)972{973 return crypto_ipsec_check_assoclen(req->assoclen) ?:974 crypto_rfc4543_crypt(req, true);975}976 977static int crypto_rfc4543_decrypt(struct aead_request *req)978{979 return crypto_ipsec_check_assoclen(req->assoclen) ?:980 crypto_rfc4543_crypt(req, false);981}982 983static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm)984{985 struct aead_instance *inst = aead_alg_instance(tfm);986 struct crypto_rfc4543_instance_ctx *ictx = aead_instance_ctx(inst);987 struct crypto_aead_spawn *spawn = &ictx->aead;988 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);989 struct crypto_aead *aead;990 struct crypto_sync_skcipher *null;991 unsigned long align;992 int err = 0;993 994 aead = crypto_spawn_aead(spawn);995 if (IS_ERR(aead))996 return PTR_ERR(aead);997 998 null = crypto_get_default_null_skcipher();999 err = PTR_ERR(null);1000 if (IS_ERR(null))1001 goto err_free_aead;1002 1003 ctx->child = aead;1004 ctx->null = null;1005 1006 align = crypto_aead_alignmask(aead);1007 align &= ~(crypto_tfm_ctx_alignment() - 1);1008 crypto_aead_set_reqsize(1009 tfm,1010 sizeof(struct crypto_rfc4543_req_ctx) +1011 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) +1012 align + GCM_AES_IV_SIZE);1013 1014 return 0;1015 1016err_free_aead:1017 crypto_free_aead(aead);1018 return err;1019}1020 1021static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm)1022{1023 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);1024 1025 crypto_free_aead(ctx->child);1026 crypto_put_default_null_skcipher();1027}1028 1029static void crypto_rfc4543_free(struct aead_instance *inst)1030{1031 struct crypto_rfc4543_instance_ctx *ctx = aead_instance_ctx(inst);1032 1033 crypto_drop_aead(&ctx->aead);1034 1035 kfree(inst);1036}1037 1038static int crypto_rfc4543_create(struct crypto_template *tmpl,1039 struct rtattr **tb)1040{1041 u32 mask;1042 struct aead_instance *inst;1043 struct aead_alg *alg;1044 struct crypto_rfc4543_instance_ctx *ctx;1045 int err;1046 1047 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask);1048 if (err)1049 return err;1050 1051 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);1052 if (!inst)1053 return -ENOMEM;1054 1055 ctx = aead_instance_ctx(inst);1056 err = crypto_grab_aead(&ctx->aead, aead_crypto_instance(inst),1057 crypto_attr_alg_name(tb[1]), 0, mask);1058 if (err)1059 goto err_free_inst;1060 1061 alg = crypto_spawn_aead_alg(&ctx->aead);1062 1063 err = -EINVAL;1064 1065 /* Underlying IV size must be 12. */1066 if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE)1067 goto err_free_inst;1068 1069 /* Not a stream cipher? */1070 if (alg->base.cra_blocksize != 1)1071 goto err_free_inst;1072 1073 err = -ENAMETOOLONG;1074 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,1075 "rfc4543(%s)", alg->base.cra_name) >=1076 CRYPTO_MAX_ALG_NAME ||1077 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,1078 "rfc4543(%s)", alg->base.cra_driver_name) >=1079 CRYPTO_MAX_ALG_NAME)1080 goto err_free_inst;1081 1082 inst->alg.base.cra_priority = alg->base.cra_priority;1083 inst->alg.base.cra_blocksize = 1;1084 inst->alg.base.cra_alignmask = alg->base.cra_alignmask;1085 1086 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx);1087 1088 inst->alg.ivsize = GCM_RFC4543_IV_SIZE;1089 inst->alg.chunksize = crypto_aead_alg_chunksize(alg);1090 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);1091 1092 inst->alg.init = crypto_rfc4543_init_tfm;1093 inst->alg.exit = crypto_rfc4543_exit_tfm;1094 1095 inst->alg.setkey = crypto_rfc4543_setkey;1096 inst->alg.setauthsize = crypto_rfc4543_setauthsize;1097 inst->alg.encrypt = crypto_rfc4543_encrypt;1098 inst->alg.decrypt = crypto_rfc4543_decrypt;1099 1100 inst->free = crypto_rfc4543_free;1101 1102 err = aead_register_instance(tmpl, inst);1103 if (err) {1104err_free_inst:1105 crypto_rfc4543_free(inst);1106 }1107 return err;1108}1109 1110static struct crypto_template crypto_gcm_tmpls[] = {1111 {1112 .name = "gcm_base",1113 .create = crypto_gcm_base_create,1114 .module = THIS_MODULE,1115 }, {1116 .name = "gcm",1117 .create = crypto_gcm_create,1118 .module = THIS_MODULE,1119 }, {1120 .name = "rfc4106",1121 .create = crypto_rfc4106_create,1122 .module = THIS_MODULE,1123 }, {1124 .name = "rfc4543",1125 .create = crypto_rfc4543_create,1126 .module = THIS_MODULE,1127 },1128};1129 1130static int __init crypto_gcm_module_init(void)1131{1132 int err;1133 1134 gcm_zeroes = kzalloc(sizeof(*gcm_zeroes), GFP_KERNEL);1135 if (!gcm_zeroes)1136 return -ENOMEM;1137 1138 sg_init_one(&gcm_zeroes->sg, gcm_zeroes->buf, sizeof(gcm_zeroes->buf));1139 1140 err = crypto_register_templates(crypto_gcm_tmpls,1141 ARRAY_SIZE(crypto_gcm_tmpls));1142 if (err)1143 kfree(gcm_zeroes);1144 1145 return err;1146}1147 1148static void __exit crypto_gcm_module_exit(void)1149{1150 kfree(gcm_zeroes);1151 crypto_unregister_templates(crypto_gcm_tmpls,1152 ARRAY_SIZE(crypto_gcm_tmpls));1153}1154 1155subsys_initcall(crypto_gcm_module_init);1156module_exit(crypto_gcm_module_exit);1157 1158MODULE_LICENSE("GPL");1159MODULE_DESCRIPTION("Galois/Counter Mode");1160MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");1161MODULE_ALIAS_CRYPTO("gcm_base");1162MODULE_ALIAS_CRYPTO("rfc4106");1163MODULE_ALIAS_CRYPTO("rfc4543");1164MODULE_ALIAS_CRYPTO("gcm");1165