564 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * AES CCM routines supporting the Power 7+ Nest Accelerators driver4 *5 * Copyright (C) 2012 International Business Machines Inc.6 *7 * Author: Kent Yoder <yoder1@us.ibm.com>8 */9 10#include <crypto/internal/aead.h>11#include <crypto/aes.h>12#include <crypto/algapi.h>13#include <crypto/scatterwalk.h>14#include <linux/module.h>15#include <linux/types.h>16#include <linux/crypto.h>17#include <asm/vio.h>18 19#include "nx_csbcpb.h"20#include "nx.h"21 22 23static int ccm_aes_nx_set_key(struct crypto_aead *tfm,24 const u8 *in_key,25 unsigned int key_len)26{27 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);28 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;29 struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;30 31 nx_ctx_init(nx_ctx, HCOP_FC_AES);32 33 switch (key_len) {34 case AES_KEYSIZE_128:35 NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);36 NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_128);37 nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];38 break;39 default:40 return -EINVAL;41 }42 43 csbcpb->cpb.hdr.mode = NX_MODE_AES_CCM;44 memcpy(csbcpb->cpb.aes_ccm.key, in_key, key_len);45 46 csbcpb_aead->cpb.hdr.mode = NX_MODE_AES_CCA;47 memcpy(csbcpb_aead->cpb.aes_cca.key, in_key, key_len);48 49 return 0;50 51}52 53static int ccm4309_aes_nx_set_key(struct crypto_aead *tfm,54 const u8 *in_key,55 unsigned int key_len)56{57 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);58 59 if (key_len < 3)60 return -EINVAL;61 62 key_len -= 3;63 64 memcpy(nx_ctx->priv.ccm.nonce, in_key + key_len, 3);65 66 return ccm_aes_nx_set_key(tfm, in_key, key_len);67}68 69static int ccm_aes_nx_setauthsize(struct crypto_aead *tfm,70 unsigned int authsize)71{72 switch (authsize) {73 case 4:74 case 6:75 case 8:76 case 10:77 case 12:78 case 14:79 case 16:80 break;81 default:82 return -EINVAL;83 }84 85 return 0;86}87 88static int ccm4309_aes_nx_setauthsize(struct crypto_aead *tfm,89 unsigned int authsize)90{91 switch (authsize) {92 case 8:93 case 12:94 case 16:95 break;96 default:97 return -EINVAL;98 }99 100 return 0;101}102 103/* taken from crypto/ccm.c */104static int set_msg_len(u8 *block, unsigned int msglen, int csize)105{106 __be32 data;107 108 memset(block, 0, csize);109 block += csize;110 111 if (csize >= 4)112 csize = 4;113 else if (msglen > (unsigned int)(1 << (8 * csize)))114 return -EOVERFLOW;115 116 data = cpu_to_be32(msglen);117 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);118 119 return 0;120}121 122/* taken from crypto/ccm.c */123static inline int crypto_ccm_check_iv(const u8 *iv)124{125 /* 2 <= L <= 8, so 1 <= L' <= 7. */126 if (1 > iv[0] || iv[0] > 7)127 return -EINVAL;128 129 return 0;130}131 132/* based on code from crypto/ccm.c */133static int generate_b0(u8 *iv, unsigned int assoclen, unsigned int authsize,134 unsigned int cryptlen, u8 *b0)135{136 unsigned int l, lp, m = authsize;137 138 memcpy(b0, iv, 16);139 140 lp = b0[0];141 l = lp + 1;142 143 /* set m, bits 3-5 */144 *b0 |= (8 * ((m - 2) / 2));145 146 /* set adata, bit 6, if associated data is used */147 if (assoclen)148 *b0 |= 64;149 150 return set_msg_len(b0 + 16 - l, cryptlen, l);151}152 153static int generate_pat(u8 *iv,154 struct aead_request *req,155 struct nx_crypto_ctx *nx_ctx,156 unsigned int authsize,157 unsigned int nbytes,158 unsigned int assoclen,159 u8 *out)160{161 struct nx_sg *nx_insg = nx_ctx->in_sg;162 struct nx_sg *nx_outsg = nx_ctx->out_sg;163 unsigned int iauth_len = 0;164 u8 tmp[16], *b1 = NULL, *b0 = NULL, *result = NULL;165 int rc;166 unsigned int max_sg_len;167 168 /* zero the ctr value */169 memset(iv + 15 - iv[0], 0, iv[0] + 1);170 171 /* page 78 of nx_wb.pdf has,172 * Note: RFC3610 allows the AAD data to be up to 2^64 -1 bytes173 * in length. If a full message is used, the AES CCA implementation174 * restricts the maximum AAD length to 2^32 -1 bytes.175 * If partial messages are used, the implementation supports176 * 2^64 -1 bytes maximum AAD length.177 *178 * However, in the cryptoapi's aead_request structure,179 * assoclen is an unsigned int, thus it cannot hold a length180 * value greater than 2^32 - 1.181 * Thus the AAD is further constrained by this and is never182 * greater than 2^32.183 */184 185 if (!assoclen) {186 b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;187 } else if (assoclen <= 14) {188 /* if associated data is 14 bytes or less, we do 1 GCM189 * operation on 2 AES blocks, B0 (stored in the csbcpb) and B1,190 * which is fed in through the source buffers here */191 b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;192 b1 = nx_ctx->priv.ccm.iauth_tag;193 iauth_len = assoclen;194 } else if (assoclen <= 65280) {195 /* if associated data is less than (2^16 - 2^8), we construct196 * B1 differently and feed in the associated data to a CCA197 * operation */198 b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;199 b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;200 iauth_len = 14;201 } else {202 b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;203 b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;204 iauth_len = 10;205 }206 207 /* generate B0 */208 rc = generate_b0(iv, assoclen, authsize, nbytes, b0);209 if (rc)210 return rc;211 212 /* generate B1:213 * add control info for associated data214 * RFC 3610 and NIST Special Publication 800-38C215 */216 if (b1) {217 memset(b1, 0, 16);218 if (assoclen <= 65280) {219 *(u16 *)b1 = assoclen;220 scatterwalk_map_and_copy(b1 + 2, req->src, 0,221 iauth_len, SCATTERWALK_FROM_SG);222 } else {223 *(u16 *)b1 = (u16)(0xfffe);224 *(u32 *)&b1[2] = assoclen;225 scatterwalk_map_and_copy(b1 + 6, req->src, 0,226 iauth_len, SCATTERWALK_FROM_SG);227 }228 }229 230 /* now copy any remaining AAD to scatterlist and call nx... */231 if (!assoclen) {232 return rc;233 } else if (assoclen <= 14) {234 unsigned int len = 16;235 236 nx_insg = nx_build_sg_list(nx_insg, b1, &len, nx_ctx->ap->sglen);237 238 if (len != 16)239 return -EINVAL;240 241 nx_outsg = nx_build_sg_list(nx_outsg, tmp, &len,242 nx_ctx->ap->sglen);243 244 if (len != 16)245 return -EINVAL;246 247 /* inlen should be negative, indicating to phyp that its a248 * pointer to an sg list */249 nx_ctx->op.inlen = (nx_ctx->in_sg - nx_insg) *250 sizeof(struct nx_sg);251 nx_ctx->op.outlen = (nx_ctx->out_sg - nx_outsg) *252 sizeof(struct nx_sg);253 254 NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_ENDE_ENCRYPT;255 NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_INTERMEDIATE;256 257 result = nx_ctx->csbcpb->cpb.aes_ccm.out_pat_or_mac;258 259 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,260 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);261 if (rc)262 return rc;263 264 atomic_inc(&(nx_ctx->stats->aes_ops));265 atomic64_add(assoclen, &nx_ctx->stats->aes_bytes);266 267 } else {268 unsigned int processed = 0, to_process;269 270 processed += iauth_len;271 272 /* page_limit: number of sg entries that fit on one page */273 max_sg_len = min_t(u64, nx_ctx->ap->sglen,274 nx_driver.of.max_sg_len/sizeof(struct nx_sg));275 max_sg_len = min_t(u64, max_sg_len,276 nx_ctx->ap->databytelen/NX_PAGE_SIZE);277 278 do {279 to_process = min_t(u32, assoclen - processed,280 nx_ctx->ap->databytelen);281 282 nx_insg = nx_walk_and_build(nx_ctx->in_sg,283 nx_ctx->ap->sglen,284 req->src, processed,285 &to_process);286 287 if ((to_process + processed) < assoclen) {288 NX_CPB_FDM(nx_ctx->csbcpb_aead) |=289 NX_FDM_INTERMEDIATE;290 } else {291 NX_CPB_FDM(nx_ctx->csbcpb_aead) &=292 ~NX_FDM_INTERMEDIATE;293 }294 295 296 nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_insg) *297 sizeof(struct nx_sg);298 299 result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;300 301 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead,302 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);303 if (rc)304 return rc;305 306 memcpy(nx_ctx->csbcpb_aead->cpb.aes_cca.b0,307 nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0,308 AES_BLOCK_SIZE);309 310 NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_CONTINUATION;311 312 atomic_inc(&(nx_ctx->stats->aes_ops));313 atomic64_add(assoclen, &nx_ctx->stats->aes_bytes);314 315 processed += to_process;316 } while (processed < assoclen);317 318 result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;319 }320 321 memcpy(out, result, AES_BLOCK_SIZE);322 323 return rc;324}325 326static int ccm_nx_decrypt(struct aead_request *req,327 u8 *iv,328 unsigned int assoclen)329{330 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);331 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;332 unsigned int nbytes = req->cryptlen;333 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));334 struct nx_ccm_priv *priv = &nx_ctx->priv.ccm;335 unsigned long irq_flags;336 unsigned int processed = 0, to_process;337 int rc = -1;338 339 spin_lock_irqsave(&nx_ctx->lock, irq_flags);340 341 nbytes -= authsize;342 343 /* copy out the auth tag to compare with later */344 scatterwalk_map_and_copy(priv->oauth_tag,345 req->src, nbytes + req->assoclen, authsize,346 SCATTERWALK_FROM_SG);347 348 rc = generate_pat(iv, req, nx_ctx, authsize, nbytes, assoclen,349 csbcpb->cpb.aes_ccm.in_pat_or_b0);350 if (rc)351 goto out;352 353 do {354 355 /* to_process: the AES_BLOCK_SIZE data chunk to process in this356 * update. This value is bound by sg list limits.357 */358 to_process = nbytes - processed;359 360 if ((to_process + processed) < nbytes)361 NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;362 else363 NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;364 365 NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;366 367 rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src,368 &to_process, processed + req->assoclen,369 csbcpb->cpb.aes_ccm.iv_or_ctr);370 if (rc)371 goto out;372 373 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,374 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);375 if (rc)376 goto out;377 378 /* for partial completion, copy following for next379 * entry into loop...380 */381 memcpy(iv, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);382 memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,383 csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);384 memcpy(csbcpb->cpb.aes_ccm.in_s0,385 csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);386 387 NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;388 389 /* update stats */390 atomic_inc(&(nx_ctx->stats->aes_ops));391 atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),392 &(nx_ctx->stats->aes_bytes));393 394 processed += to_process;395 } while (processed < nbytes);396 397 rc = crypto_memneq(csbcpb->cpb.aes_ccm.out_pat_or_mac, priv->oauth_tag,398 authsize) ? -EBADMSG : 0;399out:400 spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);401 return rc;402}403 404static int ccm_nx_encrypt(struct aead_request *req,405 u8 *iv,406 unsigned int assoclen)407{408 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);409 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;410 unsigned int nbytes = req->cryptlen;411 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));412 unsigned long irq_flags;413 unsigned int processed = 0, to_process;414 int rc = -1;415 416 spin_lock_irqsave(&nx_ctx->lock, irq_flags);417 418 rc = generate_pat(iv, req, nx_ctx, authsize, nbytes, assoclen,419 csbcpb->cpb.aes_ccm.in_pat_or_b0);420 if (rc)421 goto out;422 423 do {424 /* to process: the AES_BLOCK_SIZE data chunk to process in this425 * update. This value is bound by sg list limits.426 */427 to_process = nbytes - processed;428 429 if ((to_process + processed) < nbytes)430 NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;431 else432 NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;433 434 NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;435 436 rc = nx_build_sg_lists(nx_ctx, iv, req->dst, req->src,437 &to_process, processed + req->assoclen,438 csbcpb->cpb.aes_ccm.iv_or_ctr);439 if (rc)440 goto out;441 442 rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,443 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);444 if (rc)445 goto out;446 447 /* for partial completion, copy following for next448 * entry into loop...449 */450 memcpy(iv, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);451 memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,452 csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);453 memcpy(csbcpb->cpb.aes_ccm.in_s0,454 csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);455 456 NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;457 458 /* update stats */459 atomic_inc(&(nx_ctx->stats->aes_ops));460 atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count),461 &(nx_ctx->stats->aes_bytes));462 463 processed += to_process;464 465 } while (processed < nbytes);466 467 /* copy out the auth tag */468 scatterwalk_map_and_copy(csbcpb->cpb.aes_ccm.out_pat_or_mac,469 req->dst, nbytes + req->assoclen, authsize,470 SCATTERWALK_TO_SG);471 472out:473 spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);474 return rc;475}476 477static int ccm4309_aes_nx_encrypt(struct aead_request *req)478{479 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);480 struct nx_gcm_rctx *rctx = aead_request_ctx(req);481 u8 *iv = rctx->iv;482 483 iv[0] = 3;484 memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3);485 memcpy(iv + 4, req->iv, 8);486 487 return ccm_nx_encrypt(req, iv, req->assoclen - 8);488}489 490static int ccm_aes_nx_encrypt(struct aead_request *req)491{492 int rc;493 494 rc = crypto_ccm_check_iv(req->iv);495 if (rc)496 return rc;497 498 return ccm_nx_encrypt(req, req->iv, req->assoclen);499}500 501static int ccm4309_aes_nx_decrypt(struct aead_request *req)502{503 struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);504 struct nx_gcm_rctx *rctx = aead_request_ctx(req);505 u8 *iv = rctx->iv;506 507 iv[0] = 3;508 memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3);509 memcpy(iv + 4, req->iv, 8);510 511 return ccm_nx_decrypt(req, iv, req->assoclen - 8);512}513 514static int ccm_aes_nx_decrypt(struct aead_request *req)515{516 int rc;517 518 rc = crypto_ccm_check_iv(req->iv);519 if (rc)520 return rc;521 522 return ccm_nx_decrypt(req, req->iv, req->assoclen);523}524 525struct aead_alg nx_ccm_aes_alg = {526 .base = {527 .cra_name = "ccm(aes)",528 .cra_driver_name = "ccm-aes-nx",529 .cra_priority = 300,530 .cra_flags = CRYPTO_ALG_NEED_FALLBACK,531 .cra_blocksize = 1,532 .cra_ctxsize = sizeof(struct nx_crypto_ctx),533 .cra_module = THIS_MODULE,534 },535 .init = nx_crypto_ctx_aes_ccm_init,536 .exit = nx_crypto_ctx_aead_exit,537 .ivsize = AES_BLOCK_SIZE,538 .maxauthsize = AES_BLOCK_SIZE,539 .setkey = ccm_aes_nx_set_key,540 .setauthsize = ccm_aes_nx_setauthsize,541 .encrypt = ccm_aes_nx_encrypt,542 .decrypt = ccm_aes_nx_decrypt,543};544 545struct aead_alg nx_ccm4309_aes_alg = {546 .base = {547 .cra_name = "rfc4309(ccm(aes))",548 .cra_driver_name = "rfc4309-ccm-aes-nx",549 .cra_priority = 300,550 .cra_flags = CRYPTO_ALG_NEED_FALLBACK,551 .cra_blocksize = 1,552 .cra_ctxsize = sizeof(struct nx_crypto_ctx),553 .cra_module = THIS_MODULE,554 },555 .init = nx_crypto_ctx_aes_ccm_init,556 .exit = nx_crypto_ctx_aead_exit,557 .ivsize = 8,558 .maxauthsize = AES_BLOCK_SIZE,559 .setkey = ccm4309_aes_nx_set_key,560 .setauthsize = ccm4309_aes_nx_setauthsize,561 .encrypt = ccm4309_aes_nx_encrypt,562 .decrypt = ccm4309_aes_nx_decrypt,563};564