1638 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Marvell OcteonTX CPT driver3 *4 * Copyright (C) 2019 Marvell International Ltd.5 *6 * This program is free software; you can redistribute it and/or modify7 * it under the terms of the GNU General Public License version 2 as8 * published by the Free Software Foundation.9 */10 11#include <crypto/aes.h>12#include <crypto/authenc.h>13#include <crypto/cryptd.h>14#include <crypto/des.h>15#include <crypto/internal/aead.h>16#include <crypto/sha1.h>17#include <crypto/sha2.h>18#include <crypto/xts.h>19#include <crypto/scatterwalk.h>20#include <linux/sort.h>21#include <linux/module.h>22#include "otx_cptvf.h"23#include "otx_cptvf_algs.h"24#include "otx_cptvf_reqmgr.h"25 26#define CPT_MAX_VF_NUM 6427/* Size of salt in AES GCM mode */28#define AES_GCM_SALT_SIZE 429/* Size of IV in AES GCM mode */30#define AES_GCM_IV_SIZE 831/* Size of ICV (Integrity Check Value) in AES GCM mode */32#define AES_GCM_ICV_SIZE 1633/* Offset of IV in AES GCM mode */34#define AES_GCM_IV_OFFSET 835#define CONTROL_WORD_LEN 836#define KEY2_OFFSET 4837#define DMA_MODE_FLAG(dma_mode) \38 (((dma_mode) == OTX_CPT_DMA_GATHER_SCATTER) ? (1 << 7) : 0)39 40/* Truncated SHA digest size */41#define SHA1_TRUNC_DIGEST_SIZE 1242#define SHA256_TRUNC_DIGEST_SIZE 1643#define SHA384_TRUNC_DIGEST_SIZE 2444#define SHA512_TRUNC_DIGEST_SIZE 3245 46static DEFINE_MUTEX(mutex);47static int is_crypto_registered;48 49struct cpt_device_desc {50 enum otx_cptpf_type pf_type;51 struct pci_dev *dev;52 int num_queues;53};54 55struct cpt_device_table {56 atomic_t count;57 struct cpt_device_desc desc[CPT_MAX_VF_NUM];58};59 60static struct cpt_device_table se_devices = {61 .count = ATOMIC_INIT(0)62};63 64static struct cpt_device_table ae_devices = {65 .count = ATOMIC_INIT(0)66};67 68static struct otx_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg);69 70static inline int get_se_device(struct pci_dev **pdev, int *cpu_num)71{72 int count, ret = 0;73 74 count = atomic_read(&se_devices.count);75 if (count < 1)76 return -ENODEV;77 78 *cpu_num = get_cpu();79 80 if (se_devices.desc[0].pf_type == OTX_CPT_SE) {81 /*82 * On OcteonTX platform there is one CPT instruction queue bound83 * to each VF. We get maximum performance if one CPT queue84 * is available for each cpu otherwise CPT queues need to be85 * shared between cpus.86 */87 if (*cpu_num >= count)88 *cpu_num %= count;89 *pdev = se_devices.desc[*cpu_num].dev;90 } else {91 pr_err("Unknown PF type %d\n", se_devices.desc[0].pf_type);92 ret = -EINVAL;93 }94 put_cpu();95 96 return ret;97}98 99static inline int validate_hmac_cipher_null(struct otx_cpt_req_info *cpt_req)100{101 struct otx_cpt_req_ctx *rctx;102 struct aead_request *req;103 struct crypto_aead *tfm;104 105 req = container_of(cpt_req->areq, struct aead_request, base);106 tfm = crypto_aead_reqtfm(req);107 rctx = aead_request_ctx_dma(req);108 if (memcmp(rctx->fctx.hmac.s.hmac_calc,109 rctx->fctx.hmac.s.hmac_recv,110 crypto_aead_authsize(tfm)) != 0)111 return -EBADMSG;112 113 return 0;114}115 116static void otx_cpt_aead_callback(int status, void *arg1, void *arg2)117{118 struct otx_cpt_info_buffer *cpt_info = arg2;119 struct crypto_async_request *areq = arg1;120 struct otx_cpt_req_info *cpt_req;121 struct pci_dev *pdev;122 123 if (!cpt_info)124 goto complete;125 126 cpt_req = cpt_info->req;127 if (!status) {128 /*129 * When selected cipher is NULL we need to manually130 * verify whether calculated hmac value matches131 * received hmac value132 */133 if (cpt_req->req_type == OTX_CPT_AEAD_ENC_DEC_NULL_REQ &&134 !cpt_req->is_enc)135 status = validate_hmac_cipher_null(cpt_req);136 }137 pdev = cpt_info->pdev;138 do_request_cleanup(pdev, cpt_info);139 140complete:141 if (areq)142 crypto_request_complete(areq, status);143}144 145static void output_iv_copyback(struct crypto_async_request *areq)146{147 struct otx_cpt_req_info *req_info;148 struct skcipher_request *sreq;149 struct crypto_skcipher *stfm;150 struct otx_cpt_req_ctx *rctx;151 struct otx_cpt_enc_ctx *ctx;152 u32 start, ivsize;153 154 sreq = container_of(areq, struct skcipher_request, base);155 stfm = crypto_skcipher_reqtfm(sreq);156 ctx = crypto_skcipher_ctx(stfm);157 if (ctx->cipher_type == OTX_CPT_AES_CBC ||158 ctx->cipher_type == OTX_CPT_DES3_CBC) {159 rctx = skcipher_request_ctx_dma(sreq);160 req_info = &rctx->cpt_req;161 ivsize = crypto_skcipher_ivsize(stfm);162 start = sreq->cryptlen - ivsize;163 164 if (req_info->is_enc) {165 scatterwalk_map_and_copy(sreq->iv, sreq->dst, start,166 ivsize, 0);167 } else {168 if (sreq->src != sreq->dst) {169 scatterwalk_map_and_copy(sreq->iv, sreq->src,170 start, ivsize, 0);171 } else {172 memcpy(sreq->iv, req_info->iv_out, ivsize);173 kfree(req_info->iv_out);174 }175 }176 }177}178 179static void otx_cpt_skcipher_callback(int status, void *arg1, void *arg2)180{181 struct otx_cpt_info_buffer *cpt_info = arg2;182 struct crypto_async_request *areq = arg1;183 struct pci_dev *pdev;184 185 if (areq) {186 if (!status)187 output_iv_copyback(areq);188 if (cpt_info) {189 pdev = cpt_info->pdev;190 do_request_cleanup(pdev, cpt_info);191 }192 crypto_request_complete(areq, status);193 }194}195 196static inline void update_input_data(struct otx_cpt_req_info *req_info,197 struct scatterlist *inp_sg,198 u32 nbytes, u32 *argcnt)199{200 req_info->req.dlen += nbytes;201 202 while (nbytes) {203 u32 len = min(nbytes, inp_sg->length);204 u8 *ptr = sg_virt(inp_sg);205 206 req_info->in[*argcnt].vptr = (void *)ptr;207 req_info->in[*argcnt].size = len;208 nbytes -= len;209 ++(*argcnt);210 inp_sg = sg_next(inp_sg);211 }212}213 214static inline void update_output_data(struct otx_cpt_req_info *req_info,215 struct scatterlist *outp_sg,216 u32 offset, u32 nbytes, u32 *argcnt)217{218 req_info->rlen += nbytes;219 220 while (nbytes) {221 u32 len = min(nbytes, outp_sg->length - offset);222 u8 *ptr = sg_virt(outp_sg);223 224 req_info->out[*argcnt].vptr = (void *) (ptr + offset);225 req_info->out[*argcnt].size = len;226 nbytes -= len;227 ++(*argcnt);228 offset = 0;229 outp_sg = sg_next(outp_sg);230 }231}232 233static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc,234 u32 *argcnt)235{236 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);237 struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);238 struct otx_cpt_req_info *req_info = &rctx->cpt_req;239 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);240 struct otx_cpt_enc_ctx *ctx = crypto_tfm_ctx(tfm);241 struct otx_cpt_fc_ctx *fctx = &rctx->fctx;242 int ivsize = crypto_skcipher_ivsize(stfm);243 u32 start = req->cryptlen - ivsize;244 gfp_t flags;245 246 flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?247 GFP_KERNEL : GFP_ATOMIC;248 req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER;249 req_info->ctrl.s.se_req = OTX_CPT_SE_CORE_REQ;250 251 req_info->req.opcode.s.major = OTX_CPT_MAJOR_OP_FC |252 DMA_MODE_FLAG(OTX_CPT_DMA_GATHER_SCATTER);253 if (enc) {254 req_info->req.opcode.s.minor = 2;255 } else {256 req_info->req.opcode.s.minor = 3;257 if ((ctx->cipher_type == OTX_CPT_AES_CBC ||258 ctx->cipher_type == OTX_CPT_DES3_CBC) &&259 req->src == req->dst) {260 req_info->iv_out = kmalloc(ivsize, flags);261 if (!req_info->iv_out)262 return -ENOMEM;263 264 scatterwalk_map_and_copy(req_info->iv_out, req->src,265 start, ivsize, 0);266 }267 }268 /* Encryption data length */269 req_info->req.param1 = req->cryptlen;270 /* Authentication data length */271 req_info->req.param2 = 0;272 273 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;274 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;275 fctx->enc.enc_ctrl.e.iv_source = OTX_CPT_FROM_CPTR;276 277 if (ctx->cipher_type == OTX_CPT_AES_XTS)278 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);279 else280 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);281 282 memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm));283 284 fctx->enc.enc_ctrl.flags = cpu_to_be64(fctx->enc.enc_ctrl.cflags);285 286 /*287 * Storing Packet Data Information in offset288 * Control Word First 8 bytes289 */290 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;291 req_info->in[*argcnt].size = CONTROL_WORD_LEN;292 req_info->req.dlen += CONTROL_WORD_LEN;293 ++(*argcnt);294 295 req_info->in[*argcnt].vptr = (u8 *)fctx;296 req_info->in[*argcnt].size = sizeof(struct otx_cpt_fc_ctx);297 req_info->req.dlen += sizeof(struct otx_cpt_fc_ctx);298 299 ++(*argcnt);300 301 return 0;302}303 304static inline u32 create_input_list(struct skcipher_request *req, u32 enc,305 u32 enc_iv_len)306{307 struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);308 struct otx_cpt_req_info *req_info = &rctx->cpt_req;309 u32 argcnt = 0;310 int ret;311 312 ret = create_ctx_hdr(req, enc, &argcnt);313 if (ret)314 return ret;315 316 update_input_data(req_info, req->src, req->cryptlen, &argcnt);317 req_info->incnt = argcnt;318 319 return 0;320}321 322static inline void create_output_list(struct skcipher_request *req,323 u32 enc_iv_len)324{325 struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);326 struct otx_cpt_req_info *req_info = &rctx->cpt_req;327 u32 argcnt = 0;328 329 /*330 * OUTPUT Buffer Processing331 * AES encryption/decryption output would be332 * received in the following format333 *334 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|335 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ]336 */337 update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt);338 req_info->outcnt = argcnt;339}340 341static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc)342{343 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);344 struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);345 struct otx_cpt_req_info *req_info = &rctx->cpt_req;346 u32 enc_iv_len = crypto_skcipher_ivsize(stfm);347 struct pci_dev *pdev;348 int status, cpu_num;349 350 /* Validate that request doesn't exceed maximum CPT supported size */351 if (req->cryptlen > OTX_CPT_MAX_REQ_SIZE)352 return -E2BIG;353 354 /* Clear control words */355 rctx->ctrl_word.flags = 0;356 rctx->fctx.enc.enc_ctrl.flags = 0;357 358 status = create_input_list(req, enc, enc_iv_len);359 if (status)360 return status;361 create_output_list(req, enc_iv_len);362 363 status = get_se_device(&pdev, &cpu_num);364 if (status)365 return status;366 367 req_info->callback = (void *)otx_cpt_skcipher_callback;368 req_info->areq = &req->base;369 req_info->req_type = OTX_CPT_ENC_DEC_REQ;370 req_info->is_enc = enc;371 req_info->is_trunc_hmac = false;372 req_info->ctrl.s.grp = 0;373 374 /*375 * We perform an asynchronous send and once376 * the request is completed the driver would377 * intimate through registered call back functions378 */379 status = otx_cpt_do_request(pdev, req_info, cpu_num);380 381 return status;382}383 384static int otx_cpt_skcipher_encrypt(struct skcipher_request *req)385{386 return cpt_enc_dec(req, true);387}388 389static int otx_cpt_skcipher_decrypt(struct skcipher_request *req)390{391 return cpt_enc_dec(req, false);392}393 394static int otx_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm,395 const u8 *key, u32 keylen)396{397 struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);398 const u8 *key2 = key + (keylen / 2);399 const u8 *key1 = key;400 int ret;401 402 ret = xts_verify_key(tfm, key, keylen);403 if (ret)404 return ret;405 ctx->key_len = keylen;406 memcpy(ctx->enc_key, key1, keylen / 2);407 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);408 ctx->cipher_type = OTX_CPT_AES_XTS;409 switch (ctx->key_len) {410 case 2 * AES_KEYSIZE_128:411 ctx->key_type = OTX_CPT_AES_128_BIT;412 break;413 case 2 * AES_KEYSIZE_256:414 ctx->key_type = OTX_CPT_AES_256_BIT;415 break;416 default:417 return -EINVAL;418 }419 420 return 0;421}422 423static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key,424 u32 keylen, u8 cipher_type)425{426 struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);427 428 if (keylen != DES3_EDE_KEY_SIZE)429 return -EINVAL;430 431 ctx->key_len = keylen;432 ctx->cipher_type = cipher_type;433 434 memcpy(ctx->enc_key, key, keylen);435 436 return 0;437}438 439static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,440 u32 keylen, u8 cipher_type)441{442 struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);443 444 switch (keylen) {445 case AES_KEYSIZE_128:446 ctx->key_type = OTX_CPT_AES_128_BIT;447 break;448 case AES_KEYSIZE_192:449 ctx->key_type = OTX_CPT_AES_192_BIT;450 break;451 case AES_KEYSIZE_256:452 ctx->key_type = OTX_CPT_AES_256_BIT;453 break;454 default:455 return -EINVAL;456 }457 ctx->key_len = keylen;458 ctx->cipher_type = cipher_type;459 460 memcpy(ctx->enc_key, key, keylen);461 462 return 0;463}464 465static int otx_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm,466 const u8 *key, u32 keylen)467{468 return cpt_aes_setkey(tfm, key, keylen, OTX_CPT_AES_CBC);469}470 471static int otx_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm,472 const u8 *key, u32 keylen)473{474 return cpt_aes_setkey(tfm, key, keylen, OTX_CPT_AES_ECB);475}476 477static int otx_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm,478 const u8 *key, u32 keylen)479{480 return cpt_des_setkey(tfm, key, keylen, OTX_CPT_DES3_CBC);481}482 483static int otx_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm,484 const u8 *key, u32 keylen)485{486 return cpt_des_setkey(tfm, key, keylen, OTX_CPT_DES3_ECB);487}488 489static int otx_cpt_enc_dec_init(struct crypto_skcipher *tfm)490{491 struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);492 493 memset(ctx, 0, sizeof(*ctx));494 /*495 * Additional memory for skcipher_request is496 * allocated since the cryptd daemon uses497 * this memory for request_ctx information498 */499 crypto_skcipher_set_reqsize_dma(500 tfm, sizeof(struct otx_cpt_req_ctx) +501 sizeof(struct skcipher_request));502 503 return 0;504}505 506static int cpt_aead_init(struct crypto_aead *tfm, u8 cipher_type, u8 mac_type)507{508 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);509 510 ctx->cipher_type = cipher_type;511 ctx->mac_type = mac_type;512 513 switch (ctx->mac_type) {514 case OTX_CPT_SHA1:515 ctx->hashalg = crypto_alloc_shash("sha1", 0, 0);516 break;517 518 case OTX_CPT_SHA256:519 ctx->hashalg = crypto_alloc_shash("sha256", 0, 0);520 break;521 522 case OTX_CPT_SHA384:523 ctx->hashalg = crypto_alloc_shash("sha384", 0, 0);524 break;525 526 case OTX_CPT_SHA512:527 ctx->hashalg = crypto_alloc_shash("sha512", 0, 0);528 break;529 }530 531 if (IS_ERR(ctx->hashalg))532 return PTR_ERR(ctx->hashalg);533 534 crypto_aead_set_reqsize_dma(tfm, sizeof(struct otx_cpt_req_ctx));535 536 if (!ctx->hashalg)537 return 0;538 539 /*540 * When selected cipher is NULL we use HMAC opcode instead of541 * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms542 * for calculating ipad and opad543 */544 if (ctx->cipher_type != OTX_CPT_CIPHER_NULL) {545 int ss = crypto_shash_statesize(ctx->hashalg);546 547 ctx->ipad = kzalloc(ss, GFP_KERNEL);548 if (!ctx->ipad) {549 crypto_free_shash(ctx->hashalg);550 return -ENOMEM;551 }552 553 ctx->opad = kzalloc(ss, GFP_KERNEL);554 if (!ctx->opad) {555 kfree(ctx->ipad);556 crypto_free_shash(ctx->hashalg);557 return -ENOMEM;558 }559 }560 561 ctx->sdesc = alloc_sdesc(ctx->hashalg);562 if (!ctx->sdesc) {563 kfree(ctx->opad);564 kfree(ctx->ipad);565 crypto_free_shash(ctx->hashalg);566 return -ENOMEM;567 }568 569 return 0;570}571 572static int otx_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm)573{574 return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA1);575}576 577static int otx_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm)578{579 return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA256);580}581 582static int otx_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm)583{584 return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA384);585}586 587static int otx_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm)588{589 return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA512);590}591 592static int otx_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm)593{594 return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA1);595}596 597static int otx_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm)598{599 return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA256);600}601 602static int otx_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm)603{604 return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA384);605}606 607static int otx_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm)608{609 return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA512);610}611 612static int otx_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)613{614 return cpt_aead_init(tfm, OTX_CPT_AES_GCM, OTX_CPT_MAC_NULL);615}616 617static void otx_cpt_aead_exit(struct crypto_aead *tfm)618{619 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);620 621 kfree(ctx->ipad);622 kfree(ctx->opad);623 crypto_free_shash(ctx->hashalg);624 kfree(ctx->sdesc);625}626 627/*628 * This is the Integrity Check Value validation (aka the authentication tag629 * length)630 */631static int otx_cpt_aead_set_authsize(struct crypto_aead *tfm,632 unsigned int authsize)633{634 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);635 636 switch (ctx->mac_type) {637 case OTX_CPT_SHA1:638 if (authsize != SHA1_DIGEST_SIZE &&639 authsize != SHA1_TRUNC_DIGEST_SIZE)640 return -EINVAL;641 642 if (authsize == SHA1_TRUNC_DIGEST_SIZE)643 ctx->is_trunc_hmac = true;644 break;645 646 case OTX_CPT_SHA256:647 if (authsize != SHA256_DIGEST_SIZE &&648 authsize != SHA256_TRUNC_DIGEST_SIZE)649 return -EINVAL;650 651 if (authsize == SHA256_TRUNC_DIGEST_SIZE)652 ctx->is_trunc_hmac = true;653 break;654 655 case OTX_CPT_SHA384:656 if (authsize != SHA384_DIGEST_SIZE &&657 authsize != SHA384_TRUNC_DIGEST_SIZE)658 return -EINVAL;659 660 if (authsize == SHA384_TRUNC_DIGEST_SIZE)661 ctx->is_trunc_hmac = true;662 break;663 664 case OTX_CPT_SHA512:665 if (authsize != SHA512_DIGEST_SIZE &&666 authsize != SHA512_TRUNC_DIGEST_SIZE)667 return -EINVAL;668 669 if (authsize == SHA512_TRUNC_DIGEST_SIZE)670 ctx->is_trunc_hmac = true;671 break;672 673 case OTX_CPT_MAC_NULL:674 if (ctx->cipher_type == OTX_CPT_AES_GCM) {675 if (authsize != AES_GCM_ICV_SIZE)676 return -EINVAL;677 } else678 return -EINVAL;679 break;680 681 default:682 return -EINVAL;683 }684 685 tfm->authsize = authsize;686 return 0;687}688 689static struct otx_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg)690{691 struct otx_cpt_sdesc *sdesc;692 int size;693 694 size = sizeof(struct shash_desc) + crypto_shash_descsize(alg);695 sdesc = kmalloc(size, GFP_KERNEL);696 if (!sdesc)697 return NULL;698 699 sdesc->shash.tfm = alg;700 701 return sdesc;702}703 704static inline void swap_data32(void *buf, u32 len)705{706 cpu_to_be32_array(buf, buf, len / 4);707}708 709static inline void swap_data64(void *buf, u32 len)710{711 __be64 *dst = buf;712 u64 *src = buf;713 int i = 0;714 715 for (i = 0 ; i < len / 8; i++, src++, dst++)716 *dst = cpu_to_be64p(src);717}718 719static int swap_pad(u8 mac_type, u8 *pad)720{721 struct sha512_state *sha512;722 struct sha256_state *sha256;723 struct sha1_state *sha1;724 725 switch (mac_type) {726 case OTX_CPT_SHA1:727 sha1 = (struct sha1_state *)pad;728 swap_data32(sha1->state, SHA1_DIGEST_SIZE);729 break;730 731 case OTX_CPT_SHA256:732 sha256 = (struct sha256_state *)pad;733 swap_data32(sha256->state, SHA256_DIGEST_SIZE);734 break;735 736 case OTX_CPT_SHA384:737 case OTX_CPT_SHA512:738 sha512 = (struct sha512_state *)pad;739 swap_data64(sha512->state, SHA512_DIGEST_SIZE);740 break;741 742 default:743 return -EINVAL;744 }745 746 return 0;747}748 749static int aead_hmac_init(struct crypto_aead *cipher,750 struct crypto_authenc_keys *keys)751{752 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);753 int ds = crypto_shash_digestsize(ctx->hashalg);754 int bs = crypto_shash_blocksize(ctx->hashalg);755 int authkeylen = keys->authkeylen;756 u8 *ipad = NULL, *opad = NULL;757 int icount = 0;758 int ret;759 760 if (authkeylen > bs) {761 ret = crypto_shash_digest(&ctx->sdesc->shash, keys->authkey,762 authkeylen, ctx->key);763 if (ret)764 return ret;765 authkeylen = ds;766 } else767 memcpy(ctx->key, keys->authkey, authkeylen);768 769 ctx->enc_key_len = keys->enckeylen;770 ctx->auth_key_len = authkeylen;771 772 if (ctx->cipher_type == OTX_CPT_CIPHER_NULL)773 return keys->enckeylen ? -EINVAL : 0;774 775 switch (keys->enckeylen) {776 case AES_KEYSIZE_128:777 ctx->key_type = OTX_CPT_AES_128_BIT;778 break;779 case AES_KEYSIZE_192:780 ctx->key_type = OTX_CPT_AES_192_BIT;781 break;782 case AES_KEYSIZE_256:783 ctx->key_type = OTX_CPT_AES_256_BIT;784 break;785 default:786 /* Invalid key length */787 return -EINVAL;788 }789 790 memcpy(ctx->key + authkeylen, keys->enckey, keys->enckeylen);791 792 ipad = ctx->ipad;793 opad = ctx->opad;794 795 memcpy(ipad, ctx->key, authkeylen);796 memset(ipad + authkeylen, 0, bs - authkeylen);797 memcpy(opad, ipad, bs);798 799 for (icount = 0; icount < bs; icount++) {800 ipad[icount] ^= 0x36;801 opad[icount] ^= 0x5c;802 }803 804 /*805 * Partial Hash calculated from the software806 * algorithm is retrieved for IPAD & OPAD807 */808 809 /* IPAD Calculation */810 crypto_shash_init(&ctx->sdesc->shash);811 crypto_shash_update(&ctx->sdesc->shash, ipad, bs);812 crypto_shash_export(&ctx->sdesc->shash, ipad);813 ret = swap_pad(ctx->mac_type, ipad);814 if (ret)815 goto calc_fail;816 817 /* OPAD Calculation */818 crypto_shash_init(&ctx->sdesc->shash);819 crypto_shash_update(&ctx->sdesc->shash, opad, bs);820 crypto_shash_export(&ctx->sdesc->shash, opad);821 ret = swap_pad(ctx->mac_type, opad);822 823calc_fail:824 return ret;825}826 827static int otx_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher,828 const unsigned char *key,829 unsigned int keylen)830{831 struct crypto_authenc_keys authenc_keys;832 int status;833 834 status = crypto_authenc_extractkeys(&authenc_keys, key, keylen);835 if (status)836 goto badkey;837 838 status = aead_hmac_init(cipher, &authenc_keys);839 840badkey:841 return status;842}843 844static int otx_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher,845 const unsigned char *key,846 unsigned int keylen)847{848 return otx_cpt_aead_cbc_aes_sha_setkey(cipher, key, keylen);849}850 851static int otx_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,852 const unsigned char *key,853 unsigned int keylen)854{855 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);856 857 /*858 * For aes gcm we expect to get encryption key (16, 24, 32 bytes)859 * and salt (4 bytes)860 */861 switch (keylen) {862 case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE:863 ctx->key_type = OTX_CPT_AES_128_BIT;864 ctx->enc_key_len = AES_KEYSIZE_128;865 break;866 case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE:867 ctx->key_type = OTX_CPT_AES_192_BIT;868 ctx->enc_key_len = AES_KEYSIZE_192;869 break;870 case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE:871 ctx->key_type = OTX_CPT_AES_256_BIT;872 ctx->enc_key_len = AES_KEYSIZE_256;873 break;874 default:875 /* Invalid key and salt length */876 return -EINVAL;877 }878 879 /* Store encryption key and salt */880 memcpy(ctx->key, key, keylen);881 882 return 0;883}884 885static inline u32 create_aead_ctx_hdr(struct aead_request *req, u32 enc,886 u32 *argcnt)887{888 struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);889 struct crypto_aead *tfm = crypto_aead_reqtfm(req);890 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);891 struct otx_cpt_req_info *req_info = &rctx->cpt_req;892 struct otx_cpt_fc_ctx *fctx = &rctx->fctx;893 int mac_len = crypto_aead_authsize(tfm);894 int ds;895 896 rctx->ctrl_word.e.enc_data_offset = req->assoclen;897 898 switch (ctx->cipher_type) {899 case OTX_CPT_AES_CBC:900 fctx->enc.enc_ctrl.e.iv_source = OTX_CPT_FROM_CPTR;901 /* Copy encryption key to context */902 memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len,903 ctx->enc_key_len);904 /* Copy IV to context */905 memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm));906 907 ds = crypto_shash_digestsize(ctx->hashalg);908 if (ctx->mac_type == OTX_CPT_SHA384)909 ds = SHA512_DIGEST_SIZE;910 if (ctx->ipad)911 memcpy(fctx->hmac.e.ipad, ctx->ipad, ds);912 if (ctx->opad)913 memcpy(fctx->hmac.e.opad, ctx->opad, ds);914 break;915 916 case OTX_CPT_AES_GCM:917 fctx->enc.enc_ctrl.e.iv_source = OTX_CPT_FROM_DPTR;918 /* Copy encryption key to context */919 memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len);920 /* Copy salt to context */921 memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len,922 AES_GCM_SALT_SIZE);923 924 rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET;925 break;926 927 default:928 /* Unknown cipher type */929 return -EINVAL;930 }931 rctx->ctrl_word.flags = cpu_to_be64(rctx->ctrl_word.cflags);932 933 req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER;934 req_info->ctrl.s.se_req = OTX_CPT_SE_CORE_REQ;935 req_info->req.opcode.s.major = OTX_CPT_MAJOR_OP_FC |936 DMA_MODE_FLAG(OTX_CPT_DMA_GATHER_SCATTER);937 if (enc) {938 req_info->req.opcode.s.minor = 2;939 req_info->req.param1 = req->cryptlen;940 req_info->req.param2 = req->cryptlen + req->assoclen;941 } else {942 req_info->req.opcode.s.minor = 3;943 req_info->req.param1 = req->cryptlen - mac_len;944 req_info->req.param2 = req->cryptlen + req->assoclen - mac_len;945 }946 947 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;948 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;949 fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type;950 fctx->enc.enc_ctrl.e.mac_len = mac_len;951 fctx->enc.enc_ctrl.flags = cpu_to_be64(fctx->enc.enc_ctrl.cflags);952 953 /*954 * Storing Packet Data Information in offset955 * Control Word First 8 bytes956 */957 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;958 req_info->in[*argcnt].size = CONTROL_WORD_LEN;959 req_info->req.dlen += CONTROL_WORD_LEN;960 ++(*argcnt);961 962 req_info->in[*argcnt].vptr = (u8 *)fctx;963 req_info->in[*argcnt].size = sizeof(struct otx_cpt_fc_ctx);964 req_info->req.dlen += sizeof(struct otx_cpt_fc_ctx);965 ++(*argcnt);966 967 return 0;968}969 970static inline u32 create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,971 u32 enc)972{973 struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);974 struct crypto_aead *tfm = crypto_aead_reqtfm(req);975 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);976 struct otx_cpt_req_info *req_info = &rctx->cpt_req;977 978 req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER;979 req_info->ctrl.s.se_req = OTX_CPT_SE_CORE_REQ;980 req_info->req.opcode.s.major = OTX_CPT_MAJOR_OP_HMAC |981 DMA_MODE_FLAG(OTX_CPT_DMA_GATHER_SCATTER);982 req_info->is_trunc_hmac = ctx->is_trunc_hmac;983 984 req_info->req.opcode.s.minor = 0;985 req_info->req.param1 = ctx->auth_key_len;986 req_info->req.param2 = ctx->mac_type << 8;987 988 /* Add authentication key */989 req_info->in[*argcnt].vptr = ctx->key;990 req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8);991 req_info->req.dlen += round_up(ctx->auth_key_len, 8);992 ++(*argcnt);993 994 return 0;995}996 997static inline u32 create_aead_input_list(struct aead_request *req, u32 enc)998{999 struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);1000 struct otx_cpt_req_info *req_info = &rctx->cpt_req;1001 u32 inputlen = req->cryptlen + req->assoclen;1002 u32 status, argcnt = 0;1003 1004 status = create_aead_ctx_hdr(req, enc, &argcnt);1005 if (status)1006 return status;1007 update_input_data(req_info, req->src, inputlen, &argcnt);1008 req_info->incnt = argcnt;1009 1010 return 0;1011}1012 1013static inline u32 create_aead_output_list(struct aead_request *req, u32 enc,1014 u32 mac_len)1015{1016 struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);1017 struct otx_cpt_req_info *req_info = &rctx->cpt_req;1018 u32 argcnt = 0, outputlen = 0;1019 1020 if (enc)1021 outputlen = req->cryptlen + req->assoclen + mac_len;1022 else1023 outputlen = req->cryptlen + req->assoclen - mac_len;1024 1025 update_output_data(req_info, req->dst, 0, outputlen, &argcnt);1026 req_info->outcnt = argcnt;1027 1028 return 0;1029}1030 1031static inline u32 create_aead_null_input_list(struct aead_request *req,1032 u32 enc, u32 mac_len)1033{1034 struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);1035 struct otx_cpt_req_info *req_info = &rctx->cpt_req;1036 u32 inputlen, argcnt = 0;1037 1038 if (enc)1039 inputlen = req->cryptlen + req->assoclen;1040 else1041 inputlen = req->cryptlen + req->assoclen - mac_len;1042 1043 create_hmac_ctx_hdr(req, &argcnt, enc);1044 update_input_data(req_info, req->src, inputlen, &argcnt);1045 req_info->incnt = argcnt;1046 1047 return 0;1048}1049 1050static inline u32 create_aead_null_output_list(struct aead_request *req,1051 u32 enc, u32 mac_len)1052{1053 struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);1054 struct otx_cpt_req_info *req_info = &rctx->cpt_req;1055 struct scatterlist *dst;1056 u8 *ptr = NULL;1057 int argcnt = 0, status, offset;1058 u32 inputlen;1059 1060 if (enc)1061 inputlen = req->cryptlen + req->assoclen;1062 else1063 inputlen = req->cryptlen + req->assoclen - mac_len;1064 1065 /*1066 * If source and destination are different1067 * then copy payload to destination1068 */1069 if (req->src != req->dst) {1070 1071 ptr = kmalloc(inputlen, (req_info->areq->flags &1072 CRYPTO_TFM_REQ_MAY_SLEEP) ?1073 GFP_KERNEL : GFP_ATOMIC);1074 if (!ptr) {1075 status = -ENOMEM;1076 goto error;1077 }1078 1079 status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr,1080 inputlen);1081 if (status != inputlen) {1082 status = -EINVAL;1083 goto error_free;1084 }1085 status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr,1086 inputlen);1087 if (status != inputlen) {1088 status = -EINVAL;1089 goto error_free;1090 }1091 kfree(ptr);1092 }1093 1094 if (enc) {1095 /*1096 * In an encryption scenario hmac needs1097 * to be appended after payload1098 */1099 dst = req->dst;1100 offset = inputlen;1101 while (offset >= dst->length) {1102 offset -= dst->length;1103 dst = sg_next(dst);1104 if (!dst) {1105 status = -ENOENT;1106 goto error;1107 }1108 }1109 1110 update_output_data(req_info, dst, offset, mac_len, &argcnt);1111 } else {1112 /*1113 * In a decryption scenario calculated hmac for received1114 * payload needs to be compare with hmac received1115 */1116 status = sg_copy_buffer(req->src, sg_nents(req->src),1117 rctx->fctx.hmac.s.hmac_recv, mac_len,1118 inputlen, true);1119 if (status != mac_len) {1120 status = -EINVAL;1121 goto error;1122 }1123 1124 req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc;1125 req_info->out[argcnt].size = mac_len;1126 argcnt++;1127 }1128 1129 req_info->outcnt = argcnt;1130 return 0;1131 1132error_free:1133 kfree(ptr);1134error:1135 return status;1136}1137 1138static u32 cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc)1139{1140 struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);1141 struct otx_cpt_req_info *req_info = &rctx->cpt_req;1142 struct crypto_aead *tfm = crypto_aead_reqtfm(req);1143 struct pci_dev *pdev;1144 u32 status, cpu_num;1145 1146 /* Clear control words */1147 rctx->ctrl_word.flags = 0;1148 rctx->fctx.enc.enc_ctrl.flags = 0;1149 1150 req_info->callback = otx_cpt_aead_callback;1151 req_info->areq = &req->base;1152 req_info->req_type = reg_type;1153 req_info->is_enc = enc;1154 req_info->is_trunc_hmac = false;1155 1156 switch (reg_type) {1157 case OTX_CPT_AEAD_ENC_DEC_REQ:1158 status = create_aead_input_list(req, enc);1159 if (status)1160 return status;1161 status = create_aead_output_list(req, enc,1162 crypto_aead_authsize(tfm));1163 if (status)1164 return status;1165 break;1166 1167 case OTX_CPT_AEAD_ENC_DEC_NULL_REQ:1168 status = create_aead_null_input_list(req, enc,1169 crypto_aead_authsize(tfm));1170 if (status)1171 return status;1172 status = create_aead_null_output_list(req, enc,1173 crypto_aead_authsize(tfm));1174 if (status)1175 return status;1176 break;1177 1178 default:1179 return -EINVAL;1180 }1181 1182 /* Validate that request doesn't exceed maximum CPT supported size */1183 if (req_info->req.param1 > OTX_CPT_MAX_REQ_SIZE ||1184 req_info->req.param2 > OTX_CPT_MAX_REQ_SIZE)1185 return -E2BIG;1186 1187 status = get_se_device(&pdev, &cpu_num);1188 if (status)1189 return status;1190 1191 req_info->ctrl.s.grp = 0;1192 1193 status = otx_cpt_do_request(pdev, req_info, cpu_num);1194 /*1195 * We perform an asynchronous send and once1196 * the request is completed the driver would1197 * intimate through registered call back functions1198 */1199 return status;1200}1201 1202static int otx_cpt_aead_encrypt(struct aead_request *req)1203{1204 return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_REQ, true);1205}1206 1207static int otx_cpt_aead_decrypt(struct aead_request *req)1208{1209 return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_REQ, false);1210}1211 1212static int otx_cpt_aead_null_encrypt(struct aead_request *req)1213{1214 return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_NULL_REQ, true);1215}1216 1217static int otx_cpt_aead_null_decrypt(struct aead_request *req)1218{1219 return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_NULL_REQ, false);1220}1221 1222static struct skcipher_alg otx_cpt_skciphers[] = { {1223 .base.cra_name = "xts(aes)",1224 .base.cra_driver_name = "cpt_xts_aes",1225 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1226 .base.cra_blocksize = AES_BLOCK_SIZE,1227 .base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx),1228 .base.cra_alignmask = 7,1229 .base.cra_priority = 4001,1230 .base.cra_module = THIS_MODULE,1231 1232 .init = otx_cpt_enc_dec_init,1233 .ivsize = AES_BLOCK_SIZE,1234 .min_keysize = 2 * AES_MIN_KEY_SIZE,1235 .max_keysize = 2 * AES_MAX_KEY_SIZE,1236 .setkey = otx_cpt_skcipher_xts_setkey,1237 .encrypt = otx_cpt_skcipher_encrypt,1238 .decrypt = otx_cpt_skcipher_decrypt,1239}, {1240 .base.cra_name = "cbc(aes)",1241 .base.cra_driver_name = "cpt_cbc_aes",1242 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1243 .base.cra_blocksize = AES_BLOCK_SIZE,1244 .base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx),1245 .base.cra_alignmask = 7,1246 .base.cra_priority = 4001,1247 .base.cra_module = THIS_MODULE,1248 1249 .init = otx_cpt_enc_dec_init,1250 .ivsize = AES_BLOCK_SIZE,1251 .min_keysize = AES_MIN_KEY_SIZE,1252 .max_keysize = AES_MAX_KEY_SIZE,1253 .setkey = otx_cpt_skcipher_cbc_aes_setkey,1254 .encrypt = otx_cpt_skcipher_encrypt,1255 .decrypt = otx_cpt_skcipher_decrypt,1256}, {1257 .base.cra_name = "ecb(aes)",1258 .base.cra_driver_name = "cpt_ecb_aes",1259 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1260 .base.cra_blocksize = AES_BLOCK_SIZE,1261 .base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx),1262 .base.cra_alignmask = 7,1263 .base.cra_priority = 4001,1264 .base.cra_module = THIS_MODULE,1265 1266 .init = otx_cpt_enc_dec_init,1267 .ivsize = 0,1268 .min_keysize = AES_MIN_KEY_SIZE,1269 .max_keysize = AES_MAX_KEY_SIZE,1270 .setkey = otx_cpt_skcipher_ecb_aes_setkey,1271 .encrypt = otx_cpt_skcipher_encrypt,1272 .decrypt = otx_cpt_skcipher_decrypt,1273}, {1274 .base.cra_name = "cbc(des3_ede)",1275 .base.cra_driver_name = "cpt_cbc_des3_ede",1276 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1277 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,1278 .base.cra_ctxsize = sizeof(struct otx_cpt_des3_ctx),1279 .base.cra_alignmask = 7,1280 .base.cra_priority = 4001,1281 .base.cra_module = THIS_MODULE,1282 1283 .init = otx_cpt_enc_dec_init,1284 .min_keysize = DES3_EDE_KEY_SIZE,1285 .max_keysize = DES3_EDE_KEY_SIZE,1286 .ivsize = DES_BLOCK_SIZE,1287 .setkey = otx_cpt_skcipher_cbc_des3_setkey,1288 .encrypt = otx_cpt_skcipher_encrypt,1289 .decrypt = otx_cpt_skcipher_decrypt,1290}, {1291 .base.cra_name = "ecb(des3_ede)",1292 .base.cra_driver_name = "cpt_ecb_des3_ede",1293 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1294 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,1295 .base.cra_ctxsize = sizeof(struct otx_cpt_des3_ctx),1296 .base.cra_alignmask = 7,1297 .base.cra_priority = 4001,1298 .base.cra_module = THIS_MODULE,1299 1300 .init = otx_cpt_enc_dec_init,1301 .min_keysize = DES3_EDE_KEY_SIZE,1302 .max_keysize = DES3_EDE_KEY_SIZE,1303 .ivsize = 0,1304 .setkey = otx_cpt_skcipher_ecb_des3_setkey,1305 .encrypt = otx_cpt_skcipher_encrypt,1306 .decrypt = otx_cpt_skcipher_decrypt,1307} };1308 1309static struct aead_alg otx_cpt_aeads[] = { {1310 .base = {1311 .cra_name = "authenc(hmac(sha1),cbc(aes))",1312 .cra_driver_name = "cpt_hmac_sha1_cbc_aes",1313 .cra_blocksize = AES_BLOCK_SIZE,1314 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1315 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1316 .cra_priority = 4001,1317 .cra_alignmask = 0,1318 .cra_module = THIS_MODULE,1319 },1320 .init = otx_cpt_aead_cbc_aes_sha1_init,1321 .exit = otx_cpt_aead_exit,1322 .setkey = otx_cpt_aead_cbc_aes_sha_setkey,1323 .setauthsize = otx_cpt_aead_set_authsize,1324 .encrypt = otx_cpt_aead_encrypt,1325 .decrypt = otx_cpt_aead_decrypt,1326 .ivsize = AES_BLOCK_SIZE,1327 .maxauthsize = SHA1_DIGEST_SIZE,1328}, {1329 .base = {1330 .cra_name = "authenc(hmac(sha256),cbc(aes))",1331 .cra_driver_name = "cpt_hmac_sha256_cbc_aes",1332 .cra_blocksize = AES_BLOCK_SIZE,1333 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1334 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1335 .cra_priority = 4001,1336 .cra_alignmask = 0,1337 .cra_module = THIS_MODULE,1338 },1339 .init = otx_cpt_aead_cbc_aes_sha256_init,1340 .exit = otx_cpt_aead_exit,1341 .setkey = otx_cpt_aead_cbc_aes_sha_setkey,1342 .setauthsize = otx_cpt_aead_set_authsize,1343 .encrypt = otx_cpt_aead_encrypt,1344 .decrypt = otx_cpt_aead_decrypt,1345 .ivsize = AES_BLOCK_SIZE,1346 .maxauthsize = SHA256_DIGEST_SIZE,1347}, {1348 .base = {1349 .cra_name = "authenc(hmac(sha384),cbc(aes))",1350 .cra_driver_name = "cpt_hmac_sha384_cbc_aes",1351 .cra_blocksize = AES_BLOCK_SIZE,1352 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1353 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1354 .cra_priority = 4001,1355 .cra_alignmask = 0,1356 .cra_module = THIS_MODULE,1357 },1358 .init = otx_cpt_aead_cbc_aes_sha384_init,1359 .exit = otx_cpt_aead_exit,1360 .setkey = otx_cpt_aead_cbc_aes_sha_setkey,1361 .setauthsize = otx_cpt_aead_set_authsize,1362 .encrypt = otx_cpt_aead_encrypt,1363 .decrypt = otx_cpt_aead_decrypt,1364 .ivsize = AES_BLOCK_SIZE,1365 .maxauthsize = SHA384_DIGEST_SIZE,1366}, {1367 .base = {1368 .cra_name = "authenc(hmac(sha512),cbc(aes))",1369 .cra_driver_name = "cpt_hmac_sha512_cbc_aes",1370 .cra_blocksize = AES_BLOCK_SIZE,1371 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1372 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1373 .cra_priority = 4001,1374 .cra_alignmask = 0,1375 .cra_module = THIS_MODULE,1376 },1377 .init = otx_cpt_aead_cbc_aes_sha512_init,1378 .exit = otx_cpt_aead_exit,1379 .setkey = otx_cpt_aead_cbc_aes_sha_setkey,1380 .setauthsize = otx_cpt_aead_set_authsize,1381 .encrypt = otx_cpt_aead_encrypt,1382 .decrypt = otx_cpt_aead_decrypt,1383 .ivsize = AES_BLOCK_SIZE,1384 .maxauthsize = SHA512_DIGEST_SIZE,1385}, {1386 .base = {1387 .cra_name = "authenc(hmac(sha1),ecb(cipher_null))",1388 .cra_driver_name = "cpt_hmac_sha1_ecb_null",1389 .cra_blocksize = 1,1390 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1391 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1392 .cra_priority = 4001,1393 .cra_alignmask = 0,1394 .cra_module = THIS_MODULE,1395 },1396 .init = otx_cpt_aead_ecb_null_sha1_init,1397 .exit = otx_cpt_aead_exit,1398 .setkey = otx_cpt_aead_ecb_null_sha_setkey,1399 .setauthsize = otx_cpt_aead_set_authsize,1400 .encrypt = otx_cpt_aead_null_encrypt,1401 .decrypt = otx_cpt_aead_null_decrypt,1402 .ivsize = 0,1403 .maxauthsize = SHA1_DIGEST_SIZE,1404}, {1405 .base = {1406 .cra_name = "authenc(hmac(sha256),ecb(cipher_null))",1407 .cra_driver_name = "cpt_hmac_sha256_ecb_null",1408 .cra_blocksize = 1,1409 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1410 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1411 .cra_priority = 4001,1412 .cra_alignmask = 0,1413 .cra_module = THIS_MODULE,1414 },1415 .init = otx_cpt_aead_ecb_null_sha256_init,1416 .exit = otx_cpt_aead_exit,1417 .setkey = otx_cpt_aead_ecb_null_sha_setkey,1418 .setauthsize = otx_cpt_aead_set_authsize,1419 .encrypt = otx_cpt_aead_null_encrypt,1420 .decrypt = otx_cpt_aead_null_decrypt,1421 .ivsize = 0,1422 .maxauthsize = SHA256_DIGEST_SIZE,1423}, {1424 .base = {1425 .cra_name = "authenc(hmac(sha384),ecb(cipher_null))",1426 .cra_driver_name = "cpt_hmac_sha384_ecb_null",1427 .cra_blocksize = 1,1428 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1429 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1430 .cra_priority = 4001,1431 .cra_alignmask = 0,1432 .cra_module = THIS_MODULE,1433 },1434 .init = otx_cpt_aead_ecb_null_sha384_init,1435 .exit = otx_cpt_aead_exit,1436 .setkey = otx_cpt_aead_ecb_null_sha_setkey,1437 .setauthsize = otx_cpt_aead_set_authsize,1438 .encrypt = otx_cpt_aead_null_encrypt,1439 .decrypt = otx_cpt_aead_null_decrypt,1440 .ivsize = 0,1441 .maxauthsize = SHA384_DIGEST_SIZE,1442}, {1443 .base = {1444 .cra_name = "authenc(hmac(sha512),ecb(cipher_null))",1445 .cra_driver_name = "cpt_hmac_sha512_ecb_null",1446 .cra_blocksize = 1,1447 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1448 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1449 .cra_priority = 4001,1450 .cra_alignmask = 0,1451 .cra_module = THIS_MODULE,1452 },1453 .init = otx_cpt_aead_ecb_null_sha512_init,1454 .exit = otx_cpt_aead_exit,1455 .setkey = otx_cpt_aead_ecb_null_sha_setkey,1456 .setauthsize = otx_cpt_aead_set_authsize,1457 .encrypt = otx_cpt_aead_null_encrypt,1458 .decrypt = otx_cpt_aead_null_decrypt,1459 .ivsize = 0,1460 .maxauthsize = SHA512_DIGEST_SIZE,1461}, {1462 .base = {1463 .cra_name = "rfc4106(gcm(aes))",1464 .cra_driver_name = "cpt_rfc4106_gcm_aes",1465 .cra_blocksize = 1,1466 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1467 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1468 .cra_priority = 4001,1469 .cra_alignmask = 0,1470 .cra_module = THIS_MODULE,1471 },1472 .init = otx_cpt_aead_gcm_aes_init,1473 .exit = otx_cpt_aead_exit,1474 .setkey = otx_cpt_aead_gcm_aes_setkey,1475 .setauthsize = otx_cpt_aead_set_authsize,1476 .encrypt = otx_cpt_aead_encrypt,1477 .decrypt = otx_cpt_aead_decrypt,1478 .ivsize = AES_GCM_IV_SIZE,1479 .maxauthsize = AES_GCM_ICV_SIZE,1480} };1481 1482static inline int is_any_alg_used(void)1483{1484 int i;1485 1486 for (i = 0; i < ARRAY_SIZE(otx_cpt_skciphers); i++)1487 if (refcount_read(&otx_cpt_skciphers[i].base.cra_refcnt) != 1)1488 return true;1489 for (i = 0; i < ARRAY_SIZE(otx_cpt_aeads); i++)1490 if (refcount_read(&otx_cpt_aeads[i].base.cra_refcnt) != 1)1491 return true;1492 return false;1493}1494 1495static inline int cpt_register_algs(void)1496{1497 int i, err = 0;1498 1499 if (!IS_ENABLED(CONFIG_DM_CRYPT)) {1500 for (i = 0; i < ARRAY_SIZE(otx_cpt_skciphers); i++)1501 otx_cpt_skciphers[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;1502 1503 err = crypto_register_skciphers(otx_cpt_skciphers,1504 ARRAY_SIZE(otx_cpt_skciphers));1505 if (err)1506 return err;1507 }1508 1509 for (i = 0; i < ARRAY_SIZE(otx_cpt_aeads); i++)1510 otx_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;1511 1512 err = crypto_register_aeads(otx_cpt_aeads, ARRAY_SIZE(otx_cpt_aeads));1513 if (err) {1514 crypto_unregister_skciphers(otx_cpt_skciphers,1515 ARRAY_SIZE(otx_cpt_skciphers));1516 return err;1517 }1518 1519 return 0;1520}1521 1522static inline void cpt_unregister_algs(void)1523{1524 crypto_unregister_skciphers(otx_cpt_skciphers,1525 ARRAY_SIZE(otx_cpt_skciphers));1526 crypto_unregister_aeads(otx_cpt_aeads, ARRAY_SIZE(otx_cpt_aeads));1527}1528 1529static int compare_func(const void *lptr, const void *rptr)1530{1531 struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr;1532 struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr;1533 1534 if (ldesc->dev->devfn < rdesc->dev->devfn)1535 return -1;1536 if (ldesc->dev->devfn > rdesc->dev->devfn)1537 return 1;1538 return 0;1539}1540 1541int otx_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,1542 enum otx_cptpf_type pf_type,1543 enum otx_cptvf_type engine_type,1544 int num_queues, int num_devices)1545{1546 int ret = 0;1547 int count;1548 1549 mutex_lock(&mutex);1550 switch (engine_type) {1551 case OTX_CPT_SE_TYPES:1552 count = atomic_read(&se_devices.count);1553 if (count >= CPT_MAX_VF_NUM) {1554 dev_err(&pdev->dev, "No space to add a new device\n");1555 ret = -ENOSPC;1556 goto err;1557 }1558 se_devices.desc[count].pf_type = pf_type;1559 se_devices.desc[count].num_queues = num_queues;1560 se_devices.desc[count++].dev = pdev;1561 atomic_inc(&se_devices.count);1562 1563 if (atomic_read(&se_devices.count) == num_devices &&1564 is_crypto_registered == false) {1565 if (cpt_register_algs()) {1566 dev_err(&pdev->dev,1567 "Error in registering crypto algorithms\n");1568 ret = -EINVAL;1569 goto err;1570 }1571 try_module_get(mod);1572 is_crypto_registered = true;1573 }1574 sort(se_devices.desc, count, sizeof(struct cpt_device_desc),1575 compare_func, NULL);1576 break;1577 1578 case OTX_CPT_AE_TYPES:1579 count = atomic_read(&ae_devices.count);1580 if (count >= CPT_MAX_VF_NUM) {1581 dev_err(&pdev->dev, "No space to a add new device\n");1582 ret = -ENOSPC;1583 goto err;1584 }1585 ae_devices.desc[count].pf_type = pf_type;1586 ae_devices.desc[count].num_queues = num_queues;1587 ae_devices.desc[count++].dev = pdev;1588 atomic_inc(&ae_devices.count);1589 sort(ae_devices.desc, count, sizeof(struct cpt_device_desc),1590 compare_func, NULL);1591 break;1592 1593 default:1594 dev_err(&pdev->dev, "Unknown VF type %d\n", engine_type);1595 ret = BAD_OTX_CPTVF_TYPE;1596 }1597err:1598 mutex_unlock(&mutex);1599 return ret;1600}1601 1602void otx_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod,1603 enum otx_cptvf_type engine_type)1604{1605 struct cpt_device_table *dev_tbl;1606 bool dev_found = false;1607 int i, j, count;1608 1609 mutex_lock(&mutex);1610 1611 dev_tbl = (engine_type == OTX_CPT_AE_TYPES) ? &ae_devices : &se_devices;1612 count = atomic_read(&dev_tbl->count);1613 for (i = 0; i < count; i++)1614 if (pdev == dev_tbl->desc[i].dev) {1615 for (j = i; j < count-1; j++)1616 dev_tbl->desc[j] = dev_tbl->desc[j+1];1617 dev_found = true;1618 break;1619 }1620 1621 if (!dev_found) {1622 dev_err(&pdev->dev, "%s device not found\n", __func__);1623 goto exit;1624 }1625 1626 if (engine_type != OTX_CPT_AE_TYPES) {1627 if (atomic_dec_and_test(&se_devices.count) &&1628 !is_any_alg_used()) {1629 cpt_unregister_algs();1630 module_put(mod);1631 is_crypto_registered = false;1632 }1633 } else1634 atomic_dec(&ae_devices.count);1635exit:1636 mutex_unlock(&mutex);1637}1638