1699 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright (C) 2020 Marvell. */3 4#include <crypto/aes.h>5#include <crypto/authenc.h>6#include <crypto/cryptd.h>7#include <crypto/des.h>8#include <crypto/internal/aead.h>9#include <crypto/sha1.h>10#include <crypto/sha2.h>11#include <crypto/xts.h>12#include <crypto/gcm.h>13#include <crypto/scatterwalk.h>14#include <linux/sort.h>15#include <linux/module.h>16#include "otx2_cptvf.h"17#include "otx2_cptvf_algs.h"18#include "otx2_cpt_reqmgr.h"19#include "cn10k_cpt.h"20 21/* Size of salt in AES GCM mode */22#define AES_GCM_SALT_SIZE 423/* Size of IV in AES GCM mode */24#define AES_GCM_IV_SIZE 825/* Size of ICV (Integrity Check Value) in AES GCM mode */26#define AES_GCM_ICV_SIZE 1627/* Offset of IV in AES GCM mode */28#define AES_GCM_IV_OFFSET 829#define CONTROL_WORD_LEN 830#define KEY2_OFFSET 4831#define DMA_MODE_FLAG(dma_mode) \32 (((dma_mode) == OTX2_CPT_DMA_MODE_SG) ? (1 << 7) : 0)33 34/* Truncated SHA digest size */35#define SHA1_TRUNC_DIGEST_SIZE 1236#define SHA256_TRUNC_DIGEST_SIZE 1637#define SHA384_TRUNC_DIGEST_SIZE 2438#define SHA512_TRUNC_DIGEST_SIZE 3239 40static DEFINE_MUTEX(mutex);41static int is_crypto_registered;42 43struct cpt_device_desc {44 struct pci_dev *dev;45 int num_queues;46};47 48struct cpt_device_table {49 atomic_t count;50 struct cpt_device_desc desc[OTX2_CPT_MAX_LFS_NUM];51};52 53static struct cpt_device_table se_devices = {54 .count = ATOMIC_INIT(0)55};56 57static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg);58 59static inline int get_se_device(struct pci_dev **pdev, int *cpu_num)60{61 int count;62 63 count = atomic_read(&se_devices.count);64 if (count < 1)65 return -ENODEV;66 67 *cpu_num = get_cpu();68 /*69 * On OcteonTX2 platform CPT instruction queue is bound to each70 * local function LF, in turn LFs can be attached to PF71 * or VF therefore we always use first device. We get maximum72 * performance if one CPT queue is available for each cpu73 * otherwise CPT queues need to be shared between cpus.74 */75 if (*cpu_num >= se_devices.desc[0].num_queues)76 *cpu_num %= se_devices.desc[0].num_queues;77 *pdev = se_devices.desc[0].dev;78 79 put_cpu();80 81 return 0;82}83 84static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req)85{86 struct otx2_cpt_req_ctx *rctx;87 struct aead_request *req;88 struct crypto_aead *tfm;89 90 req = container_of(cpt_req->areq, struct aead_request, base);91 tfm = crypto_aead_reqtfm(req);92 rctx = aead_request_ctx_dma(req);93 if (memcmp(rctx->fctx.hmac.s.hmac_calc,94 rctx->fctx.hmac.s.hmac_recv,95 crypto_aead_authsize(tfm)) != 0)96 return -EBADMSG;97 98 return 0;99}100 101static void otx2_cpt_aead_callback(int status, void *arg1, void *arg2)102{103 struct otx2_cpt_inst_info *inst_info = arg2;104 struct crypto_async_request *areq = arg1;105 struct otx2_cpt_req_info *cpt_req;106 struct pci_dev *pdev;107 108 if (inst_info) {109 cpt_req = inst_info->req;110 if (!status) {111 /*112 * When selected cipher is NULL we need to manually113 * verify whether calculated hmac value matches114 * received hmac value115 */116 if (cpt_req->req_type ==117 OTX2_CPT_AEAD_ENC_DEC_NULL_REQ &&118 !cpt_req->is_enc)119 status = validate_hmac_cipher_null(cpt_req);120 }121 pdev = inst_info->pdev;122 otx2_cpt_info_destroy(pdev, inst_info);123 }124 if (areq)125 crypto_request_complete(areq, status);126}127 128static void output_iv_copyback(struct crypto_async_request *areq)129{130 struct otx2_cpt_req_info *req_info;131 struct otx2_cpt_req_ctx *rctx;132 struct skcipher_request *sreq;133 struct crypto_skcipher *stfm;134 struct otx2_cpt_enc_ctx *ctx;135 u32 start, ivsize;136 137 sreq = container_of(areq, struct skcipher_request, base);138 stfm = crypto_skcipher_reqtfm(sreq);139 ctx = crypto_skcipher_ctx(stfm);140 if (ctx->cipher_type == OTX2_CPT_AES_CBC ||141 ctx->cipher_type == OTX2_CPT_DES3_CBC) {142 rctx = skcipher_request_ctx_dma(sreq);143 req_info = &rctx->cpt_req;144 ivsize = crypto_skcipher_ivsize(stfm);145 start = sreq->cryptlen - ivsize;146 147 if (req_info->is_enc) {148 scatterwalk_map_and_copy(sreq->iv, sreq->dst, start,149 ivsize, 0);150 } else {151 if (sreq->src != sreq->dst) {152 scatterwalk_map_and_copy(sreq->iv, sreq->src,153 start, ivsize, 0);154 } else {155 memcpy(sreq->iv, req_info->iv_out, ivsize);156 kfree(req_info->iv_out);157 }158 }159 }160}161 162static void otx2_cpt_skcipher_callback(int status, void *arg1, void *arg2)163{164 struct otx2_cpt_inst_info *inst_info = arg2;165 struct crypto_async_request *areq = arg1;166 struct pci_dev *pdev;167 168 if (areq) {169 if (!status)170 output_iv_copyback(areq);171 if (inst_info) {172 pdev = inst_info->pdev;173 otx2_cpt_info_destroy(pdev, inst_info);174 }175 crypto_request_complete(areq, status);176 }177}178 179static inline void update_input_data(struct otx2_cpt_req_info *req_info,180 struct scatterlist *inp_sg,181 u32 nbytes, u32 *argcnt)182{183 req_info->req.dlen += nbytes;184 185 while (nbytes) {186 u32 len = (nbytes < inp_sg->length) ? nbytes : inp_sg->length;187 u8 *ptr = sg_virt(inp_sg);188 189 req_info->in[*argcnt].vptr = (void *)ptr;190 req_info->in[*argcnt].size = len;191 nbytes -= len;192 ++(*argcnt);193 inp_sg = sg_next(inp_sg);194 }195}196 197static inline void update_output_data(struct otx2_cpt_req_info *req_info,198 struct scatterlist *outp_sg,199 u32 offset, u32 nbytes, u32 *argcnt)200{201 u32 len, sg_len;202 u8 *ptr;203 204 req_info->rlen += nbytes;205 206 while (nbytes) {207 sg_len = outp_sg->length - offset;208 len = (nbytes < sg_len) ? nbytes : sg_len;209 ptr = sg_virt(outp_sg);210 211 req_info->out[*argcnt].vptr = (void *) (ptr + offset);212 req_info->out[*argcnt].size = len;213 nbytes -= len;214 ++(*argcnt);215 offset = 0;216 outp_sg = sg_next(outp_sg);217 }218}219 220static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc,221 u32 *argcnt)222{223 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);224 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);225 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);226 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;227 struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;228 int ivsize = crypto_skcipher_ivsize(stfm);229 u32 start = req->cryptlen - ivsize;230 gfp_t flags;231 232 flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?233 GFP_KERNEL : GFP_ATOMIC;234 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;235 req_info->ctrl.s.se_req = 1;236 237 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |238 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);239 if (enc) {240 req_info->req.opcode.s.minor = 2;241 } else {242 req_info->req.opcode.s.minor = 3;243 if ((ctx->cipher_type == OTX2_CPT_AES_CBC ||244 ctx->cipher_type == OTX2_CPT_DES3_CBC) &&245 req->src == req->dst) {246 req_info->iv_out = kmalloc(ivsize, flags);247 if (!req_info->iv_out)248 return -ENOMEM;249 250 scatterwalk_map_and_copy(req_info->iv_out, req->src,251 start, ivsize, 0);252 }253 }254 /* Encryption data length */255 req_info->req.param1 = req->cryptlen;256 /* Authentication data length */257 req_info->req.param2 = 0;258 259 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;260 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;261 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;262 263 if (ctx->cipher_type == OTX2_CPT_AES_XTS)264 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);265 else266 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);267 268 memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm));269 270 cpu_to_be64s(&fctx->enc.enc_ctrl.u);271 272 /*273 * Storing Packet Data Information in offset274 * Control Word First 8 bytes275 */276 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;277 req_info->in[*argcnt].size = CONTROL_WORD_LEN;278 req_info->req.dlen += CONTROL_WORD_LEN;279 ++(*argcnt);280 281 req_info->in[*argcnt].vptr = (u8 *)fctx;282 req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);283 req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);284 285 ++(*argcnt);286 287 return 0;288}289 290static inline int create_input_list(struct skcipher_request *req, u32 enc,291 u32 enc_iv_len)292{293 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);294 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;295 u32 argcnt = 0;296 int ret;297 298 ret = create_ctx_hdr(req, enc, &argcnt);299 if (ret)300 return ret;301 302 update_input_data(req_info, req->src, req->cryptlen, &argcnt);303 req_info->in_cnt = argcnt;304 305 return 0;306}307 308static inline void create_output_list(struct skcipher_request *req,309 u32 enc_iv_len)310{311 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);312 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;313 u32 argcnt = 0;314 315 /*316 * OUTPUT Buffer Processing317 * AES encryption/decryption output would be318 * received in the following format319 *320 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|321 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ]322 */323 update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt);324 req_info->out_cnt = argcnt;325}326 327static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc)328{329 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);330 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);331 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);332 int ret;333 334 if (ctx->fbk_cipher) {335 skcipher_request_set_tfm(&rctx->sk_fbk_req, ctx->fbk_cipher);336 skcipher_request_set_callback(&rctx->sk_fbk_req,337 req->base.flags,338 req->base.complete,339 req->base.data);340 skcipher_request_set_crypt(&rctx->sk_fbk_req, req->src,341 req->dst, req->cryptlen, req->iv);342 ret = is_enc ? crypto_skcipher_encrypt(&rctx->sk_fbk_req) :343 crypto_skcipher_decrypt(&rctx->sk_fbk_req);344 } else {345 ret = -EINVAL;346 }347 return ret;348}349 350static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc)351{352 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);353 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);354 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);355 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;356 u32 enc_iv_len = crypto_skcipher_ivsize(stfm);357 struct pci_dev *pdev;358 int status, cpu_num;359 360 if (req->cryptlen == 0)361 return 0;362 363 if (!IS_ALIGNED(req->cryptlen, ctx->enc_align_len))364 return -EINVAL;365 366 if (req->cryptlen > OTX2_CPT_MAX_REQ_SIZE)367 return skcipher_do_fallback(req, enc);368 369 /* Clear control words */370 rctx->ctrl_word.flags = 0;371 rctx->fctx.enc.enc_ctrl.u = 0;372 373 status = create_input_list(req, enc, enc_iv_len);374 if (status)375 return status;376 create_output_list(req, enc_iv_len);377 378 status = get_se_device(&pdev, &cpu_num);379 if (status)380 return status;381 382 req_info->callback = otx2_cpt_skcipher_callback;383 req_info->areq = &req->base;384 req_info->req_type = OTX2_CPT_ENC_DEC_REQ;385 req_info->is_enc = enc;386 req_info->is_trunc_hmac = false;387 req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);388 389 req_info->req.cptr = ctx->er_ctx.hw_ctx;390 req_info->req.cptr_dma = ctx->er_ctx.cptr_dma;391 392 /*393 * We perform an asynchronous send and once394 * the request is completed the driver would395 * intimate through registered call back functions396 */397 status = otx2_cpt_do_request(pdev, req_info, cpu_num);398 399 return status;400}401 402static int otx2_cpt_skcipher_encrypt(struct skcipher_request *req)403{404 return cpt_enc_dec(req, true);405}406 407static int otx2_cpt_skcipher_decrypt(struct skcipher_request *req)408{409 return cpt_enc_dec(req, false);410}411 412static int otx2_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm,413 const u8 *key, u32 keylen)414{415 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);416 const u8 *key2 = key + (keylen / 2);417 const u8 *key1 = key;418 int ret;419 420 ret = xts_verify_key(tfm, key, keylen);421 if (ret)422 return ret;423 ctx->key_len = keylen;424 ctx->enc_align_len = 1;425 memcpy(ctx->enc_key, key1, keylen / 2);426 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);427 ctx->cipher_type = OTX2_CPT_AES_XTS;428 switch (ctx->key_len) {429 case 2 * AES_KEYSIZE_128:430 ctx->key_type = OTX2_CPT_AES_128_BIT;431 break;432 case 2 * AES_KEYSIZE_192:433 ctx->key_type = OTX2_CPT_AES_192_BIT;434 break;435 case 2 * AES_KEYSIZE_256:436 ctx->key_type = OTX2_CPT_AES_256_BIT;437 break;438 default:439 return -EINVAL;440 }441 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);442}443 444static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key,445 u32 keylen, u8 cipher_type)446{447 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);448 449 if (keylen != DES3_EDE_KEY_SIZE)450 return -EINVAL;451 452 ctx->key_len = keylen;453 ctx->cipher_type = cipher_type;454 ctx->enc_align_len = 8;455 456 memcpy(ctx->enc_key, key, keylen);457 458 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);459}460 461static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,462 u32 keylen, u8 cipher_type)463{464 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);465 466 switch (keylen) {467 case AES_KEYSIZE_128:468 ctx->key_type = OTX2_CPT_AES_128_BIT;469 break;470 case AES_KEYSIZE_192:471 ctx->key_type = OTX2_CPT_AES_192_BIT;472 break;473 case AES_KEYSIZE_256:474 ctx->key_type = OTX2_CPT_AES_256_BIT;475 break;476 default:477 return -EINVAL;478 }479 if (cipher_type == OTX2_CPT_AES_CBC || cipher_type == OTX2_CPT_AES_ECB)480 ctx->enc_align_len = 16;481 else482 ctx->enc_align_len = 1;483 484 ctx->key_len = keylen;485 ctx->cipher_type = cipher_type;486 487 memcpy(ctx->enc_key, key, keylen);488 489 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);490}491 492static int otx2_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm,493 const u8 *key, u32 keylen)494{495 return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_CBC);496}497 498static int otx2_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm,499 const u8 *key, u32 keylen)500{501 return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_ECB);502}503 504static int otx2_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm,505 const u8 *key, u32 keylen)506{507 return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_CBC);508}509 510static int otx2_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm,511 const u8 *key, u32 keylen)512{513 return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_ECB);514}515 516static int cpt_skcipher_fallback_init(struct otx2_cpt_enc_ctx *ctx,517 struct crypto_alg *alg)518{519 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {520 ctx->fbk_cipher =521 crypto_alloc_skcipher(alg->cra_name, 0,522 CRYPTO_ALG_ASYNC |523 CRYPTO_ALG_NEED_FALLBACK);524 if (IS_ERR(ctx->fbk_cipher)) {525 pr_err("%s() failed to allocate fallback for %s\n",526 __func__, alg->cra_name);527 return PTR_ERR(ctx->fbk_cipher);528 }529 }530 return 0;531}532 533static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm)534{535 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);536 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);537 struct crypto_alg *alg = tfm->__crt_alg;538 struct pci_dev *pdev;539 int ret, cpu_num;540 541 memset(ctx, 0, sizeof(*ctx));542 /*543 * Additional memory for skcipher_request is544 * allocated since the cryptd daemon uses545 * this memory for request_ctx information546 */547 crypto_skcipher_set_reqsize_dma(548 stfm, sizeof(struct otx2_cpt_req_ctx) +549 sizeof(struct skcipher_request));550 551 ret = get_se_device(&pdev, &cpu_num);552 if (ret)553 return ret;554 555 ctx->pdev = pdev;556 ret = cn10k_cpt_hw_ctx_init(pdev, &ctx->er_ctx);557 if (ret)558 return ret;559 560 return cpt_skcipher_fallback_init(ctx, alg);561}562 563static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm)564{565 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);566 567 if (ctx->fbk_cipher) {568 crypto_free_skcipher(ctx->fbk_cipher);569 ctx->fbk_cipher = NULL;570 }571 cn10k_cpt_hw_ctx_clear(ctx->pdev, &ctx->er_ctx);572}573 574static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx,575 struct crypto_alg *alg)576{577 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {578 ctx->fbk_cipher =579 crypto_alloc_aead(alg->cra_name, 0,580 CRYPTO_ALG_ASYNC |581 CRYPTO_ALG_NEED_FALLBACK);582 if (IS_ERR(ctx->fbk_cipher)) {583 pr_err("%s() failed to allocate fallback for %s\n",584 __func__, alg->cra_name);585 return PTR_ERR(ctx->fbk_cipher);586 }587 }588 return 0;589}590 591static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type)592{593 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(atfm);594 struct crypto_tfm *tfm = crypto_aead_tfm(atfm);595 struct crypto_alg *alg = tfm->__crt_alg;596 struct pci_dev *pdev;597 int ret, cpu_num;598 599 ctx->cipher_type = cipher_type;600 ctx->mac_type = mac_type;601 602 switch (ctx->mac_type) {603 case OTX2_CPT_SHA1:604 ctx->hashalg = crypto_alloc_shash("sha1", 0, 0);605 break;606 607 case OTX2_CPT_SHA256:608 ctx->hashalg = crypto_alloc_shash("sha256", 0, 0);609 break;610 611 case OTX2_CPT_SHA384:612 ctx->hashalg = crypto_alloc_shash("sha384", 0, 0);613 break;614 615 case OTX2_CPT_SHA512:616 ctx->hashalg = crypto_alloc_shash("sha512", 0, 0);617 break;618 }619 620 if (IS_ERR(ctx->hashalg))621 return PTR_ERR(ctx->hashalg);622 623 if (ctx->hashalg) {624 ctx->sdesc = alloc_sdesc(ctx->hashalg);625 if (!ctx->sdesc) {626 crypto_free_shash(ctx->hashalg);627 return -ENOMEM;628 }629 }630 631 /*632 * When selected cipher is NULL we use HMAC opcode instead of633 * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms634 * for calculating ipad and opad635 */636 if (ctx->cipher_type != OTX2_CPT_CIPHER_NULL && ctx->hashalg) {637 int ss = crypto_shash_statesize(ctx->hashalg);638 639 ctx->ipad = kzalloc(ss, GFP_KERNEL);640 if (!ctx->ipad) {641 kfree(ctx->sdesc);642 crypto_free_shash(ctx->hashalg);643 return -ENOMEM;644 }645 646 ctx->opad = kzalloc(ss, GFP_KERNEL);647 if (!ctx->opad) {648 kfree(ctx->ipad);649 kfree(ctx->sdesc);650 crypto_free_shash(ctx->hashalg);651 return -ENOMEM;652 }653 }654 switch (ctx->cipher_type) {655 case OTX2_CPT_AES_CBC:656 case OTX2_CPT_AES_ECB:657 ctx->enc_align_len = 16;658 break;659 case OTX2_CPT_DES3_CBC:660 case OTX2_CPT_DES3_ECB:661 ctx->enc_align_len = 8;662 break;663 case OTX2_CPT_AES_GCM:664 case OTX2_CPT_CIPHER_NULL:665 ctx->enc_align_len = 1;666 break;667 }668 crypto_aead_set_reqsize_dma(atfm, sizeof(struct otx2_cpt_req_ctx));669 670 ret = get_se_device(&pdev, &cpu_num);671 if (ret)672 return ret;673 674 ctx->pdev = pdev;675 ret = cn10k_cpt_hw_ctx_init(pdev, &ctx->er_ctx);676 if (ret)677 return ret;678 679 return cpt_aead_fallback_init(ctx, alg);680}681 682static int otx2_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm)683{684 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA1);685}686 687static int otx2_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm)688{689 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA256);690}691 692static int otx2_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm)693{694 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA384);695}696 697static int otx2_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm)698{699 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA512);700}701 702static int otx2_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm)703{704 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA1);705}706 707static int otx2_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm)708{709 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA256);710}711 712static int otx2_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm)713{714 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA384);715}716 717static int otx2_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm)718{719 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA512);720}721 722static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)723{724 return cpt_aead_init(tfm, OTX2_CPT_AES_GCM, OTX2_CPT_MAC_NULL);725}726 727static void otx2_cpt_aead_exit(struct crypto_aead *tfm)728{729 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);730 731 kfree(ctx->ipad);732 kfree(ctx->opad);733 crypto_free_shash(ctx->hashalg);734 kfree(ctx->sdesc);735 736 if (ctx->fbk_cipher) {737 crypto_free_aead(ctx->fbk_cipher);738 ctx->fbk_cipher = NULL;739 }740 cn10k_cpt_hw_ctx_clear(ctx->pdev, &ctx->er_ctx);741}742 743static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm,744 unsigned int authsize)745{746 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);747 748 if (crypto_rfc4106_check_authsize(authsize))749 return -EINVAL;750 751 tfm->authsize = authsize;752 /* Set authsize for fallback case */753 if (ctx->fbk_cipher)754 ctx->fbk_cipher->authsize = authsize;755 756 return 0;757}758 759static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm,760 unsigned int authsize)761{762 tfm->authsize = authsize;763 764 return 0;765}766 767static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm,768 unsigned int authsize)769{770 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);771 772 ctx->is_trunc_hmac = true;773 tfm->authsize = authsize;774 775 return 0;776}777 778static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg)779{780 struct otx2_cpt_sdesc *sdesc;781 int size;782 783 size = sizeof(struct shash_desc) + crypto_shash_descsize(alg);784 sdesc = kmalloc(size, GFP_KERNEL);785 if (!sdesc)786 return NULL;787 788 sdesc->shash.tfm = alg;789 790 return sdesc;791}792 793static inline void swap_data32(void *buf, u32 len)794{795 cpu_to_be32_array(buf, buf, len / 4);796}797 798static inline void swap_data64(void *buf, u32 len)799{800 u64 *src = buf;801 int i = 0;802 803 for (i = 0 ; i < len / 8; i++, src++)804 cpu_to_be64s(src);805}806 807static int swap_pad(u8 mac_type, u8 *pad)808{809 struct sha512_state *sha512;810 struct sha256_state *sha256;811 struct sha1_state *sha1;812 813 switch (mac_type) {814 case OTX2_CPT_SHA1:815 sha1 = (struct sha1_state *)pad;816 swap_data32(sha1->state, SHA1_DIGEST_SIZE);817 break;818 819 case OTX2_CPT_SHA256:820 sha256 = (struct sha256_state *)pad;821 swap_data32(sha256->state, SHA256_DIGEST_SIZE);822 break;823 824 case OTX2_CPT_SHA384:825 case OTX2_CPT_SHA512:826 sha512 = (struct sha512_state *)pad;827 swap_data64(sha512->state, SHA512_DIGEST_SIZE);828 break;829 830 default:831 return -EINVAL;832 }833 834 return 0;835}836 837static int aead_hmac_init(struct crypto_aead *cipher,838 struct crypto_authenc_keys *keys)839{840 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);841 int ds = crypto_shash_digestsize(ctx->hashalg);842 int bs = crypto_shash_blocksize(ctx->hashalg);843 int authkeylen = keys->authkeylen;844 u8 *ipad = NULL, *opad = NULL;845 int icount = 0;846 int ret;847 848 if (authkeylen > bs) {849 ret = crypto_shash_digest(&ctx->sdesc->shash, keys->authkey,850 authkeylen, ctx->key);851 if (ret)852 goto calc_fail;853 854 authkeylen = ds;855 } else856 memcpy(ctx->key, keys->authkey, authkeylen);857 858 ctx->enc_key_len = keys->enckeylen;859 ctx->auth_key_len = authkeylen;860 861 if (ctx->cipher_type == OTX2_CPT_CIPHER_NULL)862 return keys->enckeylen ? -EINVAL : 0;863 864 switch (keys->enckeylen) {865 case AES_KEYSIZE_128:866 ctx->key_type = OTX2_CPT_AES_128_BIT;867 break;868 case AES_KEYSIZE_192:869 ctx->key_type = OTX2_CPT_AES_192_BIT;870 break;871 case AES_KEYSIZE_256:872 ctx->key_type = OTX2_CPT_AES_256_BIT;873 break;874 default:875 /* Invalid key length */876 return -EINVAL;877 }878 879 memcpy(ctx->key + authkeylen, keys->enckey, keys->enckeylen);880 881 ipad = ctx->ipad;882 opad = ctx->opad;883 884 memcpy(ipad, ctx->key, authkeylen);885 memset(ipad + authkeylen, 0, bs - authkeylen);886 memcpy(opad, ipad, bs);887 888 for (icount = 0; icount < bs; icount++) {889 ipad[icount] ^= 0x36;890 opad[icount] ^= 0x5c;891 }892 893 /*894 * Partial Hash calculated from the software895 * algorithm is retrieved for IPAD & OPAD896 */897 898 /* IPAD Calculation */899 crypto_shash_init(&ctx->sdesc->shash);900 crypto_shash_update(&ctx->sdesc->shash, ipad, bs);901 crypto_shash_export(&ctx->sdesc->shash, ipad);902 ret = swap_pad(ctx->mac_type, ipad);903 if (ret)904 goto calc_fail;905 906 /* OPAD Calculation */907 crypto_shash_init(&ctx->sdesc->shash);908 crypto_shash_update(&ctx->sdesc->shash, opad, bs);909 crypto_shash_export(&ctx->sdesc->shash, opad);910 ret = swap_pad(ctx->mac_type, opad);911 912calc_fail:913 return ret;914}915 916static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher,917 const unsigned char *key,918 unsigned int keylen)919{920 struct crypto_authenc_keys authenc_keys;921 922 return crypto_authenc_extractkeys(&authenc_keys, key, keylen) ?:923 aead_hmac_init(cipher, &authenc_keys);924}925 926static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher,927 const unsigned char *key,928 unsigned int keylen)929{930 return otx2_cpt_aead_cbc_aes_sha_setkey(cipher, key, keylen);931}932 933static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,934 const unsigned char *key,935 unsigned int keylen)936{937 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);938 939 /*940 * For aes gcm we expect to get encryption key (16, 24, 32 bytes)941 * and salt (4 bytes)942 */943 switch (keylen) {944 case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE:945 ctx->key_type = OTX2_CPT_AES_128_BIT;946 ctx->enc_key_len = AES_KEYSIZE_128;947 break;948 case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE:949 ctx->key_type = OTX2_CPT_AES_192_BIT;950 ctx->enc_key_len = AES_KEYSIZE_192;951 break;952 case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE:953 ctx->key_type = OTX2_CPT_AES_256_BIT;954 ctx->enc_key_len = AES_KEYSIZE_256;955 break;956 default:957 /* Invalid key and salt length */958 return -EINVAL;959 }960 961 /* Store encryption key and salt */962 memcpy(ctx->key, key, keylen);963 964 return crypto_aead_setkey(ctx->fbk_cipher, key, keylen);965}966 967static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc,968 u32 *argcnt)969{970 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);971 struct crypto_aead *tfm = crypto_aead_reqtfm(req);972 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);973 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;974 struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;975 int mac_len = crypto_aead_authsize(tfm);976 int ds;977 978 rctx->ctrl_word.e.enc_data_offset = req->assoclen;979 980 switch (ctx->cipher_type) {981 case OTX2_CPT_AES_CBC:982 if (req->assoclen > 248 || !IS_ALIGNED(req->assoclen, 8))983 return -EINVAL;984 985 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;986 /* Copy encryption key to context */987 memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len,988 ctx->enc_key_len);989 /* Copy IV to context */990 memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm));991 992 ds = crypto_shash_digestsize(ctx->hashalg);993 if (ctx->mac_type == OTX2_CPT_SHA384)994 ds = SHA512_DIGEST_SIZE;995 if (ctx->ipad)996 memcpy(fctx->hmac.e.ipad, ctx->ipad, ds);997 if (ctx->opad)998 memcpy(fctx->hmac.e.opad, ctx->opad, ds);999 break;1000 1001 case OTX2_CPT_AES_GCM:1002 if (crypto_ipsec_check_assoclen(req->assoclen))1003 return -EINVAL;1004 1005 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_DPTR;1006 /* Copy encryption key to context */1007 memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len);1008 /* Copy salt to context */1009 memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len,1010 AES_GCM_SALT_SIZE);1011 1012 rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET;1013 break;1014 1015 default:1016 /* Unknown cipher type */1017 return -EINVAL;1018 }1019 cpu_to_be64s(&rctx->ctrl_word.flags);1020 1021 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;1022 req_info->ctrl.s.se_req = 1;1023 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |1024 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);1025 if (enc) {1026 req_info->req.opcode.s.minor = 2;1027 req_info->req.param1 = req->cryptlen;1028 req_info->req.param2 = req->cryptlen + req->assoclen;1029 } else {1030 req_info->req.opcode.s.minor = 3;1031 req_info->req.param1 = req->cryptlen - mac_len;1032 req_info->req.param2 = req->cryptlen + req->assoclen - mac_len;1033 }1034 1035 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;1036 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;1037 fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type;1038 fctx->enc.enc_ctrl.e.mac_len = mac_len;1039 cpu_to_be64s(&fctx->enc.enc_ctrl.u);1040 1041 /*1042 * Storing Packet Data Information in offset1043 * Control Word First 8 bytes1044 */1045 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;1046 req_info->in[*argcnt].size = CONTROL_WORD_LEN;1047 req_info->req.dlen += CONTROL_WORD_LEN;1048 ++(*argcnt);1049 1050 req_info->in[*argcnt].vptr = (u8 *)fctx;1051 req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);1052 req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);1053 ++(*argcnt);1054 1055 return 0;1056}1057 1058static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,1059 u32 enc)1060{1061 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);1062 struct crypto_aead *tfm = crypto_aead_reqtfm(req);1063 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);1064 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;1065 1066 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;1067 req_info->ctrl.s.se_req = 1;1068 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_HMAC |1069 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);1070 req_info->is_trunc_hmac = ctx->is_trunc_hmac;1071 1072 req_info->req.opcode.s.minor = 0;1073 req_info->req.param1 = ctx->auth_key_len;1074 req_info->req.param2 = ctx->mac_type << 8;1075 1076 /* Add authentication key */1077 req_info->in[*argcnt].vptr = ctx->key;1078 req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8);1079 req_info->req.dlen += round_up(ctx->auth_key_len, 8);1080 ++(*argcnt);1081}1082 1083static inline int create_aead_input_list(struct aead_request *req, u32 enc)1084{1085 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);1086 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;1087 u32 inputlen = req->cryptlen + req->assoclen;1088 u32 status, argcnt = 0;1089 1090 status = create_aead_ctx_hdr(req, enc, &argcnt);1091 if (status)1092 return status;1093 update_input_data(req_info, req->src, inputlen, &argcnt);1094 req_info->in_cnt = argcnt;1095 1096 return 0;1097}1098 1099static inline void create_aead_output_list(struct aead_request *req, u32 enc,1100 u32 mac_len)1101{1102 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);1103 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;1104 u32 argcnt = 0, outputlen = 0;1105 1106 if (enc)1107 outputlen = req->cryptlen + req->assoclen + mac_len;1108 else1109 outputlen = req->cryptlen + req->assoclen - mac_len;1110 1111 update_output_data(req_info, req->dst, 0, outputlen, &argcnt);1112 req_info->out_cnt = argcnt;1113}1114 1115static inline void create_aead_null_input_list(struct aead_request *req,1116 u32 enc, u32 mac_len)1117{1118 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);1119 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;1120 u32 inputlen, argcnt = 0;1121 1122 if (enc)1123 inputlen = req->cryptlen + req->assoclen;1124 else1125 inputlen = req->cryptlen + req->assoclen - mac_len;1126 1127 create_hmac_ctx_hdr(req, &argcnt, enc);1128 update_input_data(req_info, req->src, inputlen, &argcnt);1129 req_info->in_cnt = argcnt;1130}1131 1132static inline int create_aead_null_output_list(struct aead_request *req,1133 u32 enc, u32 mac_len)1134{1135 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);1136 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;1137 struct scatterlist *dst;1138 u8 *ptr = NULL;1139 int argcnt = 0, status, offset;1140 u32 inputlen;1141 1142 if (enc)1143 inputlen = req->cryptlen + req->assoclen;1144 else1145 inputlen = req->cryptlen + req->assoclen - mac_len;1146 1147 /*1148 * If source and destination are different1149 * then copy payload to destination1150 */1151 if (req->src != req->dst) {1152 1153 ptr = kmalloc(inputlen, (req_info->areq->flags &1154 CRYPTO_TFM_REQ_MAY_SLEEP) ?1155 GFP_KERNEL : GFP_ATOMIC);1156 if (!ptr)1157 return -ENOMEM;1158 1159 status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr,1160 inputlen);1161 if (status != inputlen) {1162 status = -EINVAL;1163 goto error_free;1164 }1165 status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr,1166 inputlen);1167 if (status != inputlen) {1168 status = -EINVAL;1169 goto error_free;1170 }1171 kfree(ptr);1172 }1173 1174 if (enc) {1175 /*1176 * In an encryption scenario hmac needs1177 * to be appended after payload1178 */1179 dst = req->dst;1180 offset = inputlen;1181 while (offset >= dst->length) {1182 offset -= dst->length;1183 dst = sg_next(dst);1184 if (!dst)1185 return -ENOENT;1186 }1187 1188 update_output_data(req_info, dst, offset, mac_len, &argcnt);1189 } else {1190 /*1191 * In a decryption scenario calculated hmac for received1192 * payload needs to be compare with hmac received1193 */1194 status = sg_copy_buffer(req->src, sg_nents(req->src),1195 rctx->fctx.hmac.s.hmac_recv, mac_len,1196 inputlen, true);1197 if (status != mac_len)1198 return -EINVAL;1199 1200 req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc;1201 req_info->out[argcnt].size = mac_len;1202 argcnt++;1203 }1204 1205 req_info->out_cnt = argcnt;1206 return 0;1207 1208error_free:1209 kfree(ptr);1210 return status;1211}1212 1213static int aead_do_fallback(struct aead_request *req, bool is_enc)1214{1215 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);1216 struct crypto_aead *aead = crypto_aead_reqtfm(req);1217 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(aead);1218 int ret;1219 1220 if (ctx->fbk_cipher) {1221 /* Store the cipher tfm and then use the fallback tfm */1222 aead_request_set_tfm(&rctx->fbk_req, ctx->fbk_cipher);1223 aead_request_set_callback(&rctx->fbk_req, req->base.flags,1224 req->base.complete, req->base.data);1225 aead_request_set_crypt(&rctx->fbk_req, req->src,1226 req->dst, req->cryptlen, req->iv);1227 aead_request_set_ad(&rctx->fbk_req, req->assoclen);1228 ret = is_enc ? crypto_aead_encrypt(&rctx->fbk_req) :1229 crypto_aead_decrypt(&rctx->fbk_req);1230 } else {1231 ret = -EINVAL;1232 }1233 1234 return ret;1235}1236 1237static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc)1238{1239 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);1240 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;1241 struct crypto_aead *tfm = crypto_aead_reqtfm(req);1242 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);1243 struct pci_dev *pdev;1244 int status, cpu_num;1245 1246 /* Clear control words */1247 rctx->ctrl_word.flags = 0;1248 rctx->fctx.enc.enc_ctrl.u = 0;1249 1250 req_info->callback = otx2_cpt_aead_callback;1251 req_info->areq = &req->base;1252 req_info->req_type = reg_type;1253 req_info->is_enc = enc;1254 req_info->is_trunc_hmac = false;1255 1256 req_info->req.cptr = ctx->er_ctx.hw_ctx;1257 req_info->req.cptr_dma = ctx->er_ctx.cptr_dma;1258 1259 switch (reg_type) {1260 case OTX2_CPT_AEAD_ENC_DEC_REQ:1261 status = create_aead_input_list(req, enc);1262 if (status)1263 return status;1264 create_aead_output_list(req, enc, crypto_aead_authsize(tfm));1265 break;1266 1267 case OTX2_CPT_AEAD_ENC_DEC_NULL_REQ:1268 create_aead_null_input_list(req, enc,1269 crypto_aead_authsize(tfm));1270 status = create_aead_null_output_list(req, enc,1271 crypto_aead_authsize(tfm));1272 if (status)1273 return status;1274 break;1275 1276 default:1277 return -EINVAL;1278 }1279 if (!IS_ALIGNED(req_info->req.param1, ctx->enc_align_len))1280 return -EINVAL;1281 1282 if (!req_info->req.param2 ||1283 (req_info->req.param1 > OTX2_CPT_MAX_REQ_SIZE) ||1284 (req_info->req.param2 > OTX2_CPT_MAX_REQ_SIZE))1285 return aead_do_fallback(req, enc);1286 1287 status = get_se_device(&pdev, &cpu_num);1288 if (status)1289 return status;1290 1291 req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);1292 1293 /*1294 * We perform an asynchronous send and once1295 * the request is completed the driver would1296 * intimate through registered call back functions1297 */1298 return otx2_cpt_do_request(pdev, req_info, cpu_num);1299}1300 1301static int otx2_cpt_aead_encrypt(struct aead_request *req)1302{1303 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, true);1304}1305 1306static int otx2_cpt_aead_decrypt(struct aead_request *req)1307{1308 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, false);1309}1310 1311static int otx2_cpt_aead_null_encrypt(struct aead_request *req)1312{1313 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, true);1314}1315 1316static int otx2_cpt_aead_null_decrypt(struct aead_request *req)1317{1318 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, false);1319}1320 1321static struct skcipher_alg otx2_cpt_skciphers[] = { {1322 .base.cra_name = "xts(aes)",1323 .base.cra_driver_name = "cpt_xts_aes",1324 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,1325 .base.cra_blocksize = AES_BLOCK_SIZE,1326 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),1327 .base.cra_alignmask = 7,1328 .base.cra_priority = 4001,1329 .base.cra_module = THIS_MODULE,1330 1331 .init = otx2_cpt_enc_dec_init,1332 .exit = otx2_cpt_skcipher_exit,1333 .ivsize = AES_BLOCK_SIZE,1334 .min_keysize = 2 * AES_MIN_KEY_SIZE,1335 .max_keysize = 2 * AES_MAX_KEY_SIZE,1336 .setkey = otx2_cpt_skcipher_xts_setkey,1337 .encrypt = otx2_cpt_skcipher_encrypt,1338 .decrypt = otx2_cpt_skcipher_decrypt,1339}, {1340 .base.cra_name = "cbc(aes)",1341 .base.cra_driver_name = "cpt_cbc_aes",1342 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,1343 .base.cra_blocksize = AES_BLOCK_SIZE,1344 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),1345 .base.cra_alignmask = 7,1346 .base.cra_priority = 4001,1347 .base.cra_module = THIS_MODULE,1348 1349 .init = otx2_cpt_enc_dec_init,1350 .exit = otx2_cpt_skcipher_exit,1351 .ivsize = AES_BLOCK_SIZE,1352 .min_keysize = AES_MIN_KEY_SIZE,1353 .max_keysize = AES_MAX_KEY_SIZE,1354 .setkey = otx2_cpt_skcipher_cbc_aes_setkey,1355 .encrypt = otx2_cpt_skcipher_encrypt,1356 .decrypt = otx2_cpt_skcipher_decrypt,1357}, {1358 .base.cra_name = "ecb(aes)",1359 .base.cra_driver_name = "cpt_ecb_aes",1360 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,1361 .base.cra_blocksize = AES_BLOCK_SIZE,1362 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),1363 .base.cra_alignmask = 7,1364 .base.cra_priority = 4001,1365 .base.cra_module = THIS_MODULE,1366 1367 .init = otx2_cpt_enc_dec_init,1368 .exit = otx2_cpt_skcipher_exit,1369 .ivsize = 0,1370 .min_keysize = AES_MIN_KEY_SIZE,1371 .max_keysize = AES_MAX_KEY_SIZE,1372 .setkey = otx2_cpt_skcipher_ecb_aes_setkey,1373 .encrypt = otx2_cpt_skcipher_encrypt,1374 .decrypt = otx2_cpt_skcipher_decrypt,1375}, {1376 .base.cra_name = "cbc(des3_ede)",1377 .base.cra_driver_name = "cpt_cbc_des3_ede",1378 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,1379 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,1380 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),1381 .base.cra_alignmask = 7,1382 .base.cra_priority = 4001,1383 .base.cra_module = THIS_MODULE,1384 1385 .init = otx2_cpt_enc_dec_init,1386 .exit = otx2_cpt_skcipher_exit,1387 .min_keysize = DES3_EDE_KEY_SIZE,1388 .max_keysize = DES3_EDE_KEY_SIZE,1389 .ivsize = DES_BLOCK_SIZE,1390 .setkey = otx2_cpt_skcipher_cbc_des3_setkey,1391 .encrypt = otx2_cpt_skcipher_encrypt,1392 .decrypt = otx2_cpt_skcipher_decrypt,1393}, {1394 .base.cra_name = "ecb(des3_ede)",1395 .base.cra_driver_name = "cpt_ecb_des3_ede",1396 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,1397 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,1398 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),1399 .base.cra_alignmask = 7,1400 .base.cra_priority = 4001,1401 .base.cra_module = THIS_MODULE,1402 1403 .init = otx2_cpt_enc_dec_init,1404 .exit = otx2_cpt_skcipher_exit,1405 .min_keysize = DES3_EDE_KEY_SIZE,1406 .max_keysize = DES3_EDE_KEY_SIZE,1407 .ivsize = 0,1408 .setkey = otx2_cpt_skcipher_ecb_des3_setkey,1409 .encrypt = otx2_cpt_skcipher_encrypt,1410 .decrypt = otx2_cpt_skcipher_decrypt,1411} };1412 1413static struct aead_alg otx2_cpt_aeads[] = { {1414 .base = {1415 .cra_name = "authenc(hmac(sha1),cbc(aes))",1416 .cra_driver_name = "cpt_hmac_sha1_cbc_aes",1417 .cra_blocksize = AES_BLOCK_SIZE,1418 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,1419 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1420 .cra_priority = 4001,1421 .cra_alignmask = 0,1422 .cra_module = THIS_MODULE,1423 },1424 .init = otx2_cpt_aead_cbc_aes_sha1_init,1425 .exit = otx2_cpt_aead_exit,1426 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,1427 .setauthsize = otx2_cpt_aead_set_authsize,1428 .encrypt = otx2_cpt_aead_encrypt,1429 .decrypt = otx2_cpt_aead_decrypt,1430 .ivsize = AES_BLOCK_SIZE,1431 .maxauthsize = SHA1_DIGEST_SIZE,1432}, {1433 .base = {1434 .cra_name = "authenc(hmac(sha256),cbc(aes))",1435 .cra_driver_name = "cpt_hmac_sha256_cbc_aes",1436 .cra_blocksize = AES_BLOCK_SIZE,1437 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,1438 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1439 .cra_priority = 4001,1440 .cra_alignmask = 0,1441 .cra_module = THIS_MODULE,1442 },1443 .init = otx2_cpt_aead_cbc_aes_sha256_init,1444 .exit = otx2_cpt_aead_exit,1445 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,1446 .setauthsize = otx2_cpt_aead_set_authsize,1447 .encrypt = otx2_cpt_aead_encrypt,1448 .decrypt = otx2_cpt_aead_decrypt,1449 .ivsize = AES_BLOCK_SIZE,1450 .maxauthsize = SHA256_DIGEST_SIZE,1451}, {1452 .base = {1453 .cra_name = "authenc(hmac(sha384),cbc(aes))",1454 .cra_driver_name = "cpt_hmac_sha384_cbc_aes",1455 .cra_blocksize = AES_BLOCK_SIZE,1456 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,1457 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1458 .cra_priority = 4001,1459 .cra_alignmask = 0,1460 .cra_module = THIS_MODULE,1461 },1462 .init = otx2_cpt_aead_cbc_aes_sha384_init,1463 .exit = otx2_cpt_aead_exit,1464 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,1465 .setauthsize = otx2_cpt_aead_set_authsize,1466 .encrypt = otx2_cpt_aead_encrypt,1467 .decrypt = otx2_cpt_aead_decrypt,1468 .ivsize = AES_BLOCK_SIZE,1469 .maxauthsize = SHA384_DIGEST_SIZE,1470}, {1471 .base = {1472 .cra_name = "authenc(hmac(sha512),cbc(aes))",1473 .cra_driver_name = "cpt_hmac_sha512_cbc_aes",1474 .cra_blocksize = AES_BLOCK_SIZE,1475 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,1476 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1477 .cra_priority = 4001,1478 .cra_alignmask = 0,1479 .cra_module = THIS_MODULE,1480 },1481 .init = otx2_cpt_aead_cbc_aes_sha512_init,1482 .exit = otx2_cpt_aead_exit,1483 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,1484 .setauthsize = otx2_cpt_aead_set_authsize,1485 .encrypt = otx2_cpt_aead_encrypt,1486 .decrypt = otx2_cpt_aead_decrypt,1487 .ivsize = AES_BLOCK_SIZE,1488 .maxauthsize = SHA512_DIGEST_SIZE,1489}, {1490 .base = {1491 .cra_name = "authenc(hmac(sha1),ecb(cipher_null))",1492 .cra_driver_name = "cpt_hmac_sha1_ecb_null",1493 .cra_blocksize = 1,1494 .cra_flags = CRYPTO_ALG_ASYNC,1495 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1496 .cra_priority = 4001,1497 .cra_alignmask = 0,1498 .cra_module = THIS_MODULE,1499 },1500 .init = otx2_cpt_aead_ecb_null_sha1_init,1501 .exit = otx2_cpt_aead_exit,1502 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,1503 .setauthsize = otx2_cpt_aead_null_set_authsize,1504 .encrypt = otx2_cpt_aead_null_encrypt,1505 .decrypt = otx2_cpt_aead_null_decrypt,1506 .ivsize = 0,1507 .maxauthsize = SHA1_DIGEST_SIZE,1508}, {1509 .base = {1510 .cra_name = "authenc(hmac(sha256),ecb(cipher_null))",1511 .cra_driver_name = "cpt_hmac_sha256_ecb_null",1512 .cra_blocksize = 1,1513 .cra_flags = CRYPTO_ALG_ASYNC,1514 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1515 .cra_priority = 4001,1516 .cra_alignmask = 0,1517 .cra_module = THIS_MODULE,1518 },1519 .init = otx2_cpt_aead_ecb_null_sha256_init,1520 .exit = otx2_cpt_aead_exit,1521 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,1522 .setauthsize = otx2_cpt_aead_null_set_authsize,1523 .encrypt = otx2_cpt_aead_null_encrypt,1524 .decrypt = otx2_cpt_aead_null_decrypt,1525 .ivsize = 0,1526 .maxauthsize = SHA256_DIGEST_SIZE,1527}, {1528 .base = {1529 .cra_name = "authenc(hmac(sha384),ecb(cipher_null))",1530 .cra_driver_name = "cpt_hmac_sha384_ecb_null",1531 .cra_blocksize = 1,1532 .cra_flags = CRYPTO_ALG_ASYNC,1533 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1534 .cra_priority = 4001,1535 .cra_alignmask = 0,1536 .cra_module = THIS_MODULE,1537 },1538 .init = otx2_cpt_aead_ecb_null_sha384_init,1539 .exit = otx2_cpt_aead_exit,1540 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,1541 .setauthsize = otx2_cpt_aead_null_set_authsize,1542 .encrypt = otx2_cpt_aead_null_encrypt,1543 .decrypt = otx2_cpt_aead_null_decrypt,1544 .ivsize = 0,1545 .maxauthsize = SHA384_DIGEST_SIZE,1546}, {1547 .base = {1548 .cra_name = "authenc(hmac(sha512),ecb(cipher_null))",1549 .cra_driver_name = "cpt_hmac_sha512_ecb_null",1550 .cra_blocksize = 1,1551 .cra_flags = CRYPTO_ALG_ASYNC,1552 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1553 .cra_priority = 4001,1554 .cra_alignmask = 0,1555 .cra_module = THIS_MODULE,1556 },1557 .init = otx2_cpt_aead_ecb_null_sha512_init,1558 .exit = otx2_cpt_aead_exit,1559 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,1560 .setauthsize = otx2_cpt_aead_null_set_authsize,1561 .encrypt = otx2_cpt_aead_null_encrypt,1562 .decrypt = otx2_cpt_aead_null_decrypt,1563 .ivsize = 0,1564 .maxauthsize = SHA512_DIGEST_SIZE,1565}, {1566 .base = {1567 .cra_name = "rfc4106(gcm(aes))",1568 .cra_driver_name = "cpt_rfc4106_gcm_aes",1569 .cra_blocksize = 1,1570 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,1571 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,1572 .cra_priority = 4001,1573 .cra_alignmask = 0,1574 .cra_module = THIS_MODULE,1575 },1576 .init = otx2_cpt_aead_gcm_aes_init,1577 .exit = otx2_cpt_aead_exit,1578 .setkey = otx2_cpt_aead_gcm_aes_setkey,1579 .setauthsize = otx2_cpt_aead_gcm_set_authsize,1580 .encrypt = otx2_cpt_aead_encrypt,1581 .decrypt = otx2_cpt_aead_decrypt,1582 .ivsize = AES_GCM_IV_SIZE,1583 .maxauthsize = AES_GCM_ICV_SIZE,1584} };1585 1586static inline int cpt_register_algs(void)1587{1588 int i, err = 0;1589 1590 for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++)1591 otx2_cpt_skciphers[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;1592 1593 err = crypto_register_skciphers(otx2_cpt_skciphers,1594 ARRAY_SIZE(otx2_cpt_skciphers));1595 if (err)1596 return err;1597 1598 for (i = 0; i < ARRAY_SIZE(otx2_cpt_aeads); i++)1599 otx2_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;1600 1601 err = crypto_register_aeads(otx2_cpt_aeads,1602 ARRAY_SIZE(otx2_cpt_aeads));1603 if (err) {1604 crypto_unregister_skciphers(otx2_cpt_skciphers,1605 ARRAY_SIZE(otx2_cpt_skciphers));1606 return err;1607 }1608 1609 return 0;1610}1611 1612static inline void cpt_unregister_algs(void)1613{1614 crypto_unregister_skciphers(otx2_cpt_skciphers,1615 ARRAY_SIZE(otx2_cpt_skciphers));1616 crypto_unregister_aeads(otx2_cpt_aeads, ARRAY_SIZE(otx2_cpt_aeads));1617}1618 1619static int compare_func(const void *lptr, const void *rptr)1620{1621 const struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr;1622 const struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr;1623 1624 if (ldesc->dev->devfn < rdesc->dev->devfn)1625 return -1;1626 if (ldesc->dev->devfn > rdesc->dev->devfn)1627 return 1;1628 return 0;1629}1630 1631int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,1632 int num_queues, int num_devices)1633{1634 int ret = 0;1635 int count;1636 1637 mutex_lock(&mutex);1638 count = atomic_read(&se_devices.count);1639 if (count >= OTX2_CPT_MAX_LFS_NUM) {1640 dev_err(&pdev->dev, "No space to add a new device\n");1641 ret = -ENOSPC;1642 goto unlock;1643 }1644 se_devices.desc[count].num_queues = num_queues;1645 se_devices.desc[count++].dev = pdev;1646 atomic_inc(&se_devices.count);1647 1648 if (atomic_read(&se_devices.count) == num_devices &&1649 is_crypto_registered == false) {1650 if (cpt_register_algs()) {1651 dev_err(&pdev->dev,1652 "Error in registering crypto algorithms\n");1653 ret = -EINVAL;1654 goto unlock;1655 }1656 try_module_get(mod);1657 is_crypto_registered = true;1658 }1659 sort(se_devices.desc, count, sizeof(struct cpt_device_desc),1660 compare_func, NULL);1661 1662unlock:1663 mutex_unlock(&mutex);1664 return ret;1665}1666 1667void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod)1668{1669 struct cpt_device_table *dev_tbl;1670 bool dev_found = false;1671 int i, j, count;1672 1673 mutex_lock(&mutex);1674 1675 dev_tbl = &se_devices;1676 count = atomic_read(&dev_tbl->count);1677 for (i = 0; i < count; i++) {1678 if (pdev == dev_tbl->desc[i].dev) {1679 for (j = i; j < count-1; j++)1680 dev_tbl->desc[j] = dev_tbl->desc[j+1];1681 dev_found = true;1682 break;1683 }1684 }1685 1686 if (!dev_found) {1687 dev_err(&pdev->dev, "%s device not found\n", __func__);1688 goto unlock;1689 }1690 if (atomic_dec_and_test(&se_devices.count)) {1691 cpt_unregister_algs();1692 module_put(mod);1693 is_crypto_registered = false;1694 }1695 1696unlock:1697 mutex_unlock(&mutex);1698}1699