2744 lines · c
1// SPDX-License-Identifier: GPL-2.0+2/*3 * Freescale FSL CAAM support for crypto API over QI backend.4 * Based on caamalg.c5 *6 * Copyright 2013-2016 Freescale Semiconductor, Inc.7 * Copyright 2016-2019 NXP8 */9 10#include "compat.h"11#include "ctrl.h"12#include "regs.h"13#include "intern.h"14#include "desc_constr.h"15#include "error.h"16#include "sg_sw_qm.h"17#include "key_gen.h"18#include "qi.h"19#include "jr.h"20#include "caamalg_desc.h"21#include <crypto/xts.h>22#include <linux/unaligned.h>23#include <linux/device.h>24#include <linux/err.h>25#include <linux/dma-mapping.h>26#include <linux/kernel.h>27#include <linux/string.h>28 29/*30 * crypto alg31 */32#define CAAM_CRA_PRIORITY 200033/* max key is sum of AES_MAX_KEY_SIZE, max split key size */34#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \35 SHA512_DIGEST_SIZE * 2)36 37#define DESC_MAX_USED_BYTES (DESC_QI_AEAD_GIVENC_LEN + \38 CAAM_MAX_KEY_SIZE)39#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)40 41struct caam_alg_entry {42 int class1_alg_type;43 int class2_alg_type;44 bool rfc3686;45 bool geniv;46 bool nodkp;47};48 49struct caam_aead_alg {50 struct aead_alg aead;51 struct caam_alg_entry caam;52 bool registered;53};54 55struct caam_skcipher_alg {56 struct skcipher_alg skcipher;57 struct caam_alg_entry caam;58 bool registered;59};60 61/*62 * per-session context63 */64struct caam_ctx {65 struct device *jrdev;66 u32 sh_desc_enc[DESC_MAX_USED_LEN];67 u32 sh_desc_dec[DESC_MAX_USED_LEN];68 u8 key[CAAM_MAX_KEY_SIZE];69 dma_addr_t key_dma;70 enum dma_data_direction dir;71 struct alginfo adata;72 struct alginfo cdata;73 unsigned int authsize;74 struct device *qidev;75 spinlock_t lock; /* Protects multiple init of driver context */76 struct caam_drv_ctx *drv_ctx[NUM_OP];77 bool xts_key_fallback;78 struct crypto_skcipher *fallback;79};80 81struct caam_skcipher_req_ctx {82 struct skcipher_request fallback_req;83};84 85static int aead_set_sh_desc(struct crypto_aead *aead)86{87 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),88 typeof(*alg), aead);89 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);90 unsigned int ivsize = crypto_aead_ivsize(aead);91 u32 ctx1_iv_off = 0;92 u32 *nonce = NULL;93 unsigned int data_len[2];94 u32 inl_mask;95 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==96 OP_ALG_AAI_CTR_MOD128);97 const bool is_rfc3686 = alg->caam.rfc3686;98 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);99 100 if (!ctx->cdata.keylen || !ctx->authsize)101 return 0;102 103 /*104 * AES-CTR needs to load IV in CONTEXT1 reg105 * at an offset of 128bits (16bytes)106 * CONTEXT1[255:128] = IV107 */108 if (ctr_mode)109 ctx1_iv_off = 16;110 111 /*112 * RFC3686 specific:113 * CONTEXT1[255:128] = {NONCE, IV, COUNTER}114 */115 if (is_rfc3686) {116 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;117 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +118 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);119 }120 121 /*122 * In case |user key| > |derived key|, using DKP<imm,imm> would result123 * in invalid opcodes (last bytes of user key) in the resulting124 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key125 * addresses are needed.126 */127 ctx->adata.key_virt = ctx->key;128 ctx->adata.key_dma = ctx->key_dma;129 130 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;131 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;132 133 data_len[0] = ctx->adata.keylen_pad;134 data_len[1] = ctx->cdata.keylen;135 136 if (alg->caam.geniv)137 goto skip_enc;138 139 /* aead_encrypt shared descriptor */140 if (desc_inline_query(DESC_QI_AEAD_ENC_LEN +141 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),142 DESC_JOB_IO_LEN, data_len, &inl_mask,143 ARRAY_SIZE(data_len)) < 0)144 return -EINVAL;145 146 ctx->adata.key_inline = !!(inl_mask & 1);147 ctx->cdata.key_inline = !!(inl_mask & 2);148 149 cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,150 ivsize, ctx->authsize, is_rfc3686, nonce,151 ctx1_iv_off, true, ctrlpriv->era);152 153skip_enc:154 /* aead_decrypt shared descriptor */155 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +156 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),157 DESC_JOB_IO_LEN, data_len, &inl_mask,158 ARRAY_SIZE(data_len)) < 0)159 return -EINVAL;160 161 ctx->adata.key_inline = !!(inl_mask & 1);162 ctx->cdata.key_inline = !!(inl_mask & 2);163 164 cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata,165 ivsize, ctx->authsize, alg->caam.geniv,166 is_rfc3686, nonce, ctx1_iv_off, true,167 ctrlpriv->era);168 169 if (!alg->caam.geniv)170 goto skip_givenc;171 172 /* aead_givencrypt shared descriptor */173 if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN +174 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),175 DESC_JOB_IO_LEN, data_len, &inl_mask,176 ARRAY_SIZE(data_len)) < 0)177 return -EINVAL;178 179 ctx->adata.key_inline = !!(inl_mask & 1);180 ctx->cdata.key_inline = !!(inl_mask & 2);181 182 cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,183 ivsize, ctx->authsize, is_rfc3686, nonce,184 ctx1_iv_off, true, ctrlpriv->era);185 186skip_givenc:187 return 0;188}189 190static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)191{192 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);193 194 ctx->authsize = authsize;195 aead_set_sh_desc(authenc);196 197 return 0;198}199 200static int aead_setkey(struct crypto_aead *aead, const u8 *key,201 unsigned int keylen)202{203 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);204 struct device *jrdev = ctx->jrdev;205 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);206 struct crypto_authenc_keys keys;207 int ret = 0;208 209 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)210 goto badkey;211 212 dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",213 keys.authkeylen + keys.enckeylen, keys.enckeylen,214 keys.authkeylen);215 print_hex_dump_debug("key in @" __stringify(__LINE__)": ",216 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);217 218 /*219 * If DKP is supported, use it in the shared descriptor to generate220 * the split key.221 */222 if (ctrlpriv->era >= 6) {223 ctx->adata.keylen = keys.authkeylen;224 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &225 OP_ALG_ALGSEL_MASK);226 227 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)228 goto badkey;229 230 memcpy(ctx->key, keys.authkey, keys.authkeylen);231 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,232 keys.enckeylen);233 dma_sync_single_for_device(jrdev->parent, ctx->key_dma,234 ctx->adata.keylen_pad +235 keys.enckeylen, ctx->dir);236 goto skip_split_key;237 }238 239 ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey,240 keys.authkeylen, CAAM_MAX_KEY_SIZE -241 keys.enckeylen);242 if (ret)243 goto badkey;244 245 /* postpend encryption key to auth split key */246 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);247 dma_sync_single_for_device(jrdev->parent, ctx->key_dma,248 ctx->adata.keylen_pad + keys.enckeylen,249 ctx->dir);250 251 print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",252 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,253 ctx->adata.keylen_pad + keys.enckeylen, 1);254 255skip_split_key:256 ctx->cdata.keylen = keys.enckeylen;257 258 ret = aead_set_sh_desc(aead);259 if (ret)260 goto badkey;261 262 /* Now update the driver contexts with the new shared descriptor */263 if (ctx->drv_ctx[ENCRYPT]) {264 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],265 ctx->sh_desc_enc);266 if (ret) {267 dev_err(jrdev, "driver enc context update failed\n");268 goto badkey;269 }270 }271 272 if (ctx->drv_ctx[DECRYPT]) {273 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],274 ctx->sh_desc_dec);275 if (ret) {276 dev_err(jrdev, "driver dec context update failed\n");277 goto badkey;278 }279 }280 281 memzero_explicit(&keys, sizeof(keys));282 return ret;283badkey:284 memzero_explicit(&keys, sizeof(keys));285 return -EINVAL;286}287 288static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,289 unsigned int keylen)290{291 struct crypto_authenc_keys keys;292 int err;293 294 err = crypto_authenc_extractkeys(&keys, key, keylen);295 if (unlikely(err))296 return err;297 298 err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:299 aead_setkey(aead, key, keylen);300 301 memzero_explicit(&keys, sizeof(keys));302 return err;303}304 305static int gcm_set_sh_desc(struct crypto_aead *aead)306{307 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);308 unsigned int ivsize = crypto_aead_ivsize(aead);309 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -310 ctx->cdata.keylen;311 312 if (!ctx->cdata.keylen || !ctx->authsize)313 return 0;314 315 /*316 * Job Descriptor and Shared Descriptor317 * must fit into the 64-word Descriptor h/w Buffer318 */319 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {320 ctx->cdata.key_inline = true;321 ctx->cdata.key_virt = ctx->key;322 } else {323 ctx->cdata.key_inline = false;324 ctx->cdata.key_dma = ctx->key_dma;325 }326 327 cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,328 ctx->authsize, true);329 330 /*331 * Job Descriptor and Shared Descriptor332 * must fit into the 64-word Descriptor h/w Buffer333 */334 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {335 ctx->cdata.key_inline = true;336 ctx->cdata.key_virt = ctx->key;337 } else {338 ctx->cdata.key_inline = false;339 ctx->cdata.key_dma = ctx->key_dma;340 }341 342 cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,343 ctx->authsize, true);344 345 return 0;346}347 348static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)349{350 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);351 int err;352 353 err = crypto_gcm_check_authsize(authsize);354 if (err)355 return err;356 357 ctx->authsize = authsize;358 gcm_set_sh_desc(authenc);359 360 return 0;361}362 363static int gcm_setkey(struct crypto_aead *aead,364 const u8 *key, unsigned int keylen)365{366 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);367 struct device *jrdev = ctx->jrdev;368 int ret;369 370 ret = aes_check_keylen(keylen);371 if (ret)372 return ret;373 374 print_hex_dump_debug("key in @" __stringify(__LINE__)": ",375 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);376 377 memcpy(ctx->key, key, keylen);378 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen,379 ctx->dir);380 ctx->cdata.keylen = keylen;381 382 ret = gcm_set_sh_desc(aead);383 if (ret)384 return ret;385 386 /* Now update the driver contexts with the new shared descriptor */387 if (ctx->drv_ctx[ENCRYPT]) {388 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],389 ctx->sh_desc_enc);390 if (ret) {391 dev_err(jrdev, "driver enc context update failed\n");392 return ret;393 }394 }395 396 if (ctx->drv_ctx[DECRYPT]) {397 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],398 ctx->sh_desc_dec);399 if (ret) {400 dev_err(jrdev, "driver dec context update failed\n");401 return ret;402 }403 }404 405 return 0;406}407 408static int rfc4106_set_sh_desc(struct crypto_aead *aead)409{410 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);411 unsigned int ivsize = crypto_aead_ivsize(aead);412 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -413 ctx->cdata.keylen;414 415 if (!ctx->cdata.keylen || !ctx->authsize)416 return 0;417 418 ctx->cdata.key_virt = ctx->key;419 420 /*421 * Job Descriptor and Shared Descriptor422 * must fit into the 64-word Descriptor h/w Buffer423 */424 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {425 ctx->cdata.key_inline = true;426 } else {427 ctx->cdata.key_inline = false;428 ctx->cdata.key_dma = ctx->key_dma;429 }430 431 cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,432 ctx->authsize, true);433 434 /*435 * Job Descriptor and Shared Descriptor436 * must fit into the 64-word Descriptor h/w Buffer437 */438 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {439 ctx->cdata.key_inline = true;440 } else {441 ctx->cdata.key_inline = false;442 ctx->cdata.key_dma = ctx->key_dma;443 }444 445 cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,446 ctx->authsize, true);447 448 return 0;449}450 451static int rfc4106_setauthsize(struct crypto_aead *authenc,452 unsigned int authsize)453{454 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);455 int err;456 457 err = crypto_rfc4106_check_authsize(authsize);458 if (err)459 return err;460 461 ctx->authsize = authsize;462 rfc4106_set_sh_desc(authenc);463 464 return 0;465}466 467static int rfc4106_setkey(struct crypto_aead *aead,468 const u8 *key, unsigned int keylen)469{470 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);471 struct device *jrdev = ctx->jrdev;472 int ret;473 474 ret = aes_check_keylen(keylen - 4);475 if (ret)476 return ret;477 478 print_hex_dump_debug("key in @" __stringify(__LINE__)": ",479 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);480 481 memcpy(ctx->key, key, keylen);482 /*483 * The last four bytes of the key material are used as the salt value484 * in the nonce. Update the AES key length.485 */486 ctx->cdata.keylen = keylen - 4;487 dma_sync_single_for_device(jrdev->parent, ctx->key_dma,488 ctx->cdata.keylen, ctx->dir);489 490 ret = rfc4106_set_sh_desc(aead);491 if (ret)492 return ret;493 494 /* Now update the driver contexts with the new shared descriptor */495 if (ctx->drv_ctx[ENCRYPT]) {496 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],497 ctx->sh_desc_enc);498 if (ret) {499 dev_err(jrdev, "driver enc context update failed\n");500 return ret;501 }502 }503 504 if (ctx->drv_ctx[DECRYPT]) {505 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],506 ctx->sh_desc_dec);507 if (ret) {508 dev_err(jrdev, "driver dec context update failed\n");509 return ret;510 }511 }512 513 return 0;514}515 516static int rfc4543_set_sh_desc(struct crypto_aead *aead)517{518 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);519 unsigned int ivsize = crypto_aead_ivsize(aead);520 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -521 ctx->cdata.keylen;522 523 if (!ctx->cdata.keylen || !ctx->authsize)524 return 0;525 526 ctx->cdata.key_virt = ctx->key;527 528 /*529 * Job Descriptor and Shared Descriptor530 * must fit into the 64-word Descriptor h/w Buffer531 */532 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {533 ctx->cdata.key_inline = true;534 } else {535 ctx->cdata.key_inline = false;536 ctx->cdata.key_dma = ctx->key_dma;537 }538 539 cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,540 ctx->authsize, true);541 542 /*543 * Job Descriptor and Shared Descriptor544 * must fit into the 64-word Descriptor h/w Buffer545 */546 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {547 ctx->cdata.key_inline = true;548 } else {549 ctx->cdata.key_inline = false;550 ctx->cdata.key_dma = ctx->key_dma;551 }552 553 cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,554 ctx->authsize, true);555 556 return 0;557}558 559static int rfc4543_setauthsize(struct crypto_aead *authenc,560 unsigned int authsize)561{562 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);563 564 if (authsize != 16)565 return -EINVAL;566 567 ctx->authsize = authsize;568 rfc4543_set_sh_desc(authenc);569 570 return 0;571}572 573static int rfc4543_setkey(struct crypto_aead *aead,574 const u8 *key, unsigned int keylen)575{576 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);577 struct device *jrdev = ctx->jrdev;578 int ret;579 580 ret = aes_check_keylen(keylen - 4);581 if (ret)582 return ret;583 584 print_hex_dump_debug("key in @" __stringify(__LINE__)": ",585 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);586 587 memcpy(ctx->key, key, keylen);588 /*589 * The last four bytes of the key material are used as the salt value590 * in the nonce. Update the AES key length.591 */592 ctx->cdata.keylen = keylen - 4;593 dma_sync_single_for_device(jrdev->parent, ctx->key_dma,594 ctx->cdata.keylen, ctx->dir);595 596 ret = rfc4543_set_sh_desc(aead);597 if (ret)598 return ret;599 600 /* Now update the driver contexts with the new shared descriptor */601 if (ctx->drv_ctx[ENCRYPT]) {602 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],603 ctx->sh_desc_enc);604 if (ret) {605 dev_err(jrdev, "driver enc context update failed\n");606 return ret;607 }608 }609 610 if (ctx->drv_ctx[DECRYPT]) {611 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],612 ctx->sh_desc_dec);613 if (ret) {614 dev_err(jrdev, "driver dec context update failed\n");615 return ret;616 }617 }618 619 return 0;620}621 622static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,623 unsigned int keylen, const u32 ctx1_iv_off)624{625 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);626 struct caam_skcipher_alg *alg =627 container_of(crypto_skcipher_alg(skcipher), typeof(*alg),628 skcipher);629 struct device *jrdev = ctx->jrdev;630 unsigned int ivsize = crypto_skcipher_ivsize(skcipher);631 const bool is_rfc3686 = alg->caam.rfc3686;632 int ret = 0;633 634 print_hex_dump_debug("key in @" __stringify(__LINE__)": ",635 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);636 637 ctx->cdata.keylen = keylen;638 ctx->cdata.key_virt = key;639 ctx->cdata.key_inline = true;640 641 /* skcipher encrypt, decrypt shared descriptors */642 cnstr_shdsc_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,643 is_rfc3686, ctx1_iv_off);644 cnstr_shdsc_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,645 is_rfc3686, ctx1_iv_off);646 647 /* Now update the driver contexts with the new shared descriptor */648 if (ctx->drv_ctx[ENCRYPT]) {649 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],650 ctx->sh_desc_enc);651 if (ret) {652 dev_err(jrdev, "driver enc context update failed\n");653 return -EINVAL;654 }655 }656 657 if (ctx->drv_ctx[DECRYPT]) {658 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],659 ctx->sh_desc_dec);660 if (ret) {661 dev_err(jrdev, "driver dec context update failed\n");662 return -EINVAL;663 }664 }665 666 return ret;667}668 669static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,670 const u8 *key, unsigned int keylen)671{672 int err;673 674 err = aes_check_keylen(keylen);675 if (err)676 return err;677 678 return skcipher_setkey(skcipher, key, keylen, 0);679}680 681static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,682 const u8 *key, unsigned int keylen)683{684 u32 ctx1_iv_off;685 int err;686 687 /*688 * RFC3686 specific:689 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}690 * | *key = {KEY, NONCE}691 */692 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;693 keylen -= CTR_RFC3686_NONCE_SIZE;694 695 err = aes_check_keylen(keylen);696 if (err)697 return err;698 699 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);700}701 702static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,703 const u8 *key, unsigned int keylen)704{705 u32 ctx1_iv_off;706 int err;707 708 /*709 * AES-CTR needs to load IV in CONTEXT1 reg710 * at an offset of 128bits (16bytes)711 * CONTEXT1[255:128] = IV712 */713 ctx1_iv_off = 16;714 715 err = aes_check_keylen(keylen);716 if (err)717 return err;718 719 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);720}721 722static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,723 const u8 *key, unsigned int keylen)724{725 return verify_skcipher_des3_key(skcipher, key) ?:726 skcipher_setkey(skcipher, key, keylen, 0);727}728 729static int des_skcipher_setkey(struct crypto_skcipher *skcipher,730 const u8 *key, unsigned int keylen)731{732 return verify_skcipher_des_key(skcipher, key) ?:733 skcipher_setkey(skcipher, key, keylen, 0);734}735 736static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,737 unsigned int keylen)738{739 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);740 struct device *jrdev = ctx->jrdev;741 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);742 int ret = 0;743 int err;744 745 err = xts_verify_key(skcipher, key, keylen);746 if (err) {747 dev_dbg(jrdev, "key size mismatch\n");748 return err;749 }750 751 if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)752 ctx->xts_key_fallback = true;753 754 if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) {755 err = crypto_skcipher_setkey(ctx->fallback, key, keylen);756 if (err)757 return err;758 }759 760 ctx->cdata.keylen = keylen;761 ctx->cdata.key_virt = key;762 ctx->cdata.key_inline = true;763 764 /* xts skcipher encrypt, decrypt shared descriptors */765 cnstr_shdsc_xts_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata);766 cnstr_shdsc_xts_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata);767 768 /* Now update the driver contexts with the new shared descriptor */769 if (ctx->drv_ctx[ENCRYPT]) {770 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],771 ctx->sh_desc_enc);772 if (ret) {773 dev_err(jrdev, "driver enc context update failed\n");774 return -EINVAL;775 }776 }777 778 if (ctx->drv_ctx[DECRYPT]) {779 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],780 ctx->sh_desc_dec);781 if (ret) {782 dev_err(jrdev, "driver dec context update failed\n");783 return -EINVAL;784 }785 }786 787 return ret;788}789 790/*791 * aead_edesc - s/w-extended aead descriptor792 * @src_nents: number of segments in input scatterlist793 * @dst_nents: number of segments in output scatterlist794 * @iv_dma: dma address of iv for checking continuity and link table795 * @qm_sg_bytes: length of dma mapped h/w link table796 * @qm_sg_dma: bus physical mapped address of h/w link table797 * @assoclen: associated data length, in CAAM endianness798 * @assoclen_dma: bus physical mapped address of req->assoclen799 * @drv_req: driver-specific request structure800 * @sgt: the h/w link table, followed by IV801 */802struct aead_edesc {803 int src_nents;804 int dst_nents;805 dma_addr_t iv_dma;806 int qm_sg_bytes;807 dma_addr_t qm_sg_dma;808 unsigned int assoclen;809 dma_addr_t assoclen_dma;810 struct caam_drv_req drv_req;811 struct qm_sg_entry sgt[];812};813 814/*815 * skcipher_edesc - s/w-extended skcipher descriptor816 * @src_nents: number of segments in input scatterlist817 * @dst_nents: number of segments in output scatterlist818 * @iv_dma: dma address of iv for checking continuity and link table819 * @qm_sg_bytes: length of dma mapped h/w link table820 * @qm_sg_dma: bus physical mapped address of h/w link table821 * @drv_req: driver-specific request structure822 * @sgt: the h/w link table, followed by IV823 */824struct skcipher_edesc {825 int src_nents;826 int dst_nents;827 dma_addr_t iv_dma;828 int qm_sg_bytes;829 dma_addr_t qm_sg_dma;830 struct caam_drv_req drv_req;831 struct qm_sg_entry sgt[];832};833 834static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx,835 enum optype type)836{837 /*838 * This function is called on the fast path with values of 'type'839 * known at compile time. Invalid arguments are not expected and840 * thus no checks are made.841 */842 struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type];843 u32 *desc;844 845 if (unlikely(!drv_ctx)) {846 spin_lock(&ctx->lock);847 848 /* Read again to check if some other core init drv_ctx */849 drv_ctx = ctx->drv_ctx[type];850 if (!drv_ctx) {851 int cpu;852 853 if (type == ENCRYPT)854 desc = ctx->sh_desc_enc;855 else /* (type == DECRYPT) */856 desc = ctx->sh_desc_dec;857 858 cpu = smp_processor_id();859 drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc);860 if (!IS_ERR(drv_ctx))861 drv_ctx->op_type = type;862 863 ctx->drv_ctx[type] = drv_ctx;864 }865 866 spin_unlock(&ctx->lock);867 }868 869 return drv_ctx;870}871 872static void caam_unmap(struct device *dev, struct scatterlist *src,873 struct scatterlist *dst, int src_nents,874 int dst_nents, dma_addr_t iv_dma, int ivsize,875 enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,876 int qm_sg_bytes)877{878 if (dst != src) {879 if (src_nents)880 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);881 if (dst_nents)882 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);883 } else {884 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);885 }886 887 if (iv_dma)888 dma_unmap_single(dev, iv_dma, ivsize, iv_dir);889 if (qm_sg_bytes)890 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);891}892 893static void aead_unmap(struct device *dev,894 struct aead_edesc *edesc,895 struct aead_request *req)896{897 struct crypto_aead *aead = crypto_aead_reqtfm(req);898 int ivsize = crypto_aead_ivsize(aead);899 900 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,901 edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,902 edesc->qm_sg_bytes);903 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);904}905 906static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,907 struct skcipher_request *req)908{909 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);910 int ivsize = crypto_skcipher_ivsize(skcipher);911 912 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,913 edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,914 edesc->qm_sg_bytes);915}916 917static void aead_done(struct caam_drv_req *drv_req, u32 status)918{919 struct device *qidev;920 struct aead_edesc *edesc;921 struct aead_request *aead_req = drv_req->app_ctx;922 struct crypto_aead *aead = crypto_aead_reqtfm(aead_req);923 struct caam_ctx *caam_ctx = crypto_aead_ctx_dma(aead);924 int ecode = 0;925 926 qidev = caam_ctx->qidev;927 928 if (unlikely(status))929 ecode = caam_jr_strstatus(qidev, status);930 931 edesc = container_of(drv_req, typeof(*edesc), drv_req);932 aead_unmap(qidev, edesc, aead_req);933 934 aead_request_complete(aead_req, ecode);935 qi_cache_free(edesc);936}937 938/*939 * allocate and map the aead extended descriptor940 */941static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,942 bool encrypt)943{944 struct crypto_aead *aead = crypto_aead_reqtfm(req);945 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);946 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),947 typeof(*alg), aead);948 struct device *qidev = ctx->qidev;949 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?950 GFP_KERNEL : GFP_ATOMIC;951 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;952 int src_len, dst_len = 0;953 struct aead_edesc *edesc;954 dma_addr_t qm_sg_dma, iv_dma = 0;955 int ivsize = 0;956 unsigned int authsize = ctx->authsize;957 int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes;958 int in_len, out_len;959 struct qm_sg_entry *sg_table, *fd_sgt;960 struct caam_drv_ctx *drv_ctx;961 962 drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);963 if (IS_ERR(drv_ctx))964 return ERR_CAST(drv_ctx);965 966 /* allocate space for base edesc and hw desc commands, link tables */967 edesc = qi_cache_alloc(flags);968 if (unlikely(!edesc)) {969 dev_err(qidev, "could not allocate extended descriptor\n");970 return ERR_PTR(-ENOMEM);971 }972 973 if (likely(req->src == req->dst)) {974 src_len = req->assoclen + req->cryptlen +975 (encrypt ? authsize : 0);976 977 src_nents = sg_nents_for_len(req->src, src_len);978 if (unlikely(src_nents < 0)) {979 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",980 src_len);981 qi_cache_free(edesc);982 return ERR_PTR(src_nents);983 }984 985 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,986 DMA_BIDIRECTIONAL);987 if (unlikely(!mapped_src_nents)) {988 dev_err(qidev, "unable to map source\n");989 qi_cache_free(edesc);990 return ERR_PTR(-ENOMEM);991 }992 } else {993 src_len = req->assoclen + req->cryptlen;994 dst_len = src_len + (encrypt ? authsize : (-authsize));995 996 src_nents = sg_nents_for_len(req->src, src_len);997 if (unlikely(src_nents < 0)) {998 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",999 src_len);1000 qi_cache_free(edesc);1001 return ERR_PTR(src_nents);1002 }1003 1004 dst_nents = sg_nents_for_len(req->dst, dst_len);1005 if (unlikely(dst_nents < 0)) {1006 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",1007 dst_len);1008 qi_cache_free(edesc);1009 return ERR_PTR(dst_nents);1010 }1011 1012 if (src_nents) {1013 mapped_src_nents = dma_map_sg(qidev, req->src,1014 src_nents, DMA_TO_DEVICE);1015 if (unlikely(!mapped_src_nents)) {1016 dev_err(qidev, "unable to map source\n");1017 qi_cache_free(edesc);1018 return ERR_PTR(-ENOMEM);1019 }1020 } else {1021 mapped_src_nents = 0;1022 }1023 1024 if (dst_nents) {1025 mapped_dst_nents = dma_map_sg(qidev, req->dst,1026 dst_nents,1027 DMA_FROM_DEVICE);1028 if (unlikely(!mapped_dst_nents)) {1029 dev_err(qidev, "unable to map destination\n");1030 dma_unmap_sg(qidev, req->src, src_nents,1031 DMA_TO_DEVICE);1032 qi_cache_free(edesc);1033 return ERR_PTR(-ENOMEM);1034 }1035 } else {1036 mapped_dst_nents = 0;1037 }1038 }1039 1040 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)1041 ivsize = crypto_aead_ivsize(aead);1042 1043 /*1044 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].1045 * Input is not contiguous.1046 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond1047 * the end of the table by allocating more S/G entries. Logic:1048 * if (src != dst && output S/G)1049 * pad output S/G, if needed1050 * else if (src == dst && S/G)1051 * overlapping S/Gs; pad one of them1052 * else if (input S/G) ...1053 * pad input S/G, if needed1054 */1055 qm_sg_ents = 1 + !!ivsize + mapped_src_nents;1056 if (mapped_dst_nents > 1)1057 qm_sg_ents += pad_sg_nents(mapped_dst_nents);1058 else if ((req->src == req->dst) && (mapped_src_nents > 1))1059 qm_sg_ents = max(pad_sg_nents(qm_sg_ents),1060 1 + !!ivsize + pad_sg_nents(mapped_src_nents));1061 else1062 qm_sg_ents = pad_sg_nents(qm_sg_ents);1063 1064 sg_table = &edesc->sgt[0];1065 qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);1066 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >1067 CAAM_QI_MEMCACHE_SIZE)) {1068 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",1069 qm_sg_ents, ivsize);1070 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,1071 0, DMA_NONE, 0, 0);1072 qi_cache_free(edesc);1073 return ERR_PTR(-ENOMEM);1074 }1075 1076 if (ivsize) {1077 u8 *iv = (u8 *)(sg_table + qm_sg_ents);1078 1079 /* Make sure IV is located in a DMAable area */1080 memcpy(iv, req->iv, ivsize);1081 1082 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE);1083 if (dma_mapping_error(qidev, iv_dma)) {1084 dev_err(qidev, "unable to map IV\n");1085 caam_unmap(qidev, req->src, req->dst, src_nents,1086 dst_nents, 0, 0, DMA_NONE, 0, 0);1087 qi_cache_free(edesc);1088 return ERR_PTR(-ENOMEM);1089 }1090 }1091 1092 edesc->src_nents = src_nents;1093 edesc->dst_nents = dst_nents;1094 edesc->iv_dma = iv_dma;1095 edesc->drv_req.app_ctx = req;1096 edesc->drv_req.cbk = aead_done;1097 edesc->drv_req.drv_ctx = drv_ctx;1098 1099 edesc->assoclen = cpu_to_caam32(req->assoclen);1100 edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4,1101 DMA_TO_DEVICE);1102 if (dma_mapping_error(qidev, edesc->assoclen_dma)) {1103 dev_err(qidev, "unable to map assoclen\n");1104 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,1105 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);1106 qi_cache_free(edesc);1107 return ERR_PTR(-ENOMEM);1108 }1109 1110 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);1111 qm_sg_index++;1112 if (ivsize) {1113 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);1114 qm_sg_index++;1115 }1116 sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);1117 qm_sg_index += mapped_src_nents;1118 1119 if (mapped_dst_nents > 1)1120 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);1121 1122 qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);1123 if (dma_mapping_error(qidev, qm_sg_dma)) {1124 dev_err(qidev, "unable to map S/G table\n");1125 dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);1126 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,1127 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);1128 qi_cache_free(edesc);1129 return ERR_PTR(-ENOMEM);1130 }1131 1132 edesc->qm_sg_dma = qm_sg_dma;1133 edesc->qm_sg_bytes = qm_sg_bytes;1134 1135 out_len = req->assoclen + req->cryptlen +1136 (encrypt ? ctx->authsize : (-ctx->authsize));1137 in_len = 4 + ivsize + req->assoclen + req->cryptlen;1138 1139 fd_sgt = &edesc->drv_req.fd_sgt[0];1140 dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0);1141 1142 if (req->dst == req->src) {1143 if (mapped_src_nents == 1)1144 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src),1145 out_len, 0);1146 else1147 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma +1148 (1 + !!ivsize) * sizeof(*sg_table),1149 out_len, 0);1150 } else if (mapped_dst_nents <= 1) {1151 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len,1152 0);1153 } else {1154 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) *1155 qm_sg_index, out_len, 0);1156 }1157 1158 return edesc;1159}1160 1161static inline int aead_crypt(struct aead_request *req, bool encrypt)1162{1163 struct aead_edesc *edesc;1164 struct crypto_aead *aead = crypto_aead_reqtfm(req);1165 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);1166 int ret;1167 1168 if (unlikely(caam_congested))1169 return -EAGAIN;1170 1171 /* allocate extended descriptor */1172 edesc = aead_edesc_alloc(req, encrypt);1173 if (IS_ERR(edesc))1174 return PTR_ERR(edesc);1175 1176 /* Create and submit job descriptor */1177 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);1178 if (!ret) {1179 ret = -EINPROGRESS;1180 } else {1181 aead_unmap(ctx->qidev, edesc, req);1182 qi_cache_free(edesc);1183 }1184 1185 return ret;1186}1187 1188static int aead_encrypt(struct aead_request *req)1189{1190 return aead_crypt(req, true);1191}1192 1193static int aead_decrypt(struct aead_request *req)1194{1195 return aead_crypt(req, false);1196}1197 1198static int ipsec_gcm_encrypt(struct aead_request *req)1199{1200 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req,1201 true);1202}1203 1204static int ipsec_gcm_decrypt(struct aead_request *req)1205{1206 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req,1207 false);1208}1209 1210static inline u8 *skcipher_edesc_iv(struct skcipher_edesc *edesc)1211{1212 return PTR_ALIGN((u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,1213 dma_get_cache_alignment());1214}1215 1216static void skcipher_done(struct caam_drv_req *drv_req, u32 status)1217{1218 struct skcipher_edesc *edesc;1219 struct skcipher_request *req = drv_req->app_ctx;1220 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);1221 struct caam_ctx *caam_ctx = crypto_skcipher_ctx_dma(skcipher);1222 struct device *qidev = caam_ctx->qidev;1223 int ivsize = crypto_skcipher_ivsize(skcipher);1224 int ecode = 0;1225 1226 dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);1227 1228 edesc = container_of(drv_req, typeof(*edesc), drv_req);1229 1230 if (status)1231 ecode = caam_jr_strstatus(qidev, status);1232 1233 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",1234 DUMP_PREFIX_ADDRESS, 16, 4, req->iv,1235 edesc->src_nents > 1 ? 100 : ivsize, 1);1236 caam_dump_sg("dst @" __stringify(__LINE__)": ",1237 DUMP_PREFIX_ADDRESS, 16, 4, req->dst,1238 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);1239 1240 skcipher_unmap(qidev, edesc, req);1241 1242 /*1243 * The crypto API expects us to set the IV (req->iv) to the last1244 * ciphertext block (CBC mode) or last counter (CTR mode).1245 * This is used e.g. by the CTS mode.1246 */1247 if (!ecode)1248 memcpy(req->iv, skcipher_edesc_iv(edesc), ivsize);1249 1250 qi_cache_free(edesc);1251 skcipher_request_complete(req, ecode);1252}1253 1254static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,1255 bool encrypt)1256{1257 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);1258 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);1259 struct device *qidev = ctx->qidev;1260 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?1261 GFP_KERNEL : GFP_ATOMIC;1262 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;1263 struct skcipher_edesc *edesc;1264 dma_addr_t iv_dma;1265 u8 *iv;1266 int ivsize = crypto_skcipher_ivsize(skcipher);1267 int dst_sg_idx, qm_sg_ents, qm_sg_bytes;1268 struct qm_sg_entry *sg_table, *fd_sgt;1269 struct caam_drv_ctx *drv_ctx;1270 unsigned int len;1271 1272 drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);1273 if (IS_ERR(drv_ctx))1274 return ERR_CAST(drv_ctx);1275 1276 src_nents = sg_nents_for_len(req->src, req->cryptlen);1277 if (unlikely(src_nents < 0)) {1278 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",1279 req->cryptlen);1280 return ERR_PTR(src_nents);1281 }1282 1283 if (unlikely(req->src != req->dst)) {1284 dst_nents = sg_nents_for_len(req->dst, req->cryptlen);1285 if (unlikely(dst_nents < 0)) {1286 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",1287 req->cryptlen);1288 return ERR_PTR(dst_nents);1289 }1290 1291 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,1292 DMA_TO_DEVICE);1293 if (unlikely(!mapped_src_nents)) {1294 dev_err(qidev, "unable to map source\n");1295 return ERR_PTR(-ENOMEM);1296 }1297 1298 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents,1299 DMA_FROM_DEVICE);1300 if (unlikely(!mapped_dst_nents)) {1301 dev_err(qidev, "unable to map destination\n");1302 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE);1303 return ERR_PTR(-ENOMEM);1304 }1305 } else {1306 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,1307 DMA_BIDIRECTIONAL);1308 if (unlikely(!mapped_src_nents)) {1309 dev_err(qidev, "unable to map source\n");1310 return ERR_PTR(-ENOMEM);1311 }1312 }1313 1314 qm_sg_ents = 1 + mapped_src_nents;1315 dst_sg_idx = qm_sg_ents;1316 1317 /*1318 * Input, output HW S/G tables: [IV, src][dst, IV]1319 * IV entries point to the same buffer1320 * If src == dst, S/G entries are reused (S/G tables overlap)1321 *1322 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond1323 * the end of the table by allocating more S/G entries.1324 */1325 if (req->src != req->dst)1326 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);1327 else1328 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);1329 1330 qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);1331 1332 len = offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes;1333 len = ALIGN(len, dma_get_cache_alignment());1334 len += ivsize;1335 1336 if (unlikely(len > CAAM_QI_MEMCACHE_SIZE)) {1337 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",1338 qm_sg_ents, ivsize);1339 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,1340 0, DMA_NONE, 0, 0);1341 return ERR_PTR(-ENOMEM);1342 }1343 1344 /* allocate space for base edesc, link tables and IV */1345 edesc = qi_cache_alloc(flags);1346 if (unlikely(!edesc)) {1347 dev_err(qidev, "could not allocate extended descriptor\n");1348 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,1349 0, DMA_NONE, 0, 0);1350 return ERR_PTR(-ENOMEM);1351 }1352 1353 edesc->src_nents = src_nents;1354 edesc->dst_nents = dst_nents;1355 edesc->qm_sg_bytes = qm_sg_bytes;1356 edesc->drv_req.app_ctx = req;1357 edesc->drv_req.cbk = skcipher_done;1358 edesc->drv_req.drv_ctx = drv_ctx;1359 1360 /* Make sure IV is located in a DMAable area */1361 sg_table = &edesc->sgt[0];1362 iv = skcipher_edesc_iv(edesc);1363 memcpy(iv, req->iv, ivsize);1364 1365 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL);1366 if (dma_mapping_error(qidev, iv_dma)) {1367 dev_err(qidev, "unable to map IV\n");1368 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,1369 0, DMA_NONE, 0, 0);1370 qi_cache_free(edesc);1371 return ERR_PTR(-ENOMEM);1372 }1373 1374 edesc->iv_dma = iv_dma;1375 1376 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);1377 sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);1378 1379 if (req->src != req->dst)1380 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);1381 1382 dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,1383 ivsize, 0);1384 1385 edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes,1386 DMA_TO_DEVICE);1387 if (dma_mapping_error(qidev, edesc->qm_sg_dma)) {1388 dev_err(qidev, "unable to map S/G table\n");1389 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,1390 iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);1391 qi_cache_free(edesc);1392 return ERR_PTR(-ENOMEM);1393 }1394 1395 fd_sgt = &edesc->drv_req.fd_sgt[0];1396 1397 dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma,1398 ivsize + req->cryptlen, 0);1399 1400 if (req->src == req->dst)1401 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma +1402 sizeof(*sg_table), req->cryptlen + ivsize,1403 0);1404 else1405 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx *1406 sizeof(*sg_table), req->cryptlen + ivsize,1407 0);1408 1409 return edesc;1410}1411 1412static inline bool xts_skcipher_ivsize(struct skcipher_request *req)1413{1414 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);1415 unsigned int ivsize = crypto_skcipher_ivsize(skcipher);1416 1417 return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));1418}1419 1420static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)1421{1422 struct skcipher_edesc *edesc;1423 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);1424 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);1425 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);1426 int ret;1427 1428 /*1429 * XTS is expected to return an error even for input length = 01430 * Note that the case input length < block size will be caught during1431 * HW offloading and return an error.1432 */1433 if (!req->cryptlen && !ctx->fallback)1434 return 0;1435 1436 if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) ||1437 ctx->xts_key_fallback)) {1438 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);1439 1440 skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);1441 skcipher_request_set_callback(&rctx->fallback_req,1442 req->base.flags,1443 req->base.complete,1444 req->base.data);1445 skcipher_request_set_crypt(&rctx->fallback_req, req->src,1446 req->dst, req->cryptlen, req->iv);1447 1448 return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) :1449 crypto_skcipher_decrypt(&rctx->fallback_req);1450 }1451 1452 if (unlikely(caam_congested))1453 return -EAGAIN;1454 1455 /* allocate extended descriptor */1456 edesc = skcipher_edesc_alloc(req, encrypt);1457 if (IS_ERR(edesc))1458 return PTR_ERR(edesc);1459 1460 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);1461 if (!ret) {1462 ret = -EINPROGRESS;1463 } else {1464 skcipher_unmap(ctx->qidev, edesc, req);1465 qi_cache_free(edesc);1466 }1467 1468 return ret;1469}1470 1471static int skcipher_encrypt(struct skcipher_request *req)1472{1473 return skcipher_crypt(req, true);1474}1475 1476static int skcipher_decrypt(struct skcipher_request *req)1477{1478 return skcipher_crypt(req, false);1479}1480 1481static struct caam_skcipher_alg driver_algs[] = {1482 {1483 .skcipher = {1484 .base = {1485 .cra_name = "cbc(aes)",1486 .cra_driver_name = "cbc-aes-caam-qi",1487 .cra_blocksize = AES_BLOCK_SIZE,1488 },1489 .setkey = aes_skcipher_setkey,1490 .encrypt = skcipher_encrypt,1491 .decrypt = skcipher_decrypt,1492 .min_keysize = AES_MIN_KEY_SIZE,1493 .max_keysize = AES_MAX_KEY_SIZE,1494 .ivsize = AES_BLOCK_SIZE,1495 },1496 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,1497 },1498 {1499 .skcipher = {1500 .base = {1501 .cra_name = "cbc(des3_ede)",1502 .cra_driver_name = "cbc-3des-caam-qi",1503 .cra_blocksize = DES3_EDE_BLOCK_SIZE,1504 },1505 .setkey = des3_skcipher_setkey,1506 .encrypt = skcipher_encrypt,1507 .decrypt = skcipher_decrypt,1508 .min_keysize = DES3_EDE_KEY_SIZE,1509 .max_keysize = DES3_EDE_KEY_SIZE,1510 .ivsize = DES3_EDE_BLOCK_SIZE,1511 },1512 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,1513 },1514 {1515 .skcipher = {1516 .base = {1517 .cra_name = "cbc(des)",1518 .cra_driver_name = "cbc-des-caam-qi",1519 .cra_blocksize = DES_BLOCK_SIZE,1520 },1521 .setkey = des_skcipher_setkey,1522 .encrypt = skcipher_encrypt,1523 .decrypt = skcipher_decrypt,1524 .min_keysize = DES_KEY_SIZE,1525 .max_keysize = DES_KEY_SIZE,1526 .ivsize = DES_BLOCK_SIZE,1527 },1528 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,1529 },1530 {1531 .skcipher = {1532 .base = {1533 .cra_name = "ctr(aes)",1534 .cra_driver_name = "ctr-aes-caam-qi",1535 .cra_blocksize = 1,1536 },1537 .setkey = ctr_skcipher_setkey,1538 .encrypt = skcipher_encrypt,1539 .decrypt = skcipher_decrypt,1540 .min_keysize = AES_MIN_KEY_SIZE,1541 .max_keysize = AES_MAX_KEY_SIZE,1542 .ivsize = AES_BLOCK_SIZE,1543 .chunksize = AES_BLOCK_SIZE,1544 },1545 .caam.class1_alg_type = OP_ALG_ALGSEL_AES |1546 OP_ALG_AAI_CTR_MOD128,1547 },1548 {1549 .skcipher = {1550 .base = {1551 .cra_name = "rfc3686(ctr(aes))",1552 .cra_driver_name = "rfc3686-ctr-aes-caam-qi",1553 .cra_blocksize = 1,1554 },1555 .setkey = rfc3686_skcipher_setkey,1556 .encrypt = skcipher_encrypt,1557 .decrypt = skcipher_decrypt,1558 .min_keysize = AES_MIN_KEY_SIZE +1559 CTR_RFC3686_NONCE_SIZE,1560 .max_keysize = AES_MAX_KEY_SIZE +1561 CTR_RFC3686_NONCE_SIZE,1562 .ivsize = CTR_RFC3686_IV_SIZE,1563 .chunksize = AES_BLOCK_SIZE,1564 },1565 .caam = {1566 .class1_alg_type = OP_ALG_ALGSEL_AES |1567 OP_ALG_AAI_CTR_MOD128,1568 .rfc3686 = true,1569 },1570 },1571 {1572 .skcipher = {1573 .base = {1574 .cra_name = "xts(aes)",1575 .cra_driver_name = "xts-aes-caam-qi",1576 .cra_flags = CRYPTO_ALG_NEED_FALLBACK,1577 .cra_blocksize = AES_BLOCK_SIZE,1578 },1579 .setkey = xts_skcipher_setkey,1580 .encrypt = skcipher_encrypt,1581 .decrypt = skcipher_decrypt,1582 .min_keysize = 2 * AES_MIN_KEY_SIZE,1583 .max_keysize = 2 * AES_MAX_KEY_SIZE,1584 .ivsize = AES_BLOCK_SIZE,1585 },1586 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,1587 },1588};1589 1590static struct caam_aead_alg driver_aeads[] = {1591 {1592 .aead = {1593 .base = {1594 .cra_name = "rfc4106(gcm(aes))",1595 .cra_driver_name = "rfc4106-gcm-aes-caam-qi",1596 .cra_blocksize = 1,1597 },1598 .setkey = rfc4106_setkey,1599 .setauthsize = rfc4106_setauthsize,1600 .encrypt = ipsec_gcm_encrypt,1601 .decrypt = ipsec_gcm_decrypt,1602 .ivsize = 8,1603 .maxauthsize = AES_BLOCK_SIZE,1604 },1605 .caam = {1606 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,1607 .nodkp = true,1608 },1609 },1610 {1611 .aead = {1612 .base = {1613 .cra_name = "rfc4543(gcm(aes))",1614 .cra_driver_name = "rfc4543-gcm-aes-caam-qi",1615 .cra_blocksize = 1,1616 },1617 .setkey = rfc4543_setkey,1618 .setauthsize = rfc4543_setauthsize,1619 .encrypt = ipsec_gcm_encrypt,1620 .decrypt = ipsec_gcm_decrypt,1621 .ivsize = 8,1622 .maxauthsize = AES_BLOCK_SIZE,1623 },1624 .caam = {1625 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,1626 .nodkp = true,1627 },1628 },1629 /* Galois Counter Mode */1630 {1631 .aead = {1632 .base = {1633 .cra_name = "gcm(aes)",1634 .cra_driver_name = "gcm-aes-caam-qi",1635 .cra_blocksize = 1,1636 },1637 .setkey = gcm_setkey,1638 .setauthsize = gcm_setauthsize,1639 .encrypt = aead_encrypt,1640 .decrypt = aead_decrypt,1641 .ivsize = 12,1642 .maxauthsize = AES_BLOCK_SIZE,1643 },1644 .caam = {1645 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,1646 .nodkp = true,1647 }1648 },1649 /* single-pass ipsec_esp descriptor */1650 {1651 .aead = {1652 .base = {1653 .cra_name = "authenc(hmac(md5),cbc(aes))",1654 .cra_driver_name = "authenc-hmac-md5-"1655 "cbc-aes-caam-qi",1656 .cra_blocksize = AES_BLOCK_SIZE,1657 },1658 .setkey = aead_setkey,1659 .setauthsize = aead_setauthsize,1660 .encrypt = aead_encrypt,1661 .decrypt = aead_decrypt,1662 .ivsize = AES_BLOCK_SIZE,1663 .maxauthsize = MD5_DIGEST_SIZE,1664 },1665 .caam = {1666 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,1667 .class2_alg_type = OP_ALG_ALGSEL_MD5 |1668 OP_ALG_AAI_HMAC_PRECOMP,1669 }1670 },1671 {1672 .aead = {1673 .base = {1674 .cra_name = "echainiv(authenc(hmac(md5),"1675 "cbc(aes)))",1676 .cra_driver_name = "echainiv-authenc-hmac-md5-"1677 "cbc-aes-caam-qi",1678 .cra_blocksize = AES_BLOCK_SIZE,1679 },1680 .setkey = aead_setkey,1681 .setauthsize = aead_setauthsize,1682 .encrypt = aead_encrypt,1683 .decrypt = aead_decrypt,1684 .ivsize = AES_BLOCK_SIZE,1685 .maxauthsize = MD5_DIGEST_SIZE,1686 },1687 .caam = {1688 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,1689 .class2_alg_type = OP_ALG_ALGSEL_MD5 |1690 OP_ALG_AAI_HMAC_PRECOMP,1691 .geniv = true,1692 }1693 },1694 {1695 .aead = {1696 .base = {1697 .cra_name = "authenc(hmac(sha1),cbc(aes))",1698 .cra_driver_name = "authenc-hmac-sha1-"1699 "cbc-aes-caam-qi",1700 .cra_blocksize = AES_BLOCK_SIZE,1701 },1702 .setkey = aead_setkey,1703 .setauthsize = aead_setauthsize,1704 .encrypt = aead_encrypt,1705 .decrypt = aead_decrypt,1706 .ivsize = AES_BLOCK_SIZE,1707 .maxauthsize = SHA1_DIGEST_SIZE,1708 },1709 .caam = {1710 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,1711 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |1712 OP_ALG_AAI_HMAC_PRECOMP,1713 }1714 },1715 {1716 .aead = {1717 .base = {1718 .cra_name = "echainiv(authenc(hmac(sha1),"1719 "cbc(aes)))",1720 .cra_driver_name = "echainiv-authenc-"1721 "hmac-sha1-cbc-aes-caam-qi",1722 .cra_blocksize = AES_BLOCK_SIZE,1723 },1724 .setkey = aead_setkey,1725 .setauthsize = aead_setauthsize,1726 .encrypt = aead_encrypt,1727 .decrypt = aead_decrypt,1728 .ivsize = AES_BLOCK_SIZE,1729 .maxauthsize = SHA1_DIGEST_SIZE,1730 },1731 .caam = {1732 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,1733 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |1734 OP_ALG_AAI_HMAC_PRECOMP,1735 .geniv = true,1736 },1737 },1738 {1739 .aead = {1740 .base = {1741 .cra_name = "authenc(hmac(sha224),cbc(aes))",1742 .cra_driver_name = "authenc-hmac-sha224-"1743 "cbc-aes-caam-qi",1744 .cra_blocksize = AES_BLOCK_SIZE,1745 },1746 .setkey = aead_setkey,1747 .setauthsize = aead_setauthsize,1748 .encrypt = aead_encrypt,1749 .decrypt = aead_decrypt,1750 .ivsize = AES_BLOCK_SIZE,1751 .maxauthsize = SHA224_DIGEST_SIZE,1752 },1753 .caam = {1754 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,1755 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |1756 OP_ALG_AAI_HMAC_PRECOMP,1757 }1758 },1759 {1760 .aead = {1761 .base = {1762 .cra_name = "echainiv(authenc(hmac(sha224),"1763 "cbc(aes)))",1764 .cra_driver_name = "echainiv-authenc-"1765 "hmac-sha224-cbc-aes-caam-qi",1766 .cra_blocksize = AES_BLOCK_SIZE,1767 },1768 .setkey = aead_setkey,1769 .setauthsize = aead_setauthsize,1770 .encrypt = aead_encrypt,1771 .decrypt = aead_decrypt,1772 .ivsize = AES_BLOCK_SIZE,1773 .maxauthsize = SHA224_DIGEST_SIZE,1774 },1775 .caam = {1776 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,1777 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |1778 OP_ALG_AAI_HMAC_PRECOMP,1779 .geniv = true,1780 }1781 },1782 {1783 .aead = {1784 .base = {1785 .cra_name = "authenc(hmac(sha256),cbc(aes))",1786 .cra_driver_name = "authenc-hmac-sha256-"1787 "cbc-aes-caam-qi",1788 .cra_blocksize = AES_BLOCK_SIZE,1789 },1790 .setkey = aead_setkey,1791 .setauthsize = aead_setauthsize,1792 .encrypt = aead_encrypt,1793 .decrypt = aead_decrypt,1794 .ivsize = AES_BLOCK_SIZE,1795 .maxauthsize = SHA256_DIGEST_SIZE,1796 },1797 .caam = {1798 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,1799 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |1800 OP_ALG_AAI_HMAC_PRECOMP,1801 }1802 },1803 {1804 .aead = {1805 .base = {1806 .cra_name = "echainiv(authenc(hmac(sha256),"1807 "cbc(aes)))",1808 .cra_driver_name = "echainiv-authenc-"1809 "hmac-sha256-cbc-aes-"1810 "caam-qi",1811 .cra_blocksize = AES_BLOCK_SIZE,1812 },1813 .setkey = aead_setkey,1814 .setauthsize = aead_setauthsize,1815 .encrypt = aead_encrypt,1816 .decrypt = aead_decrypt,1817 .ivsize = AES_BLOCK_SIZE,1818 .maxauthsize = SHA256_DIGEST_SIZE,1819 },1820 .caam = {1821 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,1822 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |1823 OP_ALG_AAI_HMAC_PRECOMP,1824 .geniv = true,1825 }1826 },1827 {1828 .aead = {1829 .base = {1830 .cra_name = "authenc(hmac(sha384),cbc(aes))",1831 .cra_driver_name = "authenc-hmac-sha384-"1832 "cbc-aes-caam-qi",1833 .cra_blocksize = AES_BLOCK_SIZE,1834 },1835 .setkey = aead_setkey,1836 .setauthsize = aead_setauthsize,1837 .encrypt = aead_encrypt,1838 .decrypt = aead_decrypt,1839 .ivsize = AES_BLOCK_SIZE,1840 .maxauthsize = SHA384_DIGEST_SIZE,1841 },1842 .caam = {1843 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,1844 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |1845 OP_ALG_AAI_HMAC_PRECOMP,1846 }1847 },1848 {1849 .aead = {1850 .base = {1851 .cra_name = "echainiv(authenc(hmac(sha384),"1852 "cbc(aes)))",1853 .cra_driver_name = "echainiv-authenc-"1854 "hmac-sha384-cbc-aes-"1855 "caam-qi",1856 .cra_blocksize = AES_BLOCK_SIZE,1857 },1858 .setkey = aead_setkey,1859 .setauthsize = aead_setauthsize,1860 .encrypt = aead_encrypt,1861 .decrypt = aead_decrypt,1862 .ivsize = AES_BLOCK_SIZE,1863 .maxauthsize = SHA384_DIGEST_SIZE,1864 },1865 .caam = {1866 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,1867 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |1868 OP_ALG_AAI_HMAC_PRECOMP,1869 .geniv = true,1870 }1871 },1872 {1873 .aead = {1874 .base = {1875 .cra_name = "authenc(hmac(sha512),cbc(aes))",1876 .cra_driver_name = "authenc-hmac-sha512-"1877 "cbc-aes-caam-qi",1878 .cra_blocksize = AES_BLOCK_SIZE,1879 },1880 .setkey = aead_setkey,1881 .setauthsize = aead_setauthsize,1882 .encrypt = aead_encrypt,1883 .decrypt = aead_decrypt,1884 .ivsize = AES_BLOCK_SIZE,1885 .maxauthsize = SHA512_DIGEST_SIZE,1886 },1887 .caam = {1888 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,1889 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |1890 OP_ALG_AAI_HMAC_PRECOMP,1891 }1892 },1893 {1894 .aead = {1895 .base = {1896 .cra_name = "echainiv(authenc(hmac(sha512),"1897 "cbc(aes)))",1898 .cra_driver_name = "echainiv-authenc-"1899 "hmac-sha512-cbc-aes-"1900 "caam-qi",1901 .cra_blocksize = AES_BLOCK_SIZE,1902 },1903 .setkey = aead_setkey,1904 .setauthsize = aead_setauthsize,1905 .encrypt = aead_encrypt,1906 .decrypt = aead_decrypt,1907 .ivsize = AES_BLOCK_SIZE,1908 .maxauthsize = SHA512_DIGEST_SIZE,1909 },1910 .caam = {1911 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,1912 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |1913 OP_ALG_AAI_HMAC_PRECOMP,1914 .geniv = true,1915 }1916 },1917 {1918 .aead = {1919 .base = {1920 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",1921 .cra_driver_name = "authenc-hmac-md5-"1922 "cbc-des3_ede-caam-qi",1923 .cra_blocksize = DES3_EDE_BLOCK_SIZE,1924 },1925 .setkey = des3_aead_setkey,1926 .setauthsize = aead_setauthsize,1927 .encrypt = aead_encrypt,1928 .decrypt = aead_decrypt,1929 .ivsize = DES3_EDE_BLOCK_SIZE,1930 .maxauthsize = MD5_DIGEST_SIZE,1931 },1932 .caam = {1933 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,1934 .class2_alg_type = OP_ALG_ALGSEL_MD5 |1935 OP_ALG_AAI_HMAC_PRECOMP,1936 }1937 },1938 {1939 .aead = {1940 .base = {1941 .cra_name = "echainiv(authenc(hmac(md5),"1942 "cbc(des3_ede)))",1943 .cra_driver_name = "echainiv-authenc-hmac-md5-"1944 "cbc-des3_ede-caam-qi",1945 .cra_blocksize = DES3_EDE_BLOCK_SIZE,1946 },1947 .setkey = des3_aead_setkey,1948 .setauthsize = aead_setauthsize,1949 .encrypt = aead_encrypt,1950 .decrypt = aead_decrypt,1951 .ivsize = DES3_EDE_BLOCK_SIZE,1952 .maxauthsize = MD5_DIGEST_SIZE,1953 },1954 .caam = {1955 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,1956 .class2_alg_type = OP_ALG_ALGSEL_MD5 |1957 OP_ALG_AAI_HMAC_PRECOMP,1958 .geniv = true,1959 }1960 },1961 {1962 .aead = {1963 .base = {1964 .cra_name = "authenc(hmac(sha1),"1965 "cbc(des3_ede))",1966 .cra_driver_name = "authenc-hmac-sha1-"1967 "cbc-des3_ede-caam-qi",1968 .cra_blocksize = DES3_EDE_BLOCK_SIZE,1969 },1970 .setkey = des3_aead_setkey,1971 .setauthsize = aead_setauthsize,1972 .encrypt = aead_encrypt,1973 .decrypt = aead_decrypt,1974 .ivsize = DES3_EDE_BLOCK_SIZE,1975 .maxauthsize = SHA1_DIGEST_SIZE,1976 },1977 .caam = {1978 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,1979 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |1980 OP_ALG_AAI_HMAC_PRECOMP,1981 },1982 },1983 {1984 .aead = {1985 .base = {1986 .cra_name = "echainiv(authenc(hmac(sha1),"1987 "cbc(des3_ede)))",1988 .cra_driver_name = "echainiv-authenc-"1989 "hmac-sha1-"1990 "cbc-des3_ede-caam-qi",1991 .cra_blocksize = DES3_EDE_BLOCK_SIZE,1992 },1993 .setkey = des3_aead_setkey,1994 .setauthsize = aead_setauthsize,1995 .encrypt = aead_encrypt,1996 .decrypt = aead_decrypt,1997 .ivsize = DES3_EDE_BLOCK_SIZE,1998 .maxauthsize = SHA1_DIGEST_SIZE,1999 },2000 .caam = {2001 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,2002 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |2003 OP_ALG_AAI_HMAC_PRECOMP,2004 .geniv = true,2005 }2006 },2007 {2008 .aead = {2009 .base = {2010 .cra_name = "authenc(hmac(sha224),"2011 "cbc(des3_ede))",2012 .cra_driver_name = "authenc-hmac-sha224-"2013 "cbc-des3_ede-caam-qi",2014 .cra_blocksize = DES3_EDE_BLOCK_SIZE,2015 },2016 .setkey = des3_aead_setkey,2017 .setauthsize = aead_setauthsize,2018 .encrypt = aead_encrypt,2019 .decrypt = aead_decrypt,2020 .ivsize = DES3_EDE_BLOCK_SIZE,2021 .maxauthsize = SHA224_DIGEST_SIZE,2022 },2023 .caam = {2024 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,2025 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |2026 OP_ALG_AAI_HMAC_PRECOMP,2027 },2028 },2029 {2030 .aead = {2031 .base = {2032 .cra_name = "echainiv(authenc(hmac(sha224),"2033 "cbc(des3_ede)))",2034 .cra_driver_name = "echainiv-authenc-"2035 "hmac-sha224-"2036 "cbc-des3_ede-caam-qi",2037 .cra_blocksize = DES3_EDE_BLOCK_SIZE,2038 },2039 .setkey = des3_aead_setkey,2040 .setauthsize = aead_setauthsize,2041 .encrypt = aead_encrypt,2042 .decrypt = aead_decrypt,2043 .ivsize = DES3_EDE_BLOCK_SIZE,2044 .maxauthsize = SHA224_DIGEST_SIZE,2045 },2046 .caam = {2047 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,2048 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |2049 OP_ALG_AAI_HMAC_PRECOMP,2050 .geniv = true,2051 }2052 },2053 {2054 .aead = {2055 .base = {2056 .cra_name = "authenc(hmac(sha256),"2057 "cbc(des3_ede))",2058 .cra_driver_name = "authenc-hmac-sha256-"2059 "cbc-des3_ede-caam-qi",2060 .cra_blocksize = DES3_EDE_BLOCK_SIZE,2061 },2062 .setkey = des3_aead_setkey,2063 .setauthsize = aead_setauthsize,2064 .encrypt = aead_encrypt,2065 .decrypt = aead_decrypt,2066 .ivsize = DES3_EDE_BLOCK_SIZE,2067 .maxauthsize = SHA256_DIGEST_SIZE,2068 },2069 .caam = {2070 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,2071 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |2072 OP_ALG_AAI_HMAC_PRECOMP,2073 },2074 },2075 {2076 .aead = {2077 .base = {2078 .cra_name = "echainiv(authenc(hmac(sha256),"2079 "cbc(des3_ede)))",2080 .cra_driver_name = "echainiv-authenc-"2081 "hmac-sha256-"2082 "cbc-des3_ede-caam-qi",2083 .cra_blocksize = DES3_EDE_BLOCK_SIZE,2084 },2085 .setkey = des3_aead_setkey,2086 .setauthsize = aead_setauthsize,2087 .encrypt = aead_encrypt,2088 .decrypt = aead_decrypt,2089 .ivsize = DES3_EDE_BLOCK_SIZE,2090 .maxauthsize = SHA256_DIGEST_SIZE,2091 },2092 .caam = {2093 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,2094 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |2095 OP_ALG_AAI_HMAC_PRECOMP,2096 .geniv = true,2097 }2098 },2099 {2100 .aead = {2101 .base = {2102 .cra_name = "authenc(hmac(sha384),"2103 "cbc(des3_ede))",2104 .cra_driver_name = "authenc-hmac-sha384-"2105 "cbc-des3_ede-caam-qi",2106 .cra_blocksize = DES3_EDE_BLOCK_SIZE,2107 },2108 .setkey = des3_aead_setkey,2109 .setauthsize = aead_setauthsize,2110 .encrypt = aead_encrypt,2111 .decrypt = aead_decrypt,2112 .ivsize = DES3_EDE_BLOCK_SIZE,2113 .maxauthsize = SHA384_DIGEST_SIZE,2114 },2115 .caam = {2116 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,2117 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |2118 OP_ALG_AAI_HMAC_PRECOMP,2119 },2120 },2121 {2122 .aead = {2123 .base = {2124 .cra_name = "echainiv(authenc(hmac(sha384),"2125 "cbc(des3_ede)))",2126 .cra_driver_name = "echainiv-authenc-"2127 "hmac-sha384-"2128 "cbc-des3_ede-caam-qi",2129 .cra_blocksize = DES3_EDE_BLOCK_SIZE,2130 },2131 .setkey = des3_aead_setkey,2132 .setauthsize = aead_setauthsize,2133 .encrypt = aead_encrypt,2134 .decrypt = aead_decrypt,2135 .ivsize = DES3_EDE_BLOCK_SIZE,2136 .maxauthsize = SHA384_DIGEST_SIZE,2137 },2138 .caam = {2139 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,2140 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |2141 OP_ALG_AAI_HMAC_PRECOMP,2142 .geniv = true,2143 }2144 },2145 {2146 .aead = {2147 .base = {2148 .cra_name = "authenc(hmac(sha512),"2149 "cbc(des3_ede))",2150 .cra_driver_name = "authenc-hmac-sha512-"2151 "cbc-des3_ede-caam-qi",2152 .cra_blocksize = DES3_EDE_BLOCK_SIZE,2153 },2154 .setkey = des3_aead_setkey,2155 .setauthsize = aead_setauthsize,2156 .encrypt = aead_encrypt,2157 .decrypt = aead_decrypt,2158 .ivsize = DES3_EDE_BLOCK_SIZE,2159 .maxauthsize = SHA512_DIGEST_SIZE,2160 },2161 .caam = {2162 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,2163 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |2164 OP_ALG_AAI_HMAC_PRECOMP,2165 },2166 },2167 {2168 .aead = {2169 .base = {2170 .cra_name = "echainiv(authenc(hmac(sha512),"2171 "cbc(des3_ede)))",2172 .cra_driver_name = "echainiv-authenc-"2173 "hmac-sha512-"2174 "cbc-des3_ede-caam-qi",2175 .cra_blocksize = DES3_EDE_BLOCK_SIZE,2176 },2177 .setkey = des3_aead_setkey,2178 .setauthsize = aead_setauthsize,2179 .encrypt = aead_encrypt,2180 .decrypt = aead_decrypt,2181 .ivsize = DES3_EDE_BLOCK_SIZE,2182 .maxauthsize = SHA512_DIGEST_SIZE,2183 },2184 .caam = {2185 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,2186 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |2187 OP_ALG_AAI_HMAC_PRECOMP,2188 .geniv = true,2189 }2190 },2191 {2192 .aead = {2193 .base = {2194 .cra_name = "authenc(hmac(md5),cbc(des))",2195 .cra_driver_name = "authenc-hmac-md5-"2196 "cbc-des-caam-qi",2197 .cra_blocksize = DES_BLOCK_SIZE,2198 },2199 .setkey = aead_setkey,2200 .setauthsize = aead_setauthsize,2201 .encrypt = aead_encrypt,2202 .decrypt = aead_decrypt,2203 .ivsize = DES_BLOCK_SIZE,2204 .maxauthsize = MD5_DIGEST_SIZE,2205 },2206 .caam = {2207 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,2208 .class2_alg_type = OP_ALG_ALGSEL_MD5 |2209 OP_ALG_AAI_HMAC_PRECOMP,2210 },2211 },2212 {2213 .aead = {2214 .base = {2215 .cra_name = "echainiv(authenc(hmac(md5),"2216 "cbc(des)))",2217 .cra_driver_name = "echainiv-authenc-hmac-md5-"2218 "cbc-des-caam-qi",2219 .cra_blocksize = DES_BLOCK_SIZE,2220 },2221 .setkey = aead_setkey,2222 .setauthsize = aead_setauthsize,2223 .encrypt = aead_encrypt,2224 .decrypt = aead_decrypt,2225 .ivsize = DES_BLOCK_SIZE,2226 .maxauthsize = MD5_DIGEST_SIZE,2227 },2228 .caam = {2229 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,2230 .class2_alg_type = OP_ALG_ALGSEL_MD5 |2231 OP_ALG_AAI_HMAC_PRECOMP,2232 .geniv = true,2233 }2234 },2235 {2236 .aead = {2237 .base = {2238 .cra_name = "authenc(hmac(sha1),cbc(des))",2239 .cra_driver_name = "authenc-hmac-sha1-"2240 "cbc-des-caam-qi",2241 .cra_blocksize = DES_BLOCK_SIZE,2242 },2243 .setkey = aead_setkey,2244 .setauthsize = aead_setauthsize,2245 .encrypt = aead_encrypt,2246 .decrypt = aead_decrypt,2247 .ivsize = DES_BLOCK_SIZE,2248 .maxauthsize = SHA1_DIGEST_SIZE,2249 },2250 .caam = {2251 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,2252 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |2253 OP_ALG_AAI_HMAC_PRECOMP,2254 },2255 },2256 {2257 .aead = {2258 .base = {2259 .cra_name = "echainiv(authenc(hmac(sha1),"2260 "cbc(des)))",2261 .cra_driver_name = "echainiv-authenc-"2262 "hmac-sha1-cbc-des-caam-qi",2263 .cra_blocksize = DES_BLOCK_SIZE,2264 },2265 .setkey = aead_setkey,2266 .setauthsize = aead_setauthsize,2267 .encrypt = aead_encrypt,2268 .decrypt = aead_decrypt,2269 .ivsize = DES_BLOCK_SIZE,2270 .maxauthsize = SHA1_DIGEST_SIZE,2271 },2272 .caam = {2273 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,2274 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |2275 OP_ALG_AAI_HMAC_PRECOMP,2276 .geniv = true,2277 }2278 },2279 {2280 .aead = {2281 .base = {2282 .cra_name = "authenc(hmac(sha224),cbc(des))",2283 .cra_driver_name = "authenc-hmac-sha224-"2284 "cbc-des-caam-qi",2285 .cra_blocksize = DES_BLOCK_SIZE,2286 },2287 .setkey = aead_setkey,2288 .setauthsize = aead_setauthsize,2289 .encrypt = aead_encrypt,2290 .decrypt = aead_decrypt,2291 .ivsize = DES_BLOCK_SIZE,2292 .maxauthsize = SHA224_DIGEST_SIZE,2293 },2294 .caam = {2295 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,2296 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |2297 OP_ALG_AAI_HMAC_PRECOMP,2298 },2299 },2300 {2301 .aead = {2302 .base = {2303 .cra_name = "echainiv(authenc(hmac(sha224),"2304 "cbc(des)))",2305 .cra_driver_name = "echainiv-authenc-"2306 "hmac-sha224-cbc-des-"2307 "caam-qi",2308 .cra_blocksize = DES_BLOCK_SIZE,2309 },2310 .setkey = aead_setkey,2311 .setauthsize = aead_setauthsize,2312 .encrypt = aead_encrypt,2313 .decrypt = aead_decrypt,2314 .ivsize = DES_BLOCK_SIZE,2315 .maxauthsize = SHA224_DIGEST_SIZE,2316 },2317 .caam = {2318 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,2319 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |2320 OP_ALG_AAI_HMAC_PRECOMP,2321 .geniv = true,2322 }2323 },2324 {2325 .aead = {2326 .base = {2327 .cra_name = "authenc(hmac(sha256),cbc(des))",2328 .cra_driver_name = "authenc-hmac-sha256-"2329 "cbc-des-caam-qi",2330 .cra_blocksize = DES_BLOCK_SIZE,2331 },2332 .setkey = aead_setkey,2333 .setauthsize = aead_setauthsize,2334 .encrypt = aead_encrypt,2335 .decrypt = aead_decrypt,2336 .ivsize = DES_BLOCK_SIZE,2337 .maxauthsize = SHA256_DIGEST_SIZE,2338 },2339 .caam = {2340 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,2341 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |2342 OP_ALG_AAI_HMAC_PRECOMP,2343 },2344 },2345 {2346 .aead = {2347 .base = {2348 .cra_name = "echainiv(authenc(hmac(sha256),"2349 "cbc(des)))",2350 .cra_driver_name = "echainiv-authenc-"2351 "hmac-sha256-cbc-des-"2352 "caam-qi",2353 .cra_blocksize = DES_BLOCK_SIZE,2354 },2355 .setkey = aead_setkey,2356 .setauthsize = aead_setauthsize,2357 .encrypt = aead_encrypt,2358 .decrypt = aead_decrypt,2359 .ivsize = DES_BLOCK_SIZE,2360 .maxauthsize = SHA256_DIGEST_SIZE,2361 },2362 .caam = {2363 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,2364 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |2365 OP_ALG_AAI_HMAC_PRECOMP,2366 .geniv = true,2367 },2368 },2369 {2370 .aead = {2371 .base = {2372 .cra_name = "authenc(hmac(sha384),cbc(des))",2373 .cra_driver_name = "authenc-hmac-sha384-"2374 "cbc-des-caam-qi",2375 .cra_blocksize = DES_BLOCK_SIZE,2376 },2377 .setkey = aead_setkey,2378 .setauthsize = aead_setauthsize,2379 .encrypt = aead_encrypt,2380 .decrypt = aead_decrypt,2381 .ivsize = DES_BLOCK_SIZE,2382 .maxauthsize = SHA384_DIGEST_SIZE,2383 },2384 .caam = {2385 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,2386 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |2387 OP_ALG_AAI_HMAC_PRECOMP,2388 },2389 },2390 {2391 .aead = {2392 .base = {2393 .cra_name = "echainiv(authenc(hmac(sha384),"2394 "cbc(des)))",2395 .cra_driver_name = "echainiv-authenc-"2396 "hmac-sha384-cbc-des-"2397 "caam-qi",2398 .cra_blocksize = DES_BLOCK_SIZE,2399 },2400 .setkey = aead_setkey,2401 .setauthsize = aead_setauthsize,2402 .encrypt = aead_encrypt,2403 .decrypt = aead_decrypt,2404 .ivsize = DES_BLOCK_SIZE,2405 .maxauthsize = SHA384_DIGEST_SIZE,2406 },2407 .caam = {2408 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,2409 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |2410 OP_ALG_AAI_HMAC_PRECOMP,2411 .geniv = true,2412 }2413 },2414 {2415 .aead = {2416 .base = {2417 .cra_name = "authenc(hmac(sha512),cbc(des))",2418 .cra_driver_name = "authenc-hmac-sha512-"2419 "cbc-des-caam-qi",2420 .cra_blocksize = DES_BLOCK_SIZE,2421 },2422 .setkey = aead_setkey,2423 .setauthsize = aead_setauthsize,2424 .encrypt = aead_encrypt,2425 .decrypt = aead_decrypt,2426 .ivsize = DES_BLOCK_SIZE,2427 .maxauthsize = SHA512_DIGEST_SIZE,2428 },2429 .caam = {2430 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,2431 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |2432 OP_ALG_AAI_HMAC_PRECOMP,2433 }2434 },2435 {2436 .aead = {2437 .base = {2438 .cra_name = "echainiv(authenc(hmac(sha512),"2439 "cbc(des)))",2440 .cra_driver_name = "echainiv-authenc-"2441 "hmac-sha512-cbc-des-"2442 "caam-qi",2443 .cra_blocksize = DES_BLOCK_SIZE,2444 },2445 .setkey = aead_setkey,2446 .setauthsize = aead_setauthsize,2447 .encrypt = aead_encrypt,2448 .decrypt = aead_decrypt,2449 .ivsize = DES_BLOCK_SIZE,2450 .maxauthsize = SHA512_DIGEST_SIZE,2451 },2452 .caam = {2453 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,2454 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |2455 OP_ALG_AAI_HMAC_PRECOMP,2456 .geniv = true,2457 }2458 },2459};2460 2461static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,2462 bool uses_dkp)2463{2464 struct caam_drv_private *priv;2465 struct device *dev;2466 2467 /*2468 * distribute tfms across job rings to ensure in-order2469 * crypto request processing per tfm2470 */2471 ctx->jrdev = caam_jr_alloc();2472 if (IS_ERR(ctx->jrdev)) {2473 pr_err("Job Ring Device allocation for transform failed\n");2474 return PTR_ERR(ctx->jrdev);2475 }2476 2477 dev = ctx->jrdev->parent;2478 priv = dev_get_drvdata(dev);2479 if (priv->era >= 6 && uses_dkp)2480 ctx->dir = DMA_BIDIRECTIONAL;2481 else2482 ctx->dir = DMA_TO_DEVICE;2483 2484 ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key),2485 ctx->dir);2486 if (dma_mapping_error(dev, ctx->key_dma)) {2487 dev_err(dev, "unable to map key\n");2488 caam_jr_free(ctx->jrdev);2489 return -ENOMEM;2490 }2491 2492 /* copy descriptor header template value */2493 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;2494 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;2495 2496 ctx->qidev = dev;2497 2498 spin_lock_init(&ctx->lock);2499 ctx->drv_ctx[ENCRYPT] = NULL;2500 ctx->drv_ctx[DECRYPT] = NULL;2501 2502 return 0;2503}2504 2505static int caam_cra_init(struct crypto_skcipher *tfm)2506{2507 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);2508 struct caam_skcipher_alg *caam_alg =2509 container_of(alg, typeof(*caam_alg), skcipher);2510 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm);2511 u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;2512 int ret = 0;2513 2514 if (alg_aai == OP_ALG_AAI_XTS) {2515 const char *tfm_name = crypto_tfm_alg_name(&tfm->base);2516 struct crypto_skcipher *fallback;2517 2518 fallback = crypto_alloc_skcipher(tfm_name, 0,2519 CRYPTO_ALG_NEED_FALLBACK);2520 if (IS_ERR(fallback)) {2521 pr_err("Failed to allocate %s fallback: %ld\n",2522 tfm_name, PTR_ERR(fallback));2523 return PTR_ERR(fallback);2524 }2525 2526 ctx->fallback = fallback;2527 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) +2528 crypto_skcipher_reqsize(fallback));2529 }2530 2531 ret = caam_init_common(ctx, &caam_alg->caam, false);2532 if (ret && ctx->fallback)2533 crypto_free_skcipher(ctx->fallback);2534 2535 return ret;2536}2537 2538static int caam_aead_init(struct crypto_aead *tfm)2539{2540 struct aead_alg *alg = crypto_aead_alg(tfm);2541 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),2542 aead);2543 struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm);2544 2545 return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp);2546}2547 2548static void caam_exit_common(struct caam_ctx *ctx)2549{2550 caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]);2551 caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]);2552 2553 dma_unmap_single(ctx->jrdev->parent, ctx->key_dma, sizeof(ctx->key),2554 ctx->dir);2555 2556 caam_jr_free(ctx->jrdev);2557}2558 2559static void caam_cra_exit(struct crypto_skcipher *tfm)2560{2561 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm);2562 2563 if (ctx->fallback)2564 crypto_free_skcipher(ctx->fallback);2565 caam_exit_common(ctx);2566}2567 2568static void caam_aead_exit(struct crypto_aead *tfm)2569{2570 caam_exit_common(crypto_aead_ctx_dma(tfm));2571}2572 2573void caam_qi_algapi_exit(void)2574{2575 int i;2576 2577 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {2578 struct caam_aead_alg *t_alg = driver_aeads + i;2579 2580 if (t_alg->registered)2581 crypto_unregister_aead(&t_alg->aead);2582 }2583 2584 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {2585 struct caam_skcipher_alg *t_alg = driver_algs + i;2586 2587 if (t_alg->registered)2588 crypto_unregister_skcipher(&t_alg->skcipher);2589 }2590}2591 2592static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)2593{2594 struct skcipher_alg *alg = &t_alg->skcipher;2595 2596 alg->base.cra_module = THIS_MODULE;2597 alg->base.cra_priority = CAAM_CRA_PRIORITY;2598 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();2599 alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |2600 CRYPTO_ALG_KERN_DRIVER_ONLY);2601 2602 alg->init = caam_cra_init;2603 alg->exit = caam_cra_exit;2604}2605 2606static void caam_aead_alg_init(struct caam_aead_alg *t_alg)2607{2608 struct aead_alg *alg = &t_alg->aead;2609 2610 alg->base.cra_module = THIS_MODULE;2611 alg->base.cra_priority = CAAM_CRA_PRIORITY;2612 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();2613 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |2614 CRYPTO_ALG_KERN_DRIVER_ONLY;2615 2616 alg->init = caam_aead_init;2617 alg->exit = caam_aead_exit;2618}2619 2620int caam_qi_algapi_init(struct device *ctrldev)2621{2622 struct caam_drv_private *priv = dev_get_drvdata(ctrldev);2623 int i = 0, err = 0;2624 u32 aes_vid, aes_inst, des_inst, md_vid, md_inst;2625 unsigned int md_limit = SHA512_DIGEST_SIZE;2626 bool registered = false;2627 2628 /* Make sure this runs only on (DPAA 1.x) QI */2629 if (!priv->qi_present || caam_dpaa2)2630 return 0;2631 2632 /*2633 * Register crypto algorithms the device supports.2634 * First, detect presence and attributes of DES, AES, and MD blocks.2635 */2636 if (priv->era < 10) {2637 u32 cha_vid, cha_inst;2638 2639 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);2640 aes_vid = cha_vid & CHA_ID_LS_AES_MASK;2641 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;2642 2643 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);2644 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >>2645 CHA_ID_LS_DES_SHIFT;2646 aes_inst = cha_inst & CHA_ID_LS_AES_MASK;2647 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;2648 } else {2649 u32 aesa, mdha;2650 2651 aesa = rd_reg32(&priv->ctrl->vreg.aesa);2652 mdha = rd_reg32(&priv->ctrl->vreg.mdha);2653 2654 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;2655 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;2656 2657 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK;2658 aes_inst = aesa & CHA_VER_NUM_MASK;2659 md_inst = mdha & CHA_VER_NUM_MASK;2660 }2661 2662 /* If MD is present, limit digest size based on LP256 */2663 if (md_inst && md_vid == CHA_VER_VID_MD_LP256)2664 md_limit = SHA256_DIGEST_SIZE;2665 2666 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {2667 struct caam_skcipher_alg *t_alg = driver_algs + i;2668 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;2669 2670 /* Skip DES algorithms if not supported by device */2671 if (!des_inst &&2672 ((alg_sel == OP_ALG_ALGSEL_3DES) ||2673 (alg_sel == OP_ALG_ALGSEL_DES)))2674 continue;2675 2676 /* Skip AES algorithms if not supported by device */2677 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))2678 continue;2679 2680 caam_skcipher_alg_init(t_alg);2681 2682 err = crypto_register_skcipher(&t_alg->skcipher);2683 if (err) {2684 dev_warn(ctrldev, "%s alg registration failed\n",2685 t_alg->skcipher.base.cra_driver_name);2686 continue;2687 }2688 2689 t_alg->registered = true;2690 registered = true;2691 }2692 2693 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {2694 struct caam_aead_alg *t_alg = driver_aeads + i;2695 u32 c1_alg_sel = t_alg->caam.class1_alg_type &2696 OP_ALG_ALGSEL_MASK;2697 u32 c2_alg_sel = t_alg->caam.class2_alg_type &2698 OP_ALG_ALGSEL_MASK;2699 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;2700 2701 /* Skip DES algorithms if not supported by device */2702 if (!des_inst &&2703 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||2704 (c1_alg_sel == OP_ALG_ALGSEL_DES)))2705 continue;2706 2707 /* Skip AES algorithms if not supported by device */2708 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))2709 continue;2710 2711 /*2712 * Check support for AES algorithms not available2713 * on LP devices.2714 */2715 if (aes_vid == CHA_VER_VID_AES_LP && alg_aai == OP_ALG_AAI_GCM)2716 continue;2717 2718 /*2719 * Skip algorithms requiring message digests2720 * if MD or MD size is not supported by device.2721 */2722 if (c2_alg_sel &&2723 (!md_inst || (t_alg->aead.maxauthsize > md_limit)))2724 continue;2725 2726 caam_aead_alg_init(t_alg);2727 2728 err = crypto_register_aead(&t_alg->aead);2729 if (err) {2730 pr_warn("%s alg registration failed\n",2731 t_alg->aead.base.cra_driver_name);2732 continue;2733 }2734 2735 t_alg->registered = true;2736 registered = true;2737 }2738 2739 if (registered)2740 dev_info(ctrldev, "algorithms registered in /proc/crypto\n");2741 2742 return err;2743}2744