2569 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2019 HiSilicon Limited. */3 4#include <crypto/aes.h>5#include <crypto/aead.h>6#include <crypto/algapi.h>7#include <crypto/authenc.h>8#include <crypto/des.h>9#include <crypto/hash.h>10#include <crypto/internal/aead.h>11#include <crypto/internal/des.h>12#include <crypto/sha1.h>13#include <crypto/sha2.h>14#include <crypto/skcipher.h>15#include <crypto/xts.h>16#include <linux/crypto.h>17#include <linux/dma-mapping.h>18#include <linux/idr.h>19 20#include "sec.h"21#include "sec_crypto.h"22 23#define SEC_PRIORITY 400124#define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE)25#define SEC_XTS_MID_KEY_SIZE (3 * AES_MIN_KEY_SIZE)26#define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE)27#define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE)28#define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE)29 30/* SEC sqe(bd) bit operational relative MACRO */31#define SEC_DE_OFFSET 132#define SEC_CIPHER_OFFSET 433#define SEC_SCENE_OFFSET 334#define SEC_DST_SGL_OFFSET 235#define SEC_SRC_SGL_OFFSET 736#define SEC_CKEY_OFFSET 937#define SEC_CMODE_OFFSET 1238#define SEC_AKEY_OFFSET 539#define SEC_AEAD_ALG_OFFSET 1140#define SEC_AUTH_OFFSET 641 42#define SEC_DE_OFFSET_V3 943#define SEC_SCENE_OFFSET_V3 544#define SEC_CKEY_OFFSET_V3 1345#define SEC_CTR_CNT_OFFSET 2546#define SEC_CTR_CNT_ROLLOVER 247#define SEC_SRC_SGL_OFFSET_V3 1148#define SEC_DST_SGL_OFFSET_V3 1449#define SEC_CALG_OFFSET_V3 450#define SEC_AKEY_OFFSET_V3 951#define SEC_MAC_OFFSET_V3 452#define SEC_AUTH_ALG_OFFSET_V3 1553#define SEC_CIPHER_AUTH_V3 0xbf54#define SEC_AUTH_CIPHER_V3 0x4055#define SEC_FLAG_OFFSET 756#define SEC_FLAG_MASK 0x078057#define SEC_TYPE_MASK 0x0F58#define SEC_DONE_MASK 0x000159#define SEC_ICV_MASK 0x000E60#define SEC_SQE_LEN_RATE_MASK 0x361 62#define SEC_TOTAL_IV_SZ(depth) (SEC_IV_SIZE * (depth))63#define SEC_SGL_SGE_NR 12864#define SEC_CIPHER_AUTH 0xfe65#define SEC_AUTH_CIPHER 0x166#define SEC_MAX_MAC_LEN 6467#define SEC_MAX_AAD_LEN 6553568#define SEC_MAX_CCM_AAD_LEN 6527969#define SEC_TOTAL_MAC_SZ(depth) (SEC_MAX_MAC_LEN * (depth))70 71#define SEC_PBUF_SZ 51272#define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ73#define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE)74#define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \75 SEC_MAX_MAC_LEN * 2)76#define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG)77#define SEC_PBUF_PAGE_NUM(depth) ((depth) / SEC_PBUF_NUM)78#define SEC_PBUF_LEFT_SZ(depth) (SEC_PBUF_PKG * ((depth) - \79 SEC_PBUF_PAGE_NUM(depth) * SEC_PBUF_NUM))80#define SEC_TOTAL_PBUF_SZ(depth) (PAGE_SIZE * SEC_PBUF_PAGE_NUM(depth) + \81 SEC_PBUF_LEFT_SZ(depth))82 83#define SEC_SQE_LEN_RATE 484#define SEC_SQE_CFLAG 285#define SEC_SQE_AEAD_FLAG 386#define SEC_SQE_DONE 0x187#define SEC_ICV_ERR 0x288#define MIN_MAC_LEN 489#define MAC_LEN_MASK 0x1U90#define MAX_INPUT_DATA_LEN 0xFFFE0091#define BITS_MASK 0xFF92#define BYTE_BITS 0x893#define SEC_XTS_NAME_SZ 0x394#define IV_CM_CAL_NUM 295#define IV_CL_MASK 0x796#define IV_CL_MIN 297#define IV_CL_MID 498#define IV_CL_MAX 899#define IV_FLAGS_OFFSET 0x6100#define IV_CM_OFFSET 0x3101#define IV_LAST_BYTE1 1102#define IV_LAST_BYTE2 2103#define IV_LAST_BYTE_MASK 0xFF104#define IV_CTR_INIT 0x1105#define IV_BYTE_OFFSET 0x8106 107static DEFINE_MUTEX(sec_algs_lock);108static unsigned int sec_available_devs;109 110struct sec_skcipher {111 u64 alg_msk;112 struct skcipher_alg alg;113};114 115struct sec_aead {116 u64 alg_msk;117 struct aead_alg alg;118};119 120/* Get an en/de-cipher queue cyclically to balance load over queues of TFM */121static inline u32 sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req)122{123 if (req->c_req.encrypt)124 return (u32)atomic_inc_return(&ctx->enc_qcyclic) %125 ctx->hlf_q_num;126 127 return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num +128 ctx->hlf_q_num;129}130 131static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req)132{133 if (req->c_req.encrypt)134 atomic_dec(&ctx->enc_qcyclic);135 else136 atomic_dec(&ctx->dec_qcyclic);137}138 139static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx)140{141 int req_id;142 143 spin_lock_bh(&qp_ctx->req_lock);144 req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, qp_ctx->qp->sq_depth, GFP_ATOMIC);145 spin_unlock_bh(&qp_ctx->req_lock);146 if (unlikely(req_id < 0)) {147 dev_err(req->ctx->dev, "alloc req id fail!\n");148 return req_id;149 }150 151 req->qp_ctx = qp_ctx;152 qp_ctx->req_list[req_id] = req;153 154 return req_id;155}156 157static void sec_free_req_id(struct sec_req *req)158{159 struct sec_qp_ctx *qp_ctx = req->qp_ctx;160 int req_id = req->req_id;161 162 if (unlikely(req_id < 0 || req_id >= qp_ctx->qp->sq_depth)) {163 dev_err(req->ctx->dev, "free request id invalid!\n");164 return;165 }166 167 qp_ctx->req_list[req_id] = NULL;168 req->qp_ctx = NULL;169 170 spin_lock_bh(&qp_ctx->req_lock);171 idr_remove(&qp_ctx->req_idr, req_id);172 spin_unlock_bh(&qp_ctx->req_lock);173}174 175static u8 pre_parse_finished_bd(struct bd_status *status, void *resp)176{177 struct sec_sqe *bd = resp;178 179 status->done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK;180 status->icv = (le16_to_cpu(bd->type2.done_flag) & SEC_ICV_MASK) >> 1;181 status->flag = (le16_to_cpu(bd->type2.done_flag) &182 SEC_FLAG_MASK) >> SEC_FLAG_OFFSET;183 status->tag = le16_to_cpu(bd->type2.tag);184 status->err_type = bd->type2.error_type;185 186 return bd->type_cipher_auth & SEC_TYPE_MASK;187}188 189static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp)190{191 struct sec_sqe3 *bd3 = resp;192 193 status->done = le16_to_cpu(bd3->done_flag) & SEC_DONE_MASK;194 status->icv = (le16_to_cpu(bd3->done_flag) & SEC_ICV_MASK) >> 1;195 status->flag = (le16_to_cpu(bd3->done_flag) &196 SEC_FLAG_MASK) >> SEC_FLAG_OFFSET;197 status->tag = le64_to_cpu(bd3->tag);198 status->err_type = bd3->error_type;199 200 return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK;201}202 203static int sec_cb_status_check(struct sec_req *req,204 struct bd_status *status)205{206 struct sec_ctx *ctx = req->ctx;207 208 if (unlikely(req->err_type || status->done != SEC_SQE_DONE)) {209 dev_err_ratelimited(ctx->dev, "err_type[%d], done[%u]\n",210 req->err_type, status->done);211 return -EIO;212 }213 214 if (unlikely(ctx->alg_type == SEC_SKCIPHER)) {215 if (unlikely(status->flag != SEC_SQE_CFLAG)) {216 dev_err_ratelimited(ctx->dev, "flag[%u]\n",217 status->flag);218 return -EIO;219 }220 } else if (unlikely(ctx->alg_type == SEC_AEAD)) {221 if (unlikely(status->flag != SEC_SQE_AEAD_FLAG ||222 status->icv == SEC_ICV_ERR)) {223 dev_err_ratelimited(ctx->dev,224 "flag[%u], icv[%u]\n",225 status->flag, status->icv);226 return -EBADMSG;227 }228 }229 230 return 0;231}232 233static void sec_req_cb(struct hisi_qp *qp, void *resp)234{235 struct sec_qp_ctx *qp_ctx = qp->qp_ctx;236 struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx;237 u8 type_supported = qp_ctx->ctx->type_supported;238 struct bd_status status;239 struct sec_ctx *ctx;240 struct sec_req *req;241 int err;242 u8 type;243 244 if (type_supported == SEC_BD_TYPE2) {245 type = pre_parse_finished_bd(&status, resp);246 req = qp_ctx->req_list[status.tag];247 } else {248 type = pre_parse_finished_bd3(&status, resp);249 req = (void *)(uintptr_t)status.tag;250 }251 252 if (unlikely(type != type_supported)) {253 atomic64_inc(&dfx->err_bd_cnt);254 pr_err("err bd type [%u]\n", type);255 return;256 }257 258 if (unlikely(!req)) {259 atomic64_inc(&dfx->invalid_req_cnt);260 atomic_inc(&qp->qp_status.used);261 return;262 }263 264 req->err_type = status.err_type;265 ctx = req->ctx;266 err = sec_cb_status_check(req, &status);267 if (err)268 atomic64_inc(&dfx->done_flag_cnt);269 270 atomic64_inc(&dfx->recv_cnt);271 272 ctx->req_op->buf_unmap(ctx, req);273 274 ctx->req_op->callback(ctx, req, err);275}276 277static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req)278{279 struct sec_qp_ctx *qp_ctx = req->qp_ctx;280 int ret;281 282 if (ctx->fake_req_limit <=283 atomic_read(&qp_ctx->qp->qp_status.used) &&284 !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG))285 return -EBUSY;286 287 spin_lock_bh(&qp_ctx->req_lock);288 ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe);289 if (ctx->fake_req_limit <=290 atomic_read(&qp_ctx->qp->qp_status.used) && !ret) {291 list_add_tail(&req->backlog_head, &qp_ctx->backlog);292 atomic64_inc(&ctx->sec->debug.dfx.send_cnt);293 atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt);294 spin_unlock_bh(&qp_ctx->req_lock);295 return -EBUSY;296 }297 spin_unlock_bh(&qp_ctx->req_lock);298 299 if (unlikely(ret == -EBUSY))300 return -ENOBUFS;301 302 if (likely(!ret)) {303 ret = -EINPROGRESS;304 atomic64_inc(&ctx->sec->debug.dfx.send_cnt);305 }306 307 return ret;308}309 310/* Get DMA memory resources */311static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res)312{313 u16 q_depth = res->depth;314 int i;315 316 res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth),317 &res->c_ivin_dma, GFP_KERNEL);318 if (!res->c_ivin)319 return -ENOMEM;320 321 for (i = 1; i < q_depth; i++) {322 res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE;323 res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE;324 }325 326 return 0;327}328 329static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res)330{331 if (res->c_ivin)332 dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth),333 res->c_ivin, res->c_ivin_dma);334}335 336static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res)337{338 u16 q_depth = res->depth;339 int i;340 341 res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth),342 &res->a_ivin_dma, GFP_KERNEL);343 if (!res->a_ivin)344 return -ENOMEM;345 346 for (i = 1; i < q_depth; i++) {347 res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE;348 res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE;349 }350 351 return 0;352}353 354static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res)355{356 if (res->a_ivin)357 dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth),358 res->a_ivin, res->a_ivin_dma);359}360 361static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res)362{363 u16 q_depth = res->depth;364 int i;365 366 res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ(q_depth) << 1,367 &res->out_mac_dma, GFP_KERNEL);368 if (!res->out_mac)369 return -ENOMEM;370 371 for (i = 1; i < q_depth; i++) {372 res[i].out_mac_dma = res->out_mac_dma +373 i * (SEC_MAX_MAC_LEN << 1);374 res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1);375 }376 377 return 0;378}379 380static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res)381{382 if (res->out_mac)383 dma_free_coherent(dev, SEC_TOTAL_MAC_SZ(res->depth) << 1,384 res->out_mac, res->out_mac_dma);385}386 387static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res)388{389 if (res->pbuf)390 dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ(res->depth),391 res->pbuf, res->pbuf_dma);392}393 394/*395 * To improve performance, pbuffer is used for396 * small packets (< 512Bytes) as IOMMU translation using.397 */398static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res)399{400 u16 q_depth = res->depth;401 int size = SEC_PBUF_PAGE_NUM(q_depth);402 int pbuf_page_offset;403 int i, j, k;404 405 res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ(q_depth),406 &res->pbuf_dma, GFP_KERNEL);407 if (!res->pbuf)408 return -ENOMEM;409 410 /*411 * SEC_PBUF_PKG contains data pbuf, iv and412 * out_mac : <SEC_PBUF|SEC_IV|SEC_MAC>413 * Every PAGE contains six SEC_PBUF_PKG414 * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG415 * So we need SEC_PBUF_PAGE_NUM numbers of PAGE416 * for the SEC_TOTAL_PBUF_SZ417 */418 for (i = 0; i <= size; i++) {419 pbuf_page_offset = PAGE_SIZE * i;420 for (j = 0; j < SEC_PBUF_NUM; j++) {421 k = i * SEC_PBUF_NUM + j;422 if (k == q_depth)423 break;424 res[k].pbuf = res->pbuf +425 j * SEC_PBUF_PKG + pbuf_page_offset;426 res[k].pbuf_dma = res->pbuf_dma +427 j * SEC_PBUF_PKG + pbuf_page_offset;428 }429 }430 431 return 0;432}433 434static int sec_alg_resource_alloc(struct sec_ctx *ctx,435 struct sec_qp_ctx *qp_ctx)436{437 struct sec_alg_res *res = qp_ctx->res;438 struct device *dev = ctx->dev;439 int ret;440 441 ret = sec_alloc_civ_resource(dev, res);442 if (ret)443 return ret;444 445 if (ctx->alg_type == SEC_AEAD) {446 ret = sec_alloc_aiv_resource(dev, res);447 if (ret)448 goto alloc_aiv_fail;449 450 ret = sec_alloc_mac_resource(dev, res);451 if (ret)452 goto alloc_mac_fail;453 }454 if (ctx->pbuf_supported) {455 ret = sec_alloc_pbuf_resource(dev, res);456 if (ret) {457 dev_err(dev, "fail to alloc pbuf dma resource!\n");458 goto alloc_pbuf_fail;459 }460 }461 462 return 0;463 464alloc_pbuf_fail:465 if (ctx->alg_type == SEC_AEAD)466 sec_free_mac_resource(dev, qp_ctx->res);467alloc_mac_fail:468 if (ctx->alg_type == SEC_AEAD)469 sec_free_aiv_resource(dev, res);470alloc_aiv_fail:471 sec_free_civ_resource(dev, res);472 return ret;473}474 475static void sec_alg_resource_free(struct sec_ctx *ctx,476 struct sec_qp_ctx *qp_ctx)477{478 struct device *dev = ctx->dev;479 480 sec_free_civ_resource(dev, qp_ctx->res);481 482 if (ctx->pbuf_supported)483 sec_free_pbuf_resource(dev, qp_ctx->res);484 if (ctx->alg_type == SEC_AEAD) {485 sec_free_mac_resource(dev, qp_ctx->res);486 sec_free_aiv_resource(dev, qp_ctx->res);487 }488}489 490static int sec_alloc_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)491{492 u16 q_depth = qp_ctx->qp->sq_depth;493 struct device *dev = ctx->dev;494 int ret = -ENOMEM;495 496 qp_ctx->req_list = kcalloc(q_depth, sizeof(struct sec_req *), GFP_KERNEL);497 if (!qp_ctx->req_list)498 return ret;499 500 qp_ctx->res = kcalloc(q_depth, sizeof(struct sec_alg_res), GFP_KERNEL);501 if (!qp_ctx->res)502 goto err_free_req_list;503 qp_ctx->res->depth = q_depth;504 505 qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR);506 if (IS_ERR(qp_ctx->c_in_pool)) {507 dev_err(dev, "fail to create sgl pool for input!\n");508 goto err_free_res;509 }510 511 qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR);512 if (IS_ERR(qp_ctx->c_out_pool)) {513 dev_err(dev, "fail to create sgl pool for output!\n");514 goto err_free_c_in_pool;515 }516 517 ret = sec_alg_resource_alloc(ctx, qp_ctx);518 if (ret)519 goto err_free_c_out_pool;520 521 return 0;522 523err_free_c_out_pool:524 hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool);525err_free_c_in_pool:526 hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool);527err_free_res:528 kfree(qp_ctx->res);529err_free_req_list:530 kfree(qp_ctx->req_list);531 return ret;532}533 534static void sec_free_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)535{536 struct device *dev = ctx->dev;537 538 sec_alg_resource_free(ctx, qp_ctx);539 hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool);540 hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool);541 kfree(qp_ctx->res);542 kfree(qp_ctx->req_list);543}544 545static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id)546{547 struct sec_qp_ctx *qp_ctx;548 struct hisi_qp *qp;549 int ret;550 551 qp_ctx = &ctx->qp_ctx[qp_ctx_id];552 qp = ctx->qps[qp_ctx_id];553 qp->req_type = 0;554 qp->qp_ctx = qp_ctx;555 qp_ctx->qp = qp;556 qp_ctx->ctx = ctx;557 558 qp->req_cb = sec_req_cb;559 560 spin_lock_init(&qp_ctx->req_lock);561 idr_init(&qp_ctx->req_idr);562 INIT_LIST_HEAD(&qp_ctx->backlog);563 564 ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx);565 if (ret)566 goto err_destroy_idr;567 568 ret = hisi_qm_start_qp(qp, 0);569 if (ret < 0)570 goto err_resource_free;571 572 return 0;573 574err_resource_free:575 sec_free_qp_ctx_resource(ctx, qp_ctx);576err_destroy_idr:577 idr_destroy(&qp_ctx->req_idr);578 return ret;579}580 581static void sec_release_qp_ctx(struct sec_ctx *ctx,582 struct sec_qp_ctx *qp_ctx)583{584 hisi_qm_stop_qp(qp_ctx->qp);585 sec_free_qp_ctx_resource(ctx, qp_ctx);586 idr_destroy(&qp_ctx->req_idr);587}588 589static int sec_ctx_base_init(struct sec_ctx *ctx)590{591 struct sec_dev *sec;592 int i, ret;593 594 ctx->qps = sec_create_qps();595 if (!ctx->qps) {596 pr_err("Can not create sec qps!\n");597 return -ENODEV;598 }599 600 sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm);601 ctx->sec = sec;602 ctx->dev = &sec->qm.pdev->dev;603 ctx->hlf_q_num = sec->ctx_q_num >> 1;604 605 ctx->pbuf_supported = ctx->sec->iommu_used;606 607 /* Half of queue depth is taken as fake requests limit in the queue. */608 ctx->fake_req_limit = ctx->qps[0]->sq_depth >> 1;609 ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx),610 GFP_KERNEL);611 if (!ctx->qp_ctx) {612 ret = -ENOMEM;613 goto err_destroy_qps;614 }615 616 for (i = 0; i < sec->ctx_q_num; i++) {617 ret = sec_create_qp_ctx(ctx, i);618 if (ret)619 goto err_sec_release_qp_ctx;620 }621 622 return 0;623 624err_sec_release_qp_ctx:625 for (i = i - 1; i >= 0; i--)626 sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);627 kfree(ctx->qp_ctx);628err_destroy_qps:629 sec_destroy_qps(ctx->qps, sec->ctx_q_num);630 return ret;631}632 633static void sec_ctx_base_uninit(struct sec_ctx *ctx)634{635 int i;636 637 for (i = 0; i < ctx->sec->ctx_q_num; i++)638 sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);639 640 sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num);641 kfree(ctx->qp_ctx);642}643 644static int sec_cipher_init(struct sec_ctx *ctx)645{646 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;647 648 c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE,649 &c_ctx->c_key_dma, GFP_KERNEL);650 if (!c_ctx->c_key)651 return -ENOMEM;652 653 return 0;654}655 656static void sec_cipher_uninit(struct sec_ctx *ctx)657{658 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;659 660 memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE);661 dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE,662 c_ctx->c_key, c_ctx->c_key_dma);663}664 665static int sec_auth_init(struct sec_ctx *ctx)666{667 struct sec_auth_ctx *a_ctx = &ctx->a_ctx;668 669 a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_AKEY_SIZE,670 &a_ctx->a_key_dma, GFP_KERNEL);671 if (!a_ctx->a_key)672 return -ENOMEM;673 674 return 0;675}676 677static void sec_auth_uninit(struct sec_ctx *ctx)678{679 struct sec_auth_ctx *a_ctx = &ctx->a_ctx;680 681 memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE);682 dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE,683 a_ctx->a_key, a_ctx->a_key_dma);684}685 686static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm)687{688 const char *alg = crypto_tfm_alg_name(&tfm->base);689 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);690 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;691 692 c_ctx->fallback = false;693 694 /* Currently, only XTS mode need fallback tfm when using 192bit key */695 if (likely(strncmp(alg, "xts", SEC_XTS_NAME_SZ)))696 return 0;697 698 c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0,699 CRYPTO_ALG_NEED_FALLBACK);700 if (IS_ERR(c_ctx->fbtfm)) {701 pr_err("failed to alloc xts mode fallback tfm!\n");702 return PTR_ERR(c_ctx->fbtfm);703 }704 705 return 0;706}707 708static int sec_skcipher_init(struct crypto_skcipher *tfm)709{710 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);711 int ret;712 713 ctx->alg_type = SEC_SKCIPHER;714 crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req));715 ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm);716 if (ctx->c_ctx.ivsize > SEC_IV_SIZE) {717 pr_err("get error skcipher iv size!\n");718 return -EINVAL;719 }720 721 ret = sec_ctx_base_init(ctx);722 if (ret)723 return ret;724 725 ret = sec_cipher_init(ctx);726 if (ret)727 goto err_cipher_init;728 729 ret = sec_skcipher_fbtfm_init(tfm);730 if (ret)731 goto err_fbtfm_init;732 733 return 0;734 735err_fbtfm_init:736 sec_cipher_uninit(ctx);737err_cipher_init:738 sec_ctx_base_uninit(ctx);739 return ret;740}741 742static void sec_skcipher_uninit(struct crypto_skcipher *tfm)743{744 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);745 746 if (ctx->c_ctx.fbtfm)747 crypto_free_sync_skcipher(ctx->c_ctx.fbtfm);748 749 sec_cipher_uninit(ctx);750 sec_ctx_base_uninit(ctx);751}752 753static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key, const u32 keylen)754{755 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);756 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;757 int ret;758 759 ret = verify_skcipher_des3_key(tfm, key);760 if (ret)761 return ret;762 763 switch (keylen) {764 case SEC_DES3_2KEY_SIZE:765 c_ctx->c_key_len = SEC_CKEY_3DES_2KEY;766 break;767 case SEC_DES3_3KEY_SIZE:768 c_ctx->c_key_len = SEC_CKEY_3DES_3KEY;769 break;770 default:771 return -EINVAL;772 }773 774 return 0;775}776 777static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx,778 const u32 keylen,779 const enum sec_cmode c_mode)780{781 if (c_mode == SEC_CMODE_XTS) {782 switch (keylen) {783 case SEC_XTS_MIN_KEY_SIZE:784 c_ctx->c_key_len = SEC_CKEY_128BIT;785 break;786 case SEC_XTS_MID_KEY_SIZE:787 c_ctx->fallback = true;788 break;789 case SEC_XTS_MAX_KEY_SIZE:790 c_ctx->c_key_len = SEC_CKEY_256BIT;791 break;792 default:793 pr_err("hisi_sec2: xts mode key error!\n");794 return -EINVAL;795 }796 } else {797 if (c_ctx->c_alg == SEC_CALG_SM4 &&798 keylen != AES_KEYSIZE_128) {799 pr_err("hisi_sec2: sm4 key error!\n");800 return -EINVAL;801 } else {802 switch (keylen) {803 case AES_KEYSIZE_128:804 c_ctx->c_key_len = SEC_CKEY_128BIT;805 break;806 case AES_KEYSIZE_192:807 c_ctx->c_key_len = SEC_CKEY_192BIT;808 break;809 case AES_KEYSIZE_256:810 c_ctx->c_key_len = SEC_CKEY_256BIT;811 break;812 default:813 pr_err("hisi_sec2: aes key error!\n");814 return -EINVAL;815 }816 }817 }818 819 return 0;820}821 822static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,823 const u32 keylen, const enum sec_calg c_alg,824 const enum sec_cmode c_mode)825{826 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);827 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;828 struct device *dev = ctx->dev;829 int ret;830 831 if (c_mode == SEC_CMODE_XTS) {832 ret = xts_verify_key(tfm, key, keylen);833 if (ret) {834 dev_err(dev, "xts mode key err!\n");835 return ret;836 }837 }838 839 c_ctx->c_alg = c_alg;840 c_ctx->c_mode = c_mode;841 842 switch (c_alg) {843 case SEC_CALG_3DES:844 ret = sec_skcipher_3des_setkey(tfm, key, keylen);845 break;846 case SEC_CALG_AES:847 case SEC_CALG_SM4:848 ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode);849 break;850 default:851 dev_err(dev, "sec c_alg err!\n");852 return -EINVAL;853 }854 855 if (ret) {856 dev_err(dev, "set sec key err!\n");857 return ret;858 }859 860 memcpy(c_ctx->c_key, key, keylen);861 if (c_ctx->fallback && c_ctx->fbtfm) {862 ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen);863 if (ret) {864 dev_err(dev, "failed to set fallback skcipher key!\n");865 return ret;866 }867 }868 return 0;869}870 871#define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) \872static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\873 u32 keylen) \874{ \875 return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode); \876}877 878GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB)879GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC)880GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS)881GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR)882GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB)883GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC)884GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS)885GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC)886GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR)887 888static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req,889 struct scatterlist *src)890{891 struct sec_aead_req *a_req = &req->aead_req;892 struct aead_request *aead_req = a_req->aead_req;893 struct sec_cipher_req *c_req = &req->c_req;894 struct sec_qp_ctx *qp_ctx = req->qp_ctx;895 struct device *dev = ctx->dev;896 int copy_size, pbuf_length;897 int req_id = req->req_id;898 struct crypto_aead *tfm;899 size_t authsize;900 u8 *mac_offset;901 902 if (ctx->alg_type == SEC_AEAD)903 copy_size = aead_req->cryptlen + aead_req->assoclen;904 else905 copy_size = c_req->c_len;906 907 pbuf_length = sg_copy_to_buffer(src, sg_nents(src),908 qp_ctx->res[req_id].pbuf, copy_size);909 if (unlikely(pbuf_length != copy_size)) {910 dev_err(dev, "copy src data to pbuf error!\n");911 return -EINVAL;912 }913 if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) {914 tfm = crypto_aead_reqtfm(aead_req);915 authsize = crypto_aead_authsize(tfm);916 mac_offset = qp_ctx->res[req_id].pbuf + copy_size - authsize;917 memcpy(a_req->out_mac, mac_offset, authsize);918 }919 920 req->in_dma = qp_ctx->res[req_id].pbuf_dma;921 c_req->c_out_dma = req->in_dma;922 923 return 0;924}925 926static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req,927 struct scatterlist *dst)928{929 struct aead_request *aead_req = req->aead_req.aead_req;930 struct sec_cipher_req *c_req = &req->c_req;931 struct sec_qp_ctx *qp_ctx = req->qp_ctx;932 int copy_size, pbuf_length;933 int req_id = req->req_id;934 935 if (ctx->alg_type == SEC_AEAD)936 copy_size = c_req->c_len + aead_req->assoclen;937 else938 copy_size = c_req->c_len;939 940 pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst),941 qp_ctx->res[req_id].pbuf, copy_size);942 if (unlikely(pbuf_length != copy_size))943 dev_err(ctx->dev, "copy pbuf data to dst error!\n");944}945 946static int sec_aead_mac_init(struct sec_aead_req *req)947{948 struct aead_request *aead_req = req->aead_req;949 struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req);950 size_t authsize = crypto_aead_authsize(tfm);951 u8 *mac_out = req->out_mac;952 struct scatterlist *sgl = aead_req->src;953 size_t copy_size;954 off_t skip_size;955 956 /* Copy input mac */957 skip_size = aead_req->assoclen + aead_req->cryptlen - authsize;958 copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out,959 authsize, skip_size);960 if (unlikely(copy_size != authsize))961 return -EINVAL;962 963 return 0;964}965 966static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req,967 struct scatterlist *src, struct scatterlist *dst)968{969 struct sec_cipher_req *c_req = &req->c_req;970 struct sec_aead_req *a_req = &req->aead_req;971 struct sec_qp_ctx *qp_ctx = req->qp_ctx;972 struct sec_alg_res *res = &qp_ctx->res[req->req_id];973 struct device *dev = ctx->dev;974 int ret;975 976 if (req->use_pbuf) {977 c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET;978 c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET;979 if (ctx->alg_type == SEC_AEAD) {980 a_req->a_ivin = res->a_ivin;981 a_req->a_ivin_dma = res->a_ivin_dma;982 a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET;983 a_req->out_mac_dma = res->pbuf_dma +984 SEC_PBUF_MAC_OFFSET;985 }986 ret = sec_cipher_pbuf_map(ctx, req, src);987 988 return ret;989 }990 c_req->c_ivin = res->c_ivin;991 c_req->c_ivin_dma = res->c_ivin_dma;992 if (ctx->alg_type == SEC_AEAD) {993 a_req->a_ivin = res->a_ivin;994 a_req->a_ivin_dma = res->a_ivin_dma;995 a_req->out_mac = res->out_mac;996 a_req->out_mac_dma = res->out_mac_dma;997 }998 999 req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src,1000 qp_ctx->c_in_pool,1001 req->req_id,1002 &req->in_dma);1003 if (IS_ERR(req->in)) {1004 dev_err(dev, "fail to dma map input sgl buffers!\n");1005 return PTR_ERR(req->in);1006 }1007 1008 if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) {1009 ret = sec_aead_mac_init(a_req);1010 if (unlikely(ret)) {1011 dev_err(dev, "fail to init mac data for ICV!\n");1012 hisi_acc_sg_buf_unmap(dev, src, req->in);1013 return ret;1014 }1015 }1016 1017 if (dst == src) {1018 c_req->c_out = req->in;1019 c_req->c_out_dma = req->in_dma;1020 } else {1021 c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst,1022 qp_ctx->c_out_pool,1023 req->req_id,1024 &c_req->c_out_dma);1025 1026 if (IS_ERR(c_req->c_out)) {1027 dev_err(dev, "fail to dma map output sgl buffers!\n");1028 hisi_acc_sg_buf_unmap(dev, src, req->in);1029 return PTR_ERR(c_req->c_out);1030 }1031 }1032 1033 return 0;1034}1035 1036static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req,1037 struct scatterlist *src, struct scatterlist *dst)1038{1039 struct sec_cipher_req *c_req = &req->c_req;1040 struct device *dev = ctx->dev;1041 1042 if (req->use_pbuf) {1043 sec_cipher_pbuf_unmap(ctx, req, dst);1044 } else {1045 if (dst != src)1046 hisi_acc_sg_buf_unmap(dev, src, req->in);1047 1048 hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out);1049 }1050}1051 1052static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req)1053{1054 struct skcipher_request *sq = req->c_req.sk_req;1055 1056 return sec_cipher_map(ctx, req, sq->src, sq->dst);1057}1058 1059static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req)1060{1061 struct skcipher_request *sq = req->c_req.sk_req;1062 1063 sec_cipher_unmap(ctx, req, sq->src, sq->dst);1064}1065 1066static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx,1067 struct crypto_authenc_keys *keys)1068{1069 switch (keys->enckeylen) {1070 case AES_KEYSIZE_128:1071 c_ctx->c_key_len = SEC_CKEY_128BIT;1072 break;1073 case AES_KEYSIZE_192:1074 c_ctx->c_key_len = SEC_CKEY_192BIT;1075 break;1076 case AES_KEYSIZE_256:1077 c_ctx->c_key_len = SEC_CKEY_256BIT;1078 break;1079 default:1080 pr_err("hisi_sec2: aead aes key error!\n");1081 return -EINVAL;1082 }1083 memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen);1084 1085 return 0;1086}1087 1088static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx,1089 struct crypto_authenc_keys *keys)1090{1091 struct crypto_shash *hash_tfm = ctx->hash_tfm;1092 int blocksize, digestsize, ret;1093 1094 if (!keys->authkeylen) {1095 pr_err("hisi_sec2: aead auth key error!\n");1096 return -EINVAL;1097 }1098 1099 blocksize = crypto_shash_blocksize(hash_tfm);1100 digestsize = crypto_shash_digestsize(hash_tfm);1101 if (keys->authkeylen > blocksize) {1102 ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey,1103 keys->authkeylen, ctx->a_key);1104 if (ret) {1105 pr_err("hisi_sec2: aead auth digest error!\n");1106 return -EINVAL;1107 }1108 ctx->a_key_len = digestsize;1109 } else {1110 memcpy(ctx->a_key, keys->authkey, keys->authkeylen);1111 ctx->a_key_len = keys->authkeylen;1112 }1113 1114 return 0;1115}1116 1117static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize)1118{1119 struct crypto_tfm *tfm = crypto_aead_tfm(aead);1120 struct sec_ctx *ctx = crypto_tfm_ctx(tfm);1121 struct sec_auth_ctx *a_ctx = &ctx->a_ctx;1122 1123 if (unlikely(a_ctx->fallback_aead_tfm))1124 return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize);1125 1126 return 0;1127}1128 1129static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx,1130 struct crypto_aead *tfm, const u8 *key,1131 unsigned int keylen)1132{1133 crypto_aead_clear_flags(a_ctx->fallback_aead_tfm, CRYPTO_TFM_REQ_MASK);1134 crypto_aead_set_flags(a_ctx->fallback_aead_tfm,1135 crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK);1136 return crypto_aead_setkey(a_ctx->fallback_aead_tfm, key, keylen);1137}1138 1139static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key,1140 const u32 keylen, const enum sec_hash_alg a_alg,1141 const enum sec_calg c_alg,1142 const enum sec_mac_len mac_len,1143 const enum sec_cmode c_mode)1144{1145 struct sec_ctx *ctx = crypto_aead_ctx(tfm);1146 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;1147 struct sec_auth_ctx *a_ctx = &ctx->a_ctx;1148 struct device *dev = ctx->dev;1149 struct crypto_authenc_keys keys;1150 int ret;1151 1152 ctx->a_ctx.a_alg = a_alg;1153 ctx->c_ctx.c_alg = c_alg;1154 ctx->a_ctx.mac_len = mac_len;1155 c_ctx->c_mode = c_mode;1156 1157 if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) {1158 ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode);1159 if (ret) {1160 dev_err(dev, "set sec aes ccm cipher key err!\n");1161 return ret;1162 }1163 memcpy(c_ctx->c_key, key, keylen);1164 1165 if (unlikely(a_ctx->fallback_aead_tfm)) {1166 ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen);1167 if (ret)1168 return ret;1169 }1170 1171 return 0;1172 }1173 1174 ret = crypto_authenc_extractkeys(&keys, key, keylen);1175 if (ret)1176 goto bad_key;1177 1178 ret = sec_aead_aes_set_key(c_ctx, &keys);1179 if (ret) {1180 dev_err(dev, "set sec cipher key err!\n");1181 goto bad_key;1182 }1183 1184 ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys);1185 if (ret) {1186 dev_err(dev, "set sec auth key err!\n");1187 goto bad_key;1188 }1189 1190 if ((ctx->a_ctx.mac_len & SEC_SQE_LEN_RATE_MASK) ||1191 (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK)) {1192 ret = -EINVAL;1193 dev_err(dev, "MAC or AUTH key length error!\n");1194 goto bad_key;1195 }1196 1197 return 0;1198 1199bad_key:1200 memzero_explicit(&keys, sizeof(struct crypto_authenc_keys));1201 return ret;1202}1203 1204 1205#define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode) \1206static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, \1207 u32 keylen) \1208{ \1209 return sec_aead_setkey(tfm, key, keylen, aalg, calg, maclen, cmode);\1210}1211 1212GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1,1213 SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC)1214GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256,1215 SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC)1216GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512,1217 SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC)1218GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES,1219 SEC_HMAC_CCM_MAC, SEC_CMODE_CCM)1220GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES,1221 SEC_HMAC_GCM_MAC, SEC_CMODE_GCM)1222GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4,1223 SEC_HMAC_CCM_MAC, SEC_CMODE_CCM)1224GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4,1225 SEC_HMAC_GCM_MAC, SEC_CMODE_GCM)1226 1227static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req)1228{1229 struct aead_request *aq = req->aead_req.aead_req;1230 1231 return sec_cipher_map(ctx, req, aq->src, aq->dst);1232}1233 1234static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req)1235{1236 struct aead_request *aq = req->aead_req.aead_req;1237 1238 sec_cipher_unmap(ctx, req, aq->src, aq->dst);1239}1240 1241static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req)1242{1243 int ret;1244 1245 ret = ctx->req_op->buf_map(ctx, req);1246 if (unlikely(ret))1247 return ret;1248 1249 ctx->req_op->do_transfer(ctx, req);1250 1251 ret = ctx->req_op->bd_fill(ctx, req);1252 if (unlikely(ret))1253 goto unmap_req_buf;1254 1255 return ret;1256 1257unmap_req_buf:1258 ctx->req_op->buf_unmap(ctx, req);1259 return ret;1260}1261 1262static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req)1263{1264 ctx->req_op->buf_unmap(ctx, req);1265}1266 1267static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req)1268{1269 struct skcipher_request *sk_req = req->c_req.sk_req;1270 struct sec_cipher_req *c_req = &req->c_req;1271 1272 memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize);1273}1274 1275static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req)1276{1277 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;1278 struct sec_cipher_req *c_req = &req->c_req;1279 struct sec_sqe *sec_sqe = &req->sec_sqe;1280 u8 scene, sa_type, da_type;1281 u8 bd_type, cipher;1282 u8 de = 0;1283 1284 memset(sec_sqe, 0, sizeof(struct sec_sqe));1285 1286 sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma);1287 sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);1288 sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma);1289 sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma);1290 1291 sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) <<1292 SEC_CMODE_OFFSET);1293 sec_sqe->type2.c_alg = c_ctx->c_alg;1294 sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) <<1295 SEC_CKEY_OFFSET);1296 1297 bd_type = SEC_BD_TYPE2;1298 if (c_req->encrypt)1299 cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET;1300 else1301 cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET;1302 sec_sqe->type_cipher_auth = bd_type | cipher;1303 1304 /* Set destination and source address type */1305 if (req->use_pbuf) {1306 sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET;1307 da_type = SEC_PBUF << SEC_DST_SGL_OFFSET;1308 } else {1309 sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET;1310 da_type = SEC_SGL << SEC_DST_SGL_OFFSET;1311 }1312 1313 sec_sqe->sdm_addr_type |= da_type;1314 scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET;1315 if (req->in_dma != c_req->c_out_dma)1316 de = 0x1 << SEC_DE_OFFSET;1317 1318 sec_sqe->sds_sa_type = (de | scene | sa_type);1319 1320 sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len);1321 sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id);1322 1323 return 0;1324}1325 1326static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req)1327{1328 struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3;1329 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;1330 struct sec_cipher_req *c_req = &req->c_req;1331 u32 bd_param = 0;1332 u16 cipher;1333 1334 memset(sec_sqe3, 0, sizeof(struct sec_sqe3));1335 1336 sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma);1337 sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);1338 sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma);1339 sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma);1340 1341 sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) |1342 c_ctx->c_mode;1343 sec_sqe3->c_icv_key |= cpu_to_le16(((u16)c_ctx->c_key_len) <<1344 SEC_CKEY_OFFSET_V3);1345 1346 if (c_req->encrypt)1347 cipher = SEC_CIPHER_ENC;1348 else1349 cipher = SEC_CIPHER_DEC;1350 sec_sqe3->c_icv_key |= cpu_to_le16(cipher);1351 1352 /* Set the CTR counter mode is 128bit rollover */1353 sec_sqe3->auth_mac_key = cpu_to_le32((u32)SEC_CTR_CNT_ROLLOVER <<1354 SEC_CTR_CNT_OFFSET);1355 1356 if (req->use_pbuf) {1357 bd_param |= SEC_PBUF << SEC_SRC_SGL_OFFSET_V3;1358 bd_param |= SEC_PBUF << SEC_DST_SGL_OFFSET_V3;1359 } else {1360 bd_param |= SEC_SGL << SEC_SRC_SGL_OFFSET_V3;1361 bd_param |= SEC_SGL << SEC_DST_SGL_OFFSET_V3;1362 }1363 1364 bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3;1365 if (req->in_dma != c_req->c_out_dma)1366 bd_param |= 0x1 << SEC_DE_OFFSET_V3;1367 1368 bd_param |= SEC_BD_TYPE3;1369 sec_sqe3->bd_param = cpu_to_le32(bd_param);1370 1371 sec_sqe3->c_len_ivin |= cpu_to_le32(c_req->c_len);1372 sec_sqe3->tag = cpu_to_le64((unsigned long)req);1373 1374 return 0;1375}1376 1377/* increment counter (128-bit int) */1378static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums)1379{1380 do {1381 --bits;1382 nums += counter[bits];1383 counter[bits] = nums & BITS_MASK;1384 nums >>= BYTE_BITS;1385 } while (bits && nums);1386}1387 1388static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type)1389{1390 struct aead_request *aead_req = req->aead_req.aead_req;1391 struct skcipher_request *sk_req = req->c_req.sk_req;1392 u32 iv_size = req->ctx->c_ctx.ivsize;1393 struct scatterlist *sgl;1394 unsigned int cryptlen;1395 size_t sz;1396 u8 *iv;1397 1398 if (req->c_req.encrypt)1399 sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst;1400 else1401 sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src;1402 1403 if (alg_type == SEC_SKCIPHER) {1404 iv = sk_req->iv;1405 cryptlen = sk_req->cryptlen;1406 } else {1407 iv = aead_req->iv;1408 cryptlen = aead_req->cryptlen;1409 }1410 1411 if (req->ctx->c_ctx.c_mode == SEC_CMODE_CBC) {1412 sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size,1413 cryptlen - iv_size);1414 if (unlikely(sz != iv_size))1415 dev_err(req->ctx->dev, "copy output iv error!\n");1416 } else {1417 sz = cryptlen / iv_size;1418 if (cryptlen % iv_size)1419 sz += 1;1420 ctr_iv_inc(iv, iv_size, sz);1421 }1422}1423 1424static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx,1425 struct sec_qp_ctx *qp_ctx)1426{1427 struct sec_req *backlog_req = NULL;1428 1429 spin_lock_bh(&qp_ctx->req_lock);1430 if (ctx->fake_req_limit >=1431 atomic_read(&qp_ctx->qp->qp_status.used) &&1432 !list_empty(&qp_ctx->backlog)) {1433 backlog_req = list_first_entry(&qp_ctx->backlog,1434 typeof(*backlog_req), backlog_head);1435 list_del(&backlog_req->backlog_head);1436 }1437 spin_unlock_bh(&qp_ctx->req_lock);1438 1439 return backlog_req;1440}1441 1442static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req,1443 int err)1444{1445 struct skcipher_request *sk_req = req->c_req.sk_req;1446 struct sec_qp_ctx *qp_ctx = req->qp_ctx;1447 struct skcipher_request *backlog_sk_req;1448 struct sec_req *backlog_req;1449 1450 sec_free_req_id(req);1451 1452 /* IV output at encrypto of CBC/CTR mode */1453 if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC ||1454 ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt)1455 sec_update_iv(req, SEC_SKCIPHER);1456 1457 while (1) {1458 backlog_req = sec_back_req_clear(ctx, qp_ctx);1459 if (!backlog_req)1460 break;1461 1462 backlog_sk_req = backlog_req->c_req.sk_req;1463 skcipher_request_complete(backlog_sk_req, -EINPROGRESS);1464 atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt);1465 }1466 1467 skcipher_request_complete(sk_req, err);1468}1469 1470static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req)1471{1472 struct aead_request *aead_req = req->aead_req.aead_req;1473 struct sec_cipher_req *c_req = &req->c_req;1474 struct sec_aead_req *a_req = &req->aead_req;1475 size_t authsize = ctx->a_ctx.mac_len;1476 u32 data_size = aead_req->cryptlen;1477 u8 flage = 0;1478 u8 cm, cl;1479 1480 /* the specification has been checked in aead_iv_demension_check() */1481 cl = c_req->c_ivin[0] + 1;1482 c_req->c_ivin[ctx->c_ctx.ivsize - cl] = 0x00;1483 memset(&c_req->c_ivin[ctx->c_ctx.ivsize - cl], 0, cl);1484 c_req->c_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = IV_CTR_INIT;1485 1486 /* the last 3bit is L' */1487 flage |= c_req->c_ivin[0] & IV_CL_MASK;1488 1489 /* the M' is bit3~bit5, the Flags is bit6 */1490 cm = (authsize - IV_CM_CAL_NUM) / IV_CM_CAL_NUM;1491 flage |= cm << IV_CM_OFFSET;1492 if (aead_req->assoclen)1493 flage |= 0x01 << IV_FLAGS_OFFSET;1494 1495 memcpy(a_req->a_ivin, c_req->c_ivin, ctx->c_ctx.ivsize);1496 a_req->a_ivin[0] = flage;1497 1498 /*1499 * the last 32bit is counter's initial number,1500 * but the nonce uses the first 16bit1501 * the tail 16bit fill with the cipher length1502 */1503 if (!c_req->encrypt)1504 data_size = aead_req->cryptlen - authsize;1505 1506 a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] =1507 data_size & IV_LAST_BYTE_MASK;1508 data_size >>= IV_BYTE_OFFSET;1509 a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE2] =1510 data_size & IV_LAST_BYTE_MASK;1511}1512 1513static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req)1514{1515 struct aead_request *aead_req = req->aead_req.aead_req;1516 struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req);1517 size_t authsize = crypto_aead_authsize(tfm);1518 struct sec_cipher_req *c_req = &req->c_req;1519 struct sec_aead_req *a_req = &req->aead_req;1520 1521 memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize);1522 1523 if (ctx->c_ctx.c_mode == SEC_CMODE_CCM) {1524 /*1525 * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter},1526 * the counter must set to 0x011527 */1528 ctx->a_ctx.mac_len = authsize;1529 /* CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} */1530 set_aead_auth_iv(ctx, req);1531 }1532 1533 /* GCM 12Byte Cipher_IV == Auth_IV */1534 if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) {1535 ctx->a_ctx.mac_len = authsize;1536 memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE);1537 }1538}1539 1540static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir,1541 struct sec_req *req, struct sec_sqe *sec_sqe)1542{1543 struct sec_aead_req *a_req = &req->aead_req;1544 struct aead_request *aq = a_req->aead_req;1545 1546 /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */1547 sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)ctx->mac_len);1548 1549 /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */1550 sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr;1551 sec_sqe->type2.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma);1552 sec_sqe->type_cipher_auth |= SEC_NO_AUTH << SEC_AUTH_OFFSET;1553 1554 if (dir)1555 sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH;1556 else1557 sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER;1558 1559 sec_sqe->type2.alen_ivllen = cpu_to_le32(aq->assoclen);1560 sec_sqe->type2.auth_src_offset = cpu_to_le16(0x0);1561 sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen);1562 1563 sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma);1564}1565 1566static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir,1567 struct sec_req *req, struct sec_sqe3 *sqe3)1568{1569 struct sec_aead_req *a_req = &req->aead_req;1570 struct aead_request *aq = a_req->aead_req;1571 1572 /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */1573 sqe3->c_icv_key |= cpu_to_le16((u16)ctx->mac_len << SEC_MAC_OFFSET_V3);1574 1575 /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */1576 sqe3->a_key_addr = sqe3->c_key_addr;1577 sqe3->auth_ivin.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma);1578 sqe3->auth_mac_key |= SEC_NO_AUTH;1579 1580 if (dir)1581 sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3;1582 else1583 sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3;1584 1585 sqe3->a_len_key = cpu_to_le32(aq->assoclen);1586 sqe3->auth_src_offset = cpu_to_le16(0x0);1587 sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen);1588 sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma);1589}1590 1591static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir,1592 struct sec_req *req, struct sec_sqe *sec_sqe)1593{1594 struct sec_aead_req *a_req = &req->aead_req;1595 struct sec_cipher_req *c_req = &req->c_req;1596 struct aead_request *aq = a_req->aead_req;1597 1598 sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma);1599 1600 sec_sqe->type2.mac_key_alg =1601 cpu_to_le32(ctx->mac_len / SEC_SQE_LEN_RATE);1602 1603 sec_sqe->type2.mac_key_alg |=1604 cpu_to_le32((u32)((ctx->a_key_len) /1605 SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET);1606 1607 sec_sqe->type2.mac_key_alg |=1608 cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET);1609 1610 if (dir) {1611 sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET;1612 sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH;1613 } else {1614 sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE2 << SEC_AUTH_OFFSET;1615 sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER;1616 }1617 sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen);1618 1619 sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen);1620 1621 sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma);1622}1623 1624static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req)1625{1626 struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;1627 struct sec_sqe *sec_sqe = &req->sec_sqe;1628 int ret;1629 1630 ret = sec_skcipher_bd_fill(ctx, req);1631 if (unlikely(ret)) {1632 dev_err(ctx->dev, "skcipher bd fill is error!\n");1633 return ret;1634 }1635 1636 if (ctx->c_ctx.c_mode == SEC_CMODE_CCM ||1637 ctx->c_ctx.c_mode == SEC_CMODE_GCM)1638 sec_auth_bd_fill_xcm(auth_ctx, req->c_req.encrypt, req, sec_sqe);1639 else1640 sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe);1641 1642 return 0;1643}1644 1645static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir,1646 struct sec_req *req, struct sec_sqe3 *sqe3)1647{1648 struct sec_aead_req *a_req = &req->aead_req;1649 struct sec_cipher_req *c_req = &req->c_req;1650 struct aead_request *aq = a_req->aead_req;1651 1652 sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma);1653 1654 sqe3->auth_mac_key |=1655 cpu_to_le32((u32)(ctx->mac_len /1656 SEC_SQE_LEN_RATE) << SEC_MAC_OFFSET_V3);1657 1658 sqe3->auth_mac_key |=1659 cpu_to_le32((u32)(ctx->a_key_len /1660 SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET_V3);1661 1662 sqe3->auth_mac_key |=1663 cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3);1664 1665 if (dir) {1666 sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1);1667 sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3;1668 } else {1669 sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE2);1670 sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3;1671 }1672 sqe3->a_len_key = cpu_to_le32(c_req->c_len + aq->assoclen);1673 1674 sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen);1675 1676 sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma);1677}1678 1679static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req)1680{1681 struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;1682 struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3;1683 int ret;1684 1685 ret = sec_skcipher_bd_fill_v3(ctx, req);1686 if (unlikely(ret)) {1687 dev_err(ctx->dev, "skcipher bd3 fill is error!\n");1688 return ret;1689 }1690 1691 if (ctx->c_ctx.c_mode == SEC_CMODE_CCM ||1692 ctx->c_ctx.c_mode == SEC_CMODE_GCM)1693 sec_auth_bd_fill_xcm_v3(auth_ctx, req->c_req.encrypt,1694 req, sec_sqe3);1695 else1696 sec_auth_bd_fill_ex_v3(auth_ctx, req->c_req.encrypt,1697 req, sec_sqe3);1698 1699 return 0;1700}1701 1702static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err)1703{1704 struct aead_request *a_req = req->aead_req.aead_req;1705 struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);1706 struct sec_aead_req *aead_req = &req->aead_req;1707 struct sec_cipher_req *c_req = &req->c_req;1708 size_t authsize = crypto_aead_authsize(tfm);1709 struct sec_qp_ctx *qp_ctx = req->qp_ctx;1710 struct aead_request *backlog_aead_req;1711 struct sec_req *backlog_req;1712 size_t sz;1713 1714 if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt)1715 sec_update_iv(req, SEC_AEAD);1716 1717 /* Copy output mac */1718 if (!err && c_req->encrypt) {1719 struct scatterlist *sgl = a_req->dst;1720 1721 sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl),1722 aead_req->out_mac,1723 authsize, a_req->cryptlen +1724 a_req->assoclen);1725 if (unlikely(sz != authsize)) {1726 dev_err(c->dev, "copy out mac err!\n");1727 err = -EINVAL;1728 }1729 }1730 1731 sec_free_req_id(req);1732 1733 while (1) {1734 backlog_req = sec_back_req_clear(c, qp_ctx);1735 if (!backlog_req)1736 break;1737 1738 backlog_aead_req = backlog_req->aead_req.aead_req;1739 aead_request_complete(backlog_aead_req, -EINPROGRESS);1740 atomic64_inc(&c->sec->debug.dfx.recv_busy_cnt);1741 }1742 1743 aead_request_complete(a_req, err);1744}1745 1746static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req)1747{1748 sec_free_req_id(req);1749 sec_free_queue_id(ctx, req);1750}1751 1752static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req)1753{1754 struct sec_qp_ctx *qp_ctx;1755 int queue_id;1756 1757 /* To load balance */1758 queue_id = sec_alloc_queue_id(ctx, req);1759 qp_ctx = &ctx->qp_ctx[queue_id];1760 1761 req->req_id = sec_alloc_req_id(req, qp_ctx);1762 if (unlikely(req->req_id < 0)) {1763 sec_free_queue_id(ctx, req);1764 return req->req_id;1765 }1766 1767 return 0;1768}1769 1770static int sec_process(struct sec_ctx *ctx, struct sec_req *req)1771{1772 struct sec_cipher_req *c_req = &req->c_req;1773 int ret;1774 1775 ret = sec_request_init(ctx, req);1776 if (unlikely(ret))1777 return ret;1778 1779 ret = sec_request_transfer(ctx, req);1780 if (unlikely(ret))1781 goto err_uninit_req;1782 1783 /* Output IV as decrypto */1784 if (!req->c_req.encrypt && (ctx->c_ctx.c_mode == SEC_CMODE_CBC ||1785 ctx->c_ctx.c_mode == SEC_CMODE_CTR))1786 sec_update_iv(req, ctx->alg_type);1787 1788 ret = ctx->req_op->bd_send(ctx, req);1789 if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) ||1790 (ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {1791 dev_err_ratelimited(ctx->dev, "send sec request failed!\n");1792 goto err_send_req;1793 }1794 1795 return ret;1796 1797err_send_req:1798 /* As failing, restore the IV from user */1799 if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) {1800 if (ctx->alg_type == SEC_SKCIPHER)1801 memcpy(req->c_req.sk_req->iv, c_req->c_ivin,1802 ctx->c_ctx.ivsize);1803 else1804 memcpy(req->aead_req.aead_req->iv, c_req->c_ivin,1805 ctx->c_ctx.ivsize);1806 }1807 1808 sec_request_untransfer(ctx, req);1809err_uninit_req:1810 sec_request_uninit(ctx, req);1811 return ret;1812}1813 1814static const struct sec_req_op sec_skcipher_req_ops = {1815 .buf_map = sec_skcipher_sgl_map,1816 .buf_unmap = sec_skcipher_sgl_unmap,1817 .do_transfer = sec_skcipher_copy_iv,1818 .bd_fill = sec_skcipher_bd_fill,1819 .bd_send = sec_bd_send,1820 .callback = sec_skcipher_callback,1821 .process = sec_process,1822};1823 1824static const struct sec_req_op sec_aead_req_ops = {1825 .buf_map = sec_aead_sgl_map,1826 .buf_unmap = sec_aead_sgl_unmap,1827 .do_transfer = sec_aead_set_iv,1828 .bd_fill = sec_aead_bd_fill,1829 .bd_send = sec_bd_send,1830 .callback = sec_aead_callback,1831 .process = sec_process,1832};1833 1834static const struct sec_req_op sec_skcipher_req_ops_v3 = {1835 .buf_map = sec_skcipher_sgl_map,1836 .buf_unmap = sec_skcipher_sgl_unmap,1837 .do_transfer = sec_skcipher_copy_iv,1838 .bd_fill = sec_skcipher_bd_fill_v3,1839 .bd_send = sec_bd_send,1840 .callback = sec_skcipher_callback,1841 .process = sec_process,1842};1843 1844static const struct sec_req_op sec_aead_req_ops_v3 = {1845 .buf_map = sec_aead_sgl_map,1846 .buf_unmap = sec_aead_sgl_unmap,1847 .do_transfer = sec_aead_set_iv,1848 .bd_fill = sec_aead_bd_fill_v3,1849 .bd_send = sec_bd_send,1850 .callback = sec_aead_callback,1851 .process = sec_process,1852};1853 1854static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm)1855{1856 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);1857 int ret;1858 1859 ret = sec_skcipher_init(tfm);1860 if (ret)1861 return ret;1862 1863 if (ctx->sec->qm.ver < QM_HW_V3) {1864 ctx->type_supported = SEC_BD_TYPE2;1865 ctx->req_op = &sec_skcipher_req_ops;1866 } else {1867 ctx->type_supported = SEC_BD_TYPE3;1868 ctx->req_op = &sec_skcipher_req_ops_v3;1869 }1870 1871 return ret;1872}1873 1874static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm)1875{1876 sec_skcipher_uninit(tfm);1877}1878 1879static int sec_aead_init(struct crypto_aead *tfm)1880{1881 struct sec_ctx *ctx = crypto_aead_ctx(tfm);1882 int ret;1883 1884 crypto_aead_set_reqsize(tfm, sizeof(struct sec_req));1885 ctx->alg_type = SEC_AEAD;1886 ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm);1887 if (ctx->c_ctx.ivsize < SEC_AIV_SIZE ||1888 ctx->c_ctx.ivsize > SEC_IV_SIZE) {1889 pr_err("get error aead iv size!\n");1890 return -EINVAL;1891 }1892 1893 ret = sec_ctx_base_init(ctx);1894 if (ret)1895 return ret;1896 if (ctx->sec->qm.ver < QM_HW_V3) {1897 ctx->type_supported = SEC_BD_TYPE2;1898 ctx->req_op = &sec_aead_req_ops;1899 } else {1900 ctx->type_supported = SEC_BD_TYPE3;1901 ctx->req_op = &sec_aead_req_ops_v3;1902 }1903 1904 ret = sec_auth_init(ctx);1905 if (ret)1906 goto err_auth_init;1907 1908 ret = sec_cipher_init(ctx);1909 if (ret)1910 goto err_cipher_init;1911 1912 return ret;1913 1914err_cipher_init:1915 sec_auth_uninit(ctx);1916err_auth_init:1917 sec_ctx_base_uninit(ctx);1918 return ret;1919}1920 1921static void sec_aead_exit(struct crypto_aead *tfm)1922{1923 struct sec_ctx *ctx = crypto_aead_ctx(tfm);1924 1925 sec_cipher_uninit(ctx);1926 sec_auth_uninit(ctx);1927 sec_ctx_base_uninit(ctx);1928}1929 1930static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name)1931{1932 struct sec_ctx *ctx = crypto_aead_ctx(tfm);1933 struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;1934 int ret;1935 1936 ret = sec_aead_init(tfm);1937 if (ret) {1938 pr_err("hisi_sec2: aead init error!\n");1939 return ret;1940 }1941 1942 auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);1943 if (IS_ERR(auth_ctx->hash_tfm)) {1944 dev_err(ctx->dev, "aead alloc shash error!\n");1945 sec_aead_exit(tfm);1946 return PTR_ERR(auth_ctx->hash_tfm);1947 }1948 1949 return 0;1950}1951 1952static void sec_aead_ctx_exit(struct crypto_aead *tfm)1953{1954 struct sec_ctx *ctx = crypto_aead_ctx(tfm);1955 1956 crypto_free_shash(ctx->a_ctx.hash_tfm);1957 sec_aead_exit(tfm);1958}1959 1960static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm)1961{1962 struct aead_alg *alg = crypto_aead_alg(tfm);1963 struct sec_ctx *ctx = crypto_aead_ctx(tfm);1964 struct sec_auth_ctx *a_ctx = &ctx->a_ctx;1965 const char *aead_name = alg->base.cra_name;1966 int ret;1967 1968 ret = sec_aead_init(tfm);1969 if (ret) {1970 dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n");1971 return ret;1972 }1973 1974 a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0,1975 CRYPTO_ALG_NEED_FALLBACK |1976 CRYPTO_ALG_ASYNC);1977 if (IS_ERR(a_ctx->fallback_aead_tfm)) {1978 dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n");1979 sec_aead_exit(tfm);1980 return PTR_ERR(a_ctx->fallback_aead_tfm);1981 }1982 a_ctx->fallback = false;1983 1984 return 0;1985}1986 1987static void sec_aead_xcm_ctx_exit(struct crypto_aead *tfm)1988{1989 struct sec_ctx *ctx = crypto_aead_ctx(tfm);1990 1991 crypto_free_aead(ctx->a_ctx.fallback_aead_tfm);1992 sec_aead_exit(tfm);1993}1994 1995static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm)1996{1997 return sec_aead_ctx_init(tfm, "sha1");1998}1999 2000static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm)2001{2002 return sec_aead_ctx_init(tfm, "sha256");2003}2004 2005static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm)2006{2007 return sec_aead_ctx_init(tfm, "sha512");2008}2009 2010static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx,2011 struct sec_req *sreq)2012{2013 u32 cryptlen = sreq->c_req.sk_req->cryptlen;2014 struct device *dev = ctx->dev;2015 u8 c_mode = ctx->c_ctx.c_mode;2016 int ret = 0;2017 2018 switch (c_mode) {2019 case SEC_CMODE_XTS:2020 if (unlikely(cryptlen < AES_BLOCK_SIZE)) {2021 dev_err(dev, "skcipher XTS mode input length error!\n");2022 ret = -EINVAL;2023 }2024 break;2025 case SEC_CMODE_ECB:2026 case SEC_CMODE_CBC:2027 if (unlikely(cryptlen & (AES_BLOCK_SIZE - 1))) {2028 dev_err(dev, "skcipher AES input length error!\n");2029 ret = -EINVAL;2030 }2031 break;2032 case SEC_CMODE_CTR:2033 if (unlikely(ctx->sec->qm.ver < QM_HW_V3)) {2034 dev_err(dev, "skcipher HW version error!\n");2035 ret = -EINVAL;2036 }2037 break;2038 default:2039 ret = -EINVAL;2040 }2041 2042 return ret;2043}2044 2045static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq)2046{2047 struct skcipher_request *sk_req = sreq->c_req.sk_req;2048 struct device *dev = ctx->dev;2049 u8 c_alg = ctx->c_ctx.c_alg;2050 2051 if (unlikely(!sk_req->src || !sk_req->dst ||2052 sk_req->cryptlen > MAX_INPUT_DATA_LEN)) {2053 dev_err(dev, "skcipher input param error!\n");2054 return -EINVAL;2055 }2056 sreq->c_req.c_len = sk_req->cryptlen;2057 2058 if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ)2059 sreq->use_pbuf = true;2060 else2061 sreq->use_pbuf = false;2062 2063 if (c_alg == SEC_CALG_3DES) {2064 if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) {2065 dev_err(dev, "skcipher 3des input length error!\n");2066 return -EINVAL;2067 }2068 return 0;2069 } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) {2070 return sec_skcipher_cryptlen_check(ctx, sreq);2071 }2072 2073 dev_err(dev, "skcipher algorithm error!\n");2074 2075 return -EINVAL;2076}2077 2078static int sec_skcipher_soft_crypto(struct sec_ctx *ctx,2079 struct skcipher_request *sreq, bool encrypt)2080{2081 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;2082 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, c_ctx->fbtfm);2083 struct device *dev = ctx->dev;2084 int ret;2085 2086 if (!c_ctx->fbtfm) {2087 dev_err_ratelimited(dev, "the soft tfm isn't supported in the current system.\n");2088 return -EINVAL;2089 }2090 2091 skcipher_request_set_sync_tfm(subreq, c_ctx->fbtfm);2092 2093 /* software need sync mode to do crypto */2094 skcipher_request_set_callback(subreq, sreq->base.flags,2095 NULL, NULL);2096 skcipher_request_set_crypt(subreq, sreq->src, sreq->dst,2097 sreq->cryptlen, sreq->iv);2098 if (encrypt)2099 ret = crypto_skcipher_encrypt(subreq);2100 else2101 ret = crypto_skcipher_decrypt(subreq);2102 2103 skcipher_request_zero(subreq);2104 2105 return ret;2106}2107 2108static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt)2109{2110 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req);2111 struct sec_req *req = skcipher_request_ctx(sk_req);2112 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);2113 int ret;2114 2115 if (!sk_req->cryptlen) {2116 if (ctx->c_ctx.c_mode == SEC_CMODE_XTS)2117 return -EINVAL;2118 return 0;2119 }2120 2121 req->flag = sk_req->base.flags;2122 req->c_req.sk_req = sk_req;2123 req->c_req.encrypt = encrypt;2124 req->ctx = ctx;2125 2126 ret = sec_skcipher_param_check(ctx, req);2127 if (unlikely(ret))2128 return -EINVAL;2129 2130 if (unlikely(ctx->c_ctx.fallback))2131 return sec_skcipher_soft_crypto(ctx, sk_req, encrypt);2132 2133 return ctx->req_op->process(ctx, req);2134}2135 2136static int sec_skcipher_encrypt(struct skcipher_request *sk_req)2137{2138 return sec_skcipher_crypto(sk_req, true);2139}2140 2141static int sec_skcipher_decrypt(struct skcipher_request *sk_req)2142{2143 return sec_skcipher_crypto(sk_req, false);2144}2145 2146#define SEC_SKCIPHER_ALG(sec_cra_name, sec_set_key, \2147 sec_min_key_size, sec_max_key_size, blk_size, iv_size)\2148{\2149 .base = {\2150 .cra_name = sec_cra_name,\2151 .cra_driver_name = "hisi_sec_"sec_cra_name,\2152 .cra_priority = SEC_PRIORITY,\2153 .cra_flags = CRYPTO_ALG_ASYNC |\2154 CRYPTO_ALG_NEED_FALLBACK,\2155 .cra_blocksize = blk_size,\2156 .cra_ctxsize = sizeof(struct sec_ctx),\2157 .cra_module = THIS_MODULE,\2158 },\2159 .init = sec_skcipher_ctx_init,\2160 .exit = sec_skcipher_ctx_exit,\2161 .setkey = sec_set_key,\2162 .decrypt = sec_skcipher_decrypt,\2163 .encrypt = sec_skcipher_encrypt,\2164 .min_keysize = sec_min_key_size,\2165 .max_keysize = sec_max_key_size,\2166 .ivsize = iv_size,\2167}2168 2169static struct sec_skcipher sec_skciphers[] = {2170 {2171 .alg_msk = BIT(0),2172 .alg = SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, AES_MIN_KEY_SIZE,2173 AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, 0),2174 },2175 {2176 .alg_msk = BIT(1),2177 .alg = SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, AES_MIN_KEY_SIZE,2178 AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),2179 },2180 {2181 .alg_msk = BIT(2),2182 .alg = SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, AES_MIN_KEY_SIZE,2183 AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),2184 },2185 {2186 .alg_msk = BIT(3),2187 .alg = SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, SEC_XTS_MIN_KEY_SIZE,2188 SEC_XTS_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),2189 },2190 {2191 .alg_msk = BIT(12),2192 .alg = SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, AES_MIN_KEY_SIZE,2193 AES_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),2194 },2195 {2196 .alg_msk = BIT(13),2197 .alg = SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, AES_MIN_KEY_SIZE,2198 AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),2199 },2200 {2201 .alg_msk = BIT(14),2202 .alg = SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, SEC_XTS_MIN_KEY_SIZE,2203 SEC_XTS_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),2204 },2205 {2206 .alg_msk = BIT(23),2207 .alg = SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, SEC_DES3_3KEY_SIZE,2208 SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 0),2209 },2210 {2211 .alg_msk = BIT(24),2212 .alg = SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, SEC_DES3_3KEY_SIZE,2213 SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE,2214 DES3_EDE_BLOCK_SIZE),2215 },2216};2217 2218static int aead_iv_demension_check(struct aead_request *aead_req)2219{2220 u8 cl;2221 2222 cl = aead_req->iv[0] + 1;2223 if (cl < IV_CL_MIN || cl > IV_CL_MAX)2224 return -EINVAL;2225 2226 if (cl < IV_CL_MID && aead_req->cryptlen >> (BYTE_BITS * cl))2227 return -EOVERFLOW;2228 2229 return 0;2230}2231 2232static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq)2233{2234 struct aead_request *req = sreq->aead_req.aead_req;2235 struct crypto_aead *tfm = crypto_aead_reqtfm(req);2236 size_t authsize = crypto_aead_authsize(tfm);2237 u8 c_mode = ctx->c_ctx.c_mode;2238 struct device *dev = ctx->dev;2239 int ret;2240 2241 if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN ||2242 req->assoclen > SEC_MAX_AAD_LEN)) {2243 dev_err(dev, "aead input spec error!\n");2244 return -EINVAL;2245 }2246 2247 if (unlikely((c_mode == SEC_CMODE_GCM && authsize < DES_BLOCK_SIZE) ||2248 (c_mode == SEC_CMODE_CCM && (authsize < MIN_MAC_LEN ||2249 authsize & MAC_LEN_MASK)))) {2250 dev_err(dev, "aead input mac length error!\n");2251 return -EINVAL;2252 }2253 2254 if (c_mode == SEC_CMODE_CCM) {2255 if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN)) {2256 dev_err_ratelimited(dev, "CCM input aad parameter is too long!\n");2257 return -EINVAL;2258 }2259 ret = aead_iv_demension_check(req);2260 if (ret) {2261 dev_err(dev, "aead input iv param error!\n");2262 return ret;2263 }2264 }2265 2266 if (sreq->c_req.encrypt)2267 sreq->c_req.c_len = req->cryptlen;2268 else2269 sreq->c_req.c_len = req->cryptlen - authsize;2270 if (c_mode == SEC_CMODE_CBC) {2271 if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) {2272 dev_err(dev, "aead crypto length error!\n");2273 return -EINVAL;2274 }2275 }2276 2277 return 0;2278}2279 2280static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq)2281{2282 struct aead_request *req = sreq->aead_req.aead_req;2283 struct crypto_aead *tfm = crypto_aead_reqtfm(req);2284 size_t authsize = crypto_aead_authsize(tfm);2285 struct device *dev = ctx->dev;2286 u8 c_alg = ctx->c_ctx.c_alg;2287 2288 if (unlikely(!req->src || !req->dst)) {2289 dev_err(dev, "aead input param error!\n");2290 return -EINVAL;2291 }2292 2293 if (ctx->sec->qm.ver == QM_HW_V2) {2294 if (unlikely(!req->cryptlen || (!sreq->c_req.encrypt &&2295 req->cryptlen <= authsize))) {2296 ctx->a_ctx.fallback = true;2297 return -EINVAL;2298 }2299 }2300 2301 /* Support AES or SM4 */2302 if (unlikely(c_alg != SEC_CALG_AES && c_alg != SEC_CALG_SM4)) {2303 dev_err(dev, "aead crypto alg error!\n");2304 return -EINVAL;2305 }2306 2307 if (unlikely(sec_aead_spec_check(ctx, sreq)))2308 return -EINVAL;2309 2310 if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <=2311 SEC_PBUF_SZ)2312 sreq->use_pbuf = true;2313 else2314 sreq->use_pbuf = false;2315 2316 return 0;2317}2318 2319static int sec_aead_soft_crypto(struct sec_ctx *ctx,2320 struct aead_request *aead_req,2321 bool encrypt)2322{2323 struct sec_auth_ctx *a_ctx = &ctx->a_ctx;2324 struct device *dev = ctx->dev;2325 struct aead_request *subreq;2326 int ret;2327 2328 /* Kunpeng920 aead mode not support input 0 size */2329 if (!a_ctx->fallback_aead_tfm) {2330 dev_err(dev, "aead fallback tfm is NULL!\n");2331 return -EINVAL;2332 }2333 2334 subreq = aead_request_alloc(a_ctx->fallback_aead_tfm, GFP_KERNEL);2335 if (!subreq)2336 return -ENOMEM;2337 2338 aead_request_set_tfm(subreq, a_ctx->fallback_aead_tfm);2339 aead_request_set_callback(subreq, aead_req->base.flags,2340 aead_req->base.complete, aead_req->base.data);2341 aead_request_set_crypt(subreq, aead_req->src, aead_req->dst,2342 aead_req->cryptlen, aead_req->iv);2343 aead_request_set_ad(subreq, aead_req->assoclen);2344 2345 if (encrypt)2346 ret = crypto_aead_encrypt(subreq);2347 else2348 ret = crypto_aead_decrypt(subreq);2349 aead_request_free(subreq);2350 2351 return ret;2352}2353 2354static int sec_aead_crypto(struct aead_request *a_req, bool encrypt)2355{2356 struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);2357 struct sec_req *req = aead_request_ctx(a_req);2358 struct sec_ctx *ctx = crypto_aead_ctx(tfm);2359 int ret;2360 2361 req->flag = a_req->base.flags;2362 req->aead_req.aead_req = a_req;2363 req->c_req.encrypt = encrypt;2364 req->ctx = ctx;2365 2366 ret = sec_aead_param_check(ctx, req);2367 if (unlikely(ret)) {2368 if (ctx->a_ctx.fallback)2369 return sec_aead_soft_crypto(ctx, a_req, encrypt);2370 return -EINVAL;2371 }2372 2373 return ctx->req_op->process(ctx, req);2374}2375 2376static int sec_aead_encrypt(struct aead_request *a_req)2377{2378 return sec_aead_crypto(a_req, true);2379}2380 2381static int sec_aead_decrypt(struct aead_request *a_req)2382{2383 return sec_aead_crypto(a_req, false);2384}2385 2386#define SEC_AEAD_ALG(sec_cra_name, sec_set_key, ctx_init,\2387 ctx_exit, blk_size, iv_size, max_authsize)\2388{\2389 .base = {\2390 .cra_name = sec_cra_name,\2391 .cra_driver_name = "hisi_sec_"sec_cra_name,\2392 .cra_priority = SEC_PRIORITY,\2393 .cra_flags = CRYPTO_ALG_ASYNC |\2394 CRYPTO_ALG_NEED_FALLBACK,\2395 .cra_blocksize = blk_size,\2396 .cra_ctxsize = sizeof(struct sec_ctx),\2397 .cra_module = THIS_MODULE,\2398 },\2399 .init = ctx_init,\2400 .exit = ctx_exit,\2401 .setkey = sec_set_key,\2402 .setauthsize = sec_aead_setauthsize,\2403 .decrypt = sec_aead_decrypt,\2404 .encrypt = sec_aead_encrypt,\2405 .ivsize = iv_size,\2406 .maxauthsize = max_authsize,\2407}2408 2409static struct sec_aead sec_aeads[] = {2410 {2411 .alg_msk = BIT(6),2412 .alg = SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init,2413 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE,2414 AES_BLOCK_SIZE),2415 },2416 {2417 .alg_msk = BIT(7),2418 .alg = SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init,2419 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE,2420 AES_BLOCK_SIZE),2421 },2422 {2423 .alg_msk = BIT(17),2424 .alg = SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init,2425 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE,2426 AES_BLOCK_SIZE),2427 },2428 {2429 .alg_msk = BIT(18),2430 .alg = SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init,2431 sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE,2432 AES_BLOCK_SIZE),2433 },2434 {2435 .alg_msk = BIT(43),2436 .alg = SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", sec_setkey_aes_cbc_sha1,2437 sec_aead_sha1_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE,2438 AES_BLOCK_SIZE, SHA1_DIGEST_SIZE),2439 },2440 {2441 .alg_msk = BIT(44),2442 .alg = SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", sec_setkey_aes_cbc_sha256,2443 sec_aead_sha256_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE,2444 AES_BLOCK_SIZE, SHA256_DIGEST_SIZE),2445 },2446 {2447 .alg_msk = BIT(45),2448 .alg = SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", sec_setkey_aes_cbc_sha512,2449 sec_aead_sha512_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE,2450 AES_BLOCK_SIZE, SHA512_DIGEST_SIZE),2451 },2452};2453 2454static void sec_unregister_skcipher(u64 alg_mask, int end)2455{2456 int i;2457 2458 for (i = 0; i < end; i++)2459 if (sec_skciphers[i].alg_msk & alg_mask)2460 crypto_unregister_skcipher(&sec_skciphers[i].alg);2461}2462 2463static int sec_register_skcipher(u64 alg_mask)2464{2465 int i, ret, count;2466 2467 count = ARRAY_SIZE(sec_skciphers);2468 2469 for (i = 0; i < count; i++) {2470 if (!(sec_skciphers[i].alg_msk & alg_mask))2471 continue;2472 2473 ret = crypto_register_skcipher(&sec_skciphers[i].alg);2474 if (ret)2475 goto err;2476 }2477 2478 return 0;2479 2480err:2481 sec_unregister_skcipher(alg_mask, i);2482 2483 return ret;2484}2485 2486static void sec_unregister_aead(u64 alg_mask, int end)2487{2488 int i;2489 2490 for (i = 0; i < end; i++)2491 if (sec_aeads[i].alg_msk & alg_mask)2492 crypto_unregister_aead(&sec_aeads[i].alg);2493}2494 2495static int sec_register_aead(u64 alg_mask)2496{2497 int i, ret, count;2498 2499 count = ARRAY_SIZE(sec_aeads);2500 2501 for (i = 0; i < count; i++) {2502 if (!(sec_aeads[i].alg_msk & alg_mask))2503 continue;2504 2505 ret = crypto_register_aead(&sec_aeads[i].alg);2506 if (ret)2507 goto err;2508 }2509 2510 return 0;2511 2512err:2513 sec_unregister_aead(alg_mask, i);2514 2515 return ret;2516}2517 2518int sec_register_to_crypto(struct hisi_qm *qm)2519{2520 u64 alg_mask;2521 int ret = 0;2522 2523 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX,2524 SEC_DRV_ALG_BITMAP_LOW_IDX);2525 2526 mutex_lock(&sec_algs_lock);2527 if (sec_available_devs) {2528 sec_available_devs++;2529 goto unlock;2530 }2531 2532 ret = sec_register_skcipher(alg_mask);2533 if (ret)2534 goto unlock;2535 2536 ret = sec_register_aead(alg_mask);2537 if (ret)2538 goto unreg_skcipher;2539 2540 sec_available_devs++;2541 mutex_unlock(&sec_algs_lock);2542 2543 return 0;2544 2545unreg_skcipher:2546 sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers));2547unlock:2548 mutex_unlock(&sec_algs_lock);2549 return ret;2550}2551 2552void sec_unregister_from_crypto(struct hisi_qm *qm)2553{2554 u64 alg_mask;2555 2556 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX,2557 SEC_DRV_ALG_BITMAP_LOW_IDX);2558 2559 mutex_lock(&sec_algs_lock);2560 if (--sec_available_devs)2561 goto unlock;2562 2563 sec_unregister_aead(alg_mask, ARRAY_SIZE(sec_aeads));2564 sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers));2565 2566unlock:2567 mutex_unlock(&sec_algs_lock);2568}2569