2264 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2019 HiSilicon Limited. */3#include <crypto/akcipher.h>4#include <crypto/curve25519.h>5#include <crypto/dh.h>6#include <crypto/ecc_curve.h>7#include <crypto/ecdh.h>8#include <crypto/rng.h>9#include <crypto/internal/akcipher.h>10#include <crypto/internal/kpp.h>11#include <crypto/internal/rsa.h>12#include <crypto/kpp.h>13#include <crypto/scatterwalk.h>14#include <linux/dma-mapping.h>15#include <linux/fips.h>16#include <linux/module.h>17#include <linux/time.h>18#include "hpre.h"19 20struct hpre_ctx;21 22#define HPRE_CRYPTO_ALG_PRI 100023#define HPRE_ALIGN_SZ 6424#define HPRE_BITS_2_BYTES_SHIFT 325#define HPRE_RSA_512BITS_KSZ 6426#define HPRE_RSA_1536BITS_KSZ 19227#define HPRE_CRT_PRMS 528#define HPRE_CRT_Q 229#define HPRE_CRT_P 330#define HPRE_CRT_INV 431#define HPRE_DH_G_FLAG 0x0232#define HPRE_TRY_SEND_TIMES 10033#define HPRE_INVLD_REQ_ID (-1)34 35#define HPRE_SQE_ALG_BITS 536#define HPRE_SQE_DONE_SHIFT 3037#define HPRE_DH_MAX_P_SZ 51238 39#define HPRE_DFX_SEC_TO_US 100000040#define HPRE_DFX_US_TO_NS 100041 42/* due to nist p521 */43#define HPRE_ECC_MAX_KSZ 6644 45/* size in bytes of the n prime */46#define HPRE_ECC_NIST_P192_N_SIZE 2447#define HPRE_ECC_NIST_P256_N_SIZE 3248#define HPRE_ECC_NIST_P384_N_SIZE 4849 50/* size in bytes */51#define HPRE_ECC_HW256_KSZ_B 3252#define HPRE_ECC_HW384_KSZ_B 4853 54/* capability register mask of driver */55#define HPRE_DRV_RSA_MASK_CAP BIT(0)56#define HPRE_DRV_DH_MASK_CAP BIT(1)57#define HPRE_DRV_ECDH_MASK_CAP BIT(2)58#define HPRE_DRV_X25519_MASK_CAP BIT(5)59 60static DEFINE_MUTEX(hpre_algs_lock);61static unsigned int hpre_available_devs;62 63typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe);64 65struct hpre_rsa_ctx {66 /* low address: e--->n */67 char *pubkey;68 dma_addr_t dma_pubkey;69 70 /* low address: d--->n */71 char *prikey;72 dma_addr_t dma_prikey;73 74 /* low address: dq->dp->q->p->qinv */75 char *crt_prikey;76 dma_addr_t dma_crt_prikey;77 78 struct crypto_akcipher *soft_tfm;79};80 81struct hpre_dh_ctx {82 /*83 * If base is g we compute the public key84 * ya = g^xa mod p; [RFC2631 sec 2.1.1]85 * else if base if the counterpart public key we86 * compute the shared secret87 * ZZ = yb^xa mod p; [RFC2631 sec 2.1.1]88 * low address: d--->n, please refer to Hisilicon HPRE UM89 */90 char *xa_p;91 dma_addr_t dma_xa_p;92 93 char *g; /* m */94 dma_addr_t dma_g;95};96 97struct hpre_ecdh_ctx {98 /* low address: p->a->k->b */99 unsigned char *p;100 dma_addr_t dma_p;101 102 /* low address: x->y */103 unsigned char *g;104 dma_addr_t dma_g;105};106 107struct hpre_curve25519_ctx {108 /* low address: p->a->k */109 unsigned char *p;110 dma_addr_t dma_p;111 112 /* gx coordinate */113 unsigned char *g;114 dma_addr_t dma_g;115};116 117struct hpre_ctx {118 struct hisi_qp *qp;119 struct device *dev;120 struct hpre_asym_request **req_list;121 struct hpre *hpre;122 spinlock_t req_lock;123 unsigned int key_sz;124 bool crt_g2_mode;125 struct idr req_idr;126 union {127 struct hpre_rsa_ctx rsa;128 struct hpre_dh_ctx dh;129 struct hpre_ecdh_ctx ecdh;130 struct hpre_curve25519_ctx curve25519;131 };132 /* for ecc algorithms */133 unsigned int curve_id;134};135 136struct hpre_asym_request {137 char *src;138 char *dst;139 struct hpre_sqe req;140 struct hpre_ctx *ctx;141 union {142 struct akcipher_request *rsa;143 struct kpp_request *dh;144 struct kpp_request *ecdh;145 struct kpp_request *curve25519;146 } areq;147 int err;148 int req_id;149 hpre_cb cb;150 struct timespec64 req_time;151};152 153static inline unsigned int hpre_align_sz(void)154{155 return ((crypto_dma_align() - 1) | (HPRE_ALIGN_SZ - 1)) + 1;156}157 158static inline unsigned int hpre_align_pd(void)159{160 return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1);161}162 163static int hpre_alloc_req_id(struct hpre_ctx *ctx)164{165 unsigned long flags;166 int id;167 168 spin_lock_irqsave(&ctx->req_lock, flags);169 id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC);170 spin_unlock_irqrestore(&ctx->req_lock, flags);171 172 return id;173}174 175static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id)176{177 unsigned long flags;178 179 spin_lock_irqsave(&ctx->req_lock, flags);180 idr_remove(&ctx->req_idr, req_id);181 spin_unlock_irqrestore(&ctx->req_lock, flags);182}183 184static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req)185{186 struct hpre_ctx *ctx;187 struct hpre_dfx *dfx;188 int id;189 190 ctx = hpre_req->ctx;191 id = hpre_alloc_req_id(ctx);192 if (unlikely(id < 0))193 return -EINVAL;194 195 ctx->req_list[id] = hpre_req;196 hpre_req->req_id = id;197 198 dfx = ctx->hpre->debug.dfx;199 if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value))200 ktime_get_ts64(&hpre_req->req_time);201 202 return id;203}204 205static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req)206{207 struct hpre_ctx *ctx = hpre_req->ctx;208 int id = hpre_req->req_id;209 210 if (hpre_req->req_id >= 0) {211 hpre_req->req_id = HPRE_INVLD_REQ_ID;212 ctx->req_list[id] = NULL;213 hpre_free_req_id(ctx, id);214 }215}216 217static struct hisi_qp *hpre_get_qp_and_start(u8 type)218{219 struct hisi_qp *qp;220 int ret;221 222 qp = hpre_create_qp(type);223 if (!qp) {224 pr_err("Can not create hpre qp!\n");225 return ERR_PTR(-ENODEV);226 }227 228 ret = hisi_qm_start_qp(qp, 0);229 if (ret < 0) {230 hisi_qm_free_qps(&qp, 1);231 pci_err(qp->qm->pdev, "Can not start qp!\n");232 return ERR_PTR(-EINVAL);233 }234 235 return qp;236}237 238static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req,239 struct scatterlist *data, unsigned int len,240 int is_src, dma_addr_t *tmp)241{242 struct device *dev = hpre_req->ctx->dev;243 enum dma_data_direction dma_dir;244 245 if (is_src) {246 hpre_req->src = NULL;247 dma_dir = DMA_TO_DEVICE;248 } else {249 hpre_req->dst = NULL;250 dma_dir = DMA_FROM_DEVICE;251 }252 *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir);253 if (unlikely(dma_mapping_error(dev, *tmp))) {254 dev_err(dev, "dma map data err!\n");255 return -ENOMEM;256 }257 258 return 0;259}260 261static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req,262 struct scatterlist *data, unsigned int len,263 int is_src, dma_addr_t *tmp)264{265 struct hpre_ctx *ctx = hpre_req->ctx;266 struct device *dev = ctx->dev;267 void *ptr;268 int shift;269 270 shift = ctx->key_sz - len;271 if (unlikely(shift < 0))272 return -EINVAL;273 274 ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_ATOMIC);275 if (unlikely(!ptr))276 return -ENOMEM;277 278 if (is_src) {279 scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0);280 hpre_req->src = ptr;281 } else {282 hpre_req->dst = ptr;283 }284 285 return 0;286}287 288static int hpre_hw_data_init(struct hpre_asym_request *hpre_req,289 struct scatterlist *data, unsigned int len,290 int is_src, int is_dh)291{292 struct hpre_sqe *msg = &hpre_req->req;293 struct hpre_ctx *ctx = hpre_req->ctx;294 dma_addr_t tmp = 0;295 int ret;296 297 /* when the data is dh's source, we should format it */298 if ((sg_is_last(data) && len == ctx->key_sz) &&299 ((is_dh && !is_src) || !is_dh))300 ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp);301 else302 ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp);303 304 if (unlikely(ret))305 return ret;306 307 if (is_src)308 msg->in = cpu_to_le64(tmp);309 else310 msg->out = cpu_to_le64(tmp);311 312 return 0;313}314 315static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,316 struct hpre_asym_request *req,317 struct scatterlist *dst,318 struct scatterlist *src)319{320 struct device *dev = ctx->dev;321 struct hpre_sqe *sqe = &req->req;322 dma_addr_t tmp;323 324 tmp = le64_to_cpu(sqe->in);325 if (unlikely(dma_mapping_error(dev, tmp)))326 return;327 328 if (src) {329 if (req->src)330 dma_free_coherent(dev, ctx->key_sz, req->src, tmp);331 else332 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE);333 }334 335 tmp = le64_to_cpu(sqe->out);336 if (unlikely(dma_mapping_error(dev, tmp)))337 return;338 339 if (req->dst) {340 if (dst)341 scatterwalk_map_and_copy(req->dst, dst, 0,342 ctx->key_sz, 1);343 dma_free_coherent(dev, ctx->key_sz, req->dst, tmp);344 } else {345 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE);346 }347}348 349static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe,350 void **kreq)351{352 struct hpre_asym_request *req;353 unsigned int err, done, alg;354 int id;355 356#define HPRE_NO_HW_ERR 0357#define HPRE_HW_TASK_DONE 3358#define HREE_HW_ERR_MASK GENMASK(10, 0)359#define HREE_SQE_DONE_MASK GENMASK(1, 0)360#define HREE_ALG_TYPE_MASK GENMASK(4, 0)361 id = (int)le16_to_cpu(sqe->tag);362 req = ctx->req_list[id];363 hpre_rm_req_from_ctx(req);364 *kreq = req;365 366 err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) &367 HREE_HW_ERR_MASK;368 369 done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) &370 HREE_SQE_DONE_MASK;371 372 if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE))373 return 0;374 375 alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK;376 dev_err_ratelimited(ctx->dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n",377 alg, done, err);378 379 return -EINVAL;380}381 382static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen)383{384 struct hpre *hpre;385 386 if (!ctx || !qp || qlen < 0)387 return -EINVAL;388 389 spin_lock_init(&ctx->req_lock);390 ctx->qp = qp;391 ctx->dev = &qp->qm->pdev->dev;392 393 hpre = container_of(ctx->qp->qm, struct hpre, qm);394 ctx->hpre = hpre;395 ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL);396 if (!ctx->req_list)397 return -ENOMEM;398 ctx->key_sz = 0;399 ctx->crt_g2_mode = false;400 idr_init(&ctx->req_idr);401 402 return 0;403}404 405static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all)406{407 if (is_clear_all) {408 idr_destroy(&ctx->req_idr);409 kfree(ctx->req_list);410 hisi_qm_free_qps(&ctx->qp, 1);411 }412 413 ctx->crt_g2_mode = false;414 ctx->key_sz = 0;415}416 417static bool hpre_is_bd_timeout(struct hpre_asym_request *req,418 u64 overtime_thrhld)419{420 struct timespec64 reply_time;421 u64 time_use_us;422 423 ktime_get_ts64(&reply_time);424 time_use_us = (reply_time.tv_sec - req->req_time.tv_sec) *425 HPRE_DFX_SEC_TO_US +426 (reply_time.tv_nsec - req->req_time.tv_nsec) /427 HPRE_DFX_US_TO_NS;428 429 if (time_use_us <= overtime_thrhld)430 return false;431 432 return true;433}434 435static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp)436{437 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;438 struct hpre_asym_request *req;439 struct kpp_request *areq;440 u64 overtime_thrhld;441 int ret;442 443 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);444 areq = req->areq.dh;445 areq->dst_len = ctx->key_sz;446 447 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);448 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))449 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);450 451 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);452 kpp_request_complete(areq, ret);453 atomic64_inc(&dfx[HPRE_RECV_CNT].value);454}455 456static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp)457{458 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;459 struct hpre_asym_request *req;460 struct akcipher_request *areq;461 u64 overtime_thrhld;462 int ret;463 464 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);465 466 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);467 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))468 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);469 470 areq = req->areq.rsa;471 areq->dst_len = ctx->key_sz;472 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);473 akcipher_request_complete(areq, ret);474 atomic64_inc(&dfx[HPRE_RECV_CNT].value);475}476 477static void hpre_alg_cb(struct hisi_qp *qp, void *resp)478{479 struct hpre_ctx *ctx = qp->qp_ctx;480 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;481 struct hpre_sqe *sqe = resp;482 struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)];483 484 if (unlikely(!req)) {485 atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value);486 return;487 }488 489 req->cb(ctx, resp);490}491 492static void hpre_stop_qp_and_put(struct hisi_qp *qp)493{494 hisi_qm_stop_qp(qp);495 hisi_qm_free_qps(&qp, 1);496}497 498static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type)499{500 struct hisi_qp *qp;501 int ret;502 503 qp = hpre_get_qp_and_start(type);504 if (IS_ERR(qp))505 return PTR_ERR(qp);506 507 qp->qp_ctx = ctx;508 qp->req_cb = hpre_alg_cb;509 510 ret = hpre_ctx_set(ctx, qp, qp->sq_depth);511 if (ret)512 hpre_stop_qp_and_put(qp);513 514 return ret;515}516 517static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)518{519 struct hpre_asym_request *h_req;520 struct hpre_sqe *msg;521 int req_id;522 void *tmp;523 524 if (is_rsa) {525 struct akcipher_request *akreq = req;526 527 if (akreq->dst_len < ctx->key_sz) {528 akreq->dst_len = ctx->key_sz;529 return -EOVERFLOW;530 }531 532 tmp = akcipher_request_ctx(akreq);533 h_req = PTR_ALIGN(tmp, hpre_align_sz());534 h_req->cb = hpre_rsa_cb;535 h_req->areq.rsa = akreq;536 msg = &h_req->req;537 memset(msg, 0, sizeof(*msg));538 } else {539 struct kpp_request *kreq = req;540 541 if (kreq->dst_len < ctx->key_sz) {542 kreq->dst_len = ctx->key_sz;543 return -EOVERFLOW;544 }545 546 tmp = kpp_request_ctx(kreq);547 h_req = PTR_ALIGN(tmp, hpre_align_sz());548 h_req->cb = hpre_dh_cb;549 h_req->areq.dh = kreq;550 msg = &h_req->req;551 memset(msg, 0, sizeof(*msg));552 msg->key = cpu_to_le64(ctx->dh.dma_xa_p);553 }554 555 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);556 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);557 msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT);558 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;559 h_req->ctx = ctx;560 561 req_id = hpre_add_req_to_ctx(h_req);562 if (req_id < 0)563 return -EBUSY;564 565 msg->tag = cpu_to_le16((u16)req_id);566 567 return 0;568}569 570static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg)571{572 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;573 int ctr = 0;574 int ret;575 576 do {577 atomic64_inc(&dfx[HPRE_SEND_CNT].value);578 spin_lock_bh(&ctx->req_lock);579 ret = hisi_qp_send(ctx->qp, msg);580 spin_unlock_bh(&ctx->req_lock);581 if (ret != -EBUSY)582 break;583 atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value);584 } while (ctr++ < HPRE_TRY_SEND_TIMES);585 586 if (likely(!ret))587 return ret;588 589 if (ret != -EBUSY)590 atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value);591 592 return ret;593}594 595static int hpre_dh_compute_value(struct kpp_request *req)596{597 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);598 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);599 void *tmp = kpp_request_ctx(req);600 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());601 struct hpre_sqe *msg = &hpre_req->req;602 int ret;603 604 ret = hpre_msg_request_set(ctx, req, false);605 if (unlikely(ret))606 return ret;607 608 if (req->src) {609 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1);610 if (unlikely(ret))611 goto clear_all;612 } else {613 msg->in = cpu_to_le64(ctx->dh.dma_g);614 }615 616 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1);617 if (unlikely(ret))618 goto clear_all;619 620 if (ctx->crt_g2_mode && !req->src)621 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH_G2);622 else623 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH);624 625 /* success */626 ret = hpre_send(ctx, msg);627 if (likely(!ret))628 return -EINPROGRESS;629 630clear_all:631 hpre_rm_req_from_ctx(hpre_req);632 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);633 634 return ret;635}636 637static int hpre_is_dh_params_length_valid(unsigned int key_sz)638{639#define _HPRE_DH_GRP1 768640#define _HPRE_DH_GRP2 1024641#define _HPRE_DH_GRP5 1536642#define _HPRE_DH_GRP14 2048643#define _HPRE_DH_GRP15 3072644#define _HPRE_DH_GRP16 4096645 switch (key_sz) {646 case _HPRE_DH_GRP1:647 case _HPRE_DH_GRP2:648 case _HPRE_DH_GRP5:649 case _HPRE_DH_GRP14:650 case _HPRE_DH_GRP15:651 case _HPRE_DH_GRP16:652 return 0;653 default:654 return -EINVAL;655 }656}657 658static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params)659{660 struct device *dev = ctx->dev;661 unsigned int sz;662 663 if (params->p_size > HPRE_DH_MAX_P_SZ)664 return -EINVAL;665 666 if (hpre_is_dh_params_length_valid(params->p_size <<667 HPRE_BITS_2_BYTES_SHIFT))668 return -EINVAL;669 670 sz = ctx->key_sz = params->p_size;671 ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1,672 &ctx->dh.dma_xa_p, GFP_KERNEL);673 if (!ctx->dh.xa_p)674 return -ENOMEM;675 676 memcpy(ctx->dh.xa_p + sz, params->p, sz);677 678 /* If g equals 2 don't copy it */679 if (params->g_size == 1 && *(char *)params->g == HPRE_DH_G_FLAG) {680 ctx->crt_g2_mode = true;681 return 0;682 }683 684 ctx->dh.g = dma_alloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL);685 if (!ctx->dh.g) {686 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,687 ctx->dh.dma_xa_p);688 ctx->dh.xa_p = NULL;689 return -ENOMEM;690 }691 692 memcpy(ctx->dh.g + (sz - params->g_size), params->g, params->g_size);693 694 return 0;695}696 697static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)698{699 struct device *dev = ctx->dev;700 unsigned int sz = ctx->key_sz;701 702 if (is_clear_all)703 hisi_qm_stop_qp(ctx->qp);704 705 if (ctx->dh.g) {706 dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g);707 ctx->dh.g = NULL;708 }709 710 if (ctx->dh.xa_p) {711 memzero_explicit(ctx->dh.xa_p, sz);712 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,713 ctx->dh.dma_xa_p);714 ctx->dh.xa_p = NULL;715 }716 717 hpre_ctx_clear(ctx, is_clear_all);718}719 720static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf,721 unsigned int len)722{723 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);724 struct dh params;725 int ret;726 727 if (crypto_dh_decode_key(buf, len, ¶ms) < 0)728 return -EINVAL;729 730 /* Free old secret if any */731 hpre_dh_clear_ctx(ctx, false);732 733 ret = hpre_dh_set_params(ctx, ¶ms);734 if (ret < 0)735 goto err_clear_ctx;736 737 memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key,738 params.key_size);739 740 return 0;741 742err_clear_ctx:743 hpre_dh_clear_ctx(ctx, false);744 return ret;745}746 747static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm)748{749 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);750 751 return ctx->key_sz;752}753 754static int hpre_dh_init_tfm(struct crypto_kpp *tfm)755{756 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);757 758 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());759 760 return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);761}762 763static void hpre_dh_exit_tfm(struct crypto_kpp *tfm)764{765 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);766 767 hpre_dh_clear_ctx(ctx, true);768}769 770static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len)771{772 while (!**ptr && *len) {773 (*ptr)++;774 (*len)--;775 }776}777 778static bool hpre_rsa_key_size_is_support(unsigned int len)779{780 unsigned int bits = len << HPRE_BITS_2_BYTES_SHIFT;781 782#define _RSA_1024BITS_KEY_WDTH 1024783#define _RSA_2048BITS_KEY_WDTH 2048784#define _RSA_3072BITS_KEY_WDTH 3072785#define _RSA_4096BITS_KEY_WDTH 4096786 787 switch (bits) {788 case _RSA_1024BITS_KEY_WDTH:789 case _RSA_2048BITS_KEY_WDTH:790 case _RSA_3072BITS_KEY_WDTH:791 case _RSA_4096BITS_KEY_WDTH:792 return true;793 default:794 return false;795 }796}797 798static int hpre_rsa_enc(struct akcipher_request *req)799{800 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);801 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);802 void *tmp = akcipher_request_ctx(req);803 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());804 struct hpre_sqe *msg = &hpre_req->req;805 int ret;806 807 /* For 512 and 1536 bits key size, use soft tfm instead */808 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||809 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {810 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);811 ret = crypto_akcipher_encrypt(req);812 akcipher_request_set_tfm(req, tfm);813 return ret;814 }815 816 if (unlikely(!ctx->rsa.pubkey))817 return -EINVAL;818 819 ret = hpre_msg_request_set(ctx, req, true);820 if (unlikely(ret))821 return ret;822 823 msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT);824 msg->key = cpu_to_le64(ctx->rsa.dma_pubkey);825 826 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);827 if (unlikely(ret))828 goto clear_all;829 830 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);831 if (unlikely(ret))832 goto clear_all;833 834 /* success */835 ret = hpre_send(ctx, msg);836 if (likely(!ret))837 return -EINPROGRESS;838 839clear_all:840 hpre_rm_req_from_ctx(hpre_req);841 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);842 843 return ret;844}845 846static int hpre_rsa_dec(struct akcipher_request *req)847{848 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);849 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);850 void *tmp = akcipher_request_ctx(req);851 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());852 struct hpre_sqe *msg = &hpre_req->req;853 int ret;854 855 /* For 512 and 1536 bits key size, use soft tfm instead */856 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||857 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {858 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);859 ret = crypto_akcipher_decrypt(req);860 akcipher_request_set_tfm(req, tfm);861 return ret;862 }863 864 if (unlikely(!ctx->rsa.prikey))865 return -EINVAL;866 867 ret = hpre_msg_request_set(ctx, req, true);868 if (unlikely(ret))869 return ret;870 871 if (ctx->crt_g2_mode) {872 msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey);873 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |874 HPRE_ALG_NC_CRT);875 } else {876 msg->key = cpu_to_le64(ctx->rsa.dma_prikey);877 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |878 HPRE_ALG_NC_NCRT);879 }880 881 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);882 if (unlikely(ret))883 goto clear_all;884 885 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);886 if (unlikely(ret))887 goto clear_all;888 889 /* success */890 ret = hpre_send(ctx, msg);891 if (likely(!ret))892 return -EINPROGRESS;893 894clear_all:895 hpre_rm_req_from_ctx(hpre_req);896 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);897 898 return ret;899}900 901static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value,902 size_t vlen, bool private)903{904 const char *ptr = value;905 906 hpre_rsa_drop_leading_zeros(&ptr, &vlen);907 908 ctx->key_sz = vlen;909 910 /* if invalid key size provided, we use software tfm */911 if (!hpre_rsa_key_size_is_support(ctx->key_sz))912 return 0;913 914 ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1,915 &ctx->rsa.dma_pubkey,916 GFP_KERNEL);917 if (!ctx->rsa.pubkey)918 return -ENOMEM;919 920 if (private) {921 ctx->rsa.prikey = dma_alloc_coherent(ctx->dev, vlen << 1,922 &ctx->rsa.dma_prikey,923 GFP_KERNEL);924 if (!ctx->rsa.prikey) {925 dma_free_coherent(ctx->dev, vlen << 1,926 ctx->rsa.pubkey,927 ctx->rsa.dma_pubkey);928 ctx->rsa.pubkey = NULL;929 return -ENOMEM;930 }931 memcpy(ctx->rsa.prikey + vlen, ptr, vlen);932 }933 memcpy(ctx->rsa.pubkey + vlen, ptr, vlen);934 935 /* Using hardware HPRE to do RSA */936 return 1;937}938 939static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value,940 size_t vlen)941{942 const char *ptr = value;943 944 hpre_rsa_drop_leading_zeros(&ptr, &vlen);945 946 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)947 return -EINVAL;948 949 memcpy(ctx->rsa.pubkey + ctx->key_sz - vlen, ptr, vlen);950 951 return 0;952}953 954static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value,955 size_t vlen)956{957 const char *ptr = value;958 959 hpre_rsa_drop_leading_zeros(&ptr, &vlen);960 961 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)962 return -EINVAL;963 964 memcpy(ctx->rsa.prikey + ctx->key_sz - vlen, ptr, vlen);965 966 return 0;967}968 969static int hpre_crt_para_get(char *para, size_t para_sz,970 const char *raw, size_t raw_sz)971{972 const char *ptr = raw;973 size_t len = raw_sz;974 975 hpre_rsa_drop_leading_zeros(&ptr, &len);976 if (!len || len > para_sz)977 return -EINVAL;978 979 memcpy(para + para_sz - len, ptr, len);980 981 return 0;982}983 984static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key)985{986 unsigned int hlf_ksz = ctx->key_sz >> 1;987 struct device *dev = ctx->dev;988 u64 offset;989 int ret;990 991 ctx->rsa.crt_prikey = dma_alloc_coherent(dev, hlf_ksz * HPRE_CRT_PRMS,992 &ctx->rsa.dma_crt_prikey,993 GFP_KERNEL);994 if (!ctx->rsa.crt_prikey)995 return -ENOMEM;996 997 ret = hpre_crt_para_get(ctx->rsa.crt_prikey, hlf_ksz,998 rsa_key->dq, rsa_key->dq_sz);999 if (ret)1000 goto free_key;1001 1002 offset = hlf_ksz;1003 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,1004 rsa_key->dp, rsa_key->dp_sz);1005 if (ret)1006 goto free_key;1007 1008 offset = hlf_ksz * HPRE_CRT_Q;1009 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,1010 rsa_key->q, rsa_key->q_sz);1011 if (ret)1012 goto free_key;1013 1014 offset = hlf_ksz * HPRE_CRT_P;1015 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,1016 rsa_key->p, rsa_key->p_sz);1017 if (ret)1018 goto free_key;1019 1020 offset = hlf_ksz * HPRE_CRT_INV;1021 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,1022 rsa_key->qinv, rsa_key->qinv_sz);1023 if (ret)1024 goto free_key;1025 1026 ctx->crt_g2_mode = true;1027 1028 return 0;1029 1030free_key:1031 offset = hlf_ksz * HPRE_CRT_PRMS;1032 memzero_explicit(ctx->rsa.crt_prikey, offset);1033 dma_free_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, ctx->rsa.crt_prikey,1034 ctx->rsa.dma_crt_prikey);1035 ctx->rsa.crt_prikey = NULL;1036 ctx->crt_g2_mode = false;1037 1038 return ret;1039}1040 1041/* If it is clear all, all the resources of the QP will be cleaned. */1042static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)1043{1044 unsigned int half_key_sz = ctx->key_sz >> 1;1045 struct device *dev = ctx->dev;1046 1047 if (is_clear_all)1048 hisi_qm_stop_qp(ctx->qp);1049 1050 if (ctx->rsa.pubkey) {1051 dma_free_coherent(dev, ctx->key_sz << 1,1052 ctx->rsa.pubkey, ctx->rsa.dma_pubkey);1053 ctx->rsa.pubkey = NULL;1054 }1055 1056 if (ctx->rsa.crt_prikey) {1057 memzero_explicit(ctx->rsa.crt_prikey,1058 half_key_sz * HPRE_CRT_PRMS);1059 dma_free_coherent(dev, half_key_sz * HPRE_CRT_PRMS,1060 ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey);1061 ctx->rsa.crt_prikey = NULL;1062 }1063 1064 if (ctx->rsa.prikey) {1065 memzero_explicit(ctx->rsa.prikey, ctx->key_sz);1066 dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey,1067 ctx->rsa.dma_prikey);1068 ctx->rsa.prikey = NULL;1069 }1070 1071 hpre_ctx_clear(ctx, is_clear_all);1072}1073 1074/*1075 * we should judge if it is CRT or not,1076 * CRT: return true, N-CRT: return false .1077 */1078static bool hpre_is_crt_key(struct rsa_key *key)1079{1080 u16 len = key->p_sz + key->q_sz + key->dp_sz + key->dq_sz +1081 key->qinv_sz;1082 1083#define LEN_OF_NCRT_PARA 51084 1085 /* N-CRT less than 5 parameters */1086 return len > LEN_OF_NCRT_PARA;1087}1088 1089static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key,1090 unsigned int keylen, bool private)1091{1092 struct rsa_key rsa_key;1093 int ret;1094 1095 hpre_rsa_clear_ctx(ctx, false);1096 1097 if (private)1098 ret = rsa_parse_priv_key(&rsa_key, key, keylen);1099 else1100 ret = rsa_parse_pub_key(&rsa_key, key, keylen);1101 if (ret < 0)1102 return ret;1103 1104 ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private);1105 if (ret <= 0)1106 return ret;1107 1108 if (private) {1109 ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz);1110 if (ret < 0)1111 goto free;1112 1113 if (hpre_is_crt_key(&rsa_key)) {1114 ret = hpre_rsa_setkey_crt(ctx, &rsa_key);1115 if (ret < 0)1116 goto free;1117 }1118 }1119 1120 ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz);1121 if (ret < 0)1122 goto free;1123 1124 if ((private && !ctx->rsa.prikey) || !ctx->rsa.pubkey) {1125 ret = -EINVAL;1126 goto free;1127 }1128 1129 return 0;1130 1131free:1132 hpre_rsa_clear_ctx(ctx, false);1133 return ret;1134}1135 1136static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key,1137 unsigned int keylen)1138{1139 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);1140 int ret;1141 1142 ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen);1143 if (ret)1144 return ret;1145 1146 return hpre_rsa_setkey(ctx, key, keylen, false);1147}1148 1149static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key,1150 unsigned int keylen)1151{1152 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);1153 int ret;1154 1155 ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen);1156 if (ret)1157 return ret;1158 1159 return hpre_rsa_setkey(ctx, key, keylen, true);1160}1161 1162static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm)1163{1164 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);1165 1166 /* For 512 and 1536 bits key size, use soft tfm instead */1167 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||1168 ctx->key_sz == HPRE_RSA_1536BITS_KSZ)1169 return crypto_akcipher_maxsize(ctx->rsa.soft_tfm);1170 1171 return ctx->key_sz;1172}1173 1174static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm)1175{1176 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);1177 int ret;1178 1179 ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0);1180 if (IS_ERR(ctx->rsa.soft_tfm)) {1181 pr_err("Can not alloc_akcipher!\n");1182 return PTR_ERR(ctx->rsa.soft_tfm);1183 }1184 1185 akcipher_set_reqsize(tfm, sizeof(struct hpre_asym_request) +1186 hpre_align_pd());1187 1188 ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);1189 if (ret)1190 crypto_free_akcipher(ctx->rsa.soft_tfm);1191 1192 return ret;1193}1194 1195static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm)1196{1197 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);1198 1199 hpre_rsa_clear_ctx(ctx, true);1200 crypto_free_akcipher(ctx->rsa.soft_tfm);1201}1202 1203static void hpre_key_to_big_end(u8 *data, int len)1204{1205 int i, j;1206 1207 for (i = 0; i < len / 2; i++) {1208 j = len - i - 1;1209 swap(data[j], data[i]);1210 }1211}1212 1213static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all,1214 bool is_ecdh)1215{1216 struct device *dev = ctx->dev;1217 unsigned int sz = ctx->key_sz;1218 unsigned int shift = sz << 1;1219 1220 if (is_clear_all)1221 hisi_qm_stop_qp(ctx->qp);1222 1223 if (is_ecdh && ctx->ecdh.p) {1224 /* ecdh: p->a->k->b */1225 memzero_explicit(ctx->ecdh.p + shift, sz);1226 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);1227 ctx->ecdh.p = NULL;1228 } else if (!is_ecdh && ctx->curve25519.p) {1229 /* curve25519: p->a->k */1230 memzero_explicit(ctx->curve25519.p + shift, sz);1231 dma_free_coherent(dev, sz << 2, ctx->curve25519.p,1232 ctx->curve25519.dma_p);1233 ctx->curve25519.p = NULL;1234 }1235 1236 hpre_ctx_clear(ctx, is_clear_all);1237}1238 1239/*1240 * The bits of 192/224/256/384/521 are supported by HPRE,1241 * and convert the bits like:1242 * bits<=256, bits=256; 256<bits<=384, bits=384; 384<bits<=576, bits=576;1243 * If the parameter bit width is insufficient, then we fill in the1244 * high-order zeros by soft, so TASK_LENGTH1 is 0x3/0x5/0x8;1245 */1246static unsigned int hpre_ecdh_supported_curve(unsigned short id)1247{1248 switch (id) {1249 case ECC_CURVE_NIST_P192:1250 case ECC_CURVE_NIST_P256:1251 return HPRE_ECC_HW256_KSZ_B;1252 case ECC_CURVE_NIST_P384:1253 return HPRE_ECC_HW384_KSZ_B;1254 default:1255 break;1256 }1257 1258 return 0;1259}1260 1261static void fill_curve_param(void *addr, u64 *param, unsigned int cur_sz, u8 ndigits)1262{1263 unsigned int sz = cur_sz - (ndigits - 1) * sizeof(u64);1264 u8 i = 0;1265 1266 while (i < ndigits - 1) {1267 memcpy(addr + sizeof(u64) * i, ¶m[i], sizeof(u64));1268 i++;1269 }1270 1271 memcpy(addr + sizeof(u64) * i, ¶m[ndigits - 1], sz);1272 hpre_key_to_big_end((u8 *)addr, cur_sz);1273}1274 1275static int hpre_ecdh_fill_curve(struct hpre_ctx *ctx, struct ecdh *params,1276 unsigned int cur_sz)1277{1278 unsigned int shifta = ctx->key_sz << 1;1279 unsigned int shiftb = ctx->key_sz << 2;1280 void *p = ctx->ecdh.p + ctx->key_sz - cur_sz;1281 void *a = ctx->ecdh.p + shifta - cur_sz;1282 void *b = ctx->ecdh.p + shiftb - cur_sz;1283 void *x = ctx->ecdh.g + ctx->key_sz - cur_sz;1284 void *y = ctx->ecdh.g + shifta - cur_sz;1285 const struct ecc_curve *curve = ecc_get_curve(ctx->curve_id);1286 char *n;1287 1288 if (unlikely(!curve))1289 return -EINVAL;1290 1291 n = kzalloc(ctx->key_sz, GFP_KERNEL);1292 if (!n)1293 return -ENOMEM;1294 1295 fill_curve_param(p, curve->p, cur_sz, curve->g.ndigits);1296 fill_curve_param(a, curve->a, cur_sz, curve->g.ndigits);1297 fill_curve_param(b, curve->b, cur_sz, curve->g.ndigits);1298 fill_curve_param(x, curve->g.x, cur_sz, curve->g.ndigits);1299 fill_curve_param(y, curve->g.y, cur_sz, curve->g.ndigits);1300 fill_curve_param(n, curve->n, cur_sz, curve->g.ndigits);1301 1302 if (params->key_size == cur_sz && memcmp(params->key, n, cur_sz) >= 0) {1303 kfree(n);1304 return -EINVAL;1305 }1306 1307 kfree(n);1308 return 0;1309}1310 1311static unsigned int hpre_ecdh_get_curvesz(unsigned short id)1312{1313 switch (id) {1314 case ECC_CURVE_NIST_P192:1315 return HPRE_ECC_NIST_P192_N_SIZE;1316 case ECC_CURVE_NIST_P256:1317 return HPRE_ECC_NIST_P256_N_SIZE;1318 case ECC_CURVE_NIST_P384:1319 return HPRE_ECC_NIST_P384_N_SIZE;1320 default:1321 break;1322 }1323 1324 return 0;1325}1326 1327static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params)1328{1329 struct device *dev = ctx->dev;1330 unsigned int sz, shift, curve_sz;1331 int ret;1332 1333 ctx->key_sz = hpre_ecdh_supported_curve(ctx->curve_id);1334 if (!ctx->key_sz)1335 return -EINVAL;1336 1337 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);1338 if (!curve_sz || params->key_size > curve_sz)1339 return -EINVAL;1340 1341 sz = ctx->key_sz;1342 1343 if (!ctx->ecdh.p) {1344 ctx->ecdh.p = dma_alloc_coherent(dev, sz << 3, &ctx->ecdh.dma_p,1345 GFP_KERNEL);1346 if (!ctx->ecdh.p)1347 return -ENOMEM;1348 }1349 1350 shift = sz << 2;1351 ctx->ecdh.g = ctx->ecdh.p + shift;1352 ctx->ecdh.dma_g = ctx->ecdh.dma_p + shift;1353 1354 ret = hpre_ecdh_fill_curve(ctx, params, curve_sz);1355 if (ret) {1356 dev_err(dev, "failed to fill curve_param, ret = %d!\n", ret);1357 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);1358 ctx->ecdh.p = NULL;1359 return ret;1360 }1361 1362 return 0;1363}1364 1365static bool hpre_key_is_zero(char *key, unsigned short key_sz)1366{1367 int i;1368 1369 for (i = 0; i < key_sz; i++)1370 if (key[i])1371 return false;1372 1373 return true;1374}1375 1376static int ecdh_gen_privkey(struct hpre_ctx *ctx, struct ecdh *params)1377{1378 struct device *dev = ctx->dev;1379 int ret;1380 1381 ret = crypto_get_default_rng();1382 if (ret) {1383 dev_err(dev, "failed to get default rng, ret = %d!\n", ret);1384 return ret;1385 }1386 1387 ret = crypto_rng_get_bytes(crypto_default_rng, (u8 *)params->key,1388 params->key_size);1389 crypto_put_default_rng();1390 if (ret)1391 dev_err(dev, "failed to get rng, ret = %d!\n", ret);1392 1393 return ret;1394}1395 1396static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,1397 unsigned int len)1398{1399 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);1400 unsigned int sz, sz_shift, curve_sz;1401 struct device *dev = ctx->dev;1402 char key[HPRE_ECC_MAX_KSZ];1403 struct ecdh params;1404 int ret;1405 1406 if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) {1407 dev_err(dev, "failed to decode ecdh key!\n");1408 return -EINVAL;1409 }1410 1411 /* Use stdrng to generate private key */1412 if (!params.key || !params.key_size) {1413 params.key = key;1414 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);1415 if (!curve_sz) {1416 dev_err(dev, "Invalid curve size!\n");1417 return -EINVAL;1418 }1419 1420 params.key_size = curve_sz - 1;1421 ret = ecdh_gen_privkey(ctx, ¶ms);1422 if (ret)1423 return ret;1424 }1425 1426 if (hpre_key_is_zero(params.key, params.key_size)) {1427 dev_err(dev, "Invalid hpre key!\n");1428 return -EINVAL;1429 }1430 1431 hpre_ecc_clear_ctx(ctx, false, true);1432 1433 ret = hpre_ecdh_set_param(ctx, ¶ms);1434 if (ret < 0) {1435 dev_err(dev, "failed to set hpre param, ret = %d!\n", ret);1436 return ret;1437 }1438 1439 sz = ctx->key_sz;1440 sz_shift = (sz << 1) + sz - params.key_size;1441 memcpy(ctx->ecdh.p + sz_shift, params.key, params.key_size);1442 1443 return 0;1444}1445 1446static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx,1447 struct hpre_asym_request *req,1448 struct scatterlist *dst,1449 struct scatterlist *src)1450{1451 struct device *dev = ctx->dev;1452 struct hpre_sqe *sqe = &req->req;1453 dma_addr_t dma;1454 1455 dma = le64_to_cpu(sqe->in);1456 if (unlikely(dma_mapping_error(dev, dma)))1457 return;1458 1459 if (src && req->src)1460 dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma);1461 1462 dma = le64_to_cpu(sqe->out);1463 if (unlikely(dma_mapping_error(dev, dma)))1464 return;1465 1466 if (req->dst)1467 dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma);1468 if (dst)1469 dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE);1470}1471 1472static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp)1473{1474 unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);1475 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;1476 struct hpre_asym_request *req = NULL;1477 struct kpp_request *areq;1478 u64 overtime_thrhld;1479 char *p;1480 int ret;1481 1482 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);1483 areq = req->areq.ecdh;1484 areq->dst_len = ctx->key_sz << 1;1485 1486 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);1487 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))1488 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);1489 1490 p = sg_virt(areq->dst);1491 memmove(p, p + ctx->key_sz - curve_sz, curve_sz);1492 memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz);1493 1494 hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src);1495 kpp_request_complete(areq, ret);1496 1497 atomic64_inc(&dfx[HPRE_RECV_CNT].value);1498}1499 1500static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx,1501 struct kpp_request *req)1502{1503 struct hpre_asym_request *h_req;1504 struct hpre_sqe *msg;1505 int req_id;1506 void *tmp;1507 1508 if (req->dst_len < ctx->key_sz << 1) {1509 req->dst_len = ctx->key_sz << 1;1510 return -EINVAL;1511 }1512 1513 tmp = kpp_request_ctx(req);1514 h_req = PTR_ALIGN(tmp, hpre_align_sz());1515 h_req->cb = hpre_ecdh_cb;1516 h_req->areq.ecdh = req;1517 msg = &h_req->req;1518 memset(msg, 0, sizeof(*msg));1519 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);1520 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);1521 msg->key = cpu_to_le64(ctx->ecdh.dma_p);1522 1523 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);1524 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;1525 h_req->ctx = ctx;1526 1527 req_id = hpre_add_req_to_ctx(h_req);1528 if (req_id < 0)1529 return -EBUSY;1530 1531 msg->tag = cpu_to_le16((u16)req_id);1532 return 0;1533}1534 1535static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req,1536 struct scatterlist *data, unsigned int len)1537{1538 struct hpre_sqe *msg = &hpre_req->req;1539 struct hpre_ctx *ctx = hpre_req->ctx;1540 struct device *dev = ctx->dev;1541 unsigned int tmpshift;1542 dma_addr_t dma = 0;1543 void *ptr;1544 int shift;1545 1546 /* Src_data include gx and gy. */1547 shift = ctx->key_sz - (len >> 1);1548 if (unlikely(shift < 0))1549 return -EINVAL;1550 1551 ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL);1552 if (unlikely(!ptr))1553 return -ENOMEM;1554 1555 tmpshift = ctx->key_sz << 1;1556 scatterwalk_map_and_copy(ptr + tmpshift, data, 0, len, 0);1557 memcpy(ptr + shift, ptr + tmpshift, len >> 1);1558 memcpy(ptr + ctx->key_sz + shift, ptr + tmpshift + (len >> 1), len >> 1);1559 1560 hpre_req->src = ptr;1561 msg->in = cpu_to_le64(dma);1562 return 0;1563}1564 1565static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req,1566 struct scatterlist *data, unsigned int len)1567{1568 struct hpre_sqe *msg = &hpre_req->req;1569 struct hpre_ctx *ctx = hpre_req->ctx;1570 struct device *dev = ctx->dev;1571 dma_addr_t dma;1572 1573 if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) {1574 dev_err(dev, "data or data length is illegal!\n");1575 return -EINVAL;1576 }1577 1578 hpre_req->dst = NULL;1579 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);1580 if (unlikely(dma_mapping_error(dev, dma))) {1581 dev_err(dev, "dma map data err!\n");1582 return -ENOMEM;1583 }1584 1585 msg->out = cpu_to_le64(dma);1586 return 0;1587}1588 1589static int hpre_ecdh_compute_value(struct kpp_request *req)1590{1591 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);1592 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);1593 struct device *dev = ctx->dev;1594 void *tmp = kpp_request_ctx(req);1595 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());1596 struct hpre_sqe *msg = &hpre_req->req;1597 int ret;1598 1599 ret = hpre_ecdh_msg_request_set(ctx, req);1600 if (unlikely(ret)) {1601 dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret);1602 return ret;1603 }1604 1605 if (req->src) {1606 ret = hpre_ecdh_src_data_init(hpre_req, req->src, req->src_len);1607 if (unlikely(ret)) {1608 dev_err(dev, "failed to init src data, ret = %d!\n", ret);1609 goto clear_all;1610 }1611 } else {1612 msg->in = cpu_to_le64(ctx->ecdh.dma_g);1613 }1614 1615 ret = hpre_ecdh_dst_data_init(hpre_req, req->dst, req->dst_len);1616 if (unlikely(ret)) {1617 dev_err(dev, "failed to init dst data, ret = %d!\n", ret);1618 goto clear_all;1619 }1620 1621 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL);1622 ret = hpre_send(ctx, msg);1623 if (likely(!ret))1624 return -EINPROGRESS;1625 1626clear_all:1627 hpre_rm_req_from_ctx(hpre_req);1628 hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);1629 return ret;1630}1631 1632static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm)1633{1634 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);1635 1636 /* max size is the pub_key_size, include x and y */1637 return ctx->key_sz << 1;1638}1639 1640static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm)1641{1642 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);1643 1644 ctx->curve_id = ECC_CURVE_NIST_P192;1645 1646 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());1647 1648 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);1649}1650 1651static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm)1652{1653 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);1654 1655 ctx->curve_id = ECC_CURVE_NIST_P256;1656 1657 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());1658 1659 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);1660}1661 1662static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm)1663{1664 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);1665 1666 ctx->curve_id = ECC_CURVE_NIST_P384;1667 1668 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());1669 1670 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);1671}1672 1673static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm)1674{1675 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);1676 1677 hpre_ecc_clear_ctx(ctx, true, true);1678}1679 1680static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf,1681 unsigned int len)1682{1683 u8 secret[CURVE25519_KEY_SIZE] = { 0 };1684 unsigned int sz = ctx->key_sz;1685 const struct ecc_curve *curve;1686 unsigned int shift = sz << 1;1687 void *p;1688 1689 /*1690 * The key from 'buf' is in little-endian, we should preprocess it as1691 * the description in rfc7748: "k[0] &= 248, k[31] &= 127, k[31] |= 64",1692 * then convert it to big endian. Only in this way, the result can be1693 * the same as the software curve-25519 that exists in crypto.1694 */1695 memcpy(secret, buf, len);1696 curve25519_clamp_secret(secret);1697 hpre_key_to_big_end(secret, CURVE25519_KEY_SIZE);1698 1699 p = ctx->curve25519.p + sz - len;1700 1701 curve = ecc_get_curve25519();1702 1703 /* fill curve parameters */1704 fill_curve_param(p, curve->p, len, curve->g.ndigits);1705 fill_curve_param(p + sz, curve->a, len, curve->g.ndigits);1706 memcpy(p + shift, secret, len);1707 fill_curve_param(p + shift + sz, curve->g.x, len, curve->g.ndigits);1708 memzero_explicit(secret, CURVE25519_KEY_SIZE);1709}1710 1711static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf,1712 unsigned int len)1713{1714 struct device *dev = ctx->dev;1715 unsigned int sz = ctx->key_sz;1716 unsigned int shift = sz << 1;1717 1718 /* p->a->k->gx */1719 if (!ctx->curve25519.p) {1720 ctx->curve25519.p = dma_alloc_coherent(dev, sz << 2,1721 &ctx->curve25519.dma_p,1722 GFP_KERNEL);1723 if (!ctx->curve25519.p)1724 return -ENOMEM;1725 }1726 1727 ctx->curve25519.g = ctx->curve25519.p + shift + sz;1728 ctx->curve25519.dma_g = ctx->curve25519.dma_p + shift + sz;1729 1730 hpre_curve25519_fill_curve(ctx, buf, len);1731 1732 return 0;1733}1734 1735static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf,1736 unsigned int len)1737{1738 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);1739 struct device *dev = ctx->dev;1740 int ret = -EINVAL;1741 1742 if (len != CURVE25519_KEY_SIZE ||1743 !crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE)) {1744 dev_err(dev, "key is null or key len is not 32bytes!\n");1745 return ret;1746 }1747 1748 /* Free old secret if any */1749 hpre_ecc_clear_ctx(ctx, false, false);1750 1751 ctx->key_sz = CURVE25519_KEY_SIZE;1752 ret = hpre_curve25519_set_param(ctx, buf, CURVE25519_KEY_SIZE);1753 if (ret) {1754 dev_err(dev, "failed to set curve25519 param, ret = %d!\n", ret);1755 hpre_ecc_clear_ctx(ctx, false, false);1756 return ret;1757 }1758 1759 return 0;1760}1761 1762static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx,1763 struct hpre_asym_request *req,1764 struct scatterlist *dst,1765 struct scatterlist *src)1766{1767 struct device *dev = ctx->dev;1768 struct hpre_sqe *sqe = &req->req;1769 dma_addr_t dma;1770 1771 dma = le64_to_cpu(sqe->in);1772 if (unlikely(dma_mapping_error(dev, dma)))1773 return;1774 1775 if (src && req->src)1776 dma_free_coherent(dev, ctx->key_sz, req->src, dma);1777 1778 dma = le64_to_cpu(sqe->out);1779 if (unlikely(dma_mapping_error(dev, dma)))1780 return;1781 1782 if (req->dst)1783 dma_free_coherent(dev, ctx->key_sz, req->dst, dma);1784 if (dst)1785 dma_unmap_single(dev, dma, ctx->key_sz, DMA_FROM_DEVICE);1786}1787 1788static void hpre_curve25519_cb(struct hpre_ctx *ctx, void *resp)1789{1790 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;1791 struct hpre_asym_request *req = NULL;1792 struct kpp_request *areq;1793 u64 overtime_thrhld;1794 int ret;1795 1796 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);1797 areq = req->areq.curve25519;1798 areq->dst_len = ctx->key_sz;1799 1800 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);1801 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))1802 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);1803 1804 hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE);1805 1806 hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src);1807 kpp_request_complete(areq, ret);1808 1809 atomic64_inc(&dfx[HPRE_RECV_CNT].value);1810}1811 1812static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx,1813 struct kpp_request *req)1814{1815 struct hpre_asym_request *h_req;1816 struct hpre_sqe *msg;1817 int req_id;1818 void *tmp;1819 1820 if (unlikely(req->dst_len < ctx->key_sz)) {1821 req->dst_len = ctx->key_sz;1822 return -EINVAL;1823 }1824 1825 tmp = kpp_request_ctx(req);1826 h_req = PTR_ALIGN(tmp, hpre_align_sz());1827 h_req->cb = hpre_curve25519_cb;1828 h_req->areq.curve25519 = req;1829 msg = &h_req->req;1830 memset(msg, 0, sizeof(*msg));1831 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);1832 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);1833 msg->key = cpu_to_le64(ctx->curve25519.dma_p);1834 1835 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);1836 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;1837 h_req->ctx = ctx;1838 1839 req_id = hpre_add_req_to_ctx(h_req);1840 if (req_id < 0)1841 return -EBUSY;1842 1843 msg->tag = cpu_to_le16((u16)req_id);1844 return 0;1845}1846 1847static void hpre_curve25519_src_modulo_p(u8 *ptr)1848{1849 int i;1850 1851 for (i = 0; i < CURVE25519_KEY_SIZE - 1; i++)1852 ptr[i] = 0;1853 1854 /* The modulus is ptr's last byte minus '0xed'(last byte of p) */1855 ptr[i] -= 0xed;1856}1857 1858static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req,1859 struct scatterlist *data, unsigned int len)1860{1861 struct hpre_sqe *msg = &hpre_req->req;1862 struct hpre_ctx *ctx = hpre_req->ctx;1863 struct device *dev = ctx->dev;1864 u8 p[CURVE25519_KEY_SIZE] = { 0 };1865 const struct ecc_curve *curve;1866 dma_addr_t dma = 0;1867 u8 *ptr;1868 1869 if (len != CURVE25519_KEY_SIZE) {1870 dev_err(dev, "sourc_data len is not 32bytes, len = %u!\n", len);1871 return -EINVAL;1872 }1873 1874 ptr = dma_alloc_coherent(dev, ctx->key_sz, &dma, GFP_KERNEL);1875 if (unlikely(!ptr))1876 return -ENOMEM;1877 1878 scatterwalk_map_and_copy(ptr, data, 0, len, 0);1879 1880 if (!crypto_memneq(ptr, curve25519_null_point, CURVE25519_KEY_SIZE)) {1881 dev_err(dev, "gx is null!\n");1882 goto err;1883 }1884 1885 /*1886 * Src_data(gx) is in little-endian order, MSB in the final byte should1887 * be masked as described in RFC7748, then transform it to big-endian1888 * form, then hisi_hpre can use the data.1889 */1890 ptr[31] &= 0x7f;1891 hpre_key_to_big_end(ptr, CURVE25519_KEY_SIZE);1892 1893 curve = ecc_get_curve25519();1894 1895 fill_curve_param(p, curve->p, CURVE25519_KEY_SIZE, curve->g.ndigits);1896 1897 /*1898 * When src_data equals (2^255 - 19) ~ (2^255 - 1), it is out of p,1899 * we get its modulus to p, and then use it.1900 */1901 if (memcmp(ptr, p, ctx->key_sz) == 0) {1902 dev_err(dev, "gx is p!\n");1903 goto err;1904 } else if (memcmp(ptr, p, ctx->key_sz) > 0) {1905 hpre_curve25519_src_modulo_p(ptr);1906 }1907 1908 hpre_req->src = ptr;1909 msg->in = cpu_to_le64(dma);1910 return 0;1911 1912err:1913 dma_free_coherent(dev, ctx->key_sz, ptr, dma);1914 return -EINVAL;1915}1916 1917static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req,1918 struct scatterlist *data, unsigned int len)1919{1920 struct hpre_sqe *msg = &hpre_req->req;1921 struct hpre_ctx *ctx = hpre_req->ctx;1922 struct device *dev = ctx->dev;1923 dma_addr_t dma;1924 1925 if (!data || !sg_is_last(data) || len != ctx->key_sz) {1926 dev_err(dev, "data or data length is illegal!\n");1927 return -EINVAL;1928 }1929 1930 hpre_req->dst = NULL;1931 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);1932 if (unlikely(dma_mapping_error(dev, dma))) {1933 dev_err(dev, "dma map data err!\n");1934 return -ENOMEM;1935 }1936 1937 msg->out = cpu_to_le64(dma);1938 return 0;1939}1940 1941static int hpre_curve25519_compute_value(struct kpp_request *req)1942{1943 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);1944 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);1945 struct device *dev = ctx->dev;1946 void *tmp = kpp_request_ctx(req);1947 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());1948 struct hpre_sqe *msg = &hpre_req->req;1949 int ret;1950 1951 ret = hpre_curve25519_msg_request_set(ctx, req);1952 if (unlikely(ret)) {1953 dev_err(dev, "failed to set curve25519 request, ret = %d!\n", ret);1954 return ret;1955 }1956 1957 if (req->src) {1958 ret = hpre_curve25519_src_init(hpre_req, req->src, req->src_len);1959 if (unlikely(ret)) {1960 dev_err(dev, "failed to init src data, ret = %d!\n",1961 ret);1962 goto clear_all;1963 }1964 } else {1965 msg->in = cpu_to_le64(ctx->curve25519.dma_g);1966 }1967 1968 ret = hpre_curve25519_dst_init(hpre_req, req->dst, req->dst_len);1969 if (unlikely(ret)) {1970 dev_err(dev, "failed to init dst data, ret = %d!\n", ret);1971 goto clear_all;1972 }1973 1974 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_CURVE25519_MUL);1975 ret = hpre_send(ctx, msg);1976 if (likely(!ret))1977 return -EINPROGRESS;1978 1979clear_all:1980 hpre_rm_req_from_ctx(hpre_req);1981 hpre_curve25519_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);1982 return ret;1983}1984 1985static unsigned int hpre_curve25519_max_size(struct crypto_kpp *tfm)1986{1987 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);1988 1989 return ctx->key_sz;1990}1991 1992static int hpre_curve25519_init_tfm(struct crypto_kpp *tfm)1993{1994 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);1995 1996 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());1997 1998 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);1999}2000 2001static void hpre_curve25519_exit_tfm(struct crypto_kpp *tfm)2002{2003 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);2004 2005 hpre_ecc_clear_ctx(ctx, true, false);2006}2007 2008static struct akcipher_alg rsa = {2009 .sign = hpre_rsa_dec,2010 .verify = hpre_rsa_enc,2011 .encrypt = hpre_rsa_enc,2012 .decrypt = hpre_rsa_dec,2013 .set_pub_key = hpre_rsa_setpubkey,2014 .set_priv_key = hpre_rsa_setprivkey,2015 .max_size = hpre_rsa_max_size,2016 .init = hpre_rsa_init_tfm,2017 .exit = hpre_rsa_exit_tfm,2018 .base = {2019 .cra_ctxsize = sizeof(struct hpre_ctx),2020 .cra_priority = HPRE_CRYPTO_ALG_PRI,2021 .cra_name = "rsa",2022 .cra_driver_name = "hpre-rsa",2023 .cra_module = THIS_MODULE,2024 },2025};2026 2027static struct kpp_alg dh = {2028 .set_secret = hpre_dh_set_secret,2029 .generate_public_key = hpre_dh_compute_value,2030 .compute_shared_secret = hpre_dh_compute_value,2031 .max_size = hpre_dh_max_size,2032 .init = hpre_dh_init_tfm,2033 .exit = hpre_dh_exit_tfm,2034 .base = {2035 .cra_ctxsize = sizeof(struct hpre_ctx),2036 .cra_priority = HPRE_CRYPTO_ALG_PRI,2037 .cra_name = "dh",2038 .cra_driver_name = "hpre-dh",2039 .cra_module = THIS_MODULE,2040 },2041};2042 2043static struct kpp_alg ecdh_curves[] = {2044 {2045 .set_secret = hpre_ecdh_set_secret,2046 .generate_public_key = hpre_ecdh_compute_value,2047 .compute_shared_secret = hpre_ecdh_compute_value,2048 .max_size = hpre_ecdh_max_size,2049 .init = hpre_ecdh_nist_p192_init_tfm,2050 .exit = hpre_ecdh_exit_tfm,2051 .base = {2052 .cra_ctxsize = sizeof(struct hpre_ctx),2053 .cra_priority = HPRE_CRYPTO_ALG_PRI,2054 .cra_name = "ecdh-nist-p192",2055 .cra_driver_name = "hpre-ecdh-nist-p192",2056 .cra_module = THIS_MODULE,2057 },2058 }, {2059 .set_secret = hpre_ecdh_set_secret,2060 .generate_public_key = hpre_ecdh_compute_value,2061 .compute_shared_secret = hpre_ecdh_compute_value,2062 .max_size = hpre_ecdh_max_size,2063 .init = hpre_ecdh_nist_p256_init_tfm,2064 .exit = hpre_ecdh_exit_tfm,2065 .base = {2066 .cra_ctxsize = sizeof(struct hpre_ctx),2067 .cra_priority = HPRE_CRYPTO_ALG_PRI,2068 .cra_name = "ecdh-nist-p256",2069 .cra_driver_name = "hpre-ecdh-nist-p256",2070 .cra_module = THIS_MODULE,2071 },2072 }, {2073 .set_secret = hpre_ecdh_set_secret,2074 .generate_public_key = hpre_ecdh_compute_value,2075 .compute_shared_secret = hpre_ecdh_compute_value,2076 .max_size = hpre_ecdh_max_size,2077 .init = hpre_ecdh_nist_p384_init_tfm,2078 .exit = hpre_ecdh_exit_tfm,2079 .base = {2080 .cra_ctxsize = sizeof(struct hpre_ctx),2081 .cra_priority = HPRE_CRYPTO_ALG_PRI,2082 .cra_name = "ecdh-nist-p384",2083 .cra_driver_name = "hpre-ecdh-nist-p384",2084 .cra_module = THIS_MODULE,2085 },2086 }2087};2088 2089static struct kpp_alg curve25519_alg = {2090 .set_secret = hpre_curve25519_set_secret,2091 .generate_public_key = hpre_curve25519_compute_value,2092 .compute_shared_secret = hpre_curve25519_compute_value,2093 .max_size = hpre_curve25519_max_size,2094 .init = hpre_curve25519_init_tfm,2095 .exit = hpre_curve25519_exit_tfm,2096 .base = {2097 .cra_ctxsize = sizeof(struct hpre_ctx),2098 .cra_priority = HPRE_CRYPTO_ALG_PRI,2099 .cra_name = "curve25519",2100 .cra_driver_name = "hpre-curve25519",2101 .cra_module = THIS_MODULE,2102 },2103};2104 2105static int hpre_register_rsa(struct hisi_qm *qm)2106{2107 int ret;2108 2109 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP))2110 return 0;2111 2112 rsa.base.cra_flags = 0;2113 ret = crypto_register_akcipher(&rsa);2114 if (ret)2115 dev_err(&qm->pdev->dev, "failed to register rsa (%d)!\n", ret);2116 2117 return ret;2118}2119 2120static void hpre_unregister_rsa(struct hisi_qm *qm)2121{2122 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP))2123 return;2124 2125 crypto_unregister_akcipher(&rsa);2126}2127 2128static int hpre_register_dh(struct hisi_qm *qm)2129{2130 int ret;2131 2132 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP))2133 return 0;2134 2135 ret = crypto_register_kpp(&dh);2136 if (ret)2137 dev_err(&qm->pdev->dev, "failed to register dh (%d)!\n", ret);2138 2139 return ret;2140}2141 2142static void hpre_unregister_dh(struct hisi_qm *qm)2143{2144 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP))2145 return;2146 2147 crypto_unregister_kpp(&dh);2148}2149 2150static int hpre_register_ecdh(struct hisi_qm *qm)2151{2152 int ret, i;2153 2154 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP))2155 return 0;2156 2157 for (i = 0; i < ARRAY_SIZE(ecdh_curves); i++) {2158 ret = crypto_register_kpp(&ecdh_curves[i]);2159 if (ret) {2160 dev_err(&qm->pdev->dev, "failed to register %s (%d)!\n",2161 ecdh_curves[i].base.cra_name, ret);2162 goto unreg_kpp;2163 }2164 }2165 2166 return 0;2167 2168unreg_kpp:2169 for (--i; i >= 0; --i)2170 crypto_unregister_kpp(&ecdh_curves[i]);2171 2172 return ret;2173}2174 2175static void hpre_unregister_ecdh(struct hisi_qm *qm)2176{2177 int i;2178 2179 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP))2180 return;2181 2182 for (i = ARRAY_SIZE(ecdh_curves) - 1; i >= 0; --i)2183 crypto_unregister_kpp(&ecdh_curves[i]);2184}2185 2186static int hpre_register_x25519(struct hisi_qm *qm)2187{2188 int ret;2189 2190 if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP))2191 return 0;2192 2193 ret = crypto_register_kpp(&curve25519_alg);2194 if (ret)2195 dev_err(&qm->pdev->dev, "failed to register x25519 (%d)!\n", ret);2196 2197 return ret;2198}2199 2200static void hpre_unregister_x25519(struct hisi_qm *qm)2201{2202 if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP))2203 return;2204 2205 crypto_unregister_kpp(&curve25519_alg);2206}2207 2208int hpre_algs_register(struct hisi_qm *qm)2209{2210 int ret = 0;2211 2212 mutex_lock(&hpre_algs_lock);2213 if (hpre_available_devs) {2214 hpre_available_devs++;2215 goto unlock;2216 }2217 2218 ret = hpre_register_rsa(qm);2219 if (ret)2220 goto unlock;2221 2222 ret = hpre_register_dh(qm);2223 if (ret)2224 goto unreg_rsa;2225 2226 ret = hpre_register_ecdh(qm);2227 if (ret)2228 goto unreg_dh;2229 2230 ret = hpre_register_x25519(qm);2231 if (ret)2232 goto unreg_ecdh;2233 2234 hpre_available_devs++;2235 mutex_unlock(&hpre_algs_lock);2236 2237 return ret;2238 2239unreg_ecdh:2240 hpre_unregister_ecdh(qm);2241unreg_dh:2242 hpre_unregister_dh(qm);2243unreg_rsa:2244 hpre_unregister_rsa(qm);2245unlock:2246 mutex_unlock(&hpre_algs_lock);2247 return ret;2248}2249 2250void hpre_algs_unregister(struct hisi_qm *qm)2251{2252 mutex_lock(&hpre_algs_lock);2253 if (--hpre_available_devs)2254 goto unlock;2255 2256 hpre_unregister_x25519(qm);2257 hpre_unregister_ecdh(qm);2258 hpre_unregister_dh(qm);2259 hpre_unregister_rsa(qm);2260 2261unlock:2262 mutex_unlock(&hpre_algs_lock);2263}2264