1362 lines · c
1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)2/* Copyright(c) 2014 - 2020 Intel Corporation */3#include <linux/module.h>4#include <crypto/internal/rsa.h>5#include <crypto/internal/akcipher.h>6#include <crypto/akcipher.h>7#include <crypto/kpp.h>8#include <crypto/internal/kpp.h>9#include <crypto/dh.h>10#include <linux/dma-mapping.h>11#include <linux/fips.h>12#include <crypto/scatterwalk.h>13#include "icp_qat_fw_pke.h"14#include "adf_accel_devices.h"15#include "qat_algs_send.h"16#include "adf_transport.h"17#include "adf_common_drv.h"18#include "qat_crypto.h"19 20static DEFINE_MUTEX(algs_lock);21static unsigned int active_devs;22 23struct qat_rsa_input_params {24 union {25 struct {26 dma_addr_t m;27 dma_addr_t e;28 dma_addr_t n;29 } enc;30 struct {31 dma_addr_t c;32 dma_addr_t d;33 dma_addr_t n;34 } dec;35 struct {36 dma_addr_t c;37 dma_addr_t p;38 dma_addr_t q;39 dma_addr_t dp;40 dma_addr_t dq;41 dma_addr_t qinv;42 } dec_crt;43 u64 in_tab[8];44 };45} __packed __aligned(64);46 47struct qat_rsa_output_params {48 union {49 struct {50 dma_addr_t c;51 } enc;52 struct {53 dma_addr_t m;54 } dec;55 u64 out_tab[8];56 };57} __packed __aligned(64);58 59struct qat_rsa_ctx {60 char *n;61 char *e;62 char *d;63 char *p;64 char *q;65 char *dp;66 char *dq;67 char *qinv;68 dma_addr_t dma_n;69 dma_addr_t dma_e;70 dma_addr_t dma_d;71 dma_addr_t dma_p;72 dma_addr_t dma_q;73 dma_addr_t dma_dp;74 dma_addr_t dma_dq;75 dma_addr_t dma_qinv;76 unsigned int key_sz;77 bool crt_mode;78 struct qat_crypto_instance *inst;79} __packed __aligned(64);80 81struct qat_dh_input_params {82 union {83 struct {84 dma_addr_t b;85 dma_addr_t xa;86 dma_addr_t p;87 } in;88 struct {89 dma_addr_t xa;90 dma_addr_t p;91 } in_g2;92 u64 in_tab[8];93 };94} __packed __aligned(64);95 96struct qat_dh_output_params {97 union {98 dma_addr_t r;99 u64 out_tab[8];100 };101} __packed __aligned(64);102 103struct qat_dh_ctx {104 char *g;105 char *xa;106 char *p;107 dma_addr_t dma_g;108 dma_addr_t dma_xa;109 dma_addr_t dma_p;110 unsigned int p_size;111 bool g2;112 struct qat_crypto_instance *inst;113 struct crypto_kpp *ftfm;114 bool fallback;115} __packed __aligned(64);116 117struct qat_asym_request {118 union {119 struct qat_rsa_input_params rsa;120 struct qat_dh_input_params dh;121 } in;122 union {123 struct qat_rsa_output_params rsa;124 struct qat_dh_output_params dh;125 } out;126 dma_addr_t phy_in;127 dma_addr_t phy_out;128 char *src_align;129 char *dst_align;130 struct icp_qat_fw_pke_request req;131 union {132 struct qat_rsa_ctx *rsa;133 struct qat_dh_ctx *dh;134 } ctx;135 union {136 struct akcipher_request *rsa;137 struct kpp_request *dh;138 } areq;139 int err;140 void (*cb)(struct icp_qat_fw_pke_resp *resp);141 struct qat_alg_req alg_req;142} __aligned(64);143 144static int qat_alg_send_asym_message(struct qat_asym_request *qat_req,145 struct qat_crypto_instance *inst,146 struct crypto_async_request *base)147{148 struct qat_alg_req *alg_req = &qat_req->alg_req;149 150 alg_req->fw_req = (u32 *)&qat_req->req;151 alg_req->tx_ring = inst->pke_tx;152 alg_req->base = base;153 alg_req->backlog = &inst->backlog;154 155 return qat_alg_send_message(alg_req);156}157 158static void qat_dh_cb(struct icp_qat_fw_pke_resp *resp)159{160 struct qat_asym_request *req = (void *)(__force long)resp->opaque;161 struct kpp_request *areq = req->areq.dh;162 struct device *dev = &GET_DEV(req->ctx.dh->inst->accel_dev);163 int err = ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(164 resp->pke_resp_hdr.comn_resp_flags);165 166 err = (err == ICP_QAT_FW_COMN_STATUS_FLAG_OK) ? 0 : -EINVAL;167 168 if (areq->src) {169 dma_unmap_single(dev, req->in.dh.in.b, req->ctx.dh->p_size,170 DMA_TO_DEVICE);171 kfree_sensitive(req->src_align);172 }173 174 areq->dst_len = req->ctx.dh->p_size;175 dma_unmap_single(dev, req->out.dh.r, req->ctx.dh->p_size,176 DMA_FROM_DEVICE);177 if (req->dst_align) {178 scatterwalk_map_and_copy(req->dst_align, areq->dst, 0,179 areq->dst_len, 1);180 kfree_sensitive(req->dst_align);181 }182 183 dma_unmap_single(dev, req->phy_in, sizeof(struct qat_dh_input_params),184 DMA_TO_DEVICE);185 dma_unmap_single(dev, req->phy_out,186 sizeof(struct qat_dh_output_params),187 DMA_TO_DEVICE);188 189 kpp_request_complete(areq, err);190}191 192#define PKE_DH_1536 0x390c1a49193#define PKE_DH_G2_1536 0x2e0b1a3e194#define PKE_DH_2048 0x4d0c1a60195#define PKE_DH_G2_2048 0x3e0b1a55196#define PKE_DH_3072 0x510c1a77197#define PKE_DH_G2_3072 0x3a0b1a6c198#define PKE_DH_4096 0x690c1a8e199#define PKE_DH_G2_4096 0x4a0b1a83200 201static unsigned long qat_dh_fn_id(unsigned int len, bool g2)202{203 unsigned int bitslen = len << 3;204 205 switch (bitslen) {206 case 1536:207 return g2 ? PKE_DH_G2_1536 : PKE_DH_1536;208 case 2048:209 return g2 ? PKE_DH_G2_2048 : PKE_DH_2048;210 case 3072:211 return g2 ? PKE_DH_G2_3072 : PKE_DH_3072;212 case 4096:213 return g2 ? PKE_DH_G2_4096 : PKE_DH_4096;214 default:215 return 0;216 }217}218 219static int qat_dh_compute_value(struct kpp_request *req)220{221 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);222 struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm);223 struct qat_crypto_instance *inst = ctx->inst;224 struct device *dev = &GET_DEV(inst->accel_dev);225 struct qat_asym_request *qat_req =226 PTR_ALIGN(kpp_request_ctx(req), 64);227 struct icp_qat_fw_pke_request *msg = &qat_req->req;228 gfp_t flags = qat_algs_alloc_flags(&req->base);229 int n_input_params = 0;230 u8 *vaddr;231 int ret;232 233 if (unlikely(!ctx->xa))234 return -EINVAL;235 236 if (req->dst_len < ctx->p_size) {237 req->dst_len = ctx->p_size;238 return -EOVERFLOW;239 }240 241 if (req->src_len > ctx->p_size)242 return -EINVAL;243 244 memset(msg, '\0', sizeof(*msg));245 ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr,246 ICP_QAT_FW_COMN_REQ_FLAG_SET);247 248 msg->pke_hdr.cd_pars.func_id = qat_dh_fn_id(ctx->p_size,249 !req->src && ctx->g2);250 if (unlikely(!msg->pke_hdr.cd_pars.func_id))251 return -EINVAL;252 253 qat_req->cb = qat_dh_cb;254 qat_req->ctx.dh = ctx;255 qat_req->areq.dh = req;256 msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE;257 msg->pke_hdr.comn_req_flags =258 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT,259 QAT_COMN_CD_FLD_TYPE_64BIT_ADR);260 261 /*262 * If no source is provided use g as base263 */264 if (req->src) {265 qat_req->in.dh.in.xa = ctx->dma_xa;266 qat_req->in.dh.in.p = ctx->dma_p;267 n_input_params = 3;268 } else {269 if (ctx->g2) {270 qat_req->in.dh.in_g2.xa = ctx->dma_xa;271 qat_req->in.dh.in_g2.p = ctx->dma_p;272 n_input_params = 2;273 } else {274 qat_req->in.dh.in.b = ctx->dma_g;275 qat_req->in.dh.in.xa = ctx->dma_xa;276 qat_req->in.dh.in.p = ctx->dma_p;277 n_input_params = 3;278 }279 }280 281 ret = -ENOMEM;282 if (req->src) {283 /*284 * src can be of any size in valid range, but HW expects it to285 * be the same as modulo p so in case it is different we need286 * to allocate a new buf and copy src data.287 * In other case we just need to map the user provided buffer.288 * Also need to make sure that it is in contiguous buffer.289 */290 if (sg_is_last(req->src) && req->src_len == ctx->p_size) {291 qat_req->src_align = NULL;292 vaddr = sg_virt(req->src);293 } else {294 int shift = ctx->p_size - req->src_len;295 296 qat_req->src_align = kzalloc(ctx->p_size, flags);297 if (unlikely(!qat_req->src_align))298 return ret;299 300 scatterwalk_map_and_copy(qat_req->src_align + shift,301 req->src, 0, req->src_len, 0);302 303 vaddr = qat_req->src_align;304 }305 306 qat_req->in.dh.in.b = dma_map_single(dev, vaddr, ctx->p_size,307 DMA_TO_DEVICE);308 if (unlikely(dma_mapping_error(dev, qat_req->in.dh.in.b)))309 goto unmap_src;310 }311 /*312 * dst can be of any size in valid range, but HW expects it to be the313 * same as modulo m so in case it is different we need to allocate a314 * new buf and copy src data.315 * In other case we just need to map the user provided buffer.316 * Also need to make sure that it is in contiguous buffer.317 */318 if (sg_is_last(req->dst) && req->dst_len == ctx->p_size) {319 qat_req->dst_align = NULL;320 vaddr = sg_virt(req->dst);321 } else {322 qat_req->dst_align = kzalloc(ctx->p_size, flags);323 if (unlikely(!qat_req->dst_align))324 goto unmap_src;325 326 vaddr = qat_req->dst_align;327 }328 qat_req->out.dh.r = dma_map_single(dev, vaddr, ctx->p_size,329 DMA_FROM_DEVICE);330 if (unlikely(dma_mapping_error(dev, qat_req->out.dh.r)))331 goto unmap_dst;332 333 qat_req->in.dh.in_tab[n_input_params] = 0;334 qat_req->out.dh.out_tab[1] = 0;335 /* Mapping in.in.b or in.in_g2.xa is the same */336 qat_req->phy_in = dma_map_single(dev, &qat_req->in.dh,337 sizeof(struct qat_dh_input_params),338 DMA_TO_DEVICE);339 if (unlikely(dma_mapping_error(dev, qat_req->phy_in)))340 goto unmap_dst;341 342 qat_req->phy_out = dma_map_single(dev, &qat_req->out.dh,343 sizeof(struct qat_dh_output_params),344 DMA_TO_DEVICE);345 if (unlikely(dma_mapping_error(dev, qat_req->phy_out)))346 goto unmap_in_params;347 348 msg->pke_mid.src_data_addr = qat_req->phy_in;349 msg->pke_mid.dest_data_addr = qat_req->phy_out;350 msg->pke_mid.opaque = (u64)(__force long)qat_req;351 msg->input_param_count = n_input_params;352 msg->output_param_count = 1;353 354 ret = qat_alg_send_asym_message(qat_req, inst, &req->base);355 if (ret == -ENOSPC)356 goto unmap_all;357 358 return ret;359 360unmap_all:361 if (!dma_mapping_error(dev, qat_req->phy_out))362 dma_unmap_single(dev, qat_req->phy_out,363 sizeof(struct qat_dh_output_params),364 DMA_TO_DEVICE);365unmap_in_params:366 if (!dma_mapping_error(dev, qat_req->phy_in))367 dma_unmap_single(dev, qat_req->phy_in,368 sizeof(struct qat_dh_input_params),369 DMA_TO_DEVICE);370unmap_dst:371 if (!dma_mapping_error(dev, qat_req->out.dh.r))372 dma_unmap_single(dev, qat_req->out.dh.r, ctx->p_size,373 DMA_FROM_DEVICE);374 kfree_sensitive(qat_req->dst_align);375unmap_src:376 if (req->src) {377 if (!dma_mapping_error(dev, qat_req->in.dh.in.b))378 dma_unmap_single(dev, qat_req->in.dh.in.b,379 ctx->p_size,380 DMA_TO_DEVICE);381 kfree_sensitive(qat_req->src_align);382 }383 return ret;384}385 386static int qat_dh_generate_public_key(struct kpp_request *req)387{388 struct kpp_request *nreq = kpp_request_ctx(req);389 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);390 struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm);391 392 if (ctx->fallback) {393 memcpy(nreq, req, sizeof(*req));394 kpp_request_set_tfm(nreq, ctx->ftfm);395 return crypto_kpp_generate_public_key(nreq);396 }397 398 return qat_dh_compute_value(req);399}400 401static int qat_dh_compute_shared_secret(struct kpp_request *req)402{403 struct kpp_request *nreq = kpp_request_ctx(req);404 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);405 struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm);406 407 if (ctx->fallback) {408 memcpy(nreq, req, sizeof(*req));409 kpp_request_set_tfm(nreq, ctx->ftfm);410 return crypto_kpp_compute_shared_secret(nreq);411 }412 413 return qat_dh_compute_value(req);414}415 416static int qat_dh_check_params_length(unsigned int p_len)417{418 switch (p_len) {419 case 1536:420 case 2048:421 case 3072:422 case 4096:423 return 0;424 }425 return -EINVAL;426}427 428static int qat_dh_set_params(struct qat_dh_ctx *ctx, struct dh *params)429{430 struct qat_crypto_instance *inst = ctx->inst;431 struct device *dev = &GET_DEV(inst->accel_dev);432 433 ctx->p_size = params->p_size;434 ctx->p = dma_alloc_coherent(dev, ctx->p_size, &ctx->dma_p, GFP_KERNEL);435 if (!ctx->p)436 return -ENOMEM;437 memcpy(ctx->p, params->p, ctx->p_size);438 439 /* If g equals 2 don't copy it */440 if (params->g_size == 1 && *(char *)params->g == 0x02) {441 ctx->g2 = true;442 return 0;443 }444 445 ctx->g = dma_alloc_coherent(dev, ctx->p_size, &ctx->dma_g, GFP_KERNEL);446 if (!ctx->g)447 return -ENOMEM;448 memcpy(ctx->g + (ctx->p_size - params->g_size), params->g,449 params->g_size);450 451 return 0;452}453 454static void qat_dh_clear_ctx(struct device *dev, struct qat_dh_ctx *ctx)455{456 if (ctx->g) {457 memset(ctx->g, 0, ctx->p_size);458 dma_free_coherent(dev, ctx->p_size, ctx->g, ctx->dma_g);459 ctx->g = NULL;460 }461 if (ctx->xa) {462 memset(ctx->xa, 0, ctx->p_size);463 dma_free_coherent(dev, ctx->p_size, ctx->xa, ctx->dma_xa);464 ctx->xa = NULL;465 }466 if (ctx->p) {467 memset(ctx->p, 0, ctx->p_size);468 dma_free_coherent(dev, ctx->p_size, ctx->p, ctx->dma_p);469 ctx->p = NULL;470 }471 ctx->p_size = 0;472 ctx->g2 = false;473}474 475static int qat_dh_set_secret(struct crypto_kpp *tfm, const void *buf,476 unsigned int len)477{478 struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm);479 struct device *dev = &GET_DEV(ctx->inst->accel_dev);480 struct dh params;481 int ret;482 483 if (crypto_dh_decode_key(buf, len, ¶ms) < 0)484 return -EINVAL;485 486 if (qat_dh_check_params_length(params.p_size << 3)) {487 ctx->fallback = true;488 return crypto_kpp_set_secret(ctx->ftfm, buf, len);489 }490 491 ctx->fallback = false;492 493 /* Free old secret if any */494 qat_dh_clear_ctx(dev, ctx);495 496 ret = qat_dh_set_params(ctx, ¶ms);497 if (ret < 0)498 goto err_clear_ctx;499 500 ctx->xa = dma_alloc_coherent(dev, ctx->p_size, &ctx->dma_xa,501 GFP_KERNEL);502 if (!ctx->xa) {503 ret = -ENOMEM;504 goto err_clear_ctx;505 }506 memcpy(ctx->xa + (ctx->p_size - params.key_size), params.key,507 params.key_size);508 509 return 0;510 511err_clear_ctx:512 qat_dh_clear_ctx(dev, ctx);513 return ret;514}515 516static unsigned int qat_dh_max_size(struct crypto_kpp *tfm)517{518 struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm);519 520 if (ctx->fallback)521 return crypto_kpp_maxsize(ctx->ftfm);522 523 return ctx->p_size;524}525 526static int qat_dh_init_tfm(struct crypto_kpp *tfm)527{528 struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm);529 struct qat_crypto_instance *inst =530 qat_crypto_get_instance_node(numa_node_id());531 const char *alg = kpp_alg_name(tfm);532 unsigned int reqsize;533 534 if (!inst)535 return -EINVAL;536 537 ctx->ftfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK);538 if (IS_ERR(ctx->ftfm))539 return PTR_ERR(ctx->ftfm);540 541 crypto_kpp_set_flags(ctx->ftfm, crypto_kpp_get_flags(tfm));542 543 reqsize = max(sizeof(struct qat_asym_request) + 64,544 sizeof(struct kpp_request) + crypto_kpp_reqsize(ctx->ftfm));545 546 kpp_set_reqsize(tfm, reqsize);547 548 ctx->p_size = 0;549 ctx->g2 = false;550 ctx->inst = inst;551 return 0;552}553 554static void qat_dh_exit_tfm(struct crypto_kpp *tfm)555{556 struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm);557 struct device *dev = &GET_DEV(ctx->inst->accel_dev);558 559 if (ctx->ftfm)560 crypto_free_kpp(ctx->ftfm);561 562 qat_dh_clear_ctx(dev, ctx);563 qat_crypto_put_instance(ctx->inst);564}565 566static void qat_rsa_cb(struct icp_qat_fw_pke_resp *resp)567{568 struct qat_asym_request *req = (void *)(__force long)resp->opaque;569 struct akcipher_request *areq = req->areq.rsa;570 struct device *dev = &GET_DEV(req->ctx.rsa->inst->accel_dev);571 int err = ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(572 resp->pke_resp_hdr.comn_resp_flags);573 574 err = (err == ICP_QAT_FW_COMN_STATUS_FLAG_OK) ? 0 : -EINVAL;575 576 dma_unmap_single(dev, req->in.rsa.enc.m, req->ctx.rsa->key_sz,577 DMA_TO_DEVICE);578 579 kfree_sensitive(req->src_align);580 581 areq->dst_len = req->ctx.rsa->key_sz;582 dma_unmap_single(dev, req->out.rsa.enc.c, req->ctx.rsa->key_sz,583 DMA_FROM_DEVICE);584 if (req->dst_align) {585 scatterwalk_map_and_copy(req->dst_align, areq->dst, 0,586 areq->dst_len, 1);587 588 kfree_sensitive(req->dst_align);589 }590 591 dma_unmap_single(dev, req->phy_in, sizeof(struct qat_rsa_input_params),592 DMA_TO_DEVICE);593 dma_unmap_single(dev, req->phy_out,594 sizeof(struct qat_rsa_output_params),595 DMA_TO_DEVICE);596 597 akcipher_request_complete(areq, err);598}599 600void qat_alg_asym_callback(void *_resp)601{602 struct icp_qat_fw_pke_resp *resp = _resp;603 struct qat_asym_request *areq = (void *)(__force long)resp->opaque;604 struct qat_instance_backlog *backlog = areq->alg_req.backlog;605 606 areq->cb(resp);607 608 qat_alg_send_backlog(backlog);609}610 611#define PKE_RSA_EP_512 0x1c161b21612#define PKE_RSA_EP_1024 0x35111bf7613#define PKE_RSA_EP_1536 0x4d111cdc614#define PKE_RSA_EP_2048 0x6e111dba615#define PKE_RSA_EP_3072 0x7d111ea3616#define PKE_RSA_EP_4096 0xa5101f7e617 618static unsigned long qat_rsa_enc_fn_id(unsigned int len)619{620 unsigned int bitslen = len << 3;621 622 switch (bitslen) {623 case 512:624 return PKE_RSA_EP_512;625 case 1024:626 return PKE_RSA_EP_1024;627 case 1536:628 return PKE_RSA_EP_1536;629 case 2048:630 return PKE_RSA_EP_2048;631 case 3072:632 return PKE_RSA_EP_3072;633 case 4096:634 return PKE_RSA_EP_4096;635 default:636 return 0;637 }638}639 640#define PKE_RSA_DP1_512 0x1c161b3c641#define PKE_RSA_DP1_1024 0x35111c12642#define PKE_RSA_DP1_1536 0x4d111cf7643#define PKE_RSA_DP1_2048 0x6e111dda644#define PKE_RSA_DP1_3072 0x7d111ebe645#define PKE_RSA_DP1_4096 0xa5101f98646 647static unsigned long qat_rsa_dec_fn_id(unsigned int len)648{649 unsigned int bitslen = len << 3;650 651 switch (bitslen) {652 case 512:653 return PKE_RSA_DP1_512;654 case 1024:655 return PKE_RSA_DP1_1024;656 case 1536:657 return PKE_RSA_DP1_1536;658 case 2048:659 return PKE_RSA_DP1_2048;660 case 3072:661 return PKE_RSA_DP1_3072;662 case 4096:663 return PKE_RSA_DP1_4096;664 default:665 return 0;666 }667}668 669#define PKE_RSA_DP2_512 0x1c131b57670#define PKE_RSA_DP2_1024 0x26131c2d671#define PKE_RSA_DP2_1536 0x45111d12672#define PKE_RSA_DP2_2048 0x59121dfa673#define PKE_RSA_DP2_3072 0x81121ed9674#define PKE_RSA_DP2_4096 0xb1111fb2675 676static unsigned long qat_rsa_dec_fn_id_crt(unsigned int len)677{678 unsigned int bitslen = len << 3;679 680 switch (bitslen) {681 case 512:682 return PKE_RSA_DP2_512;683 case 1024:684 return PKE_RSA_DP2_1024;685 case 1536:686 return PKE_RSA_DP2_1536;687 case 2048:688 return PKE_RSA_DP2_2048;689 case 3072:690 return PKE_RSA_DP2_3072;691 case 4096:692 return PKE_RSA_DP2_4096;693 default:694 return 0;695 }696}697 698static int qat_rsa_enc(struct akcipher_request *req)699{700 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);701 struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);702 struct qat_crypto_instance *inst = ctx->inst;703 struct device *dev = &GET_DEV(inst->accel_dev);704 struct qat_asym_request *qat_req =705 PTR_ALIGN(akcipher_request_ctx(req), 64);706 struct icp_qat_fw_pke_request *msg = &qat_req->req;707 gfp_t flags = qat_algs_alloc_flags(&req->base);708 u8 *vaddr;709 int ret;710 711 if (unlikely(!ctx->n || !ctx->e))712 return -EINVAL;713 714 if (req->dst_len < ctx->key_sz) {715 req->dst_len = ctx->key_sz;716 return -EOVERFLOW;717 }718 719 if (req->src_len > ctx->key_sz)720 return -EINVAL;721 722 memset(msg, '\0', sizeof(*msg));723 ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr,724 ICP_QAT_FW_COMN_REQ_FLAG_SET);725 msg->pke_hdr.cd_pars.func_id = qat_rsa_enc_fn_id(ctx->key_sz);726 if (unlikely(!msg->pke_hdr.cd_pars.func_id))727 return -EINVAL;728 729 qat_req->cb = qat_rsa_cb;730 qat_req->ctx.rsa = ctx;731 qat_req->areq.rsa = req;732 msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE;733 msg->pke_hdr.comn_req_flags =734 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT,735 QAT_COMN_CD_FLD_TYPE_64BIT_ADR);736 737 qat_req->in.rsa.enc.e = ctx->dma_e;738 qat_req->in.rsa.enc.n = ctx->dma_n;739 ret = -ENOMEM;740 741 /*742 * src can be of any size in valid range, but HW expects it to be the743 * same as modulo n so in case it is different we need to allocate a744 * new buf and copy src data.745 * In other case we just need to map the user provided buffer.746 * Also need to make sure that it is in contiguous buffer.747 */748 if (sg_is_last(req->src) && req->src_len == ctx->key_sz) {749 qat_req->src_align = NULL;750 vaddr = sg_virt(req->src);751 } else {752 int shift = ctx->key_sz - req->src_len;753 754 qat_req->src_align = kzalloc(ctx->key_sz, flags);755 if (unlikely(!qat_req->src_align))756 return ret;757 758 scatterwalk_map_and_copy(qat_req->src_align + shift, req->src,759 0, req->src_len, 0);760 vaddr = qat_req->src_align;761 }762 763 qat_req->in.rsa.enc.m = dma_map_single(dev, vaddr, ctx->key_sz,764 DMA_TO_DEVICE);765 if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.enc.m)))766 goto unmap_src;767 768 if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) {769 qat_req->dst_align = NULL;770 vaddr = sg_virt(req->dst);771 } else {772 qat_req->dst_align = kzalloc(ctx->key_sz, flags);773 if (unlikely(!qat_req->dst_align))774 goto unmap_src;775 vaddr = qat_req->dst_align;776 }777 778 qat_req->out.rsa.enc.c = dma_map_single(dev, vaddr, ctx->key_sz,779 DMA_FROM_DEVICE);780 if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.enc.c)))781 goto unmap_dst;782 783 qat_req->in.rsa.in_tab[3] = 0;784 qat_req->out.rsa.out_tab[1] = 0;785 qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa,786 sizeof(struct qat_rsa_input_params),787 DMA_TO_DEVICE);788 if (unlikely(dma_mapping_error(dev, qat_req->phy_in)))789 goto unmap_dst;790 791 qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa,792 sizeof(struct qat_rsa_output_params),793 DMA_TO_DEVICE);794 if (unlikely(dma_mapping_error(dev, qat_req->phy_out)))795 goto unmap_in_params;796 797 msg->pke_mid.src_data_addr = qat_req->phy_in;798 msg->pke_mid.dest_data_addr = qat_req->phy_out;799 msg->pke_mid.opaque = (u64)(__force long)qat_req;800 msg->input_param_count = 3;801 msg->output_param_count = 1;802 803 ret = qat_alg_send_asym_message(qat_req, inst, &req->base);804 if (ret == -ENOSPC)805 goto unmap_all;806 807 return ret;808 809unmap_all:810 if (!dma_mapping_error(dev, qat_req->phy_out))811 dma_unmap_single(dev, qat_req->phy_out,812 sizeof(struct qat_rsa_output_params),813 DMA_TO_DEVICE);814unmap_in_params:815 if (!dma_mapping_error(dev, qat_req->phy_in))816 dma_unmap_single(dev, qat_req->phy_in,817 sizeof(struct qat_rsa_input_params),818 DMA_TO_DEVICE);819unmap_dst:820 if (!dma_mapping_error(dev, qat_req->out.rsa.enc.c))821 dma_unmap_single(dev, qat_req->out.rsa.enc.c,822 ctx->key_sz, DMA_FROM_DEVICE);823 kfree_sensitive(qat_req->dst_align);824unmap_src:825 if (!dma_mapping_error(dev, qat_req->in.rsa.enc.m))826 dma_unmap_single(dev, qat_req->in.rsa.enc.m, ctx->key_sz,827 DMA_TO_DEVICE);828 kfree_sensitive(qat_req->src_align);829 return ret;830}831 832static int qat_rsa_dec(struct akcipher_request *req)833{834 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);835 struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);836 struct qat_crypto_instance *inst = ctx->inst;837 struct device *dev = &GET_DEV(inst->accel_dev);838 struct qat_asym_request *qat_req =839 PTR_ALIGN(akcipher_request_ctx(req), 64);840 struct icp_qat_fw_pke_request *msg = &qat_req->req;841 gfp_t flags = qat_algs_alloc_flags(&req->base);842 u8 *vaddr;843 int ret;844 845 if (unlikely(!ctx->n || !ctx->d))846 return -EINVAL;847 848 if (req->dst_len < ctx->key_sz) {849 req->dst_len = ctx->key_sz;850 return -EOVERFLOW;851 }852 853 if (req->src_len > ctx->key_sz)854 return -EINVAL;855 856 memset(msg, '\0', sizeof(*msg));857 ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr,858 ICP_QAT_FW_COMN_REQ_FLAG_SET);859 msg->pke_hdr.cd_pars.func_id = ctx->crt_mode ?860 qat_rsa_dec_fn_id_crt(ctx->key_sz) :861 qat_rsa_dec_fn_id(ctx->key_sz);862 if (unlikely(!msg->pke_hdr.cd_pars.func_id))863 return -EINVAL;864 865 qat_req->cb = qat_rsa_cb;866 qat_req->ctx.rsa = ctx;867 qat_req->areq.rsa = req;868 msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE;869 msg->pke_hdr.comn_req_flags =870 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT,871 QAT_COMN_CD_FLD_TYPE_64BIT_ADR);872 873 if (ctx->crt_mode) {874 qat_req->in.rsa.dec_crt.p = ctx->dma_p;875 qat_req->in.rsa.dec_crt.q = ctx->dma_q;876 qat_req->in.rsa.dec_crt.dp = ctx->dma_dp;877 qat_req->in.rsa.dec_crt.dq = ctx->dma_dq;878 qat_req->in.rsa.dec_crt.qinv = ctx->dma_qinv;879 } else {880 qat_req->in.rsa.dec.d = ctx->dma_d;881 qat_req->in.rsa.dec.n = ctx->dma_n;882 }883 ret = -ENOMEM;884 885 /*886 * src can be of any size in valid range, but HW expects it to be the887 * same as modulo n so in case it is different we need to allocate a888 * new buf and copy src data.889 * In other case we just need to map the user provided buffer.890 * Also need to make sure that it is in contiguous buffer.891 */892 if (sg_is_last(req->src) && req->src_len == ctx->key_sz) {893 qat_req->src_align = NULL;894 vaddr = sg_virt(req->src);895 } else {896 int shift = ctx->key_sz - req->src_len;897 898 qat_req->src_align = kzalloc(ctx->key_sz, flags);899 if (unlikely(!qat_req->src_align))900 return ret;901 902 scatterwalk_map_and_copy(qat_req->src_align + shift, req->src,903 0, req->src_len, 0);904 vaddr = qat_req->src_align;905 }906 907 qat_req->in.rsa.dec.c = dma_map_single(dev, vaddr, ctx->key_sz,908 DMA_TO_DEVICE);909 if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.dec.c)))910 goto unmap_src;911 912 if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) {913 qat_req->dst_align = NULL;914 vaddr = sg_virt(req->dst);915 } else {916 qat_req->dst_align = kzalloc(ctx->key_sz, flags);917 if (unlikely(!qat_req->dst_align))918 goto unmap_src;919 vaddr = qat_req->dst_align;920 }921 qat_req->out.rsa.dec.m = dma_map_single(dev, vaddr, ctx->key_sz,922 DMA_FROM_DEVICE);923 if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.dec.m)))924 goto unmap_dst;925 926 if (ctx->crt_mode)927 qat_req->in.rsa.in_tab[6] = 0;928 else929 qat_req->in.rsa.in_tab[3] = 0;930 qat_req->out.rsa.out_tab[1] = 0;931 qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa,932 sizeof(struct qat_rsa_input_params),933 DMA_TO_DEVICE);934 if (unlikely(dma_mapping_error(dev, qat_req->phy_in)))935 goto unmap_dst;936 937 qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa,938 sizeof(struct qat_rsa_output_params),939 DMA_TO_DEVICE);940 if (unlikely(dma_mapping_error(dev, qat_req->phy_out)))941 goto unmap_in_params;942 943 msg->pke_mid.src_data_addr = qat_req->phy_in;944 msg->pke_mid.dest_data_addr = qat_req->phy_out;945 msg->pke_mid.opaque = (u64)(__force long)qat_req;946 if (ctx->crt_mode)947 msg->input_param_count = 6;948 else949 msg->input_param_count = 3;950 951 msg->output_param_count = 1;952 953 ret = qat_alg_send_asym_message(qat_req, inst, &req->base);954 if (ret == -ENOSPC)955 goto unmap_all;956 957 return ret;958 959unmap_all:960 if (!dma_mapping_error(dev, qat_req->phy_out))961 dma_unmap_single(dev, qat_req->phy_out,962 sizeof(struct qat_rsa_output_params),963 DMA_TO_DEVICE);964unmap_in_params:965 if (!dma_mapping_error(dev, qat_req->phy_in))966 dma_unmap_single(dev, qat_req->phy_in,967 sizeof(struct qat_rsa_input_params),968 DMA_TO_DEVICE);969unmap_dst:970 if (!dma_mapping_error(dev, qat_req->out.rsa.dec.m))971 dma_unmap_single(dev, qat_req->out.rsa.dec.m,972 ctx->key_sz, DMA_FROM_DEVICE);973 kfree_sensitive(qat_req->dst_align);974unmap_src:975 if (!dma_mapping_error(dev, qat_req->in.rsa.dec.c))976 dma_unmap_single(dev, qat_req->in.rsa.dec.c, ctx->key_sz,977 DMA_TO_DEVICE);978 kfree_sensitive(qat_req->src_align);979 return ret;980}981 982static int qat_rsa_set_n(struct qat_rsa_ctx *ctx, const char *value,983 size_t vlen)984{985 struct qat_crypto_instance *inst = ctx->inst;986 struct device *dev = &GET_DEV(inst->accel_dev);987 const char *ptr = value;988 int ret;989 990 while (!*ptr && vlen) {991 ptr++;992 vlen--;993 }994 995 ctx->key_sz = vlen;996 ret = -EINVAL;997 /* invalid key size provided */998 if (!qat_rsa_enc_fn_id(ctx->key_sz))999 goto err;1000 1001 ret = -ENOMEM;1002 ctx->n = dma_alloc_coherent(dev, ctx->key_sz, &ctx->dma_n, GFP_KERNEL);1003 if (!ctx->n)1004 goto err;1005 1006 memcpy(ctx->n, ptr, ctx->key_sz);1007 return 0;1008err:1009 ctx->key_sz = 0;1010 ctx->n = NULL;1011 return ret;1012}1013 1014static int qat_rsa_set_e(struct qat_rsa_ctx *ctx, const char *value,1015 size_t vlen)1016{1017 struct qat_crypto_instance *inst = ctx->inst;1018 struct device *dev = &GET_DEV(inst->accel_dev);1019 const char *ptr = value;1020 1021 while (!*ptr && vlen) {1022 ptr++;1023 vlen--;1024 }1025 1026 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) {1027 ctx->e = NULL;1028 return -EINVAL;1029 }1030 1031 ctx->e = dma_alloc_coherent(dev, ctx->key_sz, &ctx->dma_e, GFP_KERNEL);1032 if (!ctx->e)1033 return -ENOMEM;1034 1035 memcpy(ctx->e + (ctx->key_sz - vlen), ptr, vlen);1036 return 0;1037}1038 1039static int qat_rsa_set_d(struct qat_rsa_ctx *ctx, const char *value,1040 size_t vlen)1041{1042 struct qat_crypto_instance *inst = ctx->inst;1043 struct device *dev = &GET_DEV(inst->accel_dev);1044 const char *ptr = value;1045 int ret;1046 1047 while (!*ptr && vlen) {1048 ptr++;1049 vlen--;1050 }1051 1052 ret = -EINVAL;1053 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)1054 goto err;1055 1056 ret = -ENOMEM;1057 ctx->d = dma_alloc_coherent(dev, ctx->key_sz, &ctx->dma_d, GFP_KERNEL);1058 if (!ctx->d)1059 goto err;1060 1061 memcpy(ctx->d + (ctx->key_sz - vlen), ptr, vlen);1062 return 0;1063err:1064 ctx->d = NULL;1065 return ret;1066}1067 1068static void qat_rsa_drop_leading_zeros(const char **ptr, unsigned int *len)1069{1070 while (!**ptr && *len) {1071 (*ptr)++;1072 (*len)--;1073 }1074}1075 1076static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key)1077{1078 struct qat_crypto_instance *inst = ctx->inst;1079 struct device *dev = &GET_DEV(inst->accel_dev);1080 const char *ptr;1081 unsigned int len;1082 unsigned int half_key_sz = ctx->key_sz / 2;1083 1084 /* p */1085 ptr = rsa_key->p;1086 len = rsa_key->p_sz;1087 qat_rsa_drop_leading_zeros(&ptr, &len);1088 if (!len)1089 goto err;1090 ctx->p = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_p, GFP_KERNEL);1091 if (!ctx->p)1092 goto err;1093 memcpy(ctx->p + (half_key_sz - len), ptr, len);1094 1095 /* q */1096 ptr = rsa_key->q;1097 len = rsa_key->q_sz;1098 qat_rsa_drop_leading_zeros(&ptr, &len);1099 if (!len)1100 goto free_p;1101 ctx->q = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_q, GFP_KERNEL);1102 if (!ctx->q)1103 goto free_p;1104 memcpy(ctx->q + (half_key_sz - len), ptr, len);1105 1106 /* dp */1107 ptr = rsa_key->dp;1108 len = rsa_key->dp_sz;1109 qat_rsa_drop_leading_zeros(&ptr, &len);1110 if (!len)1111 goto free_q;1112 ctx->dp = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_dp,1113 GFP_KERNEL);1114 if (!ctx->dp)1115 goto free_q;1116 memcpy(ctx->dp + (half_key_sz - len), ptr, len);1117 1118 /* dq */1119 ptr = rsa_key->dq;1120 len = rsa_key->dq_sz;1121 qat_rsa_drop_leading_zeros(&ptr, &len);1122 if (!len)1123 goto free_dp;1124 ctx->dq = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_dq,1125 GFP_KERNEL);1126 if (!ctx->dq)1127 goto free_dp;1128 memcpy(ctx->dq + (half_key_sz - len), ptr, len);1129 1130 /* qinv */1131 ptr = rsa_key->qinv;1132 len = rsa_key->qinv_sz;1133 qat_rsa_drop_leading_zeros(&ptr, &len);1134 if (!len)1135 goto free_dq;1136 ctx->qinv = dma_alloc_coherent(dev, half_key_sz, &ctx->dma_qinv,1137 GFP_KERNEL);1138 if (!ctx->qinv)1139 goto free_dq;1140 memcpy(ctx->qinv + (half_key_sz - len), ptr, len);1141 1142 ctx->crt_mode = true;1143 return;1144 1145free_dq:1146 memset(ctx->dq, '\0', half_key_sz);1147 dma_free_coherent(dev, half_key_sz, ctx->dq, ctx->dma_dq);1148 ctx->dq = NULL;1149free_dp:1150 memset(ctx->dp, '\0', half_key_sz);1151 dma_free_coherent(dev, half_key_sz, ctx->dp, ctx->dma_dp);1152 ctx->dp = NULL;1153free_q:1154 memset(ctx->q, '\0', half_key_sz);1155 dma_free_coherent(dev, half_key_sz, ctx->q, ctx->dma_q);1156 ctx->q = NULL;1157free_p:1158 memset(ctx->p, '\0', half_key_sz);1159 dma_free_coherent(dev, half_key_sz, ctx->p, ctx->dma_p);1160 ctx->p = NULL;1161err:1162 ctx->crt_mode = false;1163}1164 1165static void qat_rsa_clear_ctx(struct device *dev, struct qat_rsa_ctx *ctx)1166{1167 unsigned int half_key_sz = ctx->key_sz / 2;1168 1169 /* Free the old key if any */1170 if (ctx->n)1171 dma_free_coherent(dev, ctx->key_sz, ctx->n, ctx->dma_n);1172 if (ctx->e)1173 dma_free_coherent(dev, ctx->key_sz, ctx->e, ctx->dma_e);1174 if (ctx->d) {1175 memset(ctx->d, '\0', ctx->key_sz);1176 dma_free_coherent(dev, ctx->key_sz, ctx->d, ctx->dma_d);1177 }1178 if (ctx->p) {1179 memset(ctx->p, '\0', half_key_sz);1180 dma_free_coherent(dev, half_key_sz, ctx->p, ctx->dma_p);1181 }1182 if (ctx->q) {1183 memset(ctx->q, '\0', half_key_sz);1184 dma_free_coherent(dev, half_key_sz, ctx->q, ctx->dma_q);1185 }1186 if (ctx->dp) {1187 memset(ctx->dp, '\0', half_key_sz);1188 dma_free_coherent(dev, half_key_sz, ctx->dp, ctx->dma_dp);1189 }1190 if (ctx->dq) {1191 memset(ctx->dq, '\0', half_key_sz);1192 dma_free_coherent(dev, half_key_sz, ctx->dq, ctx->dma_dq);1193 }1194 if (ctx->qinv) {1195 memset(ctx->qinv, '\0', half_key_sz);1196 dma_free_coherent(dev, half_key_sz, ctx->qinv, ctx->dma_qinv);1197 }1198 1199 ctx->n = NULL;1200 ctx->e = NULL;1201 ctx->d = NULL;1202 ctx->p = NULL;1203 ctx->q = NULL;1204 ctx->dp = NULL;1205 ctx->dq = NULL;1206 ctx->qinv = NULL;1207 ctx->crt_mode = false;1208 ctx->key_sz = 0;1209}1210 1211static int qat_rsa_setkey(struct crypto_akcipher *tfm, const void *key,1212 unsigned int keylen, bool private)1213{1214 struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);1215 struct device *dev = &GET_DEV(ctx->inst->accel_dev);1216 struct rsa_key rsa_key;1217 int ret;1218 1219 qat_rsa_clear_ctx(dev, ctx);1220 1221 if (private)1222 ret = rsa_parse_priv_key(&rsa_key, key, keylen);1223 else1224 ret = rsa_parse_pub_key(&rsa_key, key, keylen);1225 if (ret < 0)1226 goto free;1227 1228 ret = qat_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz);1229 if (ret < 0)1230 goto free;1231 ret = qat_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz);1232 if (ret < 0)1233 goto free;1234 if (private) {1235 ret = qat_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz);1236 if (ret < 0)1237 goto free;1238 qat_rsa_setkey_crt(ctx, &rsa_key);1239 }1240 1241 if (!ctx->n || !ctx->e) {1242 /* invalid key provided */1243 ret = -EINVAL;1244 goto free;1245 }1246 if (private && !ctx->d) {1247 /* invalid private key provided */1248 ret = -EINVAL;1249 goto free;1250 }1251 1252 return 0;1253free:1254 qat_rsa_clear_ctx(dev, ctx);1255 return ret;1256}1257 1258static int qat_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key,1259 unsigned int keylen)1260{1261 return qat_rsa_setkey(tfm, key, keylen, false);1262}1263 1264static int qat_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key,1265 unsigned int keylen)1266{1267 return qat_rsa_setkey(tfm, key, keylen, true);1268}1269 1270static unsigned int qat_rsa_max_size(struct crypto_akcipher *tfm)1271{1272 struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);1273 1274 return ctx->key_sz;1275}1276 1277static int qat_rsa_init_tfm(struct crypto_akcipher *tfm)1278{1279 struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);1280 struct qat_crypto_instance *inst =1281 qat_crypto_get_instance_node(numa_node_id());1282 1283 if (!inst)1284 return -EINVAL;1285 1286 akcipher_set_reqsize(tfm, sizeof(struct qat_asym_request) + 64);1287 1288 ctx->key_sz = 0;1289 ctx->inst = inst;1290 return 0;1291}1292 1293static void qat_rsa_exit_tfm(struct crypto_akcipher *tfm)1294{1295 struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);1296 struct device *dev = &GET_DEV(ctx->inst->accel_dev);1297 1298 qat_rsa_clear_ctx(dev, ctx);1299 qat_crypto_put_instance(ctx->inst);1300}1301 1302static struct akcipher_alg rsa = {1303 .encrypt = qat_rsa_enc,1304 .decrypt = qat_rsa_dec,1305 .set_pub_key = qat_rsa_setpubkey,1306 .set_priv_key = qat_rsa_setprivkey,1307 .max_size = qat_rsa_max_size,1308 .init = qat_rsa_init_tfm,1309 .exit = qat_rsa_exit_tfm,1310 .base = {1311 .cra_name = "rsa",1312 .cra_driver_name = "qat-rsa",1313 .cra_priority = 1000,1314 .cra_module = THIS_MODULE,1315 .cra_ctxsize = sizeof(struct qat_rsa_ctx),1316 },1317};1318 1319static struct kpp_alg dh = {1320 .set_secret = qat_dh_set_secret,1321 .generate_public_key = qat_dh_generate_public_key,1322 .compute_shared_secret = qat_dh_compute_shared_secret,1323 .max_size = qat_dh_max_size,1324 .init = qat_dh_init_tfm,1325 .exit = qat_dh_exit_tfm,1326 .base = {1327 .cra_name = "dh",1328 .cra_driver_name = "qat-dh",1329 .cra_priority = 1000,1330 .cra_module = THIS_MODULE,1331 .cra_ctxsize = sizeof(struct qat_dh_ctx),1332 .cra_flags = CRYPTO_ALG_NEED_FALLBACK,1333 },1334};1335 1336int qat_asym_algs_register(void)1337{1338 int ret = 0;1339 1340 mutex_lock(&algs_lock);1341 if (++active_devs == 1) {1342 rsa.base.cra_flags = 0;1343 ret = crypto_register_akcipher(&rsa);1344 if (ret)1345 goto unlock;1346 ret = crypto_register_kpp(&dh);1347 }1348unlock:1349 mutex_unlock(&algs_lock);1350 return ret;1351}1352 1353void qat_asym_algs_unregister(void)1354{1355 mutex_lock(&algs_lock);1356 if (--active_devs == 0) {1357 crypto_unregister_akcipher(&rsa);1358 crypto_unregister_kpp(&dh);1359 }1360 mutex_unlock(&algs_lock);1361}1362