1424 lines · c
1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)2/* Copyright(c) 2014 - 2020 Intel Corporation */3#include <linux/module.h>4#include <linux/slab.h>5#include <linux/crypto.h>6#include <crypto/internal/aead.h>7#include <crypto/internal/cipher.h>8#include <crypto/internal/skcipher.h>9#include <crypto/aes.h>10#include <crypto/sha1.h>11#include <crypto/sha2.h>12#include <crypto/hash.h>13#include <crypto/hmac.h>14#include <crypto/algapi.h>15#include <crypto/authenc.h>16#include <crypto/scatterwalk.h>17#include <crypto/xts.h>18#include <linux/dma-mapping.h>19#include "adf_accel_devices.h"20#include "qat_algs_send.h"21#include "adf_common_drv.h"22#include "qat_crypto.h"23#include "icp_qat_hw.h"24#include "icp_qat_fw.h"25#include "icp_qat_fw_la.h"26#include "qat_bl.h"27 28#define QAT_AES_HW_CONFIG_ENC(alg, mode) \29 ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \30 ICP_QAT_HW_CIPHER_NO_CONVERT, \31 ICP_QAT_HW_CIPHER_ENCRYPT)32 33#define QAT_AES_HW_CONFIG_DEC(alg, mode) \34 ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \35 ICP_QAT_HW_CIPHER_KEY_CONVERT, \36 ICP_QAT_HW_CIPHER_DECRYPT)37 38#define QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode) \39 ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \40 ICP_QAT_HW_CIPHER_NO_CONVERT, \41 ICP_QAT_HW_CIPHER_DECRYPT)42 43#define HW_CAP_AES_V2(accel_dev) \44 (GET_HW_DATA(accel_dev)->accel_capabilities_mask & \45 ICP_ACCEL_CAPABILITIES_AES_V2)46 47static DEFINE_MUTEX(algs_lock);48static unsigned int active_devs;49 50/* Common content descriptor */51struct qat_alg_cd {52 union {53 struct qat_enc { /* Encrypt content desc */54 struct icp_qat_hw_cipher_algo_blk cipher;55 struct icp_qat_hw_auth_algo_blk hash;56 } qat_enc_cd;57 struct qat_dec { /* Decrypt content desc */58 struct icp_qat_hw_auth_algo_blk hash;59 struct icp_qat_hw_cipher_algo_blk cipher;60 } qat_dec_cd;61 };62} __aligned(64);63 64struct qat_alg_aead_ctx {65 struct qat_alg_cd *enc_cd;66 struct qat_alg_cd *dec_cd;67 dma_addr_t enc_cd_paddr;68 dma_addr_t dec_cd_paddr;69 struct icp_qat_fw_la_bulk_req enc_fw_req;70 struct icp_qat_fw_la_bulk_req dec_fw_req;71 struct crypto_shash *hash_tfm;72 enum icp_qat_hw_auth_algo qat_hash_alg;73 struct qat_crypto_instance *inst;74 union {75 struct sha1_state sha1;76 struct sha256_state sha256;77 struct sha512_state sha512;78 };79 char ipad[SHA512_BLOCK_SIZE]; /* sufficient for SHA-1/SHA-256 as well */80 char opad[SHA512_BLOCK_SIZE];81};82 83struct qat_alg_skcipher_ctx {84 struct icp_qat_hw_cipher_algo_blk *enc_cd;85 struct icp_qat_hw_cipher_algo_blk *dec_cd;86 dma_addr_t enc_cd_paddr;87 dma_addr_t dec_cd_paddr;88 struct icp_qat_fw_la_bulk_req enc_fw_req;89 struct icp_qat_fw_la_bulk_req dec_fw_req;90 struct qat_crypto_instance *inst;91 struct crypto_skcipher *ftfm;92 struct crypto_cipher *tweak;93 bool fallback;94 int mode;95};96 97static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)98{99 switch (qat_hash_alg) {100 case ICP_QAT_HW_AUTH_ALGO_SHA1:101 return ICP_QAT_HW_SHA1_STATE1_SZ;102 case ICP_QAT_HW_AUTH_ALGO_SHA256:103 return ICP_QAT_HW_SHA256_STATE1_SZ;104 case ICP_QAT_HW_AUTH_ALGO_SHA512:105 return ICP_QAT_HW_SHA512_STATE1_SZ;106 default:107 return -EFAULT;108 }109}110 111static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,112 struct qat_alg_aead_ctx *ctx,113 const u8 *auth_key,114 unsigned int auth_keylen)115{116 SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);117 int block_size = crypto_shash_blocksize(ctx->hash_tfm);118 int digest_size = crypto_shash_digestsize(ctx->hash_tfm);119 __be32 *hash_state_out;120 __be64 *hash512_state_out;121 int i, offset;122 123 memset(ctx->ipad, 0, block_size);124 memset(ctx->opad, 0, block_size);125 shash->tfm = ctx->hash_tfm;126 127 if (auth_keylen > block_size) {128 int ret = crypto_shash_digest(shash, auth_key,129 auth_keylen, ctx->ipad);130 if (ret)131 return ret;132 133 memcpy(ctx->opad, ctx->ipad, digest_size);134 } else {135 memcpy(ctx->ipad, auth_key, auth_keylen);136 memcpy(ctx->opad, auth_key, auth_keylen);137 }138 139 for (i = 0; i < block_size; i++) {140 char *ipad_ptr = ctx->ipad + i;141 char *opad_ptr = ctx->opad + i;142 *ipad_ptr ^= HMAC_IPAD_VALUE;143 *opad_ptr ^= HMAC_OPAD_VALUE;144 }145 146 if (crypto_shash_init(shash))147 return -EFAULT;148 149 if (crypto_shash_update(shash, ctx->ipad, block_size))150 return -EFAULT;151 152 hash_state_out = (__be32 *)hash->sha.state1;153 hash512_state_out = (__be64 *)hash_state_out;154 155 switch (ctx->qat_hash_alg) {156 case ICP_QAT_HW_AUTH_ALGO_SHA1:157 if (crypto_shash_export(shash, &ctx->sha1))158 return -EFAULT;159 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)160 *hash_state_out = cpu_to_be32(ctx->sha1.state[i]);161 break;162 case ICP_QAT_HW_AUTH_ALGO_SHA256:163 if (crypto_shash_export(shash, &ctx->sha256))164 return -EFAULT;165 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)166 *hash_state_out = cpu_to_be32(ctx->sha256.state[i]);167 break;168 case ICP_QAT_HW_AUTH_ALGO_SHA512:169 if (crypto_shash_export(shash, &ctx->sha512))170 return -EFAULT;171 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)172 *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]);173 break;174 default:175 return -EFAULT;176 }177 178 if (crypto_shash_init(shash))179 return -EFAULT;180 181 if (crypto_shash_update(shash, ctx->opad, block_size))182 return -EFAULT;183 184 offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);185 if (offset < 0)186 return -EFAULT;187 188 hash_state_out = (__be32 *)(hash->sha.state1 + offset);189 hash512_state_out = (__be64 *)hash_state_out;190 191 switch (ctx->qat_hash_alg) {192 case ICP_QAT_HW_AUTH_ALGO_SHA1:193 if (crypto_shash_export(shash, &ctx->sha1))194 return -EFAULT;195 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)196 *hash_state_out = cpu_to_be32(ctx->sha1.state[i]);197 break;198 case ICP_QAT_HW_AUTH_ALGO_SHA256:199 if (crypto_shash_export(shash, &ctx->sha256))200 return -EFAULT;201 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)202 *hash_state_out = cpu_to_be32(ctx->sha256.state[i]);203 break;204 case ICP_QAT_HW_AUTH_ALGO_SHA512:205 if (crypto_shash_export(shash, &ctx->sha512))206 return -EFAULT;207 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)208 *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]);209 break;210 default:211 return -EFAULT;212 }213 memzero_explicit(ctx->ipad, block_size);214 memzero_explicit(ctx->opad, block_size);215 return 0;216}217 218static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)219{220 header->hdr_flags =221 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);222 header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA;223 header->comn_req_flags =224 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,225 QAT_COMN_PTR_TYPE_SGL);226 ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,227 ICP_QAT_FW_LA_PARTIAL_NONE);228 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,229 ICP_QAT_FW_CIPH_IV_16BYTE_DATA);230 ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,231 ICP_QAT_FW_LA_NO_PROTO);232 ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,233 ICP_QAT_FW_LA_NO_UPDATE_STATE);234}235 236static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,237 int alg,238 struct crypto_authenc_keys *keys,239 int mode)240{241 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);242 unsigned int digestsize = crypto_aead_authsize(aead_tfm);243 struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd;244 struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher;245 struct icp_qat_hw_auth_algo_blk *hash =246 (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx +247 sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen);248 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req;249 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;250 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;251 void *ptr = &req_tmpl->cd_ctrl;252 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;253 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;254 255 /* CD setup */256 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);257 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);258 hash->sha.inner_setup.auth_config.config =259 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,260 ctx->qat_hash_alg, digestsize);261 hash->sha.inner_setup.auth_counter.counter =262 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));263 264 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))265 return -EFAULT;266 267 /* Request setup */268 qat_alg_init_common_hdr(header);269 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;270 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,271 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);272 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,273 ICP_QAT_FW_LA_RET_AUTH_RES);274 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,275 ICP_QAT_FW_LA_NO_CMP_AUTH_RES);276 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;277 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;278 279 /* Cipher CD config setup */280 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;281 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;282 cipher_cd_ctrl->cipher_cfg_offset = 0;283 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);284 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);285 /* Auth CD config setup */286 hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3;287 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;288 hash_cd_ctrl->inner_res_sz = digestsize;289 hash_cd_ctrl->final_sz = digestsize;290 291 switch (ctx->qat_hash_alg) {292 case ICP_QAT_HW_AUTH_ALGO_SHA1:293 hash_cd_ctrl->inner_state1_sz =294 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);295 hash_cd_ctrl->inner_state2_sz =296 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);297 break;298 case ICP_QAT_HW_AUTH_ALGO_SHA256:299 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;300 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;301 break;302 case ICP_QAT_HW_AUTH_ALGO_SHA512:303 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;304 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;305 break;306 default:307 break;308 }309 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +310 ((sizeof(struct icp_qat_hw_auth_setup) +311 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);312 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);313 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);314 return 0;315}316 317static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm,318 int alg,319 struct crypto_authenc_keys *keys,320 int mode)321{322 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);323 unsigned int digestsize = crypto_aead_authsize(aead_tfm);324 struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd;325 struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash;326 struct icp_qat_hw_cipher_algo_blk *cipher =327 (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx +328 sizeof(struct icp_qat_hw_auth_setup) +329 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);330 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;331 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;332 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;333 void *ptr = &req_tmpl->cd_ctrl;334 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;335 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;336 struct icp_qat_fw_la_auth_req_params *auth_param =337 (struct icp_qat_fw_la_auth_req_params *)338 ((char *)&req_tmpl->serv_specif_rqpars +339 sizeof(struct icp_qat_fw_la_cipher_req_params));340 341 /* CD setup */342 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode);343 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);344 hash->sha.inner_setup.auth_config.config =345 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,346 ctx->qat_hash_alg,347 digestsize);348 hash->sha.inner_setup.auth_counter.counter =349 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));350 351 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))352 return -EFAULT;353 354 /* Request setup */355 qat_alg_init_common_hdr(header);356 header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;357 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,358 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);359 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,360 ICP_QAT_FW_LA_NO_RET_AUTH_RES);361 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,362 ICP_QAT_FW_LA_CMP_AUTH_RES);363 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;364 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;365 366 /* Cipher CD config setup */367 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;368 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;369 cipher_cd_ctrl->cipher_cfg_offset =370 (sizeof(struct icp_qat_hw_auth_setup) +371 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3;372 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);373 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);374 375 /* Auth CD config setup */376 hash_cd_ctrl->hash_cfg_offset = 0;377 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;378 hash_cd_ctrl->inner_res_sz = digestsize;379 hash_cd_ctrl->final_sz = digestsize;380 381 switch (ctx->qat_hash_alg) {382 case ICP_QAT_HW_AUTH_ALGO_SHA1:383 hash_cd_ctrl->inner_state1_sz =384 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);385 hash_cd_ctrl->inner_state2_sz =386 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);387 break;388 case ICP_QAT_HW_AUTH_ALGO_SHA256:389 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;390 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;391 break;392 case ICP_QAT_HW_AUTH_ALGO_SHA512:393 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;394 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;395 break;396 default:397 break;398 }399 400 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +401 ((sizeof(struct icp_qat_hw_auth_setup) +402 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);403 auth_param->auth_res_sz = digestsize;404 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);405 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);406 return 0;407}408 409static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx,410 struct icp_qat_fw_la_bulk_req *req,411 struct icp_qat_hw_cipher_algo_blk *cd,412 const u8 *key, unsigned int keylen)413{414 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;415 struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;416 struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl;417 bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);418 int mode = ctx->mode;419 420 qat_alg_init_common_hdr(header);421 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;422 cd_pars->u.s.content_desc_params_sz =423 sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3;424 425 if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) {426 ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags,427 ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE);428 429 /* Store both XTS keys in CD, only the first key is sent430 * to the HW, the second key is used for tweak calculation431 */432 memcpy(cd->ucs_aes.key, key, keylen);433 keylen = keylen / 2;434 } else if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_CTR_MODE) {435 ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags,436 ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE);437 memcpy(cd->ucs_aes.key, key, keylen);438 keylen = round_up(keylen, 16);439 } else {440 memcpy(cd->aes.key, key, keylen);441 }442 443 /* Cipher CD config setup */444 cd_ctrl->cipher_key_sz = keylen >> 3;445 cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;446 cd_ctrl->cipher_cfg_offset = 0;447 ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);448 ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);449}450 451static void qat_alg_skcipher_init_enc(struct qat_alg_skcipher_ctx *ctx,452 int alg, const u8 *key,453 unsigned int keylen, int mode)454{455 struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;456 struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;457 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;458 459 qat_alg_skcipher_init_com(ctx, req, enc_cd, key, keylen);460 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;461 enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);462}463 464static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen,465 u8 *key_reverse)466{467 struct crypto_aes_ctx aes_expanded;468 int nrounds;469 u8 *key;470 471 aes_expandkey(&aes_expanded, key_forward, keylen);472 if (keylen == AES_KEYSIZE_128) {473 nrounds = 10;474 key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds);475 memcpy(key_reverse, key, AES_BLOCK_SIZE);476 } else {477 /* AES_KEYSIZE_256 */478 nrounds = 14;479 key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds);480 memcpy(key_reverse, key, AES_BLOCK_SIZE);481 memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE,482 AES_BLOCK_SIZE);483 }484}485 486static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx,487 int alg, const u8 *key,488 unsigned int keylen, int mode)489{490 struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;491 struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;492 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;493 bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);494 495 qat_alg_skcipher_init_com(ctx, req, dec_cd, key, keylen);496 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;497 498 if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) {499 /* Key reversing not supported, set no convert */500 dec_cd->aes.cipher_config.val =501 QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode);502 503 /* In-place key reversal */504 qat_alg_xts_reverse_key(dec_cd->ucs_aes.key, keylen / 2,505 dec_cd->ucs_aes.key);506 } else if (mode != ICP_QAT_HW_CIPHER_CTR_MODE) {507 dec_cd->aes.cipher_config.val =508 QAT_AES_HW_CONFIG_DEC(alg, mode);509 } else {510 dec_cd->aes.cipher_config.val =511 QAT_AES_HW_CONFIG_ENC(alg, mode);512 }513}514 515static int qat_alg_validate_key(int key_len, int *alg, int mode)516{517 if (mode != ICP_QAT_HW_CIPHER_XTS_MODE) {518 switch (key_len) {519 case AES_KEYSIZE_128:520 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;521 break;522 case AES_KEYSIZE_192:523 *alg = ICP_QAT_HW_CIPHER_ALGO_AES192;524 break;525 case AES_KEYSIZE_256:526 *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;527 break;528 default:529 return -EINVAL;530 }531 } else {532 switch (key_len) {533 case AES_KEYSIZE_128 << 1:534 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;535 break;536 case AES_KEYSIZE_256 << 1:537 *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;538 break;539 default:540 return -EINVAL;541 }542 }543 return 0;544}545 546static int qat_alg_aead_init_sessions(struct crypto_aead *tfm, const u8 *key,547 unsigned int keylen, int mode)548{549 struct crypto_authenc_keys keys;550 int alg;551 552 if (crypto_authenc_extractkeys(&keys, key, keylen))553 goto bad_key;554 555 if (qat_alg_validate_key(keys.enckeylen, &alg, mode))556 goto bad_key;557 558 if (qat_alg_aead_init_enc_session(tfm, alg, &keys, mode))559 goto error;560 561 if (qat_alg_aead_init_dec_session(tfm, alg, &keys, mode))562 goto error;563 564 memzero_explicit(&keys, sizeof(keys));565 return 0;566bad_key:567 memzero_explicit(&keys, sizeof(keys));568 return -EINVAL;569error:570 memzero_explicit(&keys, sizeof(keys));571 return -EFAULT;572}573 574static int qat_alg_skcipher_init_sessions(struct qat_alg_skcipher_ctx *ctx,575 const u8 *key,576 unsigned int keylen,577 int mode)578{579 int alg;580 581 if (qat_alg_validate_key(keylen, &alg, mode))582 return -EINVAL;583 584 qat_alg_skcipher_init_enc(ctx, alg, key, keylen, mode);585 qat_alg_skcipher_init_dec(ctx, alg, key, keylen, mode);586 return 0;587}588 589static int qat_alg_aead_rekey(struct crypto_aead *tfm, const u8 *key,590 unsigned int keylen)591{592 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);593 594 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));595 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));596 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));597 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));598 599 return qat_alg_aead_init_sessions(tfm, key, keylen,600 ICP_QAT_HW_CIPHER_CBC_MODE);601}602 603static int qat_alg_aead_newkey(struct crypto_aead *tfm, const u8 *key,604 unsigned int keylen)605{606 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);607 struct qat_crypto_instance *inst = NULL;608 int node = numa_node_id();609 struct device *dev;610 int ret;611 612 inst = qat_crypto_get_instance_node(node);613 if (!inst)614 return -EINVAL;615 dev = &GET_DEV(inst->accel_dev);616 ctx->inst = inst;617 ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),618 &ctx->enc_cd_paddr,619 GFP_ATOMIC);620 if (!ctx->enc_cd) {621 ret = -ENOMEM;622 goto out_free_inst;623 }624 ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),625 &ctx->dec_cd_paddr,626 GFP_ATOMIC);627 if (!ctx->dec_cd) {628 ret = -ENOMEM;629 goto out_free_enc;630 }631 632 ret = qat_alg_aead_init_sessions(tfm, key, keylen,633 ICP_QAT_HW_CIPHER_CBC_MODE);634 if (ret)635 goto out_free_all;636 637 return 0;638 639out_free_all:640 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));641 dma_free_coherent(dev, sizeof(struct qat_alg_cd),642 ctx->dec_cd, ctx->dec_cd_paddr);643 ctx->dec_cd = NULL;644out_free_enc:645 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));646 dma_free_coherent(dev, sizeof(struct qat_alg_cd),647 ctx->enc_cd, ctx->enc_cd_paddr);648 ctx->enc_cd = NULL;649out_free_inst:650 ctx->inst = NULL;651 qat_crypto_put_instance(inst);652 return ret;653}654 655static int qat_alg_aead_setkey(struct crypto_aead *tfm, const u8 *key,656 unsigned int keylen)657{658 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);659 660 if (ctx->enc_cd)661 return qat_alg_aead_rekey(tfm, key, keylen);662 else663 return qat_alg_aead_newkey(tfm, key, keylen);664}665 666static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp,667 struct qat_crypto_request *qat_req)668{669 struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;670 struct qat_crypto_instance *inst = ctx->inst;671 struct aead_request *areq = qat_req->aead_req;672 u8 stat_filed = qat_resp->comn_resp.comn_status;673 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);674 675 qat_bl_free_bufl(inst->accel_dev, &qat_req->buf);676 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))677 res = -EBADMSG;678 aead_request_complete(areq, res);679}680 681static void qat_alg_update_iv_ctr_mode(struct qat_crypto_request *qat_req)682{683 struct skcipher_request *sreq = qat_req->skcipher_req;684 u64 iv_lo_prev;685 u64 iv_lo;686 u64 iv_hi;687 688 memcpy(qat_req->iv, sreq->iv, AES_BLOCK_SIZE);689 690 iv_lo = be64_to_cpu(qat_req->iv_lo);691 iv_hi = be64_to_cpu(qat_req->iv_hi);692 693 iv_lo_prev = iv_lo;694 iv_lo += DIV_ROUND_UP(sreq->cryptlen, AES_BLOCK_SIZE);695 if (iv_lo < iv_lo_prev)696 iv_hi++;697 698 qat_req->iv_lo = cpu_to_be64(iv_lo);699 qat_req->iv_hi = cpu_to_be64(iv_hi);700}701 702static void qat_alg_update_iv_cbc_mode(struct qat_crypto_request *qat_req)703{704 struct skcipher_request *sreq = qat_req->skcipher_req;705 int offset = sreq->cryptlen - AES_BLOCK_SIZE;706 struct scatterlist *sgl;707 708 if (qat_req->encryption)709 sgl = sreq->dst;710 else711 sgl = sreq->src;712 713 scatterwalk_map_and_copy(qat_req->iv, sgl, offset, AES_BLOCK_SIZE, 0);714}715 716static void qat_alg_update_iv(struct qat_crypto_request *qat_req)717{718 struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;719 struct device *dev = &GET_DEV(ctx->inst->accel_dev);720 721 switch (ctx->mode) {722 case ICP_QAT_HW_CIPHER_CTR_MODE:723 qat_alg_update_iv_ctr_mode(qat_req);724 break;725 case ICP_QAT_HW_CIPHER_CBC_MODE:726 qat_alg_update_iv_cbc_mode(qat_req);727 break;728 case ICP_QAT_HW_CIPHER_XTS_MODE:729 break;730 default:731 dev_warn(dev, "Unsupported IV update for cipher mode %d\n",732 ctx->mode);733 }734}735 736static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,737 struct qat_crypto_request *qat_req)738{739 struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;740 struct qat_crypto_instance *inst = ctx->inst;741 struct skcipher_request *sreq = qat_req->skcipher_req;742 u8 stat_filed = qat_resp->comn_resp.comn_status;743 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);744 745 qat_bl_free_bufl(inst->accel_dev, &qat_req->buf);746 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))747 res = -EINVAL;748 749 if (qat_req->encryption)750 qat_alg_update_iv(qat_req);751 752 memcpy(sreq->iv, qat_req->iv, AES_BLOCK_SIZE);753 754 skcipher_request_complete(sreq, res);755}756 757void qat_alg_callback(void *resp)758{759 struct icp_qat_fw_la_resp *qat_resp = resp;760 struct qat_crypto_request *qat_req =761 (void *)(__force long)qat_resp->opaque_data;762 struct qat_instance_backlog *backlog = qat_req->alg_req.backlog;763 764 qat_req->cb(qat_resp, qat_req);765 766 qat_alg_send_backlog(backlog);767}768 769static int qat_alg_send_sym_message(struct qat_crypto_request *qat_req,770 struct qat_crypto_instance *inst,771 struct crypto_async_request *base)772{773 struct qat_alg_req *alg_req = &qat_req->alg_req;774 775 alg_req->fw_req = (u32 *)&qat_req->req;776 alg_req->tx_ring = inst->sym_tx;777 alg_req->base = base;778 alg_req->backlog = &inst->backlog;779 780 return qat_alg_send_message(alg_req);781}782 783static int qat_alg_aead_dec(struct aead_request *areq)784{785 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);786 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);787 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);788 struct qat_crypto_request *qat_req = aead_request_ctx(areq);789 struct icp_qat_fw_la_cipher_req_params *cipher_param;790 struct icp_qat_fw_la_auth_req_params *auth_param;791 struct icp_qat_fw_la_bulk_req *msg;792 int digst_size = crypto_aead_authsize(aead_tfm);793 gfp_t f = qat_algs_alloc_flags(&areq->base);794 int ret;795 u32 cipher_len;796 797 cipher_len = areq->cryptlen - digst_size;798 if (cipher_len % AES_BLOCK_SIZE != 0)799 return -EINVAL;800 801 ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst,802 &qat_req->buf, NULL, f);803 if (unlikely(ret))804 return ret;805 806 msg = &qat_req->req;807 *msg = ctx->dec_fw_req;808 qat_req->aead_ctx = ctx;809 qat_req->aead_req = areq;810 qat_req->cb = qat_aead_alg_callback;811 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;812 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;813 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;814 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;815 cipher_param->cipher_length = cipher_len;816 cipher_param->cipher_offset = areq->assoclen;817 memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE);818 auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param));819 auth_param->auth_off = 0;820 auth_param->auth_len = areq->assoclen + cipher_param->cipher_length;821 822 ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base);823 if (ret == -ENOSPC)824 qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);825 826 return ret;827}828 829static int qat_alg_aead_enc(struct aead_request *areq)830{831 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);832 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);833 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);834 struct qat_crypto_request *qat_req = aead_request_ctx(areq);835 struct icp_qat_fw_la_cipher_req_params *cipher_param;836 struct icp_qat_fw_la_auth_req_params *auth_param;837 gfp_t f = qat_algs_alloc_flags(&areq->base);838 struct icp_qat_fw_la_bulk_req *msg;839 u8 *iv = areq->iv;840 int ret;841 842 if (areq->cryptlen % AES_BLOCK_SIZE != 0)843 return -EINVAL;844 845 ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst,846 &qat_req->buf, NULL, f);847 if (unlikely(ret))848 return ret;849 850 msg = &qat_req->req;851 *msg = ctx->enc_fw_req;852 qat_req->aead_ctx = ctx;853 qat_req->aead_req = areq;854 qat_req->cb = qat_aead_alg_callback;855 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;856 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;857 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;858 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;859 auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param));860 861 memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);862 cipher_param->cipher_length = areq->cryptlen;863 cipher_param->cipher_offset = areq->assoclen;864 865 auth_param->auth_off = 0;866 auth_param->auth_len = areq->assoclen + areq->cryptlen;867 868 ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base);869 if (ret == -ENOSPC)870 qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);871 872 return ret;873}874 875static int qat_alg_skcipher_rekey(struct qat_alg_skcipher_ctx *ctx,876 const u8 *key, unsigned int keylen,877 int mode)878{879 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));880 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));881 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));882 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));883 884 return qat_alg_skcipher_init_sessions(ctx, key, keylen, mode);885}886 887static int qat_alg_skcipher_newkey(struct qat_alg_skcipher_ctx *ctx,888 const u8 *key, unsigned int keylen,889 int mode)890{891 struct qat_crypto_instance *inst = NULL;892 struct device *dev;893 int node = numa_node_id();894 int ret;895 896 inst = qat_crypto_get_instance_node(node);897 if (!inst)898 return -EINVAL;899 dev = &GET_DEV(inst->accel_dev);900 ctx->inst = inst;901 ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),902 &ctx->enc_cd_paddr,903 GFP_ATOMIC);904 if (!ctx->enc_cd) {905 ret = -ENOMEM;906 goto out_free_instance;907 }908 ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),909 &ctx->dec_cd_paddr,910 GFP_ATOMIC);911 if (!ctx->dec_cd) {912 ret = -ENOMEM;913 goto out_free_enc;914 }915 916 ret = qat_alg_skcipher_init_sessions(ctx, key, keylen, mode);917 if (ret)918 goto out_free_all;919 920 return 0;921 922out_free_all:923 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));924 dma_free_coherent(dev, sizeof(*ctx->dec_cd),925 ctx->dec_cd, ctx->dec_cd_paddr);926 ctx->dec_cd = NULL;927out_free_enc:928 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));929 dma_free_coherent(dev, sizeof(*ctx->enc_cd),930 ctx->enc_cd, ctx->enc_cd_paddr);931 ctx->enc_cd = NULL;932out_free_instance:933 ctx->inst = NULL;934 qat_crypto_put_instance(inst);935 return ret;936}937 938static int qat_alg_skcipher_setkey(struct crypto_skcipher *tfm,939 const u8 *key, unsigned int keylen,940 int mode)941{942 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);943 944 ctx->mode = mode;945 946 if (ctx->enc_cd)947 return qat_alg_skcipher_rekey(ctx, key, keylen, mode);948 else949 return qat_alg_skcipher_newkey(ctx, key, keylen, mode);950}951 952static int qat_alg_skcipher_cbc_setkey(struct crypto_skcipher *tfm,953 const u8 *key, unsigned int keylen)954{955 return qat_alg_skcipher_setkey(tfm, key, keylen,956 ICP_QAT_HW_CIPHER_CBC_MODE);957}958 959static int qat_alg_skcipher_ctr_setkey(struct crypto_skcipher *tfm,960 const u8 *key, unsigned int keylen)961{962 return qat_alg_skcipher_setkey(tfm, key, keylen,963 ICP_QAT_HW_CIPHER_CTR_MODE);964}965 966static int qat_alg_skcipher_xts_setkey(struct crypto_skcipher *tfm,967 const u8 *key, unsigned int keylen)968{969 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);970 int ret;971 972 ret = xts_verify_key(tfm, key, keylen);973 if (ret)974 return ret;975 976 if (keylen >> 1 == AES_KEYSIZE_192) {977 ret = crypto_skcipher_setkey(ctx->ftfm, key, keylen);978 if (ret)979 return ret;980 981 ctx->fallback = true;982 983 return 0;984 }985 986 ctx->fallback = false;987 988 ret = qat_alg_skcipher_setkey(tfm, key, keylen,989 ICP_QAT_HW_CIPHER_XTS_MODE);990 if (ret)991 return ret;992 993 if (HW_CAP_AES_V2(ctx->inst->accel_dev))994 ret = crypto_cipher_setkey(ctx->tweak, key + (keylen / 2),995 keylen / 2);996 997 return ret;998}999 1000static void qat_alg_set_req_iv(struct qat_crypto_request *qat_req)1001{1002 struct icp_qat_fw_la_cipher_req_params *cipher_param;1003 struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;1004 bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);1005 u8 *iv = qat_req->skcipher_req->iv;1006 1007 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;1008 1009 if (aes_v2_capable && ctx->mode == ICP_QAT_HW_CIPHER_XTS_MODE)1010 crypto_cipher_encrypt_one(ctx->tweak,1011 (u8 *)cipher_param->u.cipher_IV_array,1012 iv);1013 else1014 memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);1015}1016 1017static int qat_alg_skcipher_encrypt(struct skcipher_request *req)1018{1019 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);1020 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);1021 struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);1022 struct qat_crypto_request *qat_req = skcipher_request_ctx(req);1023 struct icp_qat_fw_la_cipher_req_params *cipher_param;1024 gfp_t f = qat_algs_alloc_flags(&req->base);1025 struct icp_qat_fw_la_bulk_req *msg;1026 int ret;1027 1028 if (req->cryptlen == 0)1029 return 0;1030 1031 ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, req->src, req->dst,1032 &qat_req->buf, NULL, f);1033 if (unlikely(ret))1034 return ret;1035 1036 msg = &qat_req->req;1037 *msg = ctx->enc_fw_req;1038 qat_req->skcipher_ctx = ctx;1039 qat_req->skcipher_req = req;1040 qat_req->cb = qat_skcipher_alg_callback;1041 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;1042 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;1043 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;1044 qat_req->encryption = true;1045 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;1046 cipher_param->cipher_length = req->cryptlen;1047 cipher_param->cipher_offset = 0;1048 1049 qat_alg_set_req_iv(qat_req);1050 1051 ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base);1052 if (ret == -ENOSPC)1053 qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);1054 1055 return ret;1056}1057 1058static int qat_alg_skcipher_blk_encrypt(struct skcipher_request *req)1059{1060 if (req->cryptlen % AES_BLOCK_SIZE != 0)1061 return -EINVAL;1062 1063 return qat_alg_skcipher_encrypt(req);1064}1065 1066static int qat_alg_skcipher_xts_encrypt(struct skcipher_request *req)1067{1068 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);1069 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(stfm);1070 struct skcipher_request *nreq = skcipher_request_ctx(req);1071 1072 if (req->cryptlen < XTS_BLOCK_SIZE)1073 return -EINVAL;1074 1075 if (ctx->fallback) {1076 memcpy(nreq, req, sizeof(*req));1077 skcipher_request_set_tfm(nreq, ctx->ftfm);1078 return crypto_skcipher_encrypt(nreq);1079 }1080 1081 return qat_alg_skcipher_encrypt(req);1082}1083 1084static int qat_alg_skcipher_decrypt(struct skcipher_request *req)1085{1086 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);1087 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);1088 struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);1089 struct qat_crypto_request *qat_req = skcipher_request_ctx(req);1090 struct icp_qat_fw_la_cipher_req_params *cipher_param;1091 gfp_t f = qat_algs_alloc_flags(&req->base);1092 struct icp_qat_fw_la_bulk_req *msg;1093 int ret;1094 1095 if (req->cryptlen == 0)1096 return 0;1097 1098 ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, req->src, req->dst,1099 &qat_req->buf, NULL, f);1100 if (unlikely(ret))1101 return ret;1102 1103 msg = &qat_req->req;1104 *msg = ctx->dec_fw_req;1105 qat_req->skcipher_ctx = ctx;1106 qat_req->skcipher_req = req;1107 qat_req->cb = qat_skcipher_alg_callback;1108 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;1109 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;1110 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;1111 qat_req->encryption = false;1112 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;1113 cipher_param->cipher_length = req->cryptlen;1114 cipher_param->cipher_offset = 0;1115 1116 qat_alg_set_req_iv(qat_req);1117 qat_alg_update_iv(qat_req);1118 1119 ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base);1120 if (ret == -ENOSPC)1121 qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);1122 1123 return ret;1124}1125 1126static int qat_alg_skcipher_blk_decrypt(struct skcipher_request *req)1127{1128 if (req->cryptlen % AES_BLOCK_SIZE != 0)1129 return -EINVAL;1130 1131 return qat_alg_skcipher_decrypt(req);1132}1133 1134static int qat_alg_skcipher_xts_decrypt(struct skcipher_request *req)1135{1136 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);1137 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(stfm);1138 struct skcipher_request *nreq = skcipher_request_ctx(req);1139 1140 if (req->cryptlen < XTS_BLOCK_SIZE)1141 return -EINVAL;1142 1143 if (ctx->fallback) {1144 memcpy(nreq, req, sizeof(*req));1145 skcipher_request_set_tfm(nreq, ctx->ftfm);1146 return crypto_skcipher_decrypt(nreq);1147 }1148 1149 return qat_alg_skcipher_decrypt(req);1150}1151 1152static int qat_alg_aead_init(struct crypto_aead *tfm,1153 enum icp_qat_hw_auth_algo hash,1154 const char *hash_name)1155{1156 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);1157 1158 ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);1159 if (IS_ERR(ctx->hash_tfm))1160 return PTR_ERR(ctx->hash_tfm);1161 ctx->qat_hash_alg = hash;1162 crypto_aead_set_reqsize(tfm, sizeof(struct qat_crypto_request));1163 return 0;1164}1165 1166static int qat_alg_aead_sha1_init(struct crypto_aead *tfm)1167{1168 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1");1169}1170 1171static int qat_alg_aead_sha256_init(struct crypto_aead *tfm)1172{1173 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256");1174}1175 1176static int qat_alg_aead_sha512_init(struct crypto_aead *tfm)1177{1178 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512");1179}1180 1181static void qat_alg_aead_exit(struct crypto_aead *tfm)1182{1183 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);1184 struct qat_crypto_instance *inst = ctx->inst;1185 struct device *dev;1186 1187 crypto_free_shash(ctx->hash_tfm);1188 1189 if (!inst)1190 return;1191 1192 dev = &GET_DEV(inst->accel_dev);1193 if (ctx->enc_cd) {1194 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));1195 dma_free_coherent(dev, sizeof(struct qat_alg_cd),1196 ctx->enc_cd, ctx->enc_cd_paddr);1197 }1198 if (ctx->dec_cd) {1199 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));1200 dma_free_coherent(dev, sizeof(struct qat_alg_cd),1201 ctx->dec_cd, ctx->dec_cd_paddr);1202 }1203 qat_crypto_put_instance(inst);1204}1205 1206static int qat_alg_skcipher_init_tfm(struct crypto_skcipher *tfm)1207{1208 crypto_skcipher_set_reqsize(tfm, sizeof(struct qat_crypto_request));1209 return 0;1210}1211 1212static int qat_alg_skcipher_init_xts_tfm(struct crypto_skcipher *tfm)1213{1214 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);1215 int reqsize;1216 1217 ctx->ftfm = crypto_alloc_skcipher("xts(aes)", 0,1218 CRYPTO_ALG_NEED_FALLBACK);1219 if (IS_ERR(ctx->ftfm))1220 return PTR_ERR(ctx->ftfm);1221 1222 ctx->tweak = crypto_alloc_cipher("aes", 0, 0);1223 if (IS_ERR(ctx->tweak)) {1224 crypto_free_skcipher(ctx->ftfm);1225 return PTR_ERR(ctx->tweak);1226 }1227 1228 reqsize = max(sizeof(struct qat_crypto_request),1229 sizeof(struct skcipher_request) +1230 crypto_skcipher_reqsize(ctx->ftfm));1231 crypto_skcipher_set_reqsize(tfm, reqsize);1232 1233 return 0;1234}1235 1236static void qat_alg_skcipher_exit_tfm(struct crypto_skcipher *tfm)1237{1238 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);1239 struct qat_crypto_instance *inst = ctx->inst;1240 struct device *dev;1241 1242 if (!inst)1243 return;1244 1245 dev = &GET_DEV(inst->accel_dev);1246 if (ctx->enc_cd) {1247 memset(ctx->enc_cd, 0,1248 sizeof(struct icp_qat_hw_cipher_algo_blk));1249 dma_free_coherent(dev,1250 sizeof(struct icp_qat_hw_cipher_algo_blk),1251 ctx->enc_cd, ctx->enc_cd_paddr);1252 }1253 if (ctx->dec_cd) {1254 memset(ctx->dec_cd, 0,1255 sizeof(struct icp_qat_hw_cipher_algo_blk));1256 dma_free_coherent(dev,1257 sizeof(struct icp_qat_hw_cipher_algo_blk),1258 ctx->dec_cd, ctx->dec_cd_paddr);1259 }1260 qat_crypto_put_instance(inst);1261}1262 1263static void qat_alg_skcipher_exit_xts_tfm(struct crypto_skcipher *tfm)1264{1265 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);1266 1267 if (ctx->ftfm)1268 crypto_free_skcipher(ctx->ftfm);1269 1270 if (ctx->tweak)1271 crypto_free_cipher(ctx->tweak);1272 1273 qat_alg_skcipher_exit_tfm(tfm);1274}1275 1276static struct aead_alg qat_aeads[] = { {1277 .base = {1278 .cra_name = "authenc(hmac(sha1),cbc(aes))",1279 .cra_driver_name = "qat_aes_cbc_hmac_sha1",1280 .cra_priority = 4001,1281 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1282 .cra_blocksize = AES_BLOCK_SIZE,1283 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),1284 .cra_module = THIS_MODULE,1285 },1286 .init = qat_alg_aead_sha1_init,1287 .exit = qat_alg_aead_exit,1288 .setkey = qat_alg_aead_setkey,1289 .decrypt = qat_alg_aead_dec,1290 .encrypt = qat_alg_aead_enc,1291 .ivsize = AES_BLOCK_SIZE,1292 .maxauthsize = SHA1_DIGEST_SIZE,1293}, {1294 .base = {1295 .cra_name = "authenc(hmac(sha256),cbc(aes))",1296 .cra_driver_name = "qat_aes_cbc_hmac_sha256",1297 .cra_priority = 4001,1298 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1299 .cra_blocksize = AES_BLOCK_SIZE,1300 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),1301 .cra_module = THIS_MODULE,1302 },1303 .init = qat_alg_aead_sha256_init,1304 .exit = qat_alg_aead_exit,1305 .setkey = qat_alg_aead_setkey,1306 .decrypt = qat_alg_aead_dec,1307 .encrypt = qat_alg_aead_enc,1308 .ivsize = AES_BLOCK_SIZE,1309 .maxauthsize = SHA256_DIGEST_SIZE,1310}, {1311 .base = {1312 .cra_name = "authenc(hmac(sha512),cbc(aes))",1313 .cra_driver_name = "qat_aes_cbc_hmac_sha512",1314 .cra_priority = 4001,1315 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1316 .cra_blocksize = AES_BLOCK_SIZE,1317 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),1318 .cra_module = THIS_MODULE,1319 },1320 .init = qat_alg_aead_sha512_init,1321 .exit = qat_alg_aead_exit,1322 .setkey = qat_alg_aead_setkey,1323 .decrypt = qat_alg_aead_dec,1324 .encrypt = qat_alg_aead_enc,1325 .ivsize = AES_BLOCK_SIZE,1326 .maxauthsize = SHA512_DIGEST_SIZE,1327} };1328 1329static struct skcipher_alg qat_skciphers[] = { {1330 .base.cra_name = "cbc(aes)",1331 .base.cra_driver_name = "qat_aes_cbc",1332 .base.cra_priority = 4001,1333 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1334 .base.cra_blocksize = AES_BLOCK_SIZE,1335 .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),1336 .base.cra_alignmask = 0,1337 .base.cra_module = THIS_MODULE,1338 1339 .init = qat_alg_skcipher_init_tfm,1340 .exit = qat_alg_skcipher_exit_tfm,1341 .setkey = qat_alg_skcipher_cbc_setkey,1342 .decrypt = qat_alg_skcipher_blk_decrypt,1343 .encrypt = qat_alg_skcipher_blk_encrypt,1344 .min_keysize = AES_MIN_KEY_SIZE,1345 .max_keysize = AES_MAX_KEY_SIZE,1346 .ivsize = AES_BLOCK_SIZE,1347}, {1348 .base.cra_name = "ctr(aes)",1349 .base.cra_driver_name = "qat_aes_ctr",1350 .base.cra_priority = 4001,1351 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,1352 .base.cra_blocksize = 1,1353 .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),1354 .base.cra_alignmask = 0,1355 .base.cra_module = THIS_MODULE,1356 1357 .init = qat_alg_skcipher_init_tfm,1358 .exit = qat_alg_skcipher_exit_tfm,1359 .setkey = qat_alg_skcipher_ctr_setkey,1360 .decrypt = qat_alg_skcipher_decrypt,1361 .encrypt = qat_alg_skcipher_encrypt,1362 .min_keysize = AES_MIN_KEY_SIZE,1363 .max_keysize = AES_MAX_KEY_SIZE,1364 .ivsize = AES_BLOCK_SIZE,1365}, {1366 .base.cra_name = "xts(aes)",1367 .base.cra_driver_name = "qat_aes_xts",1368 .base.cra_priority = 4001,1369 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK |1370 CRYPTO_ALG_ALLOCATES_MEMORY,1371 .base.cra_blocksize = AES_BLOCK_SIZE,1372 .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),1373 .base.cra_alignmask = 0,1374 .base.cra_module = THIS_MODULE,1375 1376 .init = qat_alg_skcipher_init_xts_tfm,1377 .exit = qat_alg_skcipher_exit_xts_tfm,1378 .setkey = qat_alg_skcipher_xts_setkey,1379 .decrypt = qat_alg_skcipher_xts_decrypt,1380 .encrypt = qat_alg_skcipher_xts_encrypt,1381 .min_keysize = 2 * AES_MIN_KEY_SIZE,1382 .max_keysize = 2 * AES_MAX_KEY_SIZE,1383 .ivsize = AES_BLOCK_SIZE,1384} };1385 1386int qat_algs_register(void)1387{1388 int ret = 0;1389 1390 mutex_lock(&algs_lock);1391 if (++active_devs != 1)1392 goto unlock;1393 1394 ret = crypto_register_skciphers(qat_skciphers,1395 ARRAY_SIZE(qat_skciphers));1396 if (ret)1397 goto unlock;1398 1399 ret = crypto_register_aeads(qat_aeads, ARRAY_SIZE(qat_aeads));1400 if (ret)1401 goto unreg_algs;1402 1403unlock:1404 mutex_unlock(&algs_lock);1405 return ret;1406 1407unreg_algs:1408 crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers));1409 goto unlock;1410}1411 1412void qat_algs_unregister(void)1413{1414 mutex_lock(&algs_lock);1415 if (--active_devs != 0)1416 goto unlock;1417 1418 crypto_unregister_aeads(qat_aeads, ARRAY_SIZE(qat_aeads));1419 crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers));1420 1421unlock:1422 mutex_unlock(&algs_lock);1423}1424