530 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved.4 */5 6#include <linux/device.h>7#include <linux/dma-mapping.h>8#include <linux/interrupt.h>9#include <linux/moduleparam.h>10#include <linux/types.h>11#include <linux/errno.h>12#include <crypto/aes.h>13#include <crypto/internal/des.h>14#include <crypto/internal/skcipher.h>15 16#include "cipher.h"17 18static unsigned int aes_sw_max_len = CONFIG_CRYPTO_DEV_QCE_SW_MAX_LEN;19module_param(aes_sw_max_len, uint, 0644);20MODULE_PARM_DESC(aes_sw_max_len,21 "Only use hardware for AES requests larger than this "22 "[0=always use hardware; anything <16 breaks AES-GCM; default="23 __stringify(CONFIG_CRYPTO_DEV_QCE_SW_MAX_LEN)"]");24 25static LIST_HEAD(skcipher_algs);26 27static void qce_skcipher_done(void *data)28{29 struct crypto_async_request *async_req = data;30 struct skcipher_request *req = skcipher_request_cast(async_req);31 struct qce_cipher_reqctx *rctx = skcipher_request_ctx(req);32 struct qce_alg_template *tmpl = to_cipher_tmpl(crypto_skcipher_reqtfm(req));33 struct qce_device *qce = tmpl->qce;34 struct qce_result_dump *result_buf = qce->dma.result_buf;35 enum dma_data_direction dir_src, dir_dst;36 u32 status;37 int error;38 bool diff_dst;39 40 diff_dst = (req->src != req->dst) ? true : false;41 dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;42 dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL;43 44 error = qce_dma_terminate_all(&qce->dma);45 if (error)46 dev_dbg(qce->dev, "skcipher dma termination error (%d)\n",47 error);48 49 if (diff_dst)50 dma_unmap_sg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src);51 dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);52 53 sg_free_table(&rctx->dst_tbl);54 55 error = qce_check_status(qce, &status);56 if (error < 0)57 dev_dbg(qce->dev, "skcipher operation error (%x)\n", status);58 59 memcpy(rctx->iv, result_buf->encr_cntr_iv, rctx->ivsize);60 qce->async_req_done(tmpl->qce, error);61}62 63static int64qce_skcipher_async_req_handle(struct crypto_async_request *async_req)65{66 struct skcipher_request *req = skcipher_request_cast(async_req);67 struct qce_cipher_reqctx *rctx = skcipher_request_ctx(req);68 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);69 struct qce_alg_template *tmpl = to_cipher_tmpl(crypto_skcipher_reqtfm(req));70 struct qce_device *qce = tmpl->qce;71 enum dma_data_direction dir_src, dir_dst;72 struct scatterlist *sg;73 bool diff_dst;74 gfp_t gfp;75 int dst_nents, src_nents, ret;76 77 rctx->iv = req->iv;78 rctx->ivsize = crypto_skcipher_ivsize(skcipher);79 rctx->cryptlen = req->cryptlen;80 81 diff_dst = (req->src != req->dst) ? true : false;82 dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;83 dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL;84 85 rctx->src_nents = sg_nents_for_len(req->src, req->cryptlen);86 if (diff_dst)87 rctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen);88 else89 rctx->dst_nents = rctx->src_nents;90 if (rctx->src_nents < 0) {91 dev_err(qce->dev, "Invalid numbers of src SG.\n");92 return rctx->src_nents;93 }94 if (rctx->dst_nents < 0) {95 dev_err(qce->dev, "Invalid numbers of dst SG.\n");96 return -rctx->dst_nents;97 }98 99 rctx->dst_nents += 1;100 101 gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?102 GFP_KERNEL : GFP_ATOMIC;103 104 ret = sg_alloc_table(&rctx->dst_tbl, rctx->dst_nents, gfp);105 if (ret)106 return ret;107 108 sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ);109 110 sg = qce_sgtable_add(&rctx->dst_tbl, req->dst, req->cryptlen);111 if (IS_ERR(sg)) {112 ret = PTR_ERR(sg);113 goto error_free;114 }115 116 sg = qce_sgtable_add(&rctx->dst_tbl, &rctx->result_sg,117 QCE_RESULT_BUF_SZ);118 if (IS_ERR(sg)) {119 ret = PTR_ERR(sg);120 goto error_free;121 }122 123 sg_mark_end(sg);124 rctx->dst_sg = rctx->dst_tbl.sgl;125 126 dst_nents = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);127 if (!dst_nents) {128 ret = -EIO;129 goto error_free;130 }131 132 if (diff_dst) {133 src_nents = dma_map_sg(qce->dev, req->src, rctx->src_nents, dir_src);134 if (!src_nents) {135 ret = -EIO;136 goto error_unmap_dst;137 }138 rctx->src_sg = req->src;139 } else {140 rctx->src_sg = rctx->dst_sg;141 src_nents = dst_nents - 1;142 }143 144 ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, src_nents,145 rctx->dst_sg, dst_nents,146 qce_skcipher_done, async_req);147 if (ret)148 goto error_unmap_src;149 150 qce_dma_issue_pending(&qce->dma);151 152 ret = qce_start(async_req, tmpl->crypto_alg_type);153 if (ret)154 goto error_terminate;155 156 return 0;157 158error_terminate:159 qce_dma_terminate_all(&qce->dma);160error_unmap_src:161 if (diff_dst)162 dma_unmap_sg(qce->dev, req->src, rctx->src_nents, dir_src);163error_unmap_dst:164 dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);165error_free:166 sg_free_table(&rctx->dst_tbl);167 return ret;168}169 170static int qce_skcipher_setkey(struct crypto_skcipher *ablk, const u8 *key,171 unsigned int keylen)172{173 struct crypto_tfm *tfm = crypto_skcipher_tfm(ablk);174 struct qce_cipher_ctx *ctx = crypto_tfm_ctx(tfm);175 unsigned long flags = to_cipher_tmpl(ablk)->alg_flags;176 unsigned int __keylen;177 int ret;178 179 if (!key || !keylen)180 return -EINVAL;181 182 /*183 * AES XTS key1 = key2 not supported by crypto engine.184 * Revisit to request a fallback cipher in this case.185 */186 if (IS_XTS(flags)) {187 __keylen = keylen >> 1;188 if (!memcmp(key, key + __keylen, __keylen))189 return -ENOKEY;190 } else {191 __keylen = keylen;192 }193 194 switch (__keylen) {195 case AES_KEYSIZE_128:196 case AES_KEYSIZE_256:197 memcpy(ctx->enc_key, key, keylen);198 break;199 case AES_KEYSIZE_192:200 break;201 default:202 return -EINVAL;203 }204 205 ret = crypto_skcipher_setkey(ctx->fallback, key, keylen);206 if (!ret)207 ctx->enc_keylen = keylen;208 return ret;209}210 211static int qce_des_setkey(struct crypto_skcipher *ablk, const u8 *key,212 unsigned int keylen)213{214 struct qce_cipher_ctx *ctx = crypto_skcipher_ctx(ablk);215 int err;216 217 err = verify_skcipher_des_key(ablk, key);218 if (err)219 return err;220 221 ctx->enc_keylen = keylen;222 memcpy(ctx->enc_key, key, keylen);223 return 0;224}225 226static int qce_des3_setkey(struct crypto_skcipher *ablk, const u8 *key,227 unsigned int keylen)228{229 struct qce_cipher_ctx *ctx = crypto_skcipher_ctx(ablk);230 u32 _key[6];231 int err;232 233 err = verify_skcipher_des3_key(ablk, key);234 if (err)235 return err;236 237 /*238 * The crypto engine does not support any two keys239 * being the same for triple des algorithms. The240 * verify_skcipher_des3_key does not check for all the241 * below conditions. Return -ENOKEY in case any two keys242 * are the same. Revisit to see if a fallback cipher243 * is needed to handle this condition.244 */245 memcpy(_key, key, DES3_EDE_KEY_SIZE);246 if (!((_key[0] ^ _key[2]) | (_key[1] ^ _key[3])) ||247 !((_key[2] ^ _key[4]) | (_key[3] ^ _key[5])) ||248 !((_key[0] ^ _key[4]) | (_key[1] ^ _key[5])))249 return -ENOKEY;250 251 ctx->enc_keylen = keylen;252 memcpy(ctx->enc_key, key, keylen);253 return 0;254}255 256static int qce_skcipher_crypt(struct skcipher_request *req, int encrypt)257{258 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);259 struct qce_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);260 struct qce_cipher_reqctx *rctx = skcipher_request_ctx(req);261 struct qce_alg_template *tmpl = to_cipher_tmpl(tfm);262 unsigned int blocksize = crypto_skcipher_blocksize(tfm);263 int keylen;264 int ret;265 266 rctx->flags = tmpl->alg_flags;267 rctx->flags |= encrypt ? QCE_ENCRYPT : QCE_DECRYPT;268 keylen = IS_XTS(rctx->flags) ? ctx->enc_keylen >> 1 : ctx->enc_keylen;269 270 /* CE does not handle 0 length messages */271 if (!req->cryptlen)272 return 0;273 274 /*275 * ECB and CBC algorithms require message lengths to be276 * multiples of block size.277 */278 if (IS_ECB(rctx->flags) || IS_CBC(rctx->flags))279 if (!IS_ALIGNED(req->cryptlen, blocksize))280 return -EINVAL;281 282 /*283 * Conditions for requesting a fallback cipher284 * AES-192 (not supported by crypto engine (CE))285 * AES-XTS request with len <= 512 byte (not recommended to use CE)286 * AES-XTS request with len > QCE_SECTOR_SIZE and287 * is not a multiple of it.(Revisit this condition to check if it is288 * needed in all versions of CE)289 */290 if (IS_AES(rctx->flags) &&291 ((keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_256) ||292 (IS_XTS(rctx->flags) && ((req->cryptlen <= aes_sw_max_len) ||293 (req->cryptlen > QCE_SECTOR_SIZE &&294 req->cryptlen % QCE_SECTOR_SIZE))))) {295 skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);296 skcipher_request_set_callback(&rctx->fallback_req,297 req->base.flags,298 req->base.complete,299 req->base.data);300 skcipher_request_set_crypt(&rctx->fallback_req, req->src,301 req->dst, req->cryptlen, req->iv);302 ret = encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) :303 crypto_skcipher_decrypt(&rctx->fallback_req);304 return ret;305 }306 307 return tmpl->qce->async_req_enqueue(tmpl->qce, &req->base);308}309 310static int qce_skcipher_encrypt(struct skcipher_request *req)311{312 return qce_skcipher_crypt(req, 1);313}314 315static int qce_skcipher_decrypt(struct skcipher_request *req)316{317 return qce_skcipher_crypt(req, 0);318}319 320static int qce_skcipher_init(struct crypto_skcipher *tfm)321{322 /* take the size without the fallback skcipher_request at the end */323 crypto_skcipher_set_reqsize(tfm, offsetof(struct qce_cipher_reqctx,324 fallback_req));325 return 0;326}327 328static int qce_skcipher_init_fallback(struct crypto_skcipher *tfm)329{330 struct qce_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);331 332 ctx->fallback = crypto_alloc_skcipher(crypto_tfm_alg_name(&tfm->base),333 0, CRYPTO_ALG_NEED_FALLBACK);334 if (IS_ERR(ctx->fallback))335 return PTR_ERR(ctx->fallback);336 337 crypto_skcipher_set_reqsize(tfm, sizeof(struct qce_cipher_reqctx) +338 crypto_skcipher_reqsize(ctx->fallback));339 return 0;340}341 342static void qce_skcipher_exit(struct crypto_skcipher *tfm)343{344 struct qce_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);345 346 crypto_free_skcipher(ctx->fallback);347}348 349struct qce_skcipher_def {350 unsigned long flags;351 const char *name;352 const char *drv_name;353 unsigned int blocksize;354 unsigned int chunksize;355 unsigned int ivsize;356 unsigned int min_keysize;357 unsigned int max_keysize;358};359 360static const struct qce_skcipher_def skcipher_def[] = {361 {362 .flags = QCE_ALG_AES | QCE_MODE_ECB,363 .name = "ecb(aes)",364 .drv_name = "ecb-aes-qce",365 .blocksize = AES_BLOCK_SIZE,366 .ivsize = 0,367 .min_keysize = AES_MIN_KEY_SIZE,368 .max_keysize = AES_MAX_KEY_SIZE,369 },370 {371 .flags = QCE_ALG_AES | QCE_MODE_CBC,372 .name = "cbc(aes)",373 .drv_name = "cbc-aes-qce",374 .blocksize = AES_BLOCK_SIZE,375 .ivsize = AES_BLOCK_SIZE,376 .min_keysize = AES_MIN_KEY_SIZE,377 .max_keysize = AES_MAX_KEY_SIZE,378 },379 {380 .flags = QCE_ALG_AES | QCE_MODE_CTR,381 .name = "ctr(aes)",382 .drv_name = "ctr-aes-qce",383 .blocksize = 1,384 .chunksize = AES_BLOCK_SIZE,385 .ivsize = AES_BLOCK_SIZE,386 .min_keysize = AES_MIN_KEY_SIZE,387 .max_keysize = AES_MAX_KEY_SIZE,388 },389 {390 .flags = QCE_ALG_AES | QCE_MODE_XTS,391 .name = "xts(aes)",392 .drv_name = "xts-aes-qce",393 .blocksize = AES_BLOCK_SIZE,394 .ivsize = AES_BLOCK_SIZE,395 .min_keysize = AES_MIN_KEY_SIZE * 2,396 .max_keysize = AES_MAX_KEY_SIZE * 2,397 },398 {399 .flags = QCE_ALG_DES | QCE_MODE_ECB,400 .name = "ecb(des)",401 .drv_name = "ecb-des-qce",402 .blocksize = DES_BLOCK_SIZE,403 .ivsize = 0,404 .min_keysize = DES_KEY_SIZE,405 .max_keysize = DES_KEY_SIZE,406 },407 {408 .flags = QCE_ALG_DES | QCE_MODE_CBC,409 .name = "cbc(des)",410 .drv_name = "cbc-des-qce",411 .blocksize = DES_BLOCK_SIZE,412 .ivsize = DES_BLOCK_SIZE,413 .min_keysize = DES_KEY_SIZE,414 .max_keysize = DES_KEY_SIZE,415 },416 {417 .flags = QCE_ALG_3DES | QCE_MODE_ECB,418 .name = "ecb(des3_ede)",419 .drv_name = "ecb-3des-qce",420 .blocksize = DES3_EDE_BLOCK_SIZE,421 .ivsize = 0,422 .min_keysize = DES3_EDE_KEY_SIZE,423 .max_keysize = DES3_EDE_KEY_SIZE,424 },425 {426 .flags = QCE_ALG_3DES | QCE_MODE_CBC,427 .name = "cbc(des3_ede)",428 .drv_name = "cbc-3des-qce",429 .blocksize = DES3_EDE_BLOCK_SIZE,430 .ivsize = DES3_EDE_BLOCK_SIZE,431 .min_keysize = DES3_EDE_KEY_SIZE,432 .max_keysize = DES3_EDE_KEY_SIZE,433 },434};435 436static int qce_skcipher_register_one(const struct qce_skcipher_def *def,437 struct qce_device *qce)438{439 struct qce_alg_template *tmpl;440 struct skcipher_alg *alg;441 int ret;442 443 tmpl = kzalloc(sizeof(*tmpl), GFP_KERNEL);444 if (!tmpl)445 return -ENOMEM;446 447 alg = &tmpl->alg.skcipher;448 449 snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name);450 snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",451 def->drv_name);452 453 alg->base.cra_blocksize = def->blocksize;454 alg->chunksize = def->chunksize;455 alg->ivsize = def->ivsize;456 alg->min_keysize = def->min_keysize;457 alg->max_keysize = def->max_keysize;458 alg->setkey = IS_3DES(def->flags) ? qce_des3_setkey :459 IS_DES(def->flags) ? qce_des_setkey :460 qce_skcipher_setkey;461 alg->encrypt = qce_skcipher_encrypt;462 alg->decrypt = qce_skcipher_decrypt;463 464 alg->base.cra_priority = 300;465 alg->base.cra_flags = CRYPTO_ALG_ASYNC |466 CRYPTO_ALG_ALLOCATES_MEMORY |467 CRYPTO_ALG_KERN_DRIVER_ONLY;468 alg->base.cra_ctxsize = sizeof(struct qce_cipher_ctx);469 alg->base.cra_alignmask = 0;470 alg->base.cra_module = THIS_MODULE;471 472 if (IS_AES(def->flags)) {473 alg->base.cra_flags |= CRYPTO_ALG_NEED_FALLBACK;474 alg->init = qce_skcipher_init_fallback;475 alg->exit = qce_skcipher_exit;476 } else {477 alg->init = qce_skcipher_init;478 }479 480 INIT_LIST_HEAD(&tmpl->entry);481 tmpl->crypto_alg_type = CRYPTO_ALG_TYPE_SKCIPHER;482 tmpl->alg_flags = def->flags;483 tmpl->qce = qce;484 485 ret = crypto_register_skcipher(alg);486 if (ret) {487 dev_err(qce->dev, "%s registration failed\n", alg->base.cra_name);488 kfree(tmpl);489 return ret;490 }491 492 list_add_tail(&tmpl->entry, &skcipher_algs);493 dev_dbg(qce->dev, "%s is registered\n", alg->base.cra_name);494 return 0;495}496 497static void qce_skcipher_unregister(struct qce_device *qce)498{499 struct qce_alg_template *tmpl, *n;500 501 list_for_each_entry_safe(tmpl, n, &skcipher_algs, entry) {502 crypto_unregister_skcipher(&tmpl->alg.skcipher);503 list_del(&tmpl->entry);504 kfree(tmpl);505 }506}507 508static int qce_skcipher_register(struct qce_device *qce)509{510 int ret, i;511 512 for (i = 0; i < ARRAY_SIZE(skcipher_def); i++) {513 ret = qce_skcipher_register_one(&skcipher_def[i], qce);514 if (ret)515 goto err;516 }517 518 return 0;519err:520 qce_skcipher_unregister(qce);521 return ret;522}523 524const struct qce_algo_ops skcipher_ops = {525 .type = CRYPTO_ALG_TYPE_SKCIPHER,526 .register_algs = qce_skcipher_register,527 .unregister_algs = qce_skcipher_unregister,528 .async_req_handle = qce_skcipher_async_req_handle,529};530