532 lines · c
1// SPDX-License-Identifier: GPL-2.02#include <linux/crypto.h>3#include <linux/kernel.h>4#include <linux/module.h>5#include <linux/printk.h>6 7#include <crypto/aes.h>8#include <crypto/skcipher.h>9#include <crypto/scatterwalk.h>10#include <crypto/ctr.h>11#include <crypto/internal/des.h>12#include <crypto/xts.h>13 14#include "nitrox_dev.h"15#include "nitrox_common.h"16#include "nitrox_req.h"17 18struct nitrox_cipher {19 const char *name;20 enum flexi_cipher value;21};22 23/*24 * supported cipher list25 */26static const struct nitrox_cipher flexi_cipher_table[] = {27 { "null", CIPHER_NULL },28 { "cbc(des3_ede)", CIPHER_3DES_CBC },29 { "ecb(des3_ede)", CIPHER_3DES_ECB },30 { "cbc(aes)", CIPHER_AES_CBC },31 { "ecb(aes)", CIPHER_AES_ECB },32 { "cfb(aes)", CIPHER_AES_CFB },33 { "rfc3686(ctr(aes))", CIPHER_AES_CTR },34 { "xts(aes)", CIPHER_AES_XTS },35 { "cts(cbc(aes))", CIPHER_AES_CBC_CTS },36 { NULL, CIPHER_INVALID }37};38 39static enum flexi_cipher flexi_cipher_type(const char *name)40{41 const struct nitrox_cipher *cipher = flexi_cipher_table;42 43 while (cipher->name) {44 if (!strcmp(cipher->name, name))45 break;46 cipher++;47 }48 return cipher->value;49}50 51static void free_src_sglist(struct skcipher_request *skreq)52{53 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);54 55 kfree(nkreq->src);56}57 58static void free_dst_sglist(struct skcipher_request *skreq)59{60 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);61 62 kfree(nkreq->dst);63}64 65static void nitrox_skcipher_callback(void *arg, int err)66{67 struct skcipher_request *skreq = arg;68 69 free_src_sglist(skreq);70 free_dst_sglist(skreq);71 if (err) {72 pr_err_ratelimited("request failed status 0x%0x\n", err);73 err = -EINVAL;74 }75 76 skcipher_request_complete(skreq, err);77}78 79static void nitrox_cbc_cipher_callback(void *arg, int err)80{81 struct skcipher_request *skreq = arg;82 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);83 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq);84 int ivsize = crypto_skcipher_ivsize(cipher);85 unsigned int start = skreq->cryptlen - ivsize;86 87 if (err) {88 nitrox_skcipher_callback(arg, err);89 return;90 }91 92 if (nkreq->creq.ctrl.s.arg == ENCRYPT) {93 scatterwalk_map_and_copy(skreq->iv, skreq->dst, start, ivsize,94 0);95 } else {96 if (skreq->src != skreq->dst) {97 scatterwalk_map_and_copy(skreq->iv, skreq->src, start,98 ivsize, 0);99 } else {100 memcpy(skreq->iv, nkreq->iv_out, ivsize);101 kfree(nkreq->iv_out);102 }103 }104 105 nitrox_skcipher_callback(arg, err);106}107 108static int nitrox_skcipher_init(struct crypto_skcipher *tfm)109{110 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);111 struct crypto_ctx_hdr *chdr;112 113 /* get the first device */114 nctx->ndev = nitrox_get_first_device();115 if (!nctx->ndev)116 return -ENODEV;117 118 /* allocate nitrox crypto context */119 chdr = crypto_alloc_context(nctx->ndev);120 if (!chdr) {121 nitrox_put_device(nctx->ndev);122 return -ENOMEM;123 }124 125 nctx->callback = nitrox_skcipher_callback;126 nctx->chdr = chdr;127 nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +128 sizeof(struct ctx_hdr));129 crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(tfm) +130 sizeof(struct nitrox_kcrypt_request));131 return 0;132}133 134static int nitrox_cbc_init(struct crypto_skcipher *tfm)135{136 int err;137 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);138 139 err = nitrox_skcipher_init(tfm);140 if (err)141 return err;142 143 nctx->callback = nitrox_cbc_cipher_callback;144 return 0;145}146 147static void nitrox_skcipher_exit(struct crypto_skcipher *tfm)148{149 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);150 151 /* free the nitrox crypto context */152 if (nctx->u.ctx_handle) {153 struct flexi_crypto_context *fctx = nctx->u.fctx;154 155 memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys));156 memzero_explicit(&fctx->auth, sizeof(struct auth_keys));157 crypto_free_context((void *)nctx->chdr);158 }159 nitrox_put_device(nctx->ndev);160 161 nctx->u.ctx_handle = 0;162 nctx->ndev = NULL;163}164 165static inline int nitrox_skcipher_setkey(struct crypto_skcipher *cipher,166 int aes_keylen, const u8 *key,167 unsigned int keylen)168{169 struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);170 struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm);171 struct flexi_crypto_context *fctx;172 union fc_ctx_flags *flags;173 enum flexi_cipher cipher_type;174 const char *name;175 176 name = crypto_tfm_alg_name(tfm);177 cipher_type = flexi_cipher_type(name);178 if (unlikely(cipher_type == CIPHER_INVALID)) {179 pr_err("unsupported cipher: %s\n", name);180 return -EINVAL;181 }182 183 /* fill crypto context */184 fctx = nctx->u.fctx;185 flags = &fctx->flags;186 flags->f = 0;187 flags->w0.cipher_type = cipher_type;188 flags->w0.aes_keylen = aes_keylen;189 flags->w0.iv_source = IV_FROM_DPTR;190 flags->f = cpu_to_be64(*(u64 *)&flags->w0);191 /* copy the key to context */192 memcpy(fctx->crypto.u.key, key, keylen);193 194 return 0;195}196 197static int nitrox_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,198 unsigned int keylen)199{200 int aes_keylen;201 202 aes_keylen = flexi_aes_keylen(keylen);203 if (aes_keylen < 0)204 return -EINVAL;205 return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);206}207 208static int alloc_src_sglist(struct skcipher_request *skreq, int ivsize)209{210 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);211 int nents = sg_nents(skreq->src) + 1;212 int ret;213 214 /* Allocate buffer to hold IV and input scatterlist array */215 ret = alloc_src_req_buf(nkreq, nents, ivsize);216 if (ret)217 return ret;218 219 nitrox_creq_copy_iv(nkreq->src, skreq->iv, ivsize);220 nitrox_creq_set_src_sg(nkreq, nents, ivsize, skreq->src,221 skreq->cryptlen);222 223 return 0;224}225 226static int alloc_dst_sglist(struct skcipher_request *skreq, int ivsize)227{228 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);229 int nents = sg_nents(skreq->dst) + 3;230 int ret;231 232 /* Allocate buffer to hold ORH, COMPLETION and output scatterlist233 * array234 */235 ret = alloc_dst_req_buf(nkreq, nents);236 if (ret)237 return ret;238 239 nitrox_creq_set_orh(nkreq);240 nitrox_creq_set_comp(nkreq);241 nitrox_creq_set_dst_sg(nkreq, nents, ivsize, skreq->dst,242 skreq->cryptlen);243 244 return 0;245}246 247static int nitrox_skcipher_crypt(struct skcipher_request *skreq, bool enc)248{249 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq);250 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher);251 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);252 int ivsize = crypto_skcipher_ivsize(cipher);253 struct se_crypto_request *creq;254 int ret;255 256 creq = &nkreq->creq;257 creq->flags = skreq->base.flags;258 creq->gfp = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?259 GFP_KERNEL : GFP_ATOMIC;260 261 /* fill the request */262 creq->ctrl.value = 0;263 creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;264 creq->ctrl.s.arg = (enc ? ENCRYPT : DECRYPT);265 /* param0: length of the data to be encrypted */266 creq->gph.param0 = cpu_to_be16(skreq->cryptlen);267 creq->gph.param1 = 0;268 /* param2: encryption data offset */269 creq->gph.param2 = cpu_to_be16(ivsize);270 creq->gph.param3 = 0;271 272 creq->ctx_handle = nctx->u.ctx_handle;273 creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);274 275 ret = alloc_src_sglist(skreq, ivsize);276 if (ret)277 return ret;278 279 ret = alloc_dst_sglist(skreq, ivsize);280 if (ret) {281 free_src_sglist(skreq);282 return ret;283 }284 285 /* send the crypto request */286 return nitrox_process_se_request(nctx->ndev, creq, nctx->callback,287 skreq);288}289 290static int nitrox_cbc_decrypt(struct skcipher_request *skreq)291{292 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);293 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq);294 int ivsize = crypto_skcipher_ivsize(cipher);295 gfp_t flags = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?296 GFP_KERNEL : GFP_ATOMIC;297 unsigned int start = skreq->cryptlen - ivsize;298 299 if (skreq->src != skreq->dst)300 return nitrox_skcipher_crypt(skreq, false);301 302 nkreq->iv_out = kmalloc(ivsize, flags);303 if (!nkreq->iv_out)304 return -ENOMEM;305 306 scatterwalk_map_and_copy(nkreq->iv_out, skreq->src, start, ivsize, 0);307 return nitrox_skcipher_crypt(skreq, false);308}309 310static int nitrox_aes_encrypt(struct skcipher_request *skreq)311{312 return nitrox_skcipher_crypt(skreq, true);313}314 315static int nitrox_aes_decrypt(struct skcipher_request *skreq)316{317 return nitrox_skcipher_crypt(skreq, false);318}319 320static int nitrox_3des_setkey(struct crypto_skcipher *cipher,321 const u8 *key, unsigned int keylen)322{323 return verify_skcipher_des3_key(cipher, key) ?:324 nitrox_skcipher_setkey(cipher, 0, key, keylen);325}326 327static int nitrox_3des_encrypt(struct skcipher_request *skreq)328{329 return nitrox_skcipher_crypt(skreq, true);330}331 332static int nitrox_3des_decrypt(struct skcipher_request *skreq)333{334 return nitrox_skcipher_crypt(skreq, false);335}336 337static int nitrox_aes_xts_setkey(struct crypto_skcipher *cipher,338 const u8 *key, unsigned int keylen)339{340 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher);341 struct flexi_crypto_context *fctx;342 int aes_keylen, ret;343 344 ret = xts_verify_key(cipher, key, keylen);345 if (ret)346 return ret;347 348 keylen /= 2;349 350 aes_keylen = flexi_aes_keylen(keylen);351 if (aes_keylen < 0)352 return -EINVAL;353 354 fctx = nctx->u.fctx;355 /* copy KEY2 */356 memcpy(fctx->auth.u.key2, (key + keylen), keylen);357 358 return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);359}360 361static int nitrox_aes_ctr_rfc3686_setkey(struct crypto_skcipher *cipher,362 const u8 *key, unsigned int keylen)363{364 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher);365 struct flexi_crypto_context *fctx;366 int aes_keylen;367 368 if (keylen < CTR_RFC3686_NONCE_SIZE)369 return -EINVAL;370 371 fctx = nctx->u.fctx;372 373 memcpy(fctx->crypto.iv, key + (keylen - CTR_RFC3686_NONCE_SIZE),374 CTR_RFC3686_NONCE_SIZE);375 376 keylen -= CTR_RFC3686_NONCE_SIZE;377 378 aes_keylen = flexi_aes_keylen(keylen);379 if (aes_keylen < 0)380 return -EINVAL;381 return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);382}383 384static struct skcipher_alg nitrox_skciphers[] = { {385 .base = {386 .cra_name = "cbc(aes)",387 .cra_driver_name = "n5_cbc(aes)",388 .cra_priority = PRIO,389 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,390 .cra_blocksize = AES_BLOCK_SIZE,391 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),392 .cra_alignmask = 0,393 .cra_module = THIS_MODULE,394 },395 .min_keysize = AES_MIN_KEY_SIZE,396 .max_keysize = AES_MAX_KEY_SIZE,397 .ivsize = AES_BLOCK_SIZE,398 .setkey = nitrox_aes_setkey,399 .encrypt = nitrox_aes_encrypt,400 .decrypt = nitrox_cbc_decrypt,401 .init = nitrox_cbc_init,402 .exit = nitrox_skcipher_exit,403}, {404 .base = {405 .cra_name = "ecb(aes)",406 .cra_driver_name = "n5_ecb(aes)",407 .cra_priority = PRIO,408 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,409 .cra_blocksize = AES_BLOCK_SIZE,410 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),411 .cra_alignmask = 0,412 .cra_module = THIS_MODULE,413 },414 .min_keysize = AES_MIN_KEY_SIZE,415 .max_keysize = AES_MAX_KEY_SIZE,416 .ivsize = AES_BLOCK_SIZE,417 .setkey = nitrox_aes_setkey,418 .encrypt = nitrox_aes_encrypt,419 .decrypt = nitrox_aes_decrypt,420 .init = nitrox_skcipher_init,421 .exit = nitrox_skcipher_exit,422}, {423 .base = {424 .cra_name = "xts(aes)",425 .cra_driver_name = "n5_xts(aes)",426 .cra_priority = PRIO,427 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,428 .cra_blocksize = AES_BLOCK_SIZE,429 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),430 .cra_alignmask = 0,431 .cra_module = THIS_MODULE,432 },433 .min_keysize = 2 * AES_MIN_KEY_SIZE,434 .max_keysize = 2 * AES_MAX_KEY_SIZE,435 .ivsize = AES_BLOCK_SIZE,436 .setkey = nitrox_aes_xts_setkey,437 .encrypt = nitrox_aes_encrypt,438 .decrypt = nitrox_aes_decrypt,439 .init = nitrox_skcipher_init,440 .exit = nitrox_skcipher_exit,441}, {442 .base = {443 .cra_name = "rfc3686(ctr(aes))",444 .cra_driver_name = "n5_rfc3686(ctr(aes))",445 .cra_priority = PRIO,446 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,447 .cra_blocksize = 1,448 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),449 .cra_alignmask = 0,450 .cra_module = THIS_MODULE,451 },452 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,453 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,454 .ivsize = CTR_RFC3686_IV_SIZE,455 .init = nitrox_skcipher_init,456 .exit = nitrox_skcipher_exit,457 .setkey = nitrox_aes_ctr_rfc3686_setkey,458 .encrypt = nitrox_aes_encrypt,459 .decrypt = nitrox_aes_decrypt,460}, {461 .base = {462 .cra_name = "cts(cbc(aes))",463 .cra_driver_name = "n5_cts(cbc(aes))",464 .cra_priority = PRIO,465 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,466 .cra_blocksize = AES_BLOCK_SIZE,467 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),468 .cra_alignmask = 0,469 .cra_module = THIS_MODULE,470 },471 .min_keysize = AES_MIN_KEY_SIZE,472 .max_keysize = AES_MAX_KEY_SIZE,473 .ivsize = AES_BLOCK_SIZE,474 .setkey = nitrox_aes_setkey,475 .encrypt = nitrox_aes_encrypt,476 .decrypt = nitrox_aes_decrypt,477 .init = nitrox_skcipher_init,478 .exit = nitrox_skcipher_exit,479}, {480 .base = {481 .cra_name = "cbc(des3_ede)",482 .cra_driver_name = "n5_cbc(des3_ede)",483 .cra_priority = PRIO,484 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,485 .cra_blocksize = DES3_EDE_BLOCK_SIZE,486 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),487 .cra_alignmask = 0,488 .cra_module = THIS_MODULE,489 },490 .min_keysize = DES3_EDE_KEY_SIZE,491 .max_keysize = DES3_EDE_KEY_SIZE,492 .ivsize = DES3_EDE_BLOCK_SIZE,493 .setkey = nitrox_3des_setkey,494 .encrypt = nitrox_3des_encrypt,495 .decrypt = nitrox_cbc_decrypt,496 .init = nitrox_cbc_init,497 .exit = nitrox_skcipher_exit,498}, {499 .base = {500 .cra_name = "ecb(des3_ede)",501 .cra_driver_name = "n5_ecb(des3_ede)",502 .cra_priority = PRIO,503 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,504 .cra_blocksize = DES3_EDE_BLOCK_SIZE,505 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),506 .cra_alignmask = 0,507 .cra_module = THIS_MODULE,508 },509 .min_keysize = DES3_EDE_KEY_SIZE,510 .max_keysize = DES3_EDE_KEY_SIZE,511 .ivsize = DES3_EDE_BLOCK_SIZE,512 .setkey = nitrox_3des_setkey,513 .encrypt = nitrox_3des_encrypt,514 .decrypt = nitrox_3des_decrypt,515 .init = nitrox_skcipher_init,516 .exit = nitrox_skcipher_exit,517}518 519};520 521int nitrox_register_skciphers(void)522{523 return crypto_register_skciphers(nitrox_skciphers,524 ARRAY_SIZE(nitrox_skciphers));525}526 527void nitrox_unregister_skciphers(void)528{529 crypto_unregister_skciphers(nitrox_skciphers,530 ARRAY_SIZE(nitrox_skciphers));531}532