568 lines · c
1// SPDX-License-Identifier: GPL-2.02#include <linux/kernel.h>3#include <linux/printk.h>4#include <linux/crypto.h>5#include <linux/rtnetlink.h>6 7#include <crypto/aead.h>8#include <crypto/authenc.h>9#include <crypto/des.h>10#include <crypto/internal/aead.h>11#include <crypto/scatterwalk.h>12#include <crypto/gcm.h>13 14#include "nitrox_dev.h"15#include "nitrox_common.h"16#include "nitrox_req.h"17 18#define GCM_AES_SALT_SIZE 419 20union gph_p3 {21 struct {22#ifdef __BIG_ENDIAN_BITFIELD23 u16 iv_offset : 8;24 u16 auth_offset : 8;25#else26 u16 auth_offset : 8;27 u16 iv_offset : 8;28#endif29 };30 u16 param;31};32 33static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,34 unsigned int keylen)35{36 int aes_keylen;37 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);38 struct flexi_crypto_context *fctx;39 union fc_ctx_flags flags;40 41 aes_keylen = flexi_aes_keylen(keylen);42 if (aes_keylen < 0)43 return -EINVAL;44 45 /* fill crypto context */46 fctx = nctx->u.fctx;47 flags.fu = be64_to_cpu(fctx->flags.f);48 flags.w0.aes_keylen = aes_keylen;49 fctx->flags.f = cpu_to_be64(flags.fu);50 51 /* copy enc key to context */52 memset(&fctx->crypto, 0, sizeof(fctx->crypto));53 memcpy(fctx->crypto.u.key, key, keylen);54 55 return 0;56}57 58static int nitrox_aead_setauthsize(struct crypto_aead *aead,59 unsigned int authsize)60{61 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);62 struct flexi_crypto_context *fctx = nctx->u.fctx;63 union fc_ctx_flags flags;64 65 flags.fu = be64_to_cpu(fctx->flags.f);66 flags.w0.mac_len = authsize;67 fctx->flags.f = cpu_to_be64(flags.fu);68 69 aead->authsize = authsize;70 71 return 0;72}73 74static int nitrox_aes_gcm_setauthsize(struct crypto_aead *aead,75 unsigned int authsize)76{77 switch (authsize) {78 case 4:79 case 8:80 case 12:81 case 13:82 case 14:83 case 15:84 case 16:85 break;86 default:87 return -EINVAL;88 }89 90 return nitrox_aead_setauthsize(aead, authsize);91}92 93static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq,94 struct scatterlist *src, char *iv, int ivsize,95 int buflen)96{97 int nents = sg_nents_for_len(src, buflen);98 int ret;99 100 if (nents < 0)101 return nents;102 103 /* IV entry */104 nents += 1;105 /* Allocate buffer to hold IV and input scatterlist array */106 ret = alloc_src_req_buf(nkreq, nents, ivsize);107 if (ret)108 return ret;109 110 nitrox_creq_copy_iv(nkreq->src, iv, ivsize);111 nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen);112 113 return 0;114}115 116static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq,117 struct scatterlist *dst, int ivsize, int buflen)118{119 int nents = sg_nents_for_len(dst, buflen);120 int ret;121 122 if (nents < 0)123 return nents;124 125 /* IV, ORH, COMPLETION entries */126 nents += 3;127 /* Allocate buffer to hold ORH, COMPLETION and output scatterlist128 * array129 */130 ret = alloc_dst_req_buf(nkreq, nents);131 if (ret)132 return ret;133 134 nitrox_creq_set_orh(nkreq);135 nitrox_creq_set_comp(nkreq);136 nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen);137 138 return 0;139}140 141static void free_src_sglist(struct nitrox_kcrypt_request *nkreq)142{143 kfree(nkreq->src);144}145 146static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq)147{148 kfree(nkreq->dst);149}150 151static int nitrox_set_creq(struct nitrox_aead_rctx *rctx)152{153 struct se_crypto_request *creq = &rctx->nkreq.creq;154 union gph_p3 param3;155 int ret;156 157 creq->flags = rctx->flags;158 creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :159 GFP_ATOMIC;160 161 creq->ctrl.value = 0;162 creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;163 creq->ctrl.s.arg = rctx->ctrl_arg;164 165 creq->gph.param0 = cpu_to_be16(rctx->cryptlen);166 creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen);167 creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen);168 param3.iv_offset = 0;169 param3.auth_offset = rctx->ivsize;170 creq->gph.param3 = cpu_to_be16(param3.param);171 172 creq->ctx_handle = rctx->ctx_handle;173 creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);174 175 ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize,176 rctx->srclen);177 if (ret)178 return ret;179 180 ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize,181 rctx->dstlen);182 if (ret) {183 free_src_sglist(&rctx->nkreq);184 return ret;185 }186 187 return 0;188}189 190static void nitrox_aead_callback(void *arg, int err)191{192 struct aead_request *areq = arg;193 struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);194 195 free_src_sglist(&rctx->nkreq);196 free_dst_sglist(&rctx->nkreq);197 if (err) {198 pr_err_ratelimited("request failed status 0x%0x\n", err);199 err = -EINVAL;200 }201 202 aead_request_complete(areq, err);203}204 205static inline bool nitrox_aes_gcm_assoclen_supported(unsigned int assoclen)206{207 if (assoclen <= 512)208 return true;209 210 return false;211}212 213static int nitrox_aes_gcm_enc(struct aead_request *areq)214{215 struct crypto_aead *aead = crypto_aead_reqtfm(areq);216 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);217 struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);218 struct se_crypto_request *creq = &rctx->nkreq.creq;219 struct flexi_crypto_context *fctx = nctx->u.fctx;220 int ret;221 222 if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))223 return -EINVAL;224 225 memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);226 227 rctx->cryptlen = areq->cryptlen;228 rctx->assoclen = areq->assoclen;229 rctx->srclen = areq->assoclen + areq->cryptlen;230 rctx->dstlen = rctx->srclen + aead->authsize;231 rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];232 rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;233 rctx->flags = areq->base.flags;234 rctx->ctx_handle = nctx->u.ctx_handle;235 rctx->src = areq->src;236 rctx->dst = areq->dst;237 rctx->ctrl_arg = ENCRYPT;238 ret = nitrox_set_creq(rctx);239 if (ret)240 return ret;241 242 /* send the crypto request */243 return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,244 areq);245}246 247static int nitrox_aes_gcm_dec(struct aead_request *areq)248{249 struct crypto_aead *aead = crypto_aead_reqtfm(areq);250 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);251 struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);252 struct se_crypto_request *creq = &rctx->nkreq.creq;253 struct flexi_crypto_context *fctx = nctx->u.fctx;254 int ret;255 256 if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))257 return -EINVAL;258 259 memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);260 261 rctx->cryptlen = areq->cryptlen - aead->authsize;262 rctx->assoclen = areq->assoclen;263 rctx->srclen = areq->cryptlen + areq->assoclen;264 rctx->dstlen = rctx->srclen - aead->authsize;265 rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];266 rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;267 rctx->flags = areq->base.flags;268 rctx->ctx_handle = nctx->u.ctx_handle;269 rctx->src = areq->src;270 rctx->dst = areq->dst;271 rctx->ctrl_arg = DECRYPT;272 ret = nitrox_set_creq(rctx);273 if (ret)274 return ret;275 276 /* send the crypto request */277 return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,278 areq);279}280 281static int nitrox_aead_init(struct crypto_aead *aead)282{283 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);284 struct crypto_ctx_hdr *chdr;285 286 /* get the first device */287 nctx->ndev = nitrox_get_first_device();288 if (!nctx->ndev)289 return -ENODEV;290 291 /* allocate nitrox crypto context */292 chdr = crypto_alloc_context(nctx->ndev);293 if (!chdr) {294 nitrox_put_device(nctx->ndev);295 return -ENOMEM;296 }297 nctx->chdr = chdr;298 nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +299 sizeof(struct ctx_hdr));300 nctx->u.fctx->flags.f = 0;301 302 return 0;303}304 305static int nitrox_gcm_common_init(struct crypto_aead *aead)306{307 int ret;308 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);309 union fc_ctx_flags *flags;310 311 ret = nitrox_aead_init(aead);312 if (ret)313 return ret;314 315 flags = &nctx->u.fctx->flags;316 flags->w0.cipher_type = CIPHER_AES_GCM;317 flags->w0.hash_type = AUTH_NULL;318 flags->w0.iv_source = IV_FROM_DPTR;319 /* ask microcode to calculate ipad/opad */320 flags->w0.auth_input_type = 1;321 flags->f = cpu_to_be64(flags->fu);322 323 return 0;324}325 326static int nitrox_aes_gcm_init(struct crypto_aead *aead)327{328 int ret;329 330 ret = nitrox_gcm_common_init(aead);331 if (ret)332 return ret;333 334 crypto_aead_set_reqsize(aead,335 sizeof(struct aead_request) +336 sizeof(struct nitrox_aead_rctx));337 338 return 0;339}340 341static void nitrox_aead_exit(struct crypto_aead *aead)342{343 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);344 345 /* free the nitrox crypto context */346 if (nctx->u.ctx_handle) {347 struct flexi_crypto_context *fctx = nctx->u.fctx;348 349 memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys));350 memzero_explicit(&fctx->auth, sizeof(struct auth_keys));351 crypto_free_context((void *)nctx->chdr);352 }353 nitrox_put_device(nctx->ndev);354 355 nctx->u.ctx_handle = 0;356 nctx->ndev = NULL;357}358 359static int nitrox_rfc4106_setkey(struct crypto_aead *aead, const u8 *key,360 unsigned int keylen)361{362 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);363 struct flexi_crypto_context *fctx = nctx->u.fctx;364 int ret;365 366 if (keylen < GCM_AES_SALT_SIZE)367 return -EINVAL;368 369 keylen -= GCM_AES_SALT_SIZE;370 ret = nitrox_aes_gcm_setkey(aead, key, keylen);371 if (ret)372 return ret;373 374 memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE);375 return 0;376}377 378static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead,379 unsigned int authsize)380{381 switch (authsize) {382 case 8:383 case 12:384 case 16:385 break;386 default:387 return -EINVAL;388 }389 390 return nitrox_aead_setauthsize(aead, authsize);391}392 393static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq)394{395 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq);396 struct nitrox_aead_rctx *aead_rctx = &rctx->base;397 unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;398 struct scatterlist *sg;399 400 if (areq->assoclen != 16 && areq->assoclen != 20)401 return -EINVAL;402 403 scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0);404 sg_init_table(rctx->src, 3);405 sg_set_buf(rctx->src, rctx->assoc, assoclen);406 sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen);407 if (sg != rctx->src + 1)408 sg_chain(rctx->src, 2, sg);409 410 if (areq->src != areq->dst) {411 sg_init_table(rctx->dst, 3);412 sg_set_buf(rctx->dst, rctx->assoc, assoclen);413 sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen);414 if (sg != rctx->dst + 1)415 sg_chain(rctx->dst, 2, sg);416 }417 418 aead_rctx->src = rctx->src;419 aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst;420 421 return 0;422}423 424static void nitrox_rfc4106_callback(void *arg, int err)425{426 struct aead_request *areq = arg;427 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq);428 struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq;429 430 free_src_sglist(nkreq);431 free_dst_sglist(nkreq);432 if (err) {433 pr_err_ratelimited("request failed status 0x%0x\n", err);434 err = -EINVAL;435 }436 437 aead_request_complete(areq, err);438}439 440static int nitrox_rfc4106_enc(struct aead_request *areq)441{442 struct crypto_aead *aead = crypto_aead_reqtfm(areq);443 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);444 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq);445 struct nitrox_aead_rctx *aead_rctx = &rctx->base;446 struct se_crypto_request *creq = &aead_rctx->nkreq.creq;447 int ret;448 449 aead_rctx->cryptlen = areq->cryptlen;450 aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;451 aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen;452 aead_rctx->dstlen = aead_rctx->srclen + aead->authsize;453 aead_rctx->iv = areq->iv;454 aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;455 aead_rctx->flags = areq->base.flags;456 aead_rctx->ctx_handle = nctx->u.ctx_handle;457 aead_rctx->ctrl_arg = ENCRYPT;458 459 ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);460 if (ret)461 return ret;462 463 ret = nitrox_set_creq(aead_rctx);464 if (ret)465 return ret;466 467 /* send the crypto request */468 return nitrox_process_se_request(nctx->ndev, creq,469 nitrox_rfc4106_callback, areq);470}471 472static int nitrox_rfc4106_dec(struct aead_request *areq)473{474 struct crypto_aead *aead = crypto_aead_reqtfm(areq);475 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx_dma(aead);476 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);477 struct nitrox_aead_rctx *aead_rctx = &rctx->base;478 struct se_crypto_request *creq = &aead_rctx->nkreq.creq;479 int ret;480 481 aead_rctx->cryptlen = areq->cryptlen - aead->authsize;482 aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;483 aead_rctx->srclen =484 areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen;485 aead_rctx->dstlen = aead_rctx->srclen - aead->authsize;486 aead_rctx->iv = areq->iv;487 aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;488 aead_rctx->flags = areq->base.flags;489 aead_rctx->ctx_handle = nctx->u.ctx_handle;490 aead_rctx->ctrl_arg = DECRYPT;491 492 ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);493 if (ret)494 return ret;495 496 ret = nitrox_set_creq(aead_rctx);497 if (ret)498 return ret;499 500 /* send the crypto request */501 return nitrox_process_se_request(nctx->ndev, creq,502 nitrox_rfc4106_callback, areq);503}504 505static int nitrox_rfc4106_init(struct crypto_aead *aead)506{507 int ret;508 509 ret = nitrox_gcm_common_init(aead);510 if (ret)511 return ret;512 513 crypto_aead_set_reqsize_dma(aead, sizeof(struct aead_request) +514 sizeof(struct nitrox_rfc4106_rctx));515 516 return 0;517}518 519static struct aead_alg nitrox_aeads[] = { {520 .base = {521 .cra_name = "gcm(aes)",522 .cra_driver_name = "n5_aes_gcm",523 .cra_priority = PRIO,524 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,525 .cra_blocksize = 1,526 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),527 .cra_alignmask = 0,528 .cra_module = THIS_MODULE,529 },530 .setkey = nitrox_aes_gcm_setkey,531 .setauthsize = nitrox_aes_gcm_setauthsize,532 .encrypt = nitrox_aes_gcm_enc,533 .decrypt = nitrox_aes_gcm_dec,534 .init = nitrox_aes_gcm_init,535 .exit = nitrox_aead_exit,536 .ivsize = GCM_AES_IV_SIZE,537 .maxauthsize = AES_BLOCK_SIZE,538}, {539 .base = {540 .cra_name = "rfc4106(gcm(aes))",541 .cra_driver_name = "n5_rfc4106",542 .cra_priority = PRIO,543 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,544 .cra_blocksize = 1,545 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),546 .cra_alignmask = 0,547 .cra_module = THIS_MODULE,548 },549 .setkey = nitrox_rfc4106_setkey,550 .setauthsize = nitrox_rfc4106_setauthsize,551 .encrypt = nitrox_rfc4106_enc,552 .decrypt = nitrox_rfc4106_dec,553 .init = nitrox_rfc4106_init,554 .exit = nitrox_aead_exit,555 .ivsize = GCM_RFC4106_IV_SIZE,556 .maxauthsize = AES_BLOCK_SIZE,557} };558 559int nitrox_register_aeads(void)560{561 return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));562}563 564void nitrox_unregister_aeads(void)565{566 crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));567}568