463 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2 3/*4 * Copyright (C) 2016 Cavium, Inc.5 */6 7#include <crypto/aes.h>8#include <crypto/algapi.h>9#include <crypto/authenc.h>10#include <crypto/internal/des.h>11#include <crypto/xts.h>12#include <linux/crypto.h>13#include <linux/err.h>14#include <linux/list.h>15#include <linux/scatterlist.h>16 17#include "cptvf.h"18#include "cptvf_algs.h"19 20struct cpt_device_handle {21 void *cdev[MAX_DEVICES];22 u32 dev_count;23};24 25static struct cpt_device_handle dev_handle;26 27static void cvm_callback(u32 status, void *arg)28{29 struct crypto_async_request *req = (struct crypto_async_request *)arg;30 31 crypto_request_complete(req, !status);32}33 34static inline void update_input_iv(struct cpt_request_info *req_info,35 u8 *iv, u32 enc_iv_len,36 u32 *argcnt)37{38 /* Setting the iv information */39 req_info->in[*argcnt].vptr = (void *)iv;40 req_info->in[*argcnt].size = enc_iv_len;41 req_info->req.dlen += enc_iv_len;42 43 ++(*argcnt);44}45 46static inline void update_output_iv(struct cpt_request_info *req_info,47 u8 *iv, u32 enc_iv_len,48 u32 *argcnt)49{50 /* Setting the iv information */51 req_info->out[*argcnt].vptr = (void *)iv;52 req_info->out[*argcnt].size = enc_iv_len;53 req_info->rlen += enc_iv_len;54 55 ++(*argcnt);56}57 58static inline void update_input_data(struct cpt_request_info *req_info,59 struct scatterlist *inp_sg,60 u32 nbytes, u32 *argcnt)61{62 req_info->req.dlen += nbytes;63 64 while (nbytes) {65 u32 len = min(nbytes, inp_sg->length);66 u8 *ptr = sg_virt(inp_sg);67 68 req_info->in[*argcnt].vptr = (void *)ptr;69 req_info->in[*argcnt].size = len;70 nbytes -= len;71 72 ++(*argcnt);73 ++inp_sg;74 }75}76 77static inline void update_output_data(struct cpt_request_info *req_info,78 struct scatterlist *outp_sg,79 u32 nbytes, u32 *argcnt)80{81 req_info->rlen += nbytes;82 83 while (nbytes) {84 u32 len = min(nbytes, outp_sg->length);85 u8 *ptr = sg_virt(outp_sg);86 87 req_info->out[*argcnt].vptr = (void *)ptr;88 req_info->out[*argcnt].size = len;89 nbytes -= len;90 ++(*argcnt);91 ++outp_sg;92 }93}94 95static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc,96 u32 *argcnt)97{98 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);99 struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(tfm);100 struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);101 struct fc_context *fctx = &rctx->fctx;102 u32 enc_iv_len = crypto_skcipher_ivsize(tfm);103 struct cpt_request_info *req_info = &rctx->cpt_req;104 __be64 *ctrl_flags = NULL;105 __be64 *offset_control;106 107 req_info->ctrl.s.grp = 0;108 req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER;109 req_info->ctrl.s.se_req = SE_CORE_REQ;110 111 req_info->req.opcode.s.major = MAJOR_OP_FC |112 DMA_MODE_FLAG(DMA_GATHER_SCATTER);113 if (enc)114 req_info->req.opcode.s.minor = 2;115 else116 req_info->req.opcode.s.minor = 3;117 118 req_info->req.param1 = req->cryptlen; /* Encryption Data length */119 req_info->req.param2 = 0; /*Auth data length */120 121 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;122 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;123 fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR;124 125 if (ctx->cipher_type == AES_XTS)126 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);127 else128 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);129 ctrl_flags = (__be64 *)&fctx->enc.enc_ctrl.flags;130 *ctrl_flags = cpu_to_be64(fctx->enc.enc_ctrl.flags);131 132 offset_control = (__be64 *)&rctx->control_word;133 *offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16));134 /* Storing Packet Data Information in offset135 * Control Word First 8 bytes136 */137 req_info->in[*argcnt].vptr = (u8 *)offset_control;138 req_info->in[*argcnt].size = CONTROL_WORD_LEN;139 req_info->req.dlen += CONTROL_WORD_LEN;140 ++(*argcnt);141 142 req_info->in[*argcnt].vptr = (u8 *)fctx;143 req_info->in[*argcnt].size = sizeof(struct fc_context);144 req_info->req.dlen += sizeof(struct fc_context);145 146 ++(*argcnt);147 148 return 0;149}150 151static inline u32 create_input_list(struct skcipher_request *req, u32 enc,152 u32 enc_iv_len)153{154 struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);155 struct cpt_request_info *req_info = &rctx->cpt_req;156 u32 argcnt = 0;157 158 create_ctx_hdr(req, enc, &argcnt);159 update_input_iv(req_info, req->iv, enc_iv_len, &argcnt);160 update_input_data(req_info, req->src, req->cryptlen, &argcnt);161 req_info->incnt = argcnt;162 163 return 0;164}165 166static inline void store_cb_info(struct skcipher_request *req,167 struct cpt_request_info *req_info)168{169 req_info->callback = (void *)cvm_callback;170 req_info->callback_arg = (void *)&req->base;171}172 173static inline void create_output_list(struct skcipher_request *req,174 u32 enc_iv_len)175{176 struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);177 struct cpt_request_info *req_info = &rctx->cpt_req;178 u32 argcnt = 0;179 180 /* OUTPUT Buffer Processing181 * AES encryption/decryption output would be182 * received in the following format183 *184 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|185 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ]186 */187 /* Reading IV information */188 update_output_iv(req_info, req->iv, enc_iv_len, &argcnt);189 update_output_data(req_info, req->dst, req->cryptlen, &argcnt);190 req_info->outcnt = argcnt;191}192 193static inline int cvm_enc_dec(struct skcipher_request *req, u32 enc)194{195 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);196 struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);197 u32 enc_iv_len = crypto_skcipher_ivsize(tfm);198 struct fc_context *fctx = &rctx->fctx;199 struct cpt_request_info *req_info = &rctx->cpt_req;200 void *cdev = NULL;201 int status;202 203 memset(req_info, 0, sizeof(struct cpt_request_info));204 req_info->may_sleep = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) != 0;205 memset(fctx, 0, sizeof(struct fc_context));206 create_input_list(req, enc, enc_iv_len);207 create_output_list(req, enc_iv_len);208 store_cb_info(req, req_info);209 cdev = dev_handle.cdev[smp_processor_id()];210 status = cptvf_do_request(cdev, req_info);211 /* We perform an asynchronous send and once212 * the request is completed the driver would213 * intimate through registered call back functions214 */215 216 if (status)217 return status;218 else219 return -EINPROGRESS;220}221 222static int cvm_encrypt(struct skcipher_request *req)223{224 return cvm_enc_dec(req, true);225}226 227static int cvm_decrypt(struct skcipher_request *req)228{229 return cvm_enc_dec(req, false);230}231 232static int cvm_xts_setkey(struct crypto_skcipher *cipher, const u8 *key,233 u32 keylen)234{235 struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(cipher);236 int err;237 const u8 *key1 = key;238 const u8 *key2 = key + (keylen / 2);239 240 err = xts_verify_key(cipher, key, keylen);241 if (err)242 return err;243 ctx->key_len = keylen;244 memcpy(ctx->enc_key, key1, keylen / 2);245 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);246 ctx->cipher_type = AES_XTS;247 switch (ctx->key_len) {248 case 32:249 ctx->key_type = AES_128_BIT;250 break;251 case 64:252 ctx->key_type = AES_256_BIT;253 break;254 default:255 return -EINVAL;256 }257 258 return 0;259}260 261static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen)262{263 if ((keylen == 16) || (keylen == 24) || (keylen == 32)) {264 ctx->key_len = keylen;265 switch (ctx->key_len) {266 case 16:267 ctx->key_type = AES_128_BIT;268 break;269 case 24:270 ctx->key_type = AES_192_BIT;271 break;272 case 32:273 ctx->key_type = AES_256_BIT;274 break;275 default:276 return -EINVAL;277 }278 279 if (ctx->cipher_type == DES3_CBC)280 ctx->key_type = 0;281 282 return 0;283 }284 285 return -EINVAL;286}287 288static int cvm_setkey(struct crypto_skcipher *cipher, const u8 *key,289 u32 keylen, u8 cipher_type)290{291 struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(cipher);292 293 ctx->cipher_type = cipher_type;294 if (!cvm_validate_keylen(ctx, keylen)) {295 memcpy(ctx->enc_key, key, keylen);296 return 0;297 } else {298 return -EINVAL;299 }300}301 302static int cvm_cbc_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,303 u32 keylen)304{305 return cvm_setkey(cipher, key, keylen, AES_CBC);306}307 308static int cvm_ecb_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,309 u32 keylen)310{311 return cvm_setkey(cipher, key, keylen, AES_ECB);312}313 314static int cvm_cbc_des3_setkey(struct crypto_skcipher *cipher, const u8 *key,315 u32 keylen)316{317 return verify_skcipher_des3_key(cipher, key) ?:318 cvm_setkey(cipher, key, keylen, DES3_CBC);319}320 321static int cvm_ecb_des3_setkey(struct crypto_skcipher *cipher, const u8 *key,322 u32 keylen)323{324 return verify_skcipher_des3_key(cipher, key) ?:325 cvm_setkey(cipher, key, keylen, DES3_ECB);326}327 328static int cvm_enc_dec_init(struct crypto_skcipher *tfm)329{330 crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct cvm_req_ctx));331 332 return 0;333}334 335static struct skcipher_alg algs[] = { {336 .base.cra_flags = CRYPTO_ALG_ASYNC |337 CRYPTO_ALG_ALLOCATES_MEMORY,338 .base.cra_blocksize = AES_BLOCK_SIZE,339 .base.cra_ctxsize = sizeof(struct cvm_enc_ctx),340 .base.cra_alignmask = 7,341 .base.cra_priority = 4001,342 .base.cra_name = "xts(aes)",343 .base.cra_driver_name = "cavium-xts-aes",344 .base.cra_module = THIS_MODULE,345 346 .ivsize = AES_BLOCK_SIZE,347 .min_keysize = 2 * AES_MIN_KEY_SIZE,348 .max_keysize = 2 * AES_MAX_KEY_SIZE,349 .setkey = cvm_xts_setkey,350 .encrypt = cvm_encrypt,351 .decrypt = cvm_decrypt,352 .init = cvm_enc_dec_init,353}, {354 .base.cra_flags = CRYPTO_ALG_ASYNC |355 CRYPTO_ALG_ALLOCATES_MEMORY,356 .base.cra_blocksize = AES_BLOCK_SIZE,357 .base.cra_ctxsize = sizeof(struct cvm_enc_ctx),358 .base.cra_alignmask = 7,359 .base.cra_priority = 4001,360 .base.cra_name = "cbc(aes)",361 .base.cra_driver_name = "cavium-cbc-aes",362 .base.cra_module = THIS_MODULE,363 364 .ivsize = AES_BLOCK_SIZE,365 .min_keysize = AES_MIN_KEY_SIZE,366 .max_keysize = AES_MAX_KEY_SIZE,367 .setkey = cvm_cbc_aes_setkey,368 .encrypt = cvm_encrypt,369 .decrypt = cvm_decrypt,370 .init = cvm_enc_dec_init,371}, {372 .base.cra_flags = CRYPTO_ALG_ASYNC |373 CRYPTO_ALG_ALLOCATES_MEMORY,374 .base.cra_blocksize = AES_BLOCK_SIZE,375 .base.cra_ctxsize = sizeof(struct cvm_enc_ctx),376 .base.cra_alignmask = 7,377 .base.cra_priority = 4001,378 .base.cra_name = "ecb(aes)",379 .base.cra_driver_name = "cavium-ecb-aes",380 .base.cra_module = THIS_MODULE,381 382 .min_keysize = AES_MIN_KEY_SIZE,383 .max_keysize = AES_MAX_KEY_SIZE,384 .setkey = cvm_ecb_aes_setkey,385 .encrypt = cvm_encrypt,386 .decrypt = cvm_decrypt,387 .init = cvm_enc_dec_init,388}, {389 .base.cra_flags = CRYPTO_ALG_ASYNC |390 CRYPTO_ALG_ALLOCATES_MEMORY,391 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,392 .base.cra_ctxsize = sizeof(struct cvm_des3_ctx),393 .base.cra_alignmask = 7,394 .base.cra_priority = 4001,395 .base.cra_name = "cbc(des3_ede)",396 .base.cra_driver_name = "cavium-cbc-des3_ede",397 .base.cra_module = THIS_MODULE,398 399 .min_keysize = DES3_EDE_KEY_SIZE,400 .max_keysize = DES3_EDE_KEY_SIZE,401 .ivsize = DES_BLOCK_SIZE,402 .setkey = cvm_cbc_des3_setkey,403 .encrypt = cvm_encrypt,404 .decrypt = cvm_decrypt,405 .init = cvm_enc_dec_init,406}, {407 .base.cra_flags = CRYPTO_ALG_ASYNC |408 CRYPTO_ALG_ALLOCATES_MEMORY,409 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,410 .base.cra_ctxsize = sizeof(struct cvm_des3_ctx),411 .base.cra_alignmask = 7,412 .base.cra_priority = 4001,413 .base.cra_name = "ecb(des3_ede)",414 .base.cra_driver_name = "cavium-ecb-des3_ede",415 .base.cra_module = THIS_MODULE,416 417 .min_keysize = DES3_EDE_KEY_SIZE,418 .max_keysize = DES3_EDE_KEY_SIZE,419 .ivsize = DES_BLOCK_SIZE,420 .setkey = cvm_ecb_des3_setkey,421 .encrypt = cvm_encrypt,422 .decrypt = cvm_decrypt,423 .init = cvm_enc_dec_init,424} };425 426static inline int cav_register_algs(void)427{428 return crypto_register_skciphers(algs, ARRAY_SIZE(algs));429}430 431static inline void cav_unregister_algs(void)432{433 crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));434}435 436int cvm_crypto_init(struct cpt_vf *cptvf)437{438 struct pci_dev *pdev = cptvf->pdev;439 u32 dev_count;440 441 dev_count = dev_handle.dev_count;442 dev_handle.cdev[dev_count] = cptvf;443 dev_handle.dev_count++;444 445 if (dev_count == 3) {446 if (cav_register_algs()) {447 dev_err(&pdev->dev, "Error in registering crypto algorithms\n");448 return -EINVAL;449 }450 }451 452 return 0;453}454 455void cvm_crypto_exit(void)456{457 u32 dev_count;458 459 dev_count = --dev_handle.dev_count;460 if (!dev_count)461 cav_unregister_algs();462}463