2516 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * AMD Cryptographic Coprocessor (CCP) driver4 *5 * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.6 *7 * Author: Tom Lendacky <thomas.lendacky@amd.com>8 * Author: Gary R Hook <gary.hook@amd.com>9 */10 11#include <linux/dma-mapping.h>12#include <linux/module.h>13#include <linux/kernel.h>14#include <linux/interrupt.h>15#include <crypto/scatterwalk.h>16#include <crypto/des.h>17#include <linux/ccp.h>18 19#include "ccp-dev.h"20 21/* SHA initial context values */22static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {23 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),24 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),25 cpu_to_be32(SHA1_H4),26};27 28static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {29 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),30 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),31 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),32 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),33};34 35static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {36 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),37 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),38 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),39 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),40};41 42static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {43 cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),44 cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),45 cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),46 cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),47};48 49static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {50 cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),51 cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),52 cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),53 cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),54};55 56#define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \57 ccp_gen_jobid(ccp) : 0)58 59static u32 ccp_gen_jobid(struct ccp_device *ccp)60{61 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;62}63 64static void ccp_sg_free(struct ccp_sg_workarea *wa)65{66 if (wa->dma_count)67 dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);68 69 wa->dma_count = 0;70}71 72static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,73 struct scatterlist *sg, u64 len,74 enum dma_data_direction dma_dir)75{76 memset(wa, 0, sizeof(*wa));77 78 wa->sg = sg;79 if (!sg)80 return 0;81 82 wa->nents = sg_nents_for_len(sg, len);83 if (wa->nents < 0)84 return wa->nents;85 86 wa->bytes_left = len;87 wa->sg_used = 0;88 89 if (len == 0)90 return 0;91 92 if (dma_dir == DMA_NONE)93 return 0;94 95 wa->dma_sg = sg;96 wa->dma_sg_head = sg;97 wa->dma_dev = dev;98 wa->dma_dir = dma_dir;99 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);100 if (!wa->dma_count)101 return -ENOMEM;102 103 return 0;104}105 106static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)107{108 unsigned int nbytes = min_t(u64, len, wa->bytes_left);109 unsigned int sg_combined_len = 0;110 111 if (!wa->sg)112 return;113 114 wa->sg_used += nbytes;115 wa->bytes_left -= nbytes;116 if (wa->sg_used == sg_dma_len(wa->dma_sg)) {117 /* Advance to the next DMA scatterlist entry */118 wa->dma_sg = sg_next(wa->dma_sg);119 120 /* In the case that the DMA mapped scatterlist has entries121 * that have been merged, the non-DMA mapped scatterlist122 * must be advanced multiple times for each merged entry.123 * This ensures that the current non-DMA mapped entry124 * corresponds to the current DMA mapped entry.125 */126 do {127 sg_combined_len += wa->sg->length;128 wa->sg = sg_next(wa->sg);129 } while (wa->sg_used > sg_combined_len);130 131 wa->sg_used = 0;132 }133}134 135static void ccp_dm_free(struct ccp_dm_workarea *wa)136{137 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {138 if (wa->address)139 dma_pool_free(wa->dma_pool, wa->address,140 wa->dma.address);141 } else {142 if (wa->dma.address)143 dma_unmap_single(wa->dev, wa->dma.address, wa->length,144 wa->dma.dir);145 kfree(wa->address);146 }147 148 wa->address = NULL;149 wa->dma.address = 0;150}151 152static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,153 struct ccp_cmd_queue *cmd_q,154 unsigned int len,155 enum dma_data_direction dir)156{157 memset(wa, 0, sizeof(*wa));158 159 if (!len)160 return 0;161 162 wa->dev = cmd_q->ccp->dev;163 wa->length = len;164 165 if (len <= CCP_DMAPOOL_MAX_SIZE) {166 wa->dma_pool = cmd_q->dma_pool;167 168 wa->address = dma_pool_zalloc(wa->dma_pool, GFP_KERNEL,169 &wa->dma.address);170 if (!wa->address)171 return -ENOMEM;172 173 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;174 175 } else {176 wa->address = kzalloc(len, GFP_KERNEL);177 if (!wa->address)178 return -ENOMEM;179 180 wa->dma.address = dma_map_single(wa->dev, wa->address, len,181 dir);182 if (dma_mapping_error(wa->dev, wa->dma.address)) {183 kfree(wa->address);184 wa->address = NULL;185 return -ENOMEM;186 }187 188 wa->dma.length = len;189 }190 wa->dma.dir = dir;191 192 return 0;193}194 195static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,196 struct scatterlist *sg, unsigned int sg_offset,197 unsigned int len)198{199 WARN_ON(!wa->address);200 201 if (len > (wa->length - wa_offset))202 return -EINVAL;203 204 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,205 0);206 return 0;207}208 209static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,210 struct scatterlist *sg, unsigned int sg_offset,211 unsigned int len)212{213 WARN_ON(!wa->address);214 215 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,216 1);217}218 219static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,220 unsigned int wa_offset,221 struct scatterlist *sg,222 unsigned int sg_offset,223 unsigned int len)224{225 u8 *p, *q;226 int rc;227 228 rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);229 if (rc)230 return rc;231 232 p = wa->address + wa_offset;233 q = p + len - 1;234 while (p < q) {235 *p = *p ^ *q;236 *q = *p ^ *q;237 *p = *p ^ *q;238 p++;239 q--;240 }241 return 0;242}243 244static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,245 unsigned int wa_offset,246 struct scatterlist *sg,247 unsigned int sg_offset,248 unsigned int len)249{250 u8 *p, *q;251 252 p = wa->address + wa_offset;253 q = p + len - 1;254 while (p < q) {255 *p = *p ^ *q;256 *q = *p ^ *q;257 *p = *p ^ *q;258 p++;259 q--;260 }261 262 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);263}264 265static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)266{267 ccp_dm_free(&data->dm_wa);268 ccp_sg_free(&data->sg_wa);269}270 271static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,272 struct scatterlist *sg, u64 sg_len,273 unsigned int dm_len,274 enum dma_data_direction dir)275{276 int ret;277 278 memset(data, 0, sizeof(*data));279 280 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,281 dir);282 if (ret)283 goto e_err;284 285 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);286 if (ret)287 goto e_err;288 289 return 0;290 291e_err:292 ccp_free_data(data, cmd_q);293 294 return ret;295}296 297static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)298{299 struct ccp_sg_workarea *sg_wa = &data->sg_wa;300 struct ccp_dm_workarea *dm_wa = &data->dm_wa;301 unsigned int buf_count, nbytes;302 303 /* Clear the buffer if setting it */304 if (!from)305 memset(dm_wa->address, 0, dm_wa->length);306 307 if (!sg_wa->sg)308 return 0;309 310 /* Perform the copy operation311 * nbytes will always be <= UINT_MAX because dm_wa->length is312 * an unsigned int313 */314 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);315 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,316 nbytes, from);317 318 /* Update the structures and generate the count */319 buf_count = 0;320 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {321 nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,322 dm_wa->length - buf_count);323 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);324 325 buf_count += nbytes;326 ccp_update_sg_workarea(sg_wa, nbytes);327 }328 329 return buf_count;330}331 332static unsigned int ccp_fill_queue_buf(struct ccp_data *data)333{334 return ccp_queue_buf(data, 0);335}336 337static unsigned int ccp_empty_queue_buf(struct ccp_data *data)338{339 return ccp_queue_buf(data, 1);340}341 342static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,343 struct ccp_op *op, unsigned int block_size,344 bool blocksize_op)345{346 unsigned int sg_src_len, sg_dst_len, op_len;347 348 /* The CCP can only DMA from/to one address each per operation. This349 * requires that we find the smallest DMA area between the source350 * and destination. The resulting len values will always be <= UINT_MAX351 * because the dma length is an unsigned int.352 */353 sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;354 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);355 356 if (dst) {357 sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;358 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);359 op_len = min(sg_src_len, sg_dst_len);360 } else {361 op_len = sg_src_len;362 }363 364 /* The data operation length will be at least block_size in length365 * or the smaller of available sg room remaining for the source or366 * the destination367 */368 op_len = max(op_len, block_size);369 370 /* Unless we have to buffer data, there's no reason to wait */371 op->soc = 0;372 373 if (sg_src_len < block_size) {374 /* Not enough data in the sg element, so it375 * needs to be buffered into a blocksize chunk376 */377 int cp_len = ccp_fill_queue_buf(src);378 379 op->soc = 1;380 op->src.u.dma.address = src->dm_wa.dma.address;381 op->src.u.dma.offset = 0;382 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;383 } else {384 /* Enough data in the sg element, but we need to385 * adjust for any previously copied data386 */387 op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);388 op->src.u.dma.offset = src->sg_wa.sg_used;389 op->src.u.dma.length = op_len & ~(block_size - 1);390 391 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);392 }393 394 if (dst) {395 if (sg_dst_len < block_size) {396 /* Not enough room in the sg element or we're on the397 * last piece of data (when using padding), so the398 * output needs to be buffered into a blocksize chunk399 */400 op->soc = 1;401 op->dst.u.dma.address = dst->dm_wa.dma.address;402 op->dst.u.dma.offset = 0;403 op->dst.u.dma.length = op->src.u.dma.length;404 } else {405 /* Enough room in the sg element, but we need to406 * adjust for any previously used area407 */408 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);409 op->dst.u.dma.offset = dst->sg_wa.sg_used;410 op->dst.u.dma.length = op->src.u.dma.length;411 }412 }413}414 415static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,416 struct ccp_op *op)417{418 op->init = 0;419 420 if (dst) {421 if (op->dst.u.dma.address == dst->dm_wa.dma.address)422 ccp_empty_queue_buf(dst);423 else424 ccp_update_sg_workarea(&dst->sg_wa,425 op->dst.u.dma.length);426 }427}428 429static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,430 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,431 u32 byte_swap, bool from)432{433 struct ccp_op op;434 435 memset(&op, 0, sizeof(op));436 437 op.cmd_q = cmd_q;438 op.jobid = jobid;439 op.eom = 1;440 441 if (from) {442 op.soc = 1;443 op.src.type = CCP_MEMTYPE_SB;444 op.src.u.sb = sb;445 op.dst.type = CCP_MEMTYPE_SYSTEM;446 op.dst.u.dma.address = wa->dma.address;447 op.dst.u.dma.length = wa->length;448 } else {449 op.src.type = CCP_MEMTYPE_SYSTEM;450 op.src.u.dma.address = wa->dma.address;451 op.src.u.dma.length = wa->length;452 op.dst.type = CCP_MEMTYPE_SB;453 op.dst.u.sb = sb;454 }455 456 op.u.passthru.byte_swap = byte_swap;457 458 return cmd_q->ccp->vdata->perform->passthru(&op);459}460 461static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,462 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,463 u32 byte_swap)464{465 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);466}467 468static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,469 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,470 u32 byte_swap)471{472 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);473}474 475static noinline_for_stack int476ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)477{478 struct ccp_aes_engine *aes = &cmd->u.aes;479 struct ccp_dm_workarea key, ctx;480 struct ccp_data src;481 struct ccp_op op;482 unsigned int dm_offset;483 int ret;484 485 if (!((aes->key_len == AES_KEYSIZE_128) ||486 (aes->key_len == AES_KEYSIZE_192) ||487 (aes->key_len == AES_KEYSIZE_256)))488 return -EINVAL;489 490 if (aes->src_len & (AES_BLOCK_SIZE - 1))491 return -EINVAL;492 493 if (aes->iv_len != AES_BLOCK_SIZE)494 return -EINVAL;495 496 if (!aes->key || !aes->iv || !aes->src)497 return -EINVAL;498 499 if (aes->cmac_final) {500 if (aes->cmac_key_len != AES_BLOCK_SIZE)501 return -EINVAL;502 503 if (!aes->cmac_key)504 return -EINVAL;505 }506 507 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);508 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);509 510 ret = -EIO;511 memset(&op, 0, sizeof(op));512 op.cmd_q = cmd_q;513 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);514 op.sb_key = cmd_q->sb_key;515 op.sb_ctx = cmd_q->sb_ctx;516 op.init = 1;517 op.u.aes.type = aes->type;518 op.u.aes.mode = aes->mode;519 op.u.aes.action = aes->action;520 521 /* All supported key sizes fit in a single (32-byte) SB entry522 * and must be in little endian format. Use the 256-bit byte523 * swap passthru option to convert from big endian to little524 * endian.525 */526 ret = ccp_init_dm_workarea(&key, cmd_q,527 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,528 DMA_TO_DEVICE);529 if (ret)530 return ret;531 532 dm_offset = CCP_SB_BYTES - aes->key_len;533 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);534 if (ret)535 goto e_key;536 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,537 CCP_PASSTHRU_BYTESWAP_256BIT);538 if (ret) {539 cmd->engine_error = cmd_q->cmd_error;540 goto e_key;541 }542 543 /* The AES context fits in a single (32-byte) SB entry and544 * must be in little endian format. Use the 256-bit byte swap545 * passthru option to convert from big endian to little endian.546 */547 ret = ccp_init_dm_workarea(&ctx, cmd_q,548 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,549 DMA_BIDIRECTIONAL);550 if (ret)551 goto e_key;552 553 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;554 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);555 if (ret)556 goto e_ctx;557 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,558 CCP_PASSTHRU_BYTESWAP_256BIT);559 if (ret) {560 cmd->engine_error = cmd_q->cmd_error;561 goto e_ctx;562 }563 564 /* Send data to the CCP AES engine */565 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,566 AES_BLOCK_SIZE, DMA_TO_DEVICE);567 if (ret)568 goto e_ctx;569 570 while (src.sg_wa.bytes_left) {571 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);572 if (aes->cmac_final && !src.sg_wa.bytes_left) {573 op.eom = 1;574 575 /* Push the K1/K2 key to the CCP now */576 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,577 op.sb_ctx,578 CCP_PASSTHRU_BYTESWAP_256BIT);579 if (ret) {580 cmd->engine_error = cmd_q->cmd_error;581 goto e_src;582 }583 584 ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,585 aes->cmac_key_len);586 if (ret)587 goto e_src;588 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,589 CCP_PASSTHRU_BYTESWAP_256BIT);590 if (ret) {591 cmd->engine_error = cmd_q->cmd_error;592 goto e_src;593 }594 }595 596 ret = cmd_q->ccp->vdata->perform->aes(&op);597 if (ret) {598 cmd->engine_error = cmd_q->cmd_error;599 goto e_src;600 }601 602 ccp_process_data(&src, NULL, &op);603 }604 605 /* Retrieve the AES context - convert from LE to BE using606 * 32-byte (256-bit) byteswapping607 */608 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,609 CCP_PASSTHRU_BYTESWAP_256BIT);610 if (ret) {611 cmd->engine_error = cmd_q->cmd_error;612 goto e_src;613 }614 615 /* ...but we only need AES_BLOCK_SIZE bytes */616 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;617 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);618 619e_src:620 ccp_free_data(&src, cmd_q);621 622e_ctx:623 ccp_dm_free(&ctx);624 625e_key:626 ccp_dm_free(&key);627 628 return ret;629}630 631static noinline_for_stack int632ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)633{634 struct ccp_aes_engine *aes = &cmd->u.aes;635 struct ccp_dm_workarea key, ctx, final_wa, tag;636 struct ccp_data src, dst;637 struct ccp_data aad;638 struct ccp_op op;639 unsigned int dm_offset;640 unsigned int authsize;641 unsigned int jobid;642 unsigned int ilen;643 bool in_place = true; /* Default value */644 __be64 *final;645 int ret;646 647 struct scatterlist *p_inp, sg_inp[2];648 struct scatterlist *p_tag, sg_tag[2];649 struct scatterlist *p_outp, sg_outp[2];650 struct scatterlist *p_aad;651 652 if (!aes->iv)653 return -EINVAL;654 655 if (!((aes->key_len == AES_KEYSIZE_128) ||656 (aes->key_len == AES_KEYSIZE_192) ||657 (aes->key_len == AES_KEYSIZE_256)))658 return -EINVAL;659 660 if (!aes->key) /* Gotta have a key SGL */661 return -EINVAL;662 663 /* Zero defaults to 16 bytes, the maximum size */664 authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;665 switch (authsize) {666 case 16:667 case 15:668 case 14:669 case 13:670 case 12:671 case 8:672 case 4:673 break;674 default:675 return -EINVAL;676 }677 678 /* First, decompose the source buffer into AAD & PT,679 * and the destination buffer into AAD, CT & tag, or680 * the input into CT & tag.681 * It is expected that the input and output SGs will682 * be valid, even if the AAD and input lengths are 0.683 */684 p_aad = aes->src;685 p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);686 p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);687 if (aes->action == CCP_AES_ACTION_ENCRYPT) {688 ilen = aes->src_len;689 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);690 } else {691 /* Input length for decryption includes tag */692 ilen = aes->src_len - authsize;693 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);694 }695 696 jobid = CCP_NEW_JOBID(cmd_q->ccp);697 698 memset(&op, 0, sizeof(op));699 op.cmd_q = cmd_q;700 op.jobid = jobid;701 op.sb_key = cmd_q->sb_key; /* Pre-allocated */702 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */703 op.init = 1;704 op.u.aes.type = aes->type;705 706 /* Copy the key to the LSB */707 ret = ccp_init_dm_workarea(&key, cmd_q,708 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,709 DMA_TO_DEVICE);710 if (ret)711 return ret;712 713 dm_offset = CCP_SB_BYTES - aes->key_len;714 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);715 if (ret)716 goto e_key;717 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,718 CCP_PASSTHRU_BYTESWAP_256BIT);719 if (ret) {720 cmd->engine_error = cmd_q->cmd_error;721 goto e_key;722 }723 724 /* Copy the context (IV) to the LSB.725 * There is an assumption here that the IV is 96 bits in length, plus726 * a nonce of 32 bits. If no IV is present, use a zeroed buffer.727 */728 ret = ccp_init_dm_workarea(&ctx, cmd_q,729 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,730 DMA_BIDIRECTIONAL);731 if (ret)732 goto e_key;733 734 dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;735 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);736 if (ret)737 goto e_ctx;738 739 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,740 CCP_PASSTHRU_BYTESWAP_256BIT);741 if (ret) {742 cmd->engine_error = cmd_q->cmd_error;743 goto e_ctx;744 }745 746 op.init = 1;747 if (aes->aad_len > 0) {748 /* Step 1: Run a GHASH over the Additional Authenticated Data */749 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,750 AES_BLOCK_SIZE,751 DMA_TO_DEVICE);752 if (ret)753 goto e_ctx;754 755 op.u.aes.mode = CCP_AES_MODE_GHASH;756 op.u.aes.action = CCP_AES_GHASHAAD;757 758 while (aad.sg_wa.bytes_left) {759 ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);760 761 ret = cmd_q->ccp->vdata->perform->aes(&op);762 if (ret) {763 cmd->engine_error = cmd_q->cmd_error;764 goto e_aad;765 }766 767 ccp_process_data(&aad, NULL, &op);768 op.init = 0;769 }770 }771 772 op.u.aes.mode = CCP_AES_MODE_GCTR;773 op.u.aes.action = aes->action;774 775 if (ilen > 0) {776 /* Step 2: Run a GCTR over the plaintext */777 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;778 779 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,780 AES_BLOCK_SIZE,781 in_place ? DMA_BIDIRECTIONAL782 : DMA_TO_DEVICE);783 if (ret)784 goto e_aad;785 786 if (in_place) {787 dst = src;788 } else {789 ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,790 AES_BLOCK_SIZE, DMA_FROM_DEVICE);791 if (ret)792 goto e_src;793 }794 795 op.soc = 0;796 op.eom = 0;797 op.init = 1;798 while (src.sg_wa.bytes_left) {799 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);800 if (!src.sg_wa.bytes_left) {801 unsigned int nbytes = ilen % AES_BLOCK_SIZE;802 803 if (nbytes) {804 op.eom = 1;805 op.u.aes.size = (nbytes * 8) - 1;806 }807 }808 809 ret = cmd_q->ccp->vdata->perform->aes(&op);810 if (ret) {811 cmd->engine_error = cmd_q->cmd_error;812 goto e_dst;813 }814 815 ccp_process_data(&src, &dst, &op);816 op.init = 0;817 }818 }819 820 /* Step 3: Update the IV portion of the context with the original IV */821 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,822 CCP_PASSTHRU_BYTESWAP_256BIT);823 if (ret) {824 cmd->engine_error = cmd_q->cmd_error;825 goto e_dst;826 }827 828 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);829 if (ret)830 goto e_dst;831 832 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,833 CCP_PASSTHRU_BYTESWAP_256BIT);834 if (ret) {835 cmd->engine_error = cmd_q->cmd_error;836 goto e_dst;837 }838 839 /* Step 4: Concatenate the lengths of the AAD and source, and840 * hash that 16 byte buffer.841 */842 ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,843 DMA_BIDIRECTIONAL);844 if (ret)845 goto e_dst;846 final = (__be64 *)final_wa.address;847 final[0] = cpu_to_be64(aes->aad_len * 8);848 final[1] = cpu_to_be64(ilen * 8);849 850 memset(&op, 0, sizeof(op));851 op.cmd_q = cmd_q;852 op.jobid = jobid;853 op.sb_key = cmd_q->sb_key; /* Pre-allocated */854 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */855 op.init = 1;856 op.u.aes.type = aes->type;857 op.u.aes.mode = CCP_AES_MODE_GHASH;858 op.u.aes.action = CCP_AES_GHASHFINAL;859 op.src.type = CCP_MEMTYPE_SYSTEM;860 op.src.u.dma.address = final_wa.dma.address;861 op.src.u.dma.length = AES_BLOCK_SIZE;862 op.dst.type = CCP_MEMTYPE_SYSTEM;863 op.dst.u.dma.address = final_wa.dma.address;864 op.dst.u.dma.length = AES_BLOCK_SIZE;865 op.eom = 1;866 op.u.aes.size = 0;867 ret = cmd_q->ccp->vdata->perform->aes(&op);868 if (ret)869 goto e_final_wa;870 871 if (aes->action == CCP_AES_ACTION_ENCRYPT) {872 /* Put the ciphered tag after the ciphertext. */873 ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);874 } else {875 /* Does this ciphered tag match the input? */876 ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,877 DMA_BIDIRECTIONAL);878 if (ret)879 goto e_final_wa;880 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);881 if (ret) {882 ccp_dm_free(&tag);883 goto e_final_wa;884 }885 886 ret = crypto_memneq(tag.address, final_wa.address,887 authsize) ? -EBADMSG : 0;888 ccp_dm_free(&tag);889 }890 891e_final_wa:892 ccp_dm_free(&final_wa);893 894e_dst:895 if (ilen > 0 && !in_place)896 ccp_free_data(&dst, cmd_q);897 898e_src:899 if (ilen > 0)900 ccp_free_data(&src, cmd_q);901 902e_aad:903 if (aes->aad_len)904 ccp_free_data(&aad, cmd_q);905 906e_ctx:907 ccp_dm_free(&ctx);908 909e_key:910 ccp_dm_free(&key);911 912 return ret;913}914 915static noinline_for_stack int916ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)917{918 struct ccp_aes_engine *aes = &cmd->u.aes;919 struct ccp_dm_workarea key, ctx;920 struct ccp_data src, dst;921 struct ccp_op op;922 unsigned int dm_offset;923 bool in_place = false;924 int ret;925 926 if (!((aes->key_len == AES_KEYSIZE_128) ||927 (aes->key_len == AES_KEYSIZE_192) ||928 (aes->key_len == AES_KEYSIZE_256)))929 return -EINVAL;930 931 if (((aes->mode == CCP_AES_MODE_ECB) ||932 (aes->mode == CCP_AES_MODE_CBC)) &&933 (aes->src_len & (AES_BLOCK_SIZE - 1)))934 return -EINVAL;935 936 if (!aes->key || !aes->src || !aes->dst)937 return -EINVAL;938 939 if (aes->mode != CCP_AES_MODE_ECB) {940 if (aes->iv_len != AES_BLOCK_SIZE)941 return -EINVAL;942 943 if (!aes->iv)944 return -EINVAL;945 }946 947 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);948 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);949 950 ret = -EIO;951 memset(&op, 0, sizeof(op));952 op.cmd_q = cmd_q;953 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);954 op.sb_key = cmd_q->sb_key;955 op.sb_ctx = cmd_q->sb_ctx;956 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;957 op.u.aes.type = aes->type;958 op.u.aes.mode = aes->mode;959 op.u.aes.action = aes->action;960 961 /* All supported key sizes fit in a single (32-byte) SB entry962 * and must be in little endian format. Use the 256-bit byte963 * swap passthru option to convert from big endian to little964 * endian.965 */966 ret = ccp_init_dm_workarea(&key, cmd_q,967 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,968 DMA_TO_DEVICE);969 if (ret)970 return ret;971 972 dm_offset = CCP_SB_BYTES - aes->key_len;973 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);974 if (ret)975 goto e_key;976 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,977 CCP_PASSTHRU_BYTESWAP_256BIT);978 if (ret) {979 cmd->engine_error = cmd_q->cmd_error;980 goto e_key;981 }982 983 /* The AES context fits in a single (32-byte) SB entry and984 * must be in little endian format. Use the 256-bit byte swap985 * passthru option to convert from big endian to little endian.986 */987 ret = ccp_init_dm_workarea(&ctx, cmd_q,988 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,989 DMA_BIDIRECTIONAL);990 if (ret)991 goto e_key;992 993 if (aes->mode != CCP_AES_MODE_ECB) {994 /* Load the AES context - convert to LE */995 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;996 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);997 if (ret)998 goto e_ctx;999 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,1000 CCP_PASSTHRU_BYTESWAP_256BIT);1001 if (ret) {1002 cmd->engine_error = cmd_q->cmd_error;1003 goto e_ctx;1004 }1005 }1006 switch (aes->mode) {1007 case CCP_AES_MODE_CFB: /* CFB128 only */1008 case CCP_AES_MODE_CTR:1009 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;1010 break;1011 default:1012 op.u.aes.size = 0;1013 }1014 1015 /* Prepare the input and output data workareas. For in-place1016 * operations we need to set the dma direction to BIDIRECTIONAL1017 * and copy the src workarea to the dst workarea.1018 */1019 if (sg_virt(aes->src) == sg_virt(aes->dst))1020 in_place = true;1021 1022 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,1023 AES_BLOCK_SIZE,1024 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);1025 if (ret)1026 goto e_ctx;1027 1028 if (in_place) {1029 dst = src;1030 } else {1031 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,1032 AES_BLOCK_SIZE, DMA_FROM_DEVICE);1033 if (ret)1034 goto e_src;1035 }1036 1037 /* Send data to the CCP AES engine */1038 while (src.sg_wa.bytes_left) {1039 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);1040 if (!src.sg_wa.bytes_left) {1041 op.eom = 1;1042 1043 /* Since we don't retrieve the AES context in ECB1044 * mode we have to wait for the operation to complete1045 * on the last piece of data1046 */1047 if (aes->mode == CCP_AES_MODE_ECB)1048 op.soc = 1;1049 }1050 1051 ret = cmd_q->ccp->vdata->perform->aes(&op);1052 if (ret) {1053 cmd->engine_error = cmd_q->cmd_error;1054 goto e_dst;1055 }1056 1057 ccp_process_data(&src, &dst, &op);1058 }1059 1060 if (aes->mode != CCP_AES_MODE_ECB) {1061 /* Retrieve the AES context - convert from LE to BE using1062 * 32-byte (256-bit) byteswapping1063 */1064 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,1065 CCP_PASSTHRU_BYTESWAP_256BIT);1066 if (ret) {1067 cmd->engine_error = cmd_q->cmd_error;1068 goto e_dst;1069 }1070 1071 /* ...but we only need AES_BLOCK_SIZE bytes */1072 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;1073 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);1074 }1075 1076e_dst:1077 if (!in_place)1078 ccp_free_data(&dst, cmd_q);1079 1080e_src:1081 ccp_free_data(&src, cmd_q);1082 1083e_ctx:1084 ccp_dm_free(&ctx);1085 1086e_key:1087 ccp_dm_free(&key);1088 1089 return ret;1090}1091 1092static noinline_for_stack int1093ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)1094{1095 struct ccp_xts_aes_engine *xts = &cmd->u.xts;1096 struct ccp_dm_workarea key, ctx;1097 struct ccp_data src, dst;1098 struct ccp_op op;1099 unsigned int unit_size, dm_offset;1100 bool in_place = false;1101 unsigned int sb_count;1102 enum ccp_aes_type aestype;1103 int ret;1104 1105 switch (xts->unit_size) {1106 case CCP_XTS_AES_UNIT_SIZE_16:1107 unit_size = 16;1108 break;1109 case CCP_XTS_AES_UNIT_SIZE_512:1110 unit_size = 512;1111 break;1112 case CCP_XTS_AES_UNIT_SIZE_1024:1113 unit_size = 1024;1114 break;1115 case CCP_XTS_AES_UNIT_SIZE_2048:1116 unit_size = 2048;1117 break;1118 case CCP_XTS_AES_UNIT_SIZE_4096:1119 unit_size = 4096;1120 break;1121 1122 default:1123 return -EINVAL;1124 }1125 1126 if (xts->key_len == AES_KEYSIZE_128)1127 aestype = CCP_AES_TYPE_128;1128 else if (xts->key_len == AES_KEYSIZE_256)1129 aestype = CCP_AES_TYPE_256;1130 else1131 return -EINVAL;1132 1133 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))1134 return -EINVAL;1135 1136 if (xts->iv_len != AES_BLOCK_SIZE)1137 return -EINVAL;1138 1139 if (!xts->key || !xts->iv || !xts->src || !xts->dst)1140 return -EINVAL;1141 1142 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);1143 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);1144 1145 ret = -EIO;1146 memset(&op, 0, sizeof(op));1147 op.cmd_q = cmd_q;1148 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);1149 op.sb_key = cmd_q->sb_key;1150 op.sb_ctx = cmd_q->sb_ctx;1151 op.init = 1;1152 op.u.xts.type = aestype;1153 op.u.xts.action = xts->action;1154 op.u.xts.unit_size = xts->unit_size;1155 1156 /* A version 3 device only supports 128-bit keys, which fits into a1157 * single SB entry. A version 5 device uses a 512-bit vector, so two1158 * SB entries.1159 */1160 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))1161 sb_count = CCP_XTS_AES_KEY_SB_COUNT;1162 else1163 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;1164 ret = ccp_init_dm_workarea(&key, cmd_q,1165 sb_count * CCP_SB_BYTES,1166 DMA_TO_DEVICE);1167 if (ret)1168 return ret;1169 1170 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {1171 /* All supported key sizes must be in little endian format.1172 * Use the 256-bit byte swap passthru option to convert from1173 * big endian to little endian.1174 */1175 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;1176 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);1177 if (ret)1178 goto e_key;1179 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);1180 if (ret)1181 goto e_key;1182 } else {1183 /* Version 5 CCPs use a 512-bit space for the key: each portion1184 * occupies 256 bits, or one entire slot, and is zero-padded.1185 */1186 unsigned int pad;1187 1188 dm_offset = CCP_SB_BYTES;1189 pad = dm_offset - xts->key_len;1190 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);1191 if (ret)1192 goto e_key;1193 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,1194 xts->key_len, xts->key_len);1195 if (ret)1196 goto e_key;1197 }1198 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,1199 CCP_PASSTHRU_BYTESWAP_256BIT);1200 if (ret) {1201 cmd->engine_error = cmd_q->cmd_error;1202 goto e_key;1203 }1204 1205 /* The AES context fits in a single (32-byte) SB entry and1206 * for XTS is already in little endian format so no byte swapping1207 * is needed.1208 */1209 ret = ccp_init_dm_workarea(&ctx, cmd_q,1210 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,1211 DMA_BIDIRECTIONAL);1212 if (ret)1213 goto e_key;1214 1215 ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);1216 if (ret)1217 goto e_ctx;1218 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,1219 CCP_PASSTHRU_BYTESWAP_NOOP);1220 if (ret) {1221 cmd->engine_error = cmd_q->cmd_error;1222 goto e_ctx;1223 }1224 1225 /* Prepare the input and output data workareas. For in-place1226 * operations we need to set the dma direction to BIDIRECTIONAL1227 * and copy the src workarea to the dst workarea.1228 */1229 if (sg_virt(xts->src) == sg_virt(xts->dst))1230 in_place = true;1231 1232 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,1233 unit_size,1234 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);1235 if (ret)1236 goto e_ctx;1237 1238 if (in_place) {1239 dst = src;1240 } else {1241 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,1242 unit_size, DMA_FROM_DEVICE);1243 if (ret)1244 goto e_src;1245 }1246 1247 /* Send data to the CCP AES engine */1248 while (src.sg_wa.bytes_left) {1249 ccp_prepare_data(&src, &dst, &op, unit_size, true);1250 if (!src.sg_wa.bytes_left)1251 op.eom = 1;1252 1253 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);1254 if (ret) {1255 cmd->engine_error = cmd_q->cmd_error;1256 goto e_dst;1257 }1258 1259 ccp_process_data(&src, &dst, &op);1260 }1261 1262 /* Retrieve the AES context - convert from LE to BE using1263 * 32-byte (256-bit) byteswapping1264 */1265 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,1266 CCP_PASSTHRU_BYTESWAP_256BIT);1267 if (ret) {1268 cmd->engine_error = cmd_q->cmd_error;1269 goto e_dst;1270 }1271 1272 /* ...but we only need AES_BLOCK_SIZE bytes */1273 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;1274 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);1275 1276e_dst:1277 if (!in_place)1278 ccp_free_data(&dst, cmd_q);1279 1280e_src:1281 ccp_free_data(&src, cmd_q);1282 1283e_ctx:1284 ccp_dm_free(&ctx);1285 1286e_key:1287 ccp_dm_free(&key);1288 1289 return ret;1290}1291 1292static noinline_for_stack int1293ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)1294{1295 struct ccp_des3_engine *des3 = &cmd->u.des3;1296 1297 struct ccp_dm_workarea key, ctx;1298 struct ccp_data src, dst;1299 struct ccp_op op;1300 unsigned int dm_offset;1301 unsigned int len_singlekey;1302 bool in_place = false;1303 int ret;1304 1305 /* Error checks */1306 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))1307 return -EINVAL;1308 1309 if (!cmd_q->ccp->vdata->perform->des3)1310 return -EINVAL;1311 1312 if (des3->key_len != DES3_EDE_KEY_SIZE)1313 return -EINVAL;1314 1315 if (((des3->mode == CCP_DES3_MODE_ECB) ||1316 (des3->mode == CCP_DES3_MODE_CBC)) &&1317 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))1318 return -EINVAL;1319 1320 if (!des3->key || !des3->src || !des3->dst)1321 return -EINVAL;1322 1323 if (des3->mode != CCP_DES3_MODE_ECB) {1324 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)1325 return -EINVAL;1326 1327 if (!des3->iv)1328 return -EINVAL;1329 }1330 1331 /* Zero out all the fields of the command desc */1332 memset(&op, 0, sizeof(op));1333 1334 /* Set up the Function field */1335 op.cmd_q = cmd_q;1336 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);1337 op.sb_key = cmd_q->sb_key;1338 1339 op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;1340 op.u.des3.type = des3->type;1341 op.u.des3.mode = des3->mode;1342 op.u.des3.action = des3->action;1343 1344 /*1345 * All supported key sizes fit in a single (32-byte) KSB entry and1346 * (like AES) must be in little endian format. Use the 256-bit byte1347 * swap passthru option to convert from big endian to little endian.1348 */1349 ret = ccp_init_dm_workarea(&key, cmd_q,1350 CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,1351 DMA_TO_DEVICE);1352 if (ret)1353 return ret;1354 1355 /*1356 * The contents of the key triplet are in the reverse order of what1357 * is required by the engine. Copy the 3 pieces individually to put1358 * them where they belong.1359 */1360 dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */1361 1362 len_singlekey = des3->key_len / 3;1363 ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,1364 des3->key, 0, len_singlekey);1365 if (ret)1366 goto e_key;1367 ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,1368 des3->key, len_singlekey, len_singlekey);1369 if (ret)1370 goto e_key;1371 ret = ccp_set_dm_area(&key, dm_offset,1372 des3->key, 2 * len_singlekey, len_singlekey);1373 if (ret)1374 goto e_key;1375 1376 /* Copy the key to the SB */1377 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,1378 CCP_PASSTHRU_BYTESWAP_256BIT);1379 if (ret) {1380 cmd->engine_error = cmd_q->cmd_error;1381 goto e_key;1382 }1383 1384 /*1385 * The DES3 context fits in a single (32-byte) KSB entry and1386 * must be in little endian format. Use the 256-bit byte swap1387 * passthru option to convert from big endian to little endian.1388 */1389 if (des3->mode != CCP_DES3_MODE_ECB) {1390 op.sb_ctx = cmd_q->sb_ctx;1391 1392 ret = ccp_init_dm_workarea(&ctx, cmd_q,1393 CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,1394 DMA_BIDIRECTIONAL);1395 if (ret)1396 goto e_key;1397 1398 /* Load the context into the LSB */1399 dm_offset = CCP_SB_BYTES - des3->iv_len;1400 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,1401 des3->iv_len);1402 if (ret)1403 goto e_ctx;1404 1405 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,1406 CCP_PASSTHRU_BYTESWAP_256BIT);1407 if (ret) {1408 cmd->engine_error = cmd_q->cmd_error;1409 goto e_ctx;1410 }1411 }1412 1413 /*1414 * Prepare the input and output data workareas. For in-place1415 * operations we need to set the dma direction to BIDIRECTIONAL1416 * and copy the src workarea to the dst workarea.1417 */1418 if (sg_virt(des3->src) == sg_virt(des3->dst))1419 in_place = true;1420 1421 ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,1422 DES3_EDE_BLOCK_SIZE,1423 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);1424 if (ret)1425 goto e_ctx;1426 1427 if (in_place)1428 dst = src;1429 else {1430 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,1431 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);1432 if (ret)1433 goto e_src;1434 }1435 1436 /* Send data to the CCP DES3 engine */1437 while (src.sg_wa.bytes_left) {1438 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);1439 if (!src.sg_wa.bytes_left) {1440 op.eom = 1;1441 1442 /* Since we don't retrieve the context in ECB mode1443 * we have to wait for the operation to complete1444 * on the last piece of data1445 */1446 op.soc = 0;1447 }1448 1449 ret = cmd_q->ccp->vdata->perform->des3(&op);1450 if (ret) {1451 cmd->engine_error = cmd_q->cmd_error;1452 goto e_dst;1453 }1454 1455 ccp_process_data(&src, &dst, &op);1456 }1457 1458 if (des3->mode != CCP_DES3_MODE_ECB) {1459 /* Retrieve the context and make BE */1460 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,1461 CCP_PASSTHRU_BYTESWAP_256BIT);1462 if (ret) {1463 cmd->engine_error = cmd_q->cmd_error;1464 goto e_dst;1465 }1466 1467 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */1468 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,1469 DES3_EDE_BLOCK_SIZE);1470 }1471e_dst:1472 if (!in_place)1473 ccp_free_data(&dst, cmd_q);1474 1475e_src:1476 ccp_free_data(&src, cmd_q);1477 1478e_ctx:1479 if (des3->mode != CCP_DES3_MODE_ECB)1480 ccp_dm_free(&ctx);1481 1482e_key:1483 ccp_dm_free(&key);1484 1485 return ret;1486}1487 1488static noinline_for_stack int1489ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)1490{1491 struct ccp_sha_engine *sha = &cmd->u.sha;1492 struct ccp_dm_workarea ctx;1493 struct ccp_data src;1494 struct ccp_op op;1495 unsigned int ioffset, ooffset;1496 unsigned int digest_size;1497 int sb_count;1498 const void *init;1499 u64 block_size;1500 int ctx_size;1501 int ret;1502 1503 switch (sha->type) {1504 case CCP_SHA_TYPE_1:1505 if (sha->ctx_len < SHA1_DIGEST_SIZE)1506 return -EINVAL;1507 block_size = SHA1_BLOCK_SIZE;1508 break;1509 case CCP_SHA_TYPE_224:1510 if (sha->ctx_len < SHA224_DIGEST_SIZE)1511 return -EINVAL;1512 block_size = SHA224_BLOCK_SIZE;1513 break;1514 case CCP_SHA_TYPE_256:1515 if (sha->ctx_len < SHA256_DIGEST_SIZE)1516 return -EINVAL;1517 block_size = SHA256_BLOCK_SIZE;1518 break;1519 case CCP_SHA_TYPE_384:1520 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)1521 || sha->ctx_len < SHA384_DIGEST_SIZE)1522 return -EINVAL;1523 block_size = SHA384_BLOCK_SIZE;1524 break;1525 case CCP_SHA_TYPE_512:1526 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)1527 || sha->ctx_len < SHA512_DIGEST_SIZE)1528 return -EINVAL;1529 block_size = SHA512_BLOCK_SIZE;1530 break;1531 default:1532 return -EINVAL;1533 }1534 1535 if (!sha->ctx)1536 return -EINVAL;1537 1538 if (!sha->final && (sha->src_len & (block_size - 1)))1539 return -EINVAL;1540 1541 /* The version 3 device can't handle zero-length input */1542 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {1543 1544 if (!sha->src_len) {1545 unsigned int digest_len;1546 const u8 *sha_zero;1547 1548 /* Not final, just return */1549 if (!sha->final)1550 return 0;1551 1552 /* CCP can't do a zero length sha operation so the1553 * caller must buffer the data.1554 */1555 if (sha->msg_bits)1556 return -EINVAL;1557 1558 /* The CCP cannot perform zero-length sha operations1559 * so the caller is required to buffer data for the1560 * final operation. However, a sha operation for a1561 * message with a total length of zero is valid so1562 * known values are required to supply the result.1563 */1564 switch (sha->type) {1565 case CCP_SHA_TYPE_1:1566 sha_zero = sha1_zero_message_hash;1567 digest_len = SHA1_DIGEST_SIZE;1568 break;1569 case CCP_SHA_TYPE_224:1570 sha_zero = sha224_zero_message_hash;1571 digest_len = SHA224_DIGEST_SIZE;1572 break;1573 case CCP_SHA_TYPE_256:1574 sha_zero = sha256_zero_message_hash;1575 digest_len = SHA256_DIGEST_SIZE;1576 break;1577 default:1578 return -EINVAL;1579 }1580 1581 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,1582 digest_len, 1);1583 1584 return 0;1585 }1586 }1587 1588 /* Set variables used throughout */1589 switch (sha->type) {1590 case CCP_SHA_TYPE_1:1591 digest_size = SHA1_DIGEST_SIZE;1592 init = (void *) ccp_sha1_init;1593 ctx_size = SHA1_DIGEST_SIZE;1594 sb_count = 1;1595 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))1596 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;1597 else1598 ooffset = ioffset = 0;1599 break;1600 case CCP_SHA_TYPE_224:1601 digest_size = SHA224_DIGEST_SIZE;1602 init = (void *) ccp_sha224_init;1603 ctx_size = SHA256_DIGEST_SIZE;1604 sb_count = 1;1605 ioffset = 0;1606 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))1607 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;1608 else1609 ooffset = 0;1610 break;1611 case CCP_SHA_TYPE_256:1612 digest_size = SHA256_DIGEST_SIZE;1613 init = (void *) ccp_sha256_init;1614 ctx_size = SHA256_DIGEST_SIZE;1615 sb_count = 1;1616 ooffset = ioffset = 0;1617 break;1618 case CCP_SHA_TYPE_384:1619 digest_size = SHA384_DIGEST_SIZE;1620 init = (void *) ccp_sha384_init;1621 ctx_size = SHA512_DIGEST_SIZE;1622 sb_count = 2;1623 ioffset = 0;1624 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;1625 break;1626 case CCP_SHA_TYPE_512:1627 digest_size = SHA512_DIGEST_SIZE;1628 init = (void *) ccp_sha512_init;1629 ctx_size = SHA512_DIGEST_SIZE;1630 sb_count = 2;1631 ooffset = ioffset = 0;1632 break;1633 default:1634 ret = -EINVAL;1635 goto e_data;1636 }1637 1638 /* For zero-length plaintext the src pointer is ignored;1639 * otherwise both parts must be valid1640 */1641 if (sha->src_len && !sha->src)1642 return -EINVAL;1643 1644 memset(&op, 0, sizeof(op));1645 op.cmd_q = cmd_q;1646 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);1647 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */1648 op.u.sha.type = sha->type;1649 op.u.sha.msg_bits = sha->msg_bits;1650 1651 /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;1652 * SHA384/512 require 2 adjacent SB slots, with the right half in the1653 * first slot, and the left half in the second. Each portion must then1654 * be in little endian format: use the 256-bit byte swap option.1655 */1656 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,1657 DMA_BIDIRECTIONAL);1658 if (ret)1659 return ret;1660 if (sha->first) {1661 switch (sha->type) {1662 case CCP_SHA_TYPE_1:1663 case CCP_SHA_TYPE_224:1664 case CCP_SHA_TYPE_256:1665 memcpy(ctx.address + ioffset, init, ctx_size);1666 break;1667 case CCP_SHA_TYPE_384:1668 case CCP_SHA_TYPE_512:1669 memcpy(ctx.address + ctx_size / 2, init,1670 ctx_size / 2);1671 memcpy(ctx.address, init + ctx_size / 2,1672 ctx_size / 2);1673 break;1674 default:1675 ret = -EINVAL;1676 goto e_ctx;1677 }1678 } else {1679 /* Restore the context */1680 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,1681 sb_count * CCP_SB_BYTES);1682 if (ret)1683 goto e_ctx;1684 }1685 1686 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,1687 CCP_PASSTHRU_BYTESWAP_256BIT);1688 if (ret) {1689 cmd->engine_error = cmd_q->cmd_error;1690 goto e_ctx;1691 }1692 1693 if (sha->src) {1694 /* Send data to the CCP SHA engine; block_size is set above */1695 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,1696 block_size, DMA_TO_DEVICE);1697 if (ret)1698 goto e_ctx;1699 1700 while (src.sg_wa.bytes_left) {1701 ccp_prepare_data(&src, NULL, &op, block_size, false);1702 if (sha->final && !src.sg_wa.bytes_left)1703 op.eom = 1;1704 1705 ret = cmd_q->ccp->vdata->perform->sha(&op);1706 if (ret) {1707 cmd->engine_error = cmd_q->cmd_error;1708 goto e_data;1709 }1710 1711 ccp_process_data(&src, NULL, &op);1712 }1713 } else {1714 op.eom = 1;1715 ret = cmd_q->ccp->vdata->perform->sha(&op);1716 if (ret) {1717 cmd->engine_error = cmd_q->cmd_error;1718 goto e_data;1719 }1720 }1721 1722 /* Retrieve the SHA context - convert from LE to BE using1723 * 32-byte (256-bit) byteswapping to BE1724 */1725 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,1726 CCP_PASSTHRU_BYTESWAP_256BIT);1727 if (ret) {1728 cmd->engine_error = cmd_q->cmd_error;1729 goto e_data;1730 }1731 1732 if (sha->final) {1733 /* Finishing up, so get the digest */1734 switch (sha->type) {1735 case CCP_SHA_TYPE_1:1736 case CCP_SHA_TYPE_224:1737 case CCP_SHA_TYPE_256:1738 ccp_get_dm_area(&ctx, ooffset,1739 sha->ctx, 0,1740 digest_size);1741 break;1742 case CCP_SHA_TYPE_384:1743 case CCP_SHA_TYPE_512:1744 ccp_get_dm_area(&ctx, 0,1745 sha->ctx, LSB_ITEM_SIZE - ooffset,1746 LSB_ITEM_SIZE);1747 ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,1748 sha->ctx, 0,1749 LSB_ITEM_SIZE - ooffset);1750 break;1751 default:1752 ret = -EINVAL;1753 goto e_data;1754 }1755 } else {1756 /* Stash the context */1757 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,1758 sb_count * CCP_SB_BYTES);1759 }1760 1761 if (sha->final && sha->opad) {1762 /* HMAC operation, recursively perform final SHA */1763 struct ccp_cmd hmac_cmd;1764 struct scatterlist sg;1765 u8 *hmac_buf;1766 1767 if (sha->opad_len != block_size) {1768 ret = -EINVAL;1769 goto e_data;1770 }1771 1772 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);1773 if (!hmac_buf) {1774 ret = -ENOMEM;1775 goto e_data;1776 }1777 sg_init_one(&sg, hmac_buf, block_size + digest_size);1778 1779 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);1780 switch (sha->type) {1781 case CCP_SHA_TYPE_1:1782 case CCP_SHA_TYPE_224:1783 case CCP_SHA_TYPE_256:1784 memcpy(hmac_buf + block_size,1785 ctx.address + ooffset,1786 digest_size);1787 break;1788 case CCP_SHA_TYPE_384:1789 case CCP_SHA_TYPE_512:1790 memcpy(hmac_buf + block_size,1791 ctx.address + LSB_ITEM_SIZE + ooffset,1792 LSB_ITEM_SIZE);1793 memcpy(hmac_buf + block_size +1794 (LSB_ITEM_SIZE - ooffset),1795 ctx.address,1796 LSB_ITEM_SIZE);1797 break;1798 default:1799 kfree(hmac_buf);1800 ret = -EINVAL;1801 goto e_data;1802 }1803 1804 memset(&hmac_cmd, 0, sizeof(hmac_cmd));1805 hmac_cmd.engine = CCP_ENGINE_SHA;1806 hmac_cmd.u.sha.type = sha->type;1807 hmac_cmd.u.sha.ctx = sha->ctx;1808 hmac_cmd.u.sha.ctx_len = sha->ctx_len;1809 hmac_cmd.u.sha.src = &sg;1810 hmac_cmd.u.sha.src_len = block_size + digest_size;1811 hmac_cmd.u.sha.opad = NULL;1812 hmac_cmd.u.sha.opad_len = 0;1813 hmac_cmd.u.sha.first = 1;1814 hmac_cmd.u.sha.final = 1;1815 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;1816 1817 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);1818 if (ret)1819 cmd->engine_error = hmac_cmd.engine_error;1820 1821 kfree(hmac_buf);1822 }1823 1824e_data:1825 if (sha->src)1826 ccp_free_data(&src, cmd_q);1827 1828e_ctx:1829 ccp_dm_free(&ctx);1830 1831 return ret;1832}1833 1834static noinline_for_stack int1835ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)1836{1837 struct ccp_rsa_engine *rsa = &cmd->u.rsa;1838 struct ccp_dm_workarea exp, src, dst;1839 struct ccp_op op;1840 unsigned int sb_count, i_len, o_len;1841 int ret;1842 1843 /* Check against the maximum allowable size, in bits */1844 if (rsa->key_size > cmd_q->ccp->vdata->rsamax)1845 return -EINVAL;1846 1847 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)1848 return -EINVAL;1849 1850 memset(&op, 0, sizeof(op));1851 op.cmd_q = cmd_q;1852 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);1853 1854 /* The RSA modulus must precede the message being acted upon, so1855 * it must be copied to a DMA area where the message and the1856 * modulus can be concatenated. Therefore the input buffer1857 * length required is twice the output buffer length (which1858 * must be a multiple of 256-bits). Compute o_len, i_len in bytes.1859 * Buffer sizes must be a multiple of 32 bytes; rounding up may be1860 * required.1861 */1862 o_len = 32 * ((rsa->key_size + 255) / 256);1863 i_len = o_len * 2;1864 1865 sb_count = 0;1866 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {1867 /* sb_count is the number of storage block slots required1868 * for the modulus.1869 */1870 sb_count = o_len / CCP_SB_BYTES;1871 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,1872 sb_count);1873 if (!op.sb_key)1874 return -EIO;1875 } else {1876 /* A version 5 device allows a modulus size that will not fit1877 * in the LSB, so the command will transfer it from memory.1878 * Set the sb key to the default, even though it's not used.1879 */1880 op.sb_key = cmd_q->sb_key;1881 }1882 1883 /* The RSA exponent must be in little endian format. Reverse its1884 * byte order.1885 */1886 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);1887 if (ret)1888 goto e_sb;1889 1890 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);1891 if (ret)1892 goto e_exp;1893 1894 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {1895 /* Copy the exponent to the local storage block, using1896 * as many 32-byte blocks as were allocated above. It's1897 * already little endian, so no further change is required.1898 */1899 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,1900 CCP_PASSTHRU_BYTESWAP_NOOP);1901 if (ret) {1902 cmd->engine_error = cmd_q->cmd_error;1903 goto e_exp;1904 }1905 } else {1906 /* The exponent can be retrieved from memory via DMA. */1907 op.exp.u.dma.address = exp.dma.address;1908 op.exp.u.dma.offset = 0;1909 }1910 1911 /* Concatenate the modulus and the message. Both the modulus and1912 * the operands must be in little endian format. Since the input1913 * is in big endian format it must be converted.1914 */1915 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);1916 if (ret)1917 goto e_exp;1918 1919 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);1920 if (ret)1921 goto e_src;1922 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);1923 if (ret)1924 goto e_src;1925 1926 /* Prepare the output area for the operation */1927 ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);1928 if (ret)1929 goto e_src;1930 1931 op.soc = 1;1932 op.src.u.dma.address = src.dma.address;1933 op.src.u.dma.offset = 0;1934 op.src.u.dma.length = i_len;1935 op.dst.u.dma.address = dst.dma.address;1936 op.dst.u.dma.offset = 0;1937 op.dst.u.dma.length = o_len;1938 1939 op.u.rsa.mod_size = rsa->key_size;1940 op.u.rsa.input_len = i_len;1941 1942 ret = cmd_q->ccp->vdata->perform->rsa(&op);1943 if (ret) {1944 cmd->engine_error = cmd_q->cmd_error;1945 goto e_dst;1946 }1947 1948 ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);1949 1950e_dst:1951 ccp_dm_free(&dst);1952 1953e_src:1954 ccp_dm_free(&src);1955 1956e_exp:1957 ccp_dm_free(&exp);1958 1959e_sb:1960 if (sb_count)1961 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);1962 1963 return ret;1964}1965 1966static noinline_for_stack int1967ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)1968{1969 struct ccp_passthru_engine *pt = &cmd->u.passthru;1970 struct ccp_dm_workarea mask;1971 struct ccp_data src, dst;1972 struct ccp_op op;1973 bool in_place = false;1974 unsigned int i;1975 int ret = 0;1976 1977 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))1978 return -EINVAL;1979 1980 if (!pt->src || !pt->dst)1981 return -EINVAL;1982 1983 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {1984 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)1985 return -EINVAL;1986 if (!pt->mask)1987 return -EINVAL;1988 }1989 1990 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);1991 1992 memset(&op, 0, sizeof(op));1993 op.cmd_q = cmd_q;1994 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);1995 1996 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {1997 /* Load the mask */1998 op.sb_key = cmd_q->sb_key;1999 2000 ret = ccp_init_dm_workarea(&mask, cmd_q,2001 CCP_PASSTHRU_SB_COUNT *2002 CCP_SB_BYTES,2003 DMA_TO_DEVICE);2004 if (ret)2005 return ret;2006 2007 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);2008 if (ret)2009 goto e_mask;2010 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,2011 CCP_PASSTHRU_BYTESWAP_NOOP);2012 if (ret) {2013 cmd->engine_error = cmd_q->cmd_error;2014 goto e_mask;2015 }2016 }2017 2018 /* Prepare the input and output data workareas. For in-place2019 * operations we need to set the dma direction to BIDIRECTIONAL2020 * and copy the src workarea to the dst workarea.2021 */2022 if (sg_virt(pt->src) == sg_virt(pt->dst))2023 in_place = true;2024 2025 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,2026 CCP_PASSTHRU_MASKSIZE,2027 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);2028 if (ret)2029 goto e_mask;2030 2031 if (in_place) {2032 dst = src;2033 } else {2034 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,2035 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);2036 if (ret)2037 goto e_src;2038 }2039 2040 /* Send data to the CCP Passthru engine2041 * Because the CCP engine works on a single source and destination2042 * dma address at a time, each entry in the source scatterlist2043 * (after the dma_map_sg call) must be less than or equal to the2044 * (remaining) length in the destination scatterlist entry and the2045 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE2046 */2047 dst.sg_wa.sg_used = 0;2048 for (i = 1; i <= src.sg_wa.dma_count; i++) {2049 if (!dst.sg_wa.sg ||2050 (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {2051 ret = -EINVAL;2052 goto e_dst;2053 }2054 2055 if (i == src.sg_wa.dma_count) {2056 op.eom = 1;2057 op.soc = 1;2058 }2059 2060 op.src.type = CCP_MEMTYPE_SYSTEM;2061 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);2062 op.src.u.dma.offset = 0;2063 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);2064 2065 op.dst.type = CCP_MEMTYPE_SYSTEM;2066 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);2067 op.dst.u.dma.offset = dst.sg_wa.sg_used;2068 op.dst.u.dma.length = op.src.u.dma.length;2069 2070 ret = cmd_q->ccp->vdata->perform->passthru(&op);2071 if (ret) {2072 cmd->engine_error = cmd_q->cmd_error;2073 goto e_dst;2074 }2075 2076 dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);2077 if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {2078 dst.sg_wa.sg = sg_next(dst.sg_wa.sg);2079 dst.sg_wa.sg_used = 0;2080 }2081 src.sg_wa.sg = sg_next(src.sg_wa.sg);2082 }2083 2084e_dst:2085 if (!in_place)2086 ccp_free_data(&dst, cmd_q);2087 2088e_src:2089 ccp_free_data(&src, cmd_q);2090 2091e_mask:2092 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)2093 ccp_dm_free(&mask);2094 2095 return ret;2096}2097 2098static noinline_for_stack int2099ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,2100 struct ccp_cmd *cmd)2101{2102 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;2103 struct ccp_dm_workarea mask;2104 struct ccp_op op;2105 int ret;2106 2107 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))2108 return -EINVAL;2109 2110 if (!pt->src_dma || !pt->dst_dma)2111 return -EINVAL;2112 2113 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {2114 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)2115 return -EINVAL;2116 if (!pt->mask)2117 return -EINVAL;2118 }2119 2120 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);2121 2122 memset(&op, 0, sizeof(op));2123 op.cmd_q = cmd_q;2124 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);2125 2126 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {2127 /* Load the mask */2128 op.sb_key = cmd_q->sb_key;2129 2130 mask.length = pt->mask_len;2131 mask.dma.address = pt->mask;2132 mask.dma.length = pt->mask_len;2133 2134 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,2135 CCP_PASSTHRU_BYTESWAP_NOOP);2136 if (ret) {2137 cmd->engine_error = cmd_q->cmd_error;2138 return ret;2139 }2140 }2141 2142 /* Send data to the CCP Passthru engine */2143 op.eom = 1;2144 op.soc = 1;2145 2146 op.src.type = CCP_MEMTYPE_SYSTEM;2147 op.src.u.dma.address = pt->src_dma;2148 op.src.u.dma.offset = 0;2149 op.src.u.dma.length = pt->src_len;2150 2151 op.dst.type = CCP_MEMTYPE_SYSTEM;2152 op.dst.u.dma.address = pt->dst_dma;2153 op.dst.u.dma.offset = 0;2154 op.dst.u.dma.length = pt->src_len;2155 2156 ret = cmd_q->ccp->vdata->perform->passthru(&op);2157 if (ret)2158 cmd->engine_error = cmd_q->cmd_error;2159 2160 return ret;2161}2162 2163static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)2164{2165 struct ccp_ecc_engine *ecc = &cmd->u.ecc;2166 struct ccp_dm_workarea src, dst;2167 struct ccp_op op;2168 int ret;2169 u8 *save;2170 2171 if (!ecc->u.mm.operand_1 ||2172 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))2173 return -EINVAL;2174 2175 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)2176 if (!ecc->u.mm.operand_2 ||2177 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))2178 return -EINVAL;2179 2180 if (!ecc->u.mm.result ||2181 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))2182 return -EINVAL;2183 2184 memset(&op, 0, sizeof(op));2185 op.cmd_q = cmd_q;2186 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);2187 2188 /* Concatenate the modulus and the operands. Both the modulus and2189 * the operands must be in little endian format. Since the input2190 * is in big endian format it must be converted and placed in a2191 * fixed length buffer.2192 */2193 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,2194 DMA_TO_DEVICE);2195 if (ret)2196 return ret;2197 2198 /* Save the workarea address since it is updated in order to perform2199 * the concatenation2200 */2201 save = src.address;2202 2203 /* Copy the ECC modulus */2204 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);2205 if (ret)2206 goto e_src;2207 src.address += CCP_ECC_OPERAND_SIZE;2208 2209 /* Copy the first operand */2210 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,2211 ecc->u.mm.operand_1_len);2212 if (ret)2213 goto e_src;2214 src.address += CCP_ECC_OPERAND_SIZE;2215 2216 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {2217 /* Copy the second operand */2218 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,2219 ecc->u.mm.operand_2_len);2220 if (ret)2221 goto e_src;2222 src.address += CCP_ECC_OPERAND_SIZE;2223 }2224 2225 /* Restore the workarea address */2226 src.address = save;2227 2228 /* Prepare the output area for the operation */2229 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,2230 DMA_FROM_DEVICE);2231 if (ret)2232 goto e_src;2233 2234 op.soc = 1;2235 op.src.u.dma.address = src.dma.address;2236 op.src.u.dma.offset = 0;2237 op.src.u.dma.length = src.length;2238 op.dst.u.dma.address = dst.dma.address;2239 op.dst.u.dma.offset = 0;2240 op.dst.u.dma.length = dst.length;2241 2242 op.u.ecc.function = cmd->u.ecc.function;2243 2244 ret = cmd_q->ccp->vdata->perform->ecc(&op);2245 if (ret) {2246 cmd->engine_error = cmd_q->cmd_error;2247 goto e_dst;2248 }2249 2250 ecc->ecc_result = le16_to_cpup(2251 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));2252 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {2253 ret = -EIO;2254 goto e_dst;2255 }2256 2257 /* Save the ECC result */2258 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,2259 CCP_ECC_MODULUS_BYTES);2260 2261e_dst:2262 ccp_dm_free(&dst);2263 2264e_src:2265 ccp_dm_free(&src);2266 2267 return ret;2268}2269 2270static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)2271{2272 struct ccp_ecc_engine *ecc = &cmd->u.ecc;2273 struct ccp_dm_workarea src, dst;2274 struct ccp_op op;2275 int ret;2276 u8 *save;2277 2278 if (!ecc->u.pm.point_1.x ||2279 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||2280 !ecc->u.pm.point_1.y ||2281 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))2282 return -EINVAL;2283 2284 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {2285 if (!ecc->u.pm.point_2.x ||2286 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||2287 !ecc->u.pm.point_2.y ||2288 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))2289 return -EINVAL;2290 } else {2291 if (!ecc->u.pm.domain_a ||2292 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))2293 return -EINVAL;2294 2295 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)2296 if (!ecc->u.pm.scalar ||2297 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))2298 return -EINVAL;2299 }2300 2301 if (!ecc->u.pm.result.x ||2302 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||2303 !ecc->u.pm.result.y ||2304 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))2305 return -EINVAL;2306 2307 memset(&op, 0, sizeof(op));2308 op.cmd_q = cmd_q;2309 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);2310 2311 /* Concatenate the modulus and the operands. Both the modulus and2312 * the operands must be in little endian format. Since the input2313 * is in big endian format it must be converted and placed in a2314 * fixed length buffer.2315 */2316 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,2317 DMA_TO_DEVICE);2318 if (ret)2319 return ret;2320 2321 /* Save the workarea address since it is updated in order to perform2322 * the concatenation2323 */2324 save = src.address;2325 2326 /* Copy the ECC modulus */2327 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);2328 if (ret)2329 goto e_src;2330 src.address += CCP_ECC_OPERAND_SIZE;2331 2332 /* Copy the first point X and Y coordinate */2333 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,2334 ecc->u.pm.point_1.x_len);2335 if (ret)2336 goto e_src;2337 src.address += CCP_ECC_OPERAND_SIZE;2338 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,2339 ecc->u.pm.point_1.y_len);2340 if (ret)2341 goto e_src;2342 src.address += CCP_ECC_OPERAND_SIZE;2343 2344 /* Set the first point Z coordinate to 1 */2345 *src.address = 0x01;2346 src.address += CCP_ECC_OPERAND_SIZE;2347 2348 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {2349 /* Copy the second point X and Y coordinate */2350 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,2351 ecc->u.pm.point_2.x_len);2352 if (ret)2353 goto e_src;2354 src.address += CCP_ECC_OPERAND_SIZE;2355 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,2356 ecc->u.pm.point_2.y_len);2357 if (ret)2358 goto e_src;2359 src.address += CCP_ECC_OPERAND_SIZE;2360 2361 /* Set the second point Z coordinate to 1 */2362 *src.address = 0x01;2363 src.address += CCP_ECC_OPERAND_SIZE;2364 } else {2365 /* Copy the Domain "a" parameter */2366 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,2367 ecc->u.pm.domain_a_len);2368 if (ret)2369 goto e_src;2370 src.address += CCP_ECC_OPERAND_SIZE;2371 2372 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {2373 /* Copy the scalar value */2374 ret = ccp_reverse_set_dm_area(&src, 0,2375 ecc->u.pm.scalar, 0,2376 ecc->u.pm.scalar_len);2377 if (ret)2378 goto e_src;2379 src.address += CCP_ECC_OPERAND_SIZE;2380 }2381 }2382 2383 /* Restore the workarea address */2384 src.address = save;2385 2386 /* Prepare the output area for the operation */2387 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,2388 DMA_FROM_DEVICE);2389 if (ret)2390 goto e_src;2391 2392 op.soc = 1;2393 op.src.u.dma.address = src.dma.address;2394 op.src.u.dma.offset = 0;2395 op.src.u.dma.length = src.length;2396 op.dst.u.dma.address = dst.dma.address;2397 op.dst.u.dma.offset = 0;2398 op.dst.u.dma.length = dst.length;2399 2400 op.u.ecc.function = cmd->u.ecc.function;2401 2402 ret = cmd_q->ccp->vdata->perform->ecc(&op);2403 if (ret) {2404 cmd->engine_error = cmd_q->cmd_error;2405 goto e_dst;2406 }2407 2408 ecc->ecc_result = le16_to_cpup(2409 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));2410 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {2411 ret = -EIO;2412 goto e_dst;2413 }2414 2415 /* Save the workarea address since it is updated as we walk through2416 * to copy the point math result2417 */2418 save = dst.address;2419 2420 /* Save the ECC result X and Y coordinates */2421 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,2422 CCP_ECC_MODULUS_BYTES);2423 dst.address += CCP_ECC_OUTPUT_SIZE;2424 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,2425 CCP_ECC_MODULUS_BYTES);2426 2427 /* Restore the workarea address */2428 dst.address = save;2429 2430e_dst:2431 ccp_dm_free(&dst);2432 2433e_src:2434 ccp_dm_free(&src);2435 2436 return ret;2437}2438 2439static noinline_for_stack int2440ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)2441{2442 struct ccp_ecc_engine *ecc = &cmd->u.ecc;2443 2444 ecc->ecc_result = 0;2445 2446 if (!ecc->mod ||2447 (ecc->mod_len > CCP_ECC_MODULUS_BYTES))2448 return -EINVAL;2449 2450 switch (ecc->function) {2451 case CCP_ECC_FUNCTION_MMUL_384BIT:2452 case CCP_ECC_FUNCTION_MADD_384BIT:2453 case CCP_ECC_FUNCTION_MINV_384BIT:2454 return ccp_run_ecc_mm_cmd(cmd_q, cmd);2455 2456 case CCP_ECC_FUNCTION_PADD_384BIT:2457 case CCP_ECC_FUNCTION_PMUL_384BIT:2458 case CCP_ECC_FUNCTION_PDBL_384BIT:2459 return ccp_run_ecc_pm_cmd(cmd_q, cmd);2460 2461 default:2462 return -EINVAL;2463 }2464}2465 2466int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)2467{2468 int ret;2469 2470 cmd->engine_error = 0;2471 cmd_q->cmd_error = 0;2472 cmd_q->int_rcvd = 0;2473 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);2474 2475 switch (cmd->engine) {2476 case CCP_ENGINE_AES:2477 switch (cmd->u.aes.mode) {2478 case CCP_AES_MODE_CMAC:2479 ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);2480 break;2481 case CCP_AES_MODE_GCM:2482 ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);2483 break;2484 default:2485 ret = ccp_run_aes_cmd(cmd_q, cmd);2486 break;2487 }2488 break;2489 case CCP_ENGINE_XTS_AES_128:2490 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);2491 break;2492 case CCP_ENGINE_DES3:2493 ret = ccp_run_des3_cmd(cmd_q, cmd);2494 break;2495 case CCP_ENGINE_SHA:2496 ret = ccp_run_sha_cmd(cmd_q, cmd);2497 break;2498 case CCP_ENGINE_RSA:2499 ret = ccp_run_rsa_cmd(cmd_q, cmd);2500 break;2501 case CCP_ENGINE_PASSTHRU:2502 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)2503 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);2504 else2505 ret = ccp_run_passthru_cmd(cmd_q, cmd);2506 break;2507 case CCP_ENGINE_ECC:2508 ret = ccp_run_ecc_cmd(cmd_q, cmd);2509 break;2510 default:2511 ret = -EINVAL;2512 }2513 2514 return ret;2515}2516