1129 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * AMD Cryptographic Coprocessor (CCP) driver4 *5 * Copyright (C) 2016,2019 Advanced Micro Devices, Inc.6 *7 * Author: Gary R Hook <gary.hook@amd.com>8 */9 10#include <linux/kernel.h>11#include <linux/kthread.h>12#include <linux/dma-mapping.h>13#include <linux/interrupt.h>14#include <linux/compiler.h>15#include <linux/ccp.h>16 17#include "ccp-dev.h"18 19/* Allocate the requested number of contiguous LSB slots20 * from the LSB bitmap. Look in the private range for this21 * queue first; failing that, check the public area.22 * If no space is available, wait around.23 * Return: first slot number24 */25static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)26{27 struct ccp_device *ccp;28 int start;29 30 /* First look at the map for the queue */31 if (cmd_q->lsb >= 0) {32 start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap,33 LSB_SIZE,34 0, count, 0);35 if (start < LSB_SIZE) {36 bitmap_set(cmd_q->lsbmap, start, count);37 return start + cmd_q->lsb * LSB_SIZE;38 }39 }40 41 /* No joy; try to get an entry from the shared blocks */42 ccp = cmd_q->ccp;43 for (;;) {44 mutex_lock(&ccp->sb_mutex);45 46 start = (u32)bitmap_find_next_zero_area(ccp->lsbmap,47 MAX_LSB_CNT * LSB_SIZE,48 0,49 count, 0);50 if (start <= MAX_LSB_CNT * LSB_SIZE) {51 bitmap_set(ccp->lsbmap, start, count);52 53 mutex_unlock(&ccp->sb_mutex);54 return start;55 }56 57 ccp->sb_avail = 0;58 59 mutex_unlock(&ccp->sb_mutex);60 61 /* Wait for KSB entries to become available */62 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))63 return 0;64 }65}66 67/* Free a number of LSB slots from the bitmap, starting at68 * the indicated starting slot number.69 */70static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start,71 unsigned int count)72{73 if (!start)74 return;75 76 if (cmd_q->lsb == start) {77 /* An entry from the private LSB */78 bitmap_clear(cmd_q->lsbmap, start, count);79 } else {80 /* From the shared LSBs */81 struct ccp_device *ccp = cmd_q->ccp;82 83 mutex_lock(&ccp->sb_mutex);84 bitmap_clear(ccp->lsbmap, start, count);85 ccp->sb_avail = 1;86 mutex_unlock(&ccp->sb_mutex);87 wake_up_interruptible_all(&ccp->sb_queue);88 }89}90 91/* CCP version 5: Union to define the function field (cmd_reg1/dword0) */92union ccp_function {93 struct {94 u16 size:7;95 u16 encrypt:1;96 u16 mode:5;97 u16 type:2;98 } aes;99 struct {100 u16 size:7;101 u16 encrypt:1;102 u16 rsvd:5;103 u16 type:2;104 } aes_xts;105 struct {106 u16 size:7;107 u16 encrypt:1;108 u16 mode:5;109 u16 type:2;110 } des3;111 struct {112 u16 rsvd1:10;113 u16 type:4;114 u16 rsvd2:1;115 } sha;116 struct {117 u16 mode:3;118 u16 size:12;119 } rsa;120 struct {121 u16 byteswap:2;122 u16 bitwise:3;123 u16 reflect:2;124 u16 rsvd:8;125 } pt;126 struct {127 u16 rsvd:13;128 } zlib;129 struct {130 u16 size:10;131 u16 type:2;132 u16 mode:3;133 } ecc;134 u16 raw;135};136 137#define CCP_AES_SIZE(p) ((p)->aes.size)138#define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt)139#define CCP_AES_MODE(p) ((p)->aes.mode)140#define CCP_AES_TYPE(p) ((p)->aes.type)141#define CCP_XTS_SIZE(p) ((p)->aes_xts.size)142#define CCP_XTS_TYPE(p) ((p)->aes_xts.type)143#define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt)144#define CCP_DES3_SIZE(p) ((p)->des3.size)145#define CCP_DES3_ENCRYPT(p) ((p)->des3.encrypt)146#define CCP_DES3_MODE(p) ((p)->des3.mode)147#define CCP_DES3_TYPE(p) ((p)->des3.type)148#define CCP_SHA_TYPE(p) ((p)->sha.type)149#define CCP_RSA_SIZE(p) ((p)->rsa.size)150#define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap)151#define CCP_PT_BITWISE(p) ((p)->pt.bitwise)152#define CCP_ECC_MODE(p) ((p)->ecc.mode)153#define CCP_ECC_AFFINE(p) ((p)->ecc.one)154 155/* Word 0 */156#define CCP5_CMD_DW0(p) ((p)->dw0)157#define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc)158#define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc)159#define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init)160#define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom)161#define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function)162#define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine)163#define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot)164 165/* Word 1 */166#define CCP5_CMD_DW1(p) ((p)->length)167#define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p))168 169/* Word 2 */170#define CCP5_CMD_DW2(p) ((p)->src_lo)171#define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p))172 173/* Word 3 */174#define CCP5_CMD_DW3(p) ((p)->dw3)175#define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem)176#define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi)177#define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id)178#define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed)179 180/* Words 4/5 */181#define CCP5_CMD_DW4(p) ((p)->dw4)182#define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo)183#define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi)184#define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p))185#define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem)186#define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed)187#define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo)188#define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi)189 190/* Word 6/7 */191#define CCP5_CMD_DW6(p) ((p)->key_lo)192#define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p))193#define CCP5_CMD_DW7(p) ((p)->dw7)194#define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi)195#define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem)196 197static inline u32 low_address(unsigned long addr)198{199 return (u64)addr & 0x0ffffffff;200}201 202static inline u32 high_address(unsigned long addr)203{204 return ((u64)addr >> 32) & 0x00000ffff;205}206 207static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q)208{209 unsigned int head_idx, n;210 u32 head_lo, queue_start;211 212 queue_start = low_address(cmd_q->qdma_tail);213 head_lo = ioread32(cmd_q->reg_head_lo);214 head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc);215 216 n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1;217 218 return n % COMMANDS_PER_QUEUE; /* Always one unused spot */219}220 221static int ccp5_do_cmd(struct ccp5_desc *desc,222 struct ccp_cmd_queue *cmd_q)223{224 __le32 *mP;225 u32 *dP;226 u32 tail;227 int i;228 int ret = 0;229 230 cmd_q->total_ops++;231 232 if (CCP5_CMD_SOC(desc)) {233 CCP5_CMD_IOC(desc) = 1;234 CCP5_CMD_SOC(desc) = 0;235 }236 mutex_lock(&cmd_q->q_mutex);237 238 mP = (__le32 *)&cmd_q->qbase[cmd_q->qidx];239 dP = (u32 *)desc;240 for (i = 0; i < 8; i++)241 mP[i] = cpu_to_le32(dP[i]); /* handle endianness */242 243 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;244 245 /* The data used by this command must be flushed to memory */246 wmb();247 248 /* Write the new tail address back to the queue register */249 tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);250 iowrite32(tail, cmd_q->reg_tail_lo);251 252 /* Turn the queue back on using our cached control register */253 iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control);254 mutex_unlock(&cmd_q->q_mutex);255 256 if (CCP5_CMD_IOC(desc)) {257 /* Wait for the job to complete */258 ret = wait_event_interruptible(cmd_q->int_queue,259 cmd_q->int_rcvd);260 if (ret || cmd_q->cmd_error) {261 /* Log the error and flush the queue by262 * moving the head pointer263 */264 if (cmd_q->cmd_error)265 ccp_log_error(cmd_q->ccp,266 cmd_q->cmd_error);267 iowrite32(tail, cmd_q->reg_head_lo);268 if (!ret)269 ret = -EIO;270 }271 cmd_q->int_rcvd = 0;272 }273 274 return ret;275}276 277static int ccp5_perform_aes(struct ccp_op *op)278{279 struct ccp5_desc desc;280 union ccp_function function;281 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;282 283 op->cmd_q->total_aes_ops++;284 285 /* Zero out all the fields of the command desc */286 memset(&desc, 0, Q_DESC_SIZE);287 288 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES;289 290 CCP5_CMD_SOC(&desc) = op->soc;291 CCP5_CMD_IOC(&desc) = 1;292 CCP5_CMD_INIT(&desc) = op->init;293 CCP5_CMD_EOM(&desc) = op->eom;294 CCP5_CMD_PROT(&desc) = 0;295 296 function.raw = 0;297 CCP_AES_ENCRYPT(&function) = op->u.aes.action;298 CCP_AES_MODE(&function) = op->u.aes.mode;299 CCP_AES_TYPE(&function) = op->u.aes.type;300 CCP_AES_SIZE(&function) = op->u.aes.size;301 302 CCP5_CMD_FUNCTION(&desc) = function.raw;303 304 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;305 306 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);307 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);308 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;309 310 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);311 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);312 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;313 314 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);315 CCP5_CMD_KEY_HI(&desc) = 0;316 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;317 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;318 319 return ccp5_do_cmd(&desc, op->cmd_q);320}321 322static int ccp5_perform_xts_aes(struct ccp_op *op)323{324 struct ccp5_desc desc;325 union ccp_function function;326 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;327 328 op->cmd_q->total_xts_aes_ops++;329 330 /* Zero out all the fields of the command desc */331 memset(&desc, 0, Q_DESC_SIZE);332 333 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128;334 335 CCP5_CMD_SOC(&desc) = op->soc;336 CCP5_CMD_IOC(&desc) = 1;337 CCP5_CMD_INIT(&desc) = op->init;338 CCP5_CMD_EOM(&desc) = op->eom;339 CCP5_CMD_PROT(&desc) = 0;340 341 function.raw = 0;342 CCP_XTS_TYPE(&function) = op->u.xts.type;343 CCP_XTS_ENCRYPT(&function) = op->u.xts.action;344 CCP_XTS_SIZE(&function) = op->u.xts.unit_size;345 CCP5_CMD_FUNCTION(&desc) = function.raw;346 347 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;348 349 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);350 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);351 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;352 353 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);354 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);355 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;356 357 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);358 CCP5_CMD_KEY_HI(&desc) = 0;359 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;360 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;361 362 return ccp5_do_cmd(&desc, op->cmd_q);363}364 365static int ccp5_perform_sha(struct ccp_op *op)366{367 struct ccp5_desc desc;368 union ccp_function function;369 370 op->cmd_q->total_sha_ops++;371 372 /* Zero out all the fields of the command desc */373 memset(&desc, 0, Q_DESC_SIZE);374 375 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA;376 377 CCP5_CMD_SOC(&desc) = op->soc;378 CCP5_CMD_IOC(&desc) = 1;379 CCP5_CMD_INIT(&desc) = 1;380 CCP5_CMD_EOM(&desc) = op->eom;381 CCP5_CMD_PROT(&desc) = 0;382 383 function.raw = 0;384 CCP_SHA_TYPE(&function) = op->u.sha.type;385 CCP5_CMD_FUNCTION(&desc) = function.raw;386 387 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;388 389 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);390 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);391 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;392 393 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;394 395 if (op->eom) {396 CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits);397 CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits);398 } else {399 CCP5_CMD_SHA_LO(&desc) = 0;400 CCP5_CMD_SHA_HI(&desc) = 0;401 }402 403 return ccp5_do_cmd(&desc, op->cmd_q);404}405 406static int ccp5_perform_des3(struct ccp_op *op)407{408 struct ccp5_desc desc;409 union ccp_function function;410 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;411 412 op->cmd_q->total_3des_ops++;413 414 /* Zero out all the fields of the command desc */415 memset(&desc, 0, sizeof(struct ccp5_desc));416 417 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_DES3;418 419 CCP5_CMD_SOC(&desc) = op->soc;420 CCP5_CMD_IOC(&desc) = 1;421 CCP5_CMD_INIT(&desc) = op->init;422 CCP5_CMD_EOM(&desc) = op->eom;423 CCP5_CMD_PROT(&desc) = 0;424 425 function.raw = 0;426 CCP_DES3_ENCRYPT(&function) = op->u.des3.action;427 CCP_DES3_MODE(&function) = op->u.des3.mode;428 CCP_DES3_TYPE(&function) = op->u.des3.type;429 CCP5_CMD_FUNCTION(&desc) = function.raw;430 431 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;432 433 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);434 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);435 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;436 437 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);438 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);439 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;440 441 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);442 CCP5_CMD_KEY_HI(&desc) = 0;443 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;444 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;445 446 return ccp5_do_cmd(&desc, op->cmd_q);447}448 449static int ccp5_perform_rsa(struct ccp_op *op)450{451 struct ccp5_desc desc;452 union ccp_function function;453 454 op->cmd_q->total_rsa_ops++;455 456 /* Zero out all the fields of the command desc */457 memset(&desc, 0, Q_DESC_SIZE);458 459 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA;460 461 CCP5_CMD_SOC(&desc) = op->soc;462 CCP5_CMD_IOC(&desc) = 1;463 CCP5_CMD_INIT(&desc) = 0;464 CCP5_CMD_EOM(&desc) = 1;465 CCP5_CMD_PROT(&desc) = 0;466 467 function.raw = 0;468 CCP_RSA_SIZE(&function) = (op->u.rsa.mod_size + 7) >> 3;469 CCP5_CMD_FUNCTION(&desc) = function.raw;470 471 CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;472 473 /* Source is from external memory */474 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);475 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);476 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;477 478 /* Destination is in external memory */479 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);480 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);481 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;482 483 /* Key (Exponent) is in external memory */484 CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma);485 CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma);486 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM;487 488 return ccp5_do_cmd(&desc, op->cmd_q);489}490 491static int ccp5_perform_passthru(struct ccp_op *op)492{493 struct ccp5_desc desc;494 union ccp_function function;495 struct ccp_dma_info *saddr = &op->src.u.dma;496 struct ccp_dma_info *daddr = &op->dst.u.dma;497 498 499 op->cmd_q->total_pt_ops++;500 501 memset(&desc, 0, Q_DESC_SIZE);502 503 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU;504 505 CCP5_CMD_SOC(&desc) = 0;506 CCP5_CMD_IOC(&desc) = 1;507 CCP5_CMD_INIT(&desc) = 0;508 CCP5_CMD_EOM(&desc) = op->eom;509 CCP5_CMD_PROT(&desc) = 0;510 511 function.raw = 0;512 CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap;513 CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod;514 CCP5_CMD_FUNCTION(&desc) = function.raw;515 516 /* Length of source data is always 256 bytes */517 if (op->src.type == CCP_MEMTYPE_SYSTEM)518 CCP5_CMD_LEN(&desc) = saddr->length;519 else520 CCP5_CMD_LEN(&desc) = daddr->length;521 522 if (op->src.type == CCP_MEMTYPE_SYSTEM) {523 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);524 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);525 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;526 527 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)528 CCP5_CMD_LSB_ID(&desc) = op->sb_key;529 } else {530 u32 key_addr = op->src.u.sb * CCP_SB_BYTES;531 532 CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr);533 CCP5_CMD_SRC_HI(&desc) = 0;534 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB;535 }536 537 if (op->dst.type == CCP_MEMTYPE_SYSTEM) {538 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);539 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);540 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;541 } else {542 u32 key_addr = op->dst.u.sb * CCP_SB_BYTES;543 544 CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr);545 CCP5_CMD_DST_HI(&desc) = 0;546 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB;547 }548 549 return ccp5_do_cmd(&desc, op->cmd_q);550}551 552static int ccp5_perform_ecc(struct ccp_op *op)553{554 struct ccp5_desc desc;555 union ccp_function function;556 557 op->cmd_q->total_ecc_ops++;558 559 /* Zero out all the fields of the command desc */560 memset(&desc, 0, Q_DESC_SIZE);561 562 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC;563 564 CCP5_CMD_SOC(&desc) = 0;565 CCP5_CMD_IOC(&desc) = 1;566 CCP5_CMD_INIT(&desc) = 0;567 CCP5_CMD_EOM(&desc) = 1;568 CCP5_CMD_PROT(&desc) = 0;569 570 function.raw = 0;571 function.ecc.mode = op->u.ecc.function;572 CCP5_CMD_FUNCTION(&desc) = function.raw;573 574 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;575 576 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);577 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);578 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;579 580 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);581 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);582 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;583 584 return ccp5_do_cmd(&desc, op->cmd_q);585}586 587static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status)588{589 int q_mask = 1 << cmd_q->id;590 int queues = 0;591 int j;592 593 /* Build a bit mask to know which LSBs this queue has access to.594 * Don't bother with segment 0 as it has special privileges.595 */596 for (j = 1; j < MAX_LSB_CNT; j++) {597 if (status & q_mask)598 bitmap_set(cmd_q->lsbmask, j, 1);599 status >>= LSB_REGION_WIDTH;600 }601 queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);602 dev_dbg(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n",603 cmd_q->id, queues);604 605 return queues ? 0 : -EINVAL;606}607 608static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,609 int lsb_cnt, int n_lsbs,610 unsigned long *lsb_pub)611{612 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);613 int bitno;614 int qlsb_wgt;615 int i;616 617 /* For each queue:618 * If the count of potential LSBs available to a queue matches the619 * ordinal given to us in lsb_cnt:620 * Copy the mask of possible LSBs for this queue into "qlsb";621 * For each bit in qlsb, see if the corresponding bit in the622 * aggregation mask is set; if so, we have a match.623 * If we have a match, clear the bit in the aggregation to624 * mark it as no longer available.625 * If there is no match, clear the bit in qlsb and keep looking.626 */627 for (i = 0; i < ccp->cmd_q_count; i++) {628 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];629 630 qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);631 632 if (qlsb_wgt == lsb_cnt) {633 bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT);634 635 bitno = find_first_bit(qlsb, MAX_LSB_CNT);636 while (bitno < MAX_LSB_CNT) {637 if (test_bit(bitno, lsb_pub)) {638 /* We found an available LSB639 * that this queue can access640 */641 cmd_q->lsb = bitno;642 bitmap_clear(lsb_pub, bitno, 1);643 dev_dbg(ccp->dev,644 "Queue %d gets LSB %d\n",645 i, bitno);646 break;647 }648 bitmap_clear(qlsb, bitno, 1);649 bitno = find_first_bit(qlsb, MAX_LSB_CNT);650 }651 if (bitno >= MAX_LSB_CNT)652 return -EINVAL;653 n_lsbs--;654 }655 }656 return n_lsbs;657}658 659/* For each queue, from the most- to least-constrained:660 * find an LSB that can be assigned to the queue. If there are N queues that661 * can only use M LSBs, where N > M, fail; otherwise, every queue will get a662 * dedicated LSB. Remaining LSB regions become a shared resource.663 * If we have fewer LSBs than queues, all LSB regions become shared resources.664 */665static int ccp_assign_lsbs(struct ccp_device *ccp)666{667 DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT);668 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);669 int n_lsbs = 0;670 int bitno;671 int i, lsb_cnt;672 int rc = 0;673 674 bitmap_zero(lsb_pub, MAX_LSB_CNT);675 676 /* Create an aggregate bitmap to get a total count of available LSBs */677 for (i = 0; i < ccp->cmd_q_count; i++)678 bitmap_or(lsb_pub,679 lsb_pub, ccp->cmd_q[i].lsbmask,680 MAX_LSB_CNT);681 682 n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT);683 684 if (n_lsbs >= ccp->cmd_q_count) {685 /* We have enough LSBS to give every queue a private LSB.686 * Brute force search to start with the queues that are more687 * constrained in LSB choice. When an LSB is privately688 * assigned, it is removed from the public mask.689 * This is an ugly N squared algorithm with some optimization.690 */691 for (lsb_cnt = 1;692 n_lsbs && (lsb_cnt <= MAX_LSB_CNT);693 lsb_cnt++) {694 rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,695 lsb_pub);696 if (rc < 0)697 return -EINVAL;698 n_lsbs = rc;699 }700 }701 702 rc = 0;703 /* What's left of the LSBs, according to the public mask, now become704 * shared. Any zero bits in the lsb_pub mask represent an LSB region705 * that can't be used as a shared resource, so mark the LSB slots for706 * them as "in use".707 */708 bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT);709 710 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);711 while (bitno < MAX_LSB_CNT) {712 bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);713 bitmap_set(qlsb, bitno, 1);714 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);715 }716 717 return rc;718}719 720static void ccp5_disable_queue_interrupts(struct ccp_device *ccp)721{722 unsigned int i;723 724 for (i = 0; i < ccp->cmd_q_count; i++)725 iowrite32(0x0, ccp->cmd_q[i].reg_int_enable);726}727 728static void ccp5_enable_queue_interrupts(struct ccp_device *ccp)729{730 unsigned int i;731 732 for (i = 0; i < ccp->cmd_q_count; i++)733 iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable);734}735 736static void ccp5_irq_bh(unsigned long data)737{738 struct ccp_device *ccp = (struct ccp_device *)data;739 u32 status;740 unsigned int i;741 742 for (i = 0; i < ccp->cmd_q_count; i++) {743 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];744 745 status = ioread32(cmd_q->reg_interrupt_status);746 747 if (status) {748 cmd_q->int_status = status;749 cmd_q->q_status = ioread32(cmd_q->reg_status);750 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);751 752 /* On error, only save the first error value */753 if ((status & INT_ERROR) && !cmd_q->cmd_error)754 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);755 756 cmd_q->int_rcvd = 1;757 758 /* Acknowledge the interrupt and wake the kthread */759 iowrite32(status, cmd_q->reg_interrupt_status);760 wake_up_interruptible(&cmd_q->int_queue);761 }762 }763 ccp5_enable_queue_interrupts(ccp);764}765 766static irqreturn_t ccp5_irq_handler(int irq, void *data)767{768 struct ccp_device *ccp = (struct ccp_device *)data;769 770 ccp5_disable_queue_interrupts(ccp);771 ccp->total_interrupts++;772 if (ccp->use_tasklet)773 tasklet_schedule(&ccp->irq_tasklet);774 else775 ccp5_irq_bh((unsigned long)ccp);776 return IRQ_HANDLED;777}778 779static int ccp5_init(struct ccp_device *ccp)780{781 struct device *dev = ccp->dev;782 struct ccp_cmd_queue *cmd_q;783 struct dma_pool *dma_pool;784 char dma_pool_name[MAX_DMAPOOL_NAME_LEN];785 unsigned int qmr, i;786 u64 status;787 u32 status_lo, status_hi;788 int ret;789 790 /* Find available queues */791 qmr = ioread32(ccp->io_regs + Q_MASK_REG);792 /*793 * Check for a access to the registers. If this read returns794 * 0xffffffff, it's likely that the system is running a broken795 * BIOS which disallows access to the device. Stop here and fail796 * the initialization (but not the load, as the PSP could get797 * properly initialized).798 */799 if (qmr == 0xffffffff) {800 dev_notice(dev, "ccp: unable to access the device: you might be running a broken BIOS.\n");801 return 1;802 }803 804 for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) {805 if (!(qmr & (1 << i)))806 continue;807 808 /* Allocate a dma pool for this queue */809 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",810 ccp->name, i);811 dma_pool = dma_pool_create(dma_pool_name, dev,812 CCP_DMAPOOL_MAX_SIZE,813 CCP_DMAPOOL_ALIGN, 0);814 if (!dma_pool) {815 dev_err(dev, "unable to allocate dma pool\n");816 ret = -ENOMEM;817 goto e_pool;818 }819 820 cmd_q = &ccp->cmd_q[ccp->cmd_q_count];821 ccp->cmd_q_count++;822 823 cmd_q->ccp = ccp;824 cmd_q->id = i;825 cmd_q->dma_pool = dma_pool;826 mutex_init(&cmd_q->q_mutex);827 828 /* Page alignment satisfies our needs for N <= 128 */829 BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128);830 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);831 cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize,832 &cmd_q->qbase_dma,833 GFP_KERNEL);834 if (!cmd_q->qbase) {835 dev_err(dev, "unable to allocate command queue\n");836 ret = -ENOMEM;837 goto e_pool;838 }839 840 cmd_q->qidx = 0;841 /* Preset some register values and masks that are queue842 * number dependent843 */844 cmd_q->reg_control = ccp->io_regs +845 CMD5_Q_STATUS_INCR * (i + 1);846 cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE;847 cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE;848 cmd_q->reg_int_enable = cmd_q->reg_control +849 CMD5_Q_INT_ENABLE_BASE;850 cmd_q->reg_interrupt_status = cmd_q->reg_control +851 CMD5_Q_INTERRUPT_STATUS_BASE;852 cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE;853 cmd_q->reg_int_status = cmd_q->reg_control +854 CMD5_Q_INT_STATUS_BASE;855 cmd_q->reg_dma_status = cmd_q->reg_control +856 CMD5_Q_DMA_STATUS_BASE;857 cmd_q->reg_dma_read_status = cmd_q->reg_control +858 CMD5_Q_DMA_READ_STATUS_BASE;859 cmd_q->reg_dma_write_status = cmd_q->reg_control +860 CMD5_Q_DMA_WRITE_STATUS_BASE;861 862 init_waitqueue_head(&cmd_q->int_queue);863 864 dev_dbg(dev, "queue #%u available\n", i);865 }866 867 if (ccp->cmd_q_count == 0) {868 dev_notice(dev, "no command queues available\n");869 ret = 1;870 goto e_pool;871 }872 873 /* Turn off the queues and disable interrupts until ready */874 ccp5_disable_queue_interrupts(ccp);875 for (i = 0; i < ccp->cmd_q_count; i++) {876 cmd_q = &ccp->cmd_q[i];877 878 cmd_q->qcontrol = 0; /* Start with nothing */879 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);880 881 ioread32(cmd_q->reg_int_status);882 ioread32(cmd_q->reg_status);883 884 /* Clear the interrupt status */885 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);886 }887 888 dev_dbg(dev, "Requesting an IRQ...\n");889 /* Request an irq */890 ret = sp_request_ccp_irq(ccp->sp, ccp5_irq_handler, ccp->name, ccp);891 if (ret) {892 dev_err(dev, "unable to allocate an IRQ\n");893 goto e_pool;894 }895 /* Initialize the ISR tasklet */896 if (ccp->use_tasklet)897 tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh,898 (unsigned long)ccp);899 900 dev_dbg(dev, "Loading LSB map...\n");901 /* Copy the private LSB mask to the public registers */902 status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);903 status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);904 iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET);905 iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET);906 status = ((u64)status_hi<<30) | (u64)status_lo;907 908 dev_dbg(dev, "Configuring virtual queues...\n");909 /* Configure size of each virtual queue accessible to host */910 for (i = 0; i < ccp->cmd_q_count; i++) {911 u32 dma_addr_lo;912 u32 dma_addr_hi;913 914 cmd_q = &ccp->cmd_q[i];915 916 cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT);917 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT;918 919 cmd_q->qdma_tail = cmd_q->qbase_dma;920 dma_addr_lo = low_address(cmd_q->qdma_tail);921 iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo);922 iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo);923 924 dma_addr_hi = high_address(cmd_q->qdma_tail);925 cmd_q->qcontrol |= (dma_addr_hi << 16);926 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);927 928 /* Find the LSB regions accessible to the queue */929 ccp_find_lsb_regions(cmd_q, status);930 cmd_q->lsb = -1; /* Unassigned value */931 }932 933 dev_dbg(dev, "Assigning LSBs...\n");934 ret = ccp_assign_lsbs(ccp);935 if (ret) {936 dev_err(dev, "Unable to assign LSBs (%d)\n", ret);937 goto e_irq;938 }939 940 /* Optimization: pre-allocate LSB slots for each queue */941 for (i = 0; i < ccp->cmd_q_count; i++) {942 ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2);943 ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2);944 }945 946 dev_dbg(dev, "Starting threads...\n");947 /* Create a kthread for each queue */948 for (i = 0; i < ccp->cmd_q_count; i++) {949 struct task_struct *kthread;950 951 cmd_q = &ccp->cmd_q[i];952 953 kthread = kthread_run(ccp_cmd_queue_thread, cmd_q,954 "%s-q%u", ccp->name, cmd_q->id);955 if (IS_ERR(kthread)) {956 dev_err(dev, "error creating queue thread (%ld)\n",957 PTR_ERR(kthread));958 ret = PTR_ERR(kthread);959 goto e_kthread;960 }961 962 cmd_q->kthread = kthread;963 }964 965 dev_dbg(dev, "Enabling interrupts...\n");966 ccp5_enable_queue_interrupts(ccp);967 968 dev_dbg(dev, "Registering device...\n");969 /* Put this on the unit list to make it available */970 ccp_add_device(ccp);971 972 ret = ccp_register_rng(ccp);973 if (ret)974 goto e_kthread;975 976 /* Register the DMA engine support */977 ret = ccp_dmaengine_register(ccp);978 if (ret)979 goto e_hwrng;980 981#ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS982 /* Set up debugfs entries */983 ccp5_debugfs_setup(ccp);984#endif985 986 return 0;987 988e_hwrng:989 ccp_unregister_rng(ccp);990 991e_kthread:992 for (i = 0; i < ccp->cmd_q_count; i++)993 if (ccp->cmd_q[i].kthread)994 kthread_stop(ccp->cmd_q[i].kthread);995 996e_irq:997 sp_free_ccp_irq(ccp->sp, ccp);998 999e_pool:1000 for (i = 0; i < ccp->cmd_q_count; i++)1001 dma_pool_destroy(ccp->cmd_q[i].dma_pool);1002 1003 return ret;1004}1005 1006static void ccp5_destroy(struct ccp_device *ccp)1007{1008 struct ccp_cmd_queue *cmd_q;1009 struct ccp_cmd *cmd;1010 unsigned int i;1011 1012 /* Unregister the DMA engine */1013 ccp_dmaengine_unregister(ccp);1014 1015 /* Unregister the RNG */1016 ccp_unregister_rng(ccp);1017 1018 /* Remove this device from the list of available units first */1019 ccp_del_device(ccp);1020 1021#ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS1022 /* We're in the process of tearing down the entire driver;1023 * when all the devices are gone clean up debugfs1024 */1025 if (ccp_present())1026 ccp5_debugfs_destroy();1027#endif1028 1029 /* Disable and clear interrupts */1030 ccp5_disable_queue_interrupts(ccp);1031 for (i = 0; i < ccp->cmd_q_count; i++) {1032 cmd_q = &ccp->cmd_q[i];1033 1034 /* Turn off the run bit */1035 iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control);1036 1037 /* Clear the interrupt status */1038 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);1039 ioread32(cmd_q->reg_int_status);1040 ioread32(cmd_q->reg_status);1041 }1042 1043 /* Stop the queue kthreads */1044 for (i = 0; i < ccp->cmd_q_count; i++)1045 if (ccp->cmd_q[i].kthread)1046 kthread_stop(ccp->cmd_q[i].kthread);1047 1048 sp_free_ccp_irq(ccp->sp, ccp);1049 1050 /* Flush the cmd and backlog queue */1051 while (!list_empty(&ccp->cmd)) {1052 /* Invoke the callback directly with an error code */1053 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);1054 list_del(&cmd->entry);1055 cmd->callback(cmd->data, -ENODEV);1056 }1057 while (!list_empty(&ccp->backlog)) {1058 /* Invoke the callback directly with an error code */1059 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);1060 list_del(&cmd->entry);1061 cmd->callback(cmd->data, -ENODEV);1062 }1063}1064 1065static void ccp5_config(struct ccp_device *ccp)1066{1067 /* Public side */1068 iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);1069}1070 1071static void ccp5other_config(struct ccp_device *ccp)1072{1073 int i;1074 u32 rnd;1075 1076 /* We own all of the queues on the NTB CCP */1077 1078 iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET);1079 iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET);1080 for (i = 0; i < 12; i++) {1081 rnd = ioread32(ccp->io_regs + TRNG_OUT_REG);1082 iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET);1083 }1084 1085 iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET);1086 iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET);1087 iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET);1088 1089 iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);1090 iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);1091 1092 iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET);1093 1094 ccp5_config(ccp);1095}1096 1097/* Version 5 adds some function, but is essentially the same as v5 */1098static const struct ccp_actions ccp5_actions = {1099 .aes = ccp5_perform_aes,1100 .xts_aes = ccp5_perform_xts_aes,1101 .sha = ccp5_perform_sha,1102 .des3 = ccp5_perform_des3,1103 .rsa = ccp5_perform_rsa,1104 .passthru = ccp5_perform_passthru,1105 .ecc = ccp5_perform_ecc,1106 .sballoc = ccp_lsb_alloc,1107 .sbfree = ccp_lsb_free,1108 .init = ccp5_init,1109 .destroy = ccp5_destroy,1110 .get_free_slots = ccp5_get_free_slots,1111};1112 1113const struct ccp_vdata ccpv5a = {1114 .version = CCP_VERSION(5, 0),1115 .setup = ccp5_config,1116 .perform = &ccp5_actions,1117 .offset = 0x0,1118 .rsamax = CCP5_RSA_MAX_WIDTH,1119};1120 1121const struct ccp_vdata ccpv5b = {1122 .version = CCP_VERSION(5, 0),1123 .dma_chan_attr = DMA_PRIVATE,1124 .setup = ccp5other_config,1125 .perform = &ccp5_actions,1126 .offset = 0x0,1127 .rsamax = CCP5_RSA_MAX_WIDTH,1128};1129