598 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * AMD Cryptographic Coprocessor (CCP) driver4 *5 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.6 *7 * Author: Tom Lendacky <thomas.lendacky@amd.com>8 * Author: Gary R Hook <gary.hook@amd.com>9 */10 11#include <linux/module.h>12#include <linux/kernel.h>13#include <linux/kthread.h>14#include <linux/interrupt.h>15#include <linux/ccp.h>16 17#include "ccp-dev.h"18 19static u32 ccp_alloc_ksb(struct ccp_cmd_queue *cmd_q, unsigned int count)20{21 int start;22 struct ccp_device *ccp = cmd_q->ccp;23 24 for (;;) {25 mutex_lock(&ccp->sb_mutex);26 27 start = (u32)bitmap_find_next_zero_area(ccp->sb,28 ccp->sb_count,29 ccp->sb_start,30 count, 0);31 if (start <= ccp->sb_count) {32 bitmap_set(ccp->sb, start, count);33 34 mutex_unlock(&ccp->sb_mutex);35 break;36 }37 38 ccp->sb_avail = 0;39 40 mutex_unlock(&ccp->sb_mutex);41 42 /* Wait for KSB entries to become available */43 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))44 return 0;45 }46 47 return KSB_START + start;48}49 50static void ccp_free_ksb(struct ccp_cmd_queue *cmd_q, unsigned int start,51 unsigned int count)52{53 struct ccp_device *ccp = cmd_q->ccp;54 55 if (!start)56 return;57 58 mutex_lock(&ccp->sb_mutex);59 60 bitmap_clear(ccp->sb, start - KSB_START, count);61 62 ccp->sb_avail = 1;63 64 mutex_unlock(&ccp->sb_mutex);65 66 wake_up_interruptible_all(&ccp->sb_queue);67}68 69static unsigned int ccp_get_free_slots(struct ccp_cmd_queue *cmd_q)70{71 return CMD_Q_DEPTH(ioread32(cmd_q->reg_status));72}73 74static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)75{76 struct ccp_cmd_queue *cmd_q = op->cmd_q;77 struct ccp_device *ccp = cmd_q->ccp;78 void __iomem *cr_addr;79 u32 cr0, cmd;80 unsigned int i;81 int ret = 0;82 83 /* We could read a status register to see how many free slots84 * are actually available, but reading that register resets it85 * and you could lose some error information.86 */87 cmd_q->free_slots--;88 89 cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)90 | (op->jobid << REQ0_JOBID_SHIFT)91 | REQ0_WAIT_FOR_WRITE;92 93 if (op->soc)94 cr0 |= REQ0_STOP_ON_COMPLETE95 | REQ0_INT_ON_COMPLETE;96 97 if (op->ioc || !cmd_q->free_slots)98 cr0 |= REQ0_INT_ON_COMPLETE;99 100 /* Start at CMD_REQ1 */101 cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;102 103 mutex_lock(&ccp->req_mutex);104 105 /* Write CMD_REQ1 through CMD_REQx first */106 for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)107 iowrite32(*(cr + i), cr_addr);108 109 /* Tell the CCP to start */110 wmb();111 iowrite32(cr0, ccp->io_regs + CMD_REQ0);112 113 mutex_unlock(&ccp->req_mutex);114 115 if (cr0 & REQ0_INT_ON_COMPLETE) {116 /* Wait for the job to complete */117 ret = wait_event_interruptible(cmd_q->int_queue,118 cmd_q->int_rcvd);119 if (ret || cmd_q->cmd_error) {120 /* On error delete all related jobs from the queue */121 cmd = (cmd_q->id << DEL_Q_ID_SHIFT)122 | op->jobid;123 if (cmd_q->cmd_error)124 ccp_log_error(cmd_q->ccp,125 cmd_q->cmd_error);126 127 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);128 129 if (!ret)130 ret = -EIO;131 } else if (op->soc) {132 /* Delete just head job from the queue on SoC */133 cmd = DEL_Q_ACTIVE134 | (cmd_q->id << DEL_Q_ID_SHIFT)135 | op->jobid;136 137 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);138 }139 140 cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);141 142 cmd_q->int_rcvd = 0;143 }144 145 return ret;146}147 148static int ccp_perform_aes(struct ccp_op *op)149{150 u32 cr[6];151 152 /* Fill out the register contents for REQ1 through REQ6 */153 cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT)154 | (op->u.aes.type << REQ1_AES_TYPE_SHIFT)155 | (op->u.aes.mode << REQ1_AES_MODE_SHIFT)156 | (op->u.aes.action << REQ1_AES_ACTION_SHIFT)157 | (op->sb_key << REQ1_KEY_KSB_SHIFT);158 cr[1] = op->src.u.dma.length - 1;159 cr[2] = ccp_addr_lo(&op->src.u.dma);160 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)161 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)162 | ccp_addr_hi(&op->src.u.dma);163 cr[4] = ccp_addr_lo(&op->dst.u.dma);164 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)165 | ccp_addr_hi(&op->dst.u.dma);166 167 if (op->u.aes.mode == CCP_AES_MODE_CFB)168 cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT);169 170 if (op->eom)171 cr[0] |= REQ1_EOM;172 173 if (op->init)174 cr[0] |= REQ1_INIT;175 176 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));177}178 179static int ccp_perform_xts_aes(struct ccp_op *op)180{181 u32 cr[6];182 183 /* Fill out the register contents for REQ1 through REQ6 */184 cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)185 | (op->u.xts.action << REQ1_AES_ACTION_SHIFT)186 | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)187 | (op->sb_key << REQ1_KEY_KSB_SHIFT);188 cr[1] = op->src.u.dma.length - 1;189 cr[2] = ccp_addr_lo(&op->src.u.dma);190 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)191 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)192 | ccp_addr_hi(&op->src.u.dma);193 cr[4] = ccp_addr_lo(&op->dst.u.dma);194 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)195 | ccp_addr_hi(&op->dst.u.dma);196 197 if (op->eom)198 cr[0] |= REQ1_EOM;199 200 if (op->init)201 cr[0] |= REQ1_INIT;202 203 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));204}205 206static int ccp_perform_sha(struct ccp_op *op)207{208 u32 cr[6];209 210 /* Fill out the register contents for REQ1 through REQ6 */211 cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT)212 | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT)213 | REQ1_INIT;214 cr[1] = op->src.u.dma.length - 1;215 cr[2] = ccp_addr_lo(&op->src.u.dma);216 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)217 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)218 | ccp_addr_hi(&op->src.u.dma);219 220 if (op->eom) {221 cr[0] |= REQ1_EOM;222 cr[4] = lower_32_bits(op->u.sha.msg_bits);223 cr[5] = upper_32_bits(op->u.sha.msg_bits);224 } else {225 cr[4] = 0;226 cr[5] = 0;227 }228 229 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));230}231 232static int ccp_perform_rsa(struct ccp_op *op)233{234 u32 cr[6];235 236 /* Fill out the register contents for REQ1 through REQ6 */237 cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)238 | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)239 | (op->sb_key << REQ1_KEY_KSB_SHIFT)240 | REQ1_EOM;241 cr[1] = op->u.rsa.input_len - 1;242 cr[2] = ccp_addr_lo(&op->src.u.dma);243 cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)244 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)245 | ccp_addr_hi(&op->src.u.dma);246 cr[4] = ccp_addr_lo(&op->dst.u.dma);247 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)248 | ccp_addr_hi(&op->dst.u.dma);249 250 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));251}252 253static int ccp_perform_passthru(struct ccp_op *op)254{255 u32 cr[6];256 257 /* Fill out the register contents for REQ1 through REQ6 */258 cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT)259 | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT)260 | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT);261 262 if (op->src.type == CCP_MEMTYPE_SYSTEM)263 cr[1] = op->src.u.dma.length - 1;264 else265 cr[1] = op->dst.u.dma.length - 1;266 267 if (op->src.type == CCP_MEMTYPE_SYSTEM) {268 cr[2] = ccp_addr_lo(&op->src.u.dma);269 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)270 | ccp_addr_hi(&op->src.u.dma);271 272 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)273 cr[3] |= (op->sb_key << REQ4_KSB_SHIFT);274 } else {275 cr[2] = op->src.u.sb * CCP_SB_BYTES;276 cr[3] = (CCP_MEMTYPE_SB << REQ4_MEMTYPE_SHIFT);277 }278 279 if (op->dst.type == CCP_MEMTYPE_SYSTEM) {280 cr[4] = ccp_addr_lo(&op->dst.u.dma);281 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)282 | ccp_addr_hi(&op->dst.u.dma);283 } else {284 cr[4] = op->dst.u.sb * CCP_SB_BYTES;285 cr[5] = (CCP_MEMTYPE_SB << REQ6_MEMTYPE_SHIFT);286 }287 288 if (op->eom)289 cr[0] |= REQ1_EOM;290 291 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));292}293 294static int ccp_perform_ecc(struct ccp_op *op)295{296 u32 cr[6];297 298 /* Fill out the register contents for REQ1 through REQ6 */299 cr[0] = REQ1_ECC_AFFINE_CONVERT300 | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT)301 | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT)302 | REQ1_EOM;303 cr[1] = op->src.u.dma.length - 1;304 cr[2] = ccp_addr_lo(&op->src.u.dma);305 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)306 | ccp_addr_hi(&op->src.u.dma);307 cr[4] = ccp_addr_lo(&op->dst.u.dma);308 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)309 | ccp_addr_hi(&op->dst.u.dma);310 311 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));312}313 314static void ccp_disable_queue_interrupts(struct ccp_device *ccp)315{316 iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG);317}318 319static void ccp_enable_queue_interrupts(struct ccp_device *ccp)320{321 iowrite32(ccp->qim, ccp->io_regs + IRQ_MASK_REG);322}323 324static void ccp_irq_bh(unsigned long data)325{326 struct ccp_device *ccp = (struct ccp_device *)data;327 struct ccp_cmd_queue *cmd_q;328 u32 q_int, status;329 unsigned int i;330 331 status = ioread32(ccp->io_regs + IRQ_STATUS_REG);332 333 for (i = 0; i < ccp->cmd_q_count; i++) {334 cmd_q = &ccp->cmd_q[i];335 336 q_int = status & (cmd_q->int_ok | cmd_q->int_err);337 if (q_int) {338 cmd_q->int_status = status;339 cmd_q->q_status = ioread32(cmd_q->reg_status);340 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);341 342 /* On error, only save the first error value */343 if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error)344 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);345 346 cmd_q->int_rcvd = 1;347 348 /* Acknowledge the interrupt and wake the kthread */349 iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG);350 wake_up_interruptible(&cmd_q->int_queue);351 }352 }353 ccp_enable_queue_interrupts(ccp);354}355 356static irqreturn_t ccp_irq_handler(int irq, void *data)357{358 struct ccp_device *ccp = (struct ccp_device *)data;359 360 ccp_disable_queue_interrupts(ccp);361 if (ccp->use_tasklet)362 tasklet_schedule(&ccp->irq_tasklet);363 else364 ccp_irq_bh((unsigned long)ccp);365 366 return IRQ_HANDLED;367}368 369static int ccp_init(struct ccp_device *ccp)370{371 struct device *dev = ccp->dev;372 struct ccp_cmd_queue *cmd_q;373 struct dma_pool *dma_pool;374 char dma_pool_name[MAX_DMAPOOL_NAME_LEN];375 unsigned int qmr, i;376 int ret;377 378 /* Find available queues */379 ccp->qim = 0;380 qmr = ioread32(ccp->io_regs + Q_MASK_REG);381 for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) {382 if (!(qmr & (1 << i)))383 continue;384 385 /* Allocate a dma pool for this queue */386 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",387 ccp->name, i);388 dma_pool = dma_pool_create(dma_pool_name, dev,389 CCP_DMAPOOL_MAX_SIZE,390 CCP_DMAPOOL_ALIGN, 0);391 if (!dma_pool) {392 dev_err(dev, "unable to allocate dma pool\n");393 ret = -ENOMEM;394 goto e_pool;395 }396 397 cmd_q = &ccp->cmd_q[ccp->cmd_q_count];398 ccp->cmd_q_count++;399 400 cmd_q->ccp = ccp;401 cmd_q->id = i;402 cmd_q->dma_pool = dma_pool;403 404 /* Reserve 2 KSB regions for the queue */405 cmd_q->sb_key = KSB_START + ccp->sb_start++;406 cmd_q->sb_ctx = KSB_START + ccp->sb_start++;407 ccp->sb_count -= 2;408 409 /* Preset some register values and masks that are queue410 * number dependent411 */412 cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE +413 (CMD_Q_STATUS_INCR * i);414 cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE +415 (CMD_Q_STATUS_INCR * i);416 cmd_q->int_ok = 1 << (i * 2);417 cmd_q->int_err = 1 << ((i * 2) + 1);418 419 cmd_q->free_slots = ccp_get_free_slots(cmd_q);420 421 init_waitqueue_head(&cmd_q->int_queue);422 423 /* Build queue interrupt mask (two interrupts per queue) */424 ccp->qim |= cmd_q->int_ok | cmd_q->int_err;425 426#ifdef CONFIG_ARM64427 /* For arm64 set the recommended queue cache settings */428 iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE +429 (CMD_Q_CACHE_INC * i));430#endif431 432 dev_dbg(dev, "queue #%u available\n", i);433 }434 if (ccp->cmd_q_count == 0) {435 dev_notice(dev, "no command queues available\n");436 ret = -EIO;437 goto e_pool;438 }439 dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);440 441 /* Disable and clear interrupts until ready */442 ccp_disable_queue_interrupts(ccp);443 for (i = 0; i < ccp->cmd_q_count; i++) {444 cmd_q = &ccp->cmd_q[i];445 446 ioread32(cmd_q->reg_int_status);447 ioread32(cmd_q->reg_status);448 }449 iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG);450 451 /* Request an irq */452 ret = sp_request_ccp_irq(ccp->sp, ccp_irq_handler, ccp->name, ccp);453 if (ret) {454 dev_err(dev, "unable to allocate an IRQ\n");455 goto e_pool;456 }457 458 /* Initialize the ISR tasklet? */459 if (ccp->use_tasklet)460 tasklet_init(&ccp->irq_tasklet, ccp_irq_bh,461 (unsigned long)ccp);462 463 dev_dbg(dev, "Starting threads...\n");464 /* Create a kthread for each queue */465 for (i = 0; i < ccp->cmd_q_count; i++) {466 struct task_struct *kthread;467 468 cmd_q = &ccp->cmd_q[i];469 470 kthread = kthread_run(ccp_cmd_queue_thread, cmd_q,471 "%s-q%u", ccp->name, cmd_q->id);472 if (IS_ERR(kthread)) {473 dev_err(dev, "error creating queue thread (%ld)\n",474 PTR_ERR(kthread));475 ret = PTR_ERR(kthread);476 goto e_kthread;477 }478 479 cmd_q->kthread = kthread;480 }481 482 dev_dbg(dev, "Enabling interrupts...\n");483 /* Enable interrupts */484 ccp_enable_queue_interrupts(ccp);485 486 dev_dbg(dev, "Registering device...\n");487 ccp_add_device(ccp);488 489 ret = ccp_register_rng(ccp);490 if (ret)491 goto e_kthread;492 493 /* Register the DMA engine support */494 ret = ccp_dmaengine_register(ccp);495 if (ret)496 goto e_hwrng;497 498 return 0;499 500e_hwrng:501 ccp_unregister_rng(ccp);502 503e_kthread:504 for (i = 0; i < ccp->cmd_q_count; i++)505 if (ccp->cmd_q[i].kthread)506 kthread_stop(ccp->cmd_q[i].kthread);507 508 sp_free_ccp_irq(ccp->sp, ccp);509 510e_pool:511 for (i = 0; i < ccp->cmd_q_count; i++)512 dma_pool_destroy(ccp->cmd_q[i].dma_pool);513 514 return ret;515}516 517static void ccp_destroy(struct ccp_device *ccp)518{519 struct ccp_cmd_queue *cmd_q;520 struct ccp_cmd *cmd;521 unsigned int i;522 523 /* Unregister the DMA engine */524 ccp_dmaengine_unregister(ccp);525 526 /* Unregister the RNG */527 ccp_unregister_rng(ccp);528 529 /* Remove this device from the list of available units */530 ccp_del_device(ccp);531 532 /* Disable and clear interrupts */533 ccp_disable_queue_interrupts(ccp);534 for (i = 0; i < ccp->cmd_q_count; i++) {535 cmd_q = &ccp->cmd_q[i];536 537 ioread32(cmd_q->reg_int_status);538 ioread32(cmd_q->reg_status);539 }540 iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG);541 542 /* Stop the queue kthreads */543 for (i = 0; i < ccp->cmd_q_count; i++)544 if (ccp->cmd_q[i].kthread)545 kthread_stop(ccp->cmd_q[i].kthread);546 547 sp_free_ccp_irq(ccp->sp, ccp);548 549 for (i = 0; i < ccp->cmd_q_count; i++)550 dma_pool_destroy(ccp->cmd_q[i].dma_pool);551 552 /* Flush the cmd and backlog queue */553 while (!list_empty(&ccp->cmd)) {554 /* Invoke the callback directly with an error code */555 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);556 list_del(&cmd->entry);557 cmd->callback(cmd->data, -ENODEV);558 }559 while (!list_empty(&ccp->backlog)) {560 /* Invoke the callback directly with an error code */561 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);562 list_del(&cmd->entry);563 cmd->callback(cmd->data, -ENODEV);564 }565}566 567static const struct ccp_actions ccp3_actions = {568 .aes = ccp_perform_aes,569 .xts_aes = ccp_perform_xts_aes,570 .des3 = NULL,571 .sha = ccp_perform_sha,572 .rsa = ccp_perform_rsa,573 .passthru = ccp_perform_passthru,574 .ecc = ccp_perform_ecc,575 .sballoc = ccp_alloc_ksb,576 .sbfree = ccp_free_ksb,577 .init = ccp_init,578 .destroy = ccp_destroy,579 .get_free_slots = ccp_get_free_slots,580 .irqhandler = ccp_irq_handler,581};582 583const struct ccp_vdata ccpv3_platform = {584 .version = CCP_VERSION(3, 0),585 .setup = NULL,586 .perform = &ccp3_actions,587 .offset = 0,588 .rsamax = CCP_RSA_MAX_WIDTH,589};590 591const struct ccp_vdata ccpv3 = {592 .version = CCP_VERSION(3, 0),593 .setup = NULL,594 .perform = &ccp3_actions,595 .offset = 0x20000,596 .rsamax = CCP_RSA_MAX_WIDTH,597};598