671 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * AMD Cryptographic Coprocessor (CCP) driver4 *5 * Copyright (C) 2013,2019 Advanced Micro Devices, Inc.6 *7 * Author: Tom Lendacky <thomas.lendacky@amd.com>8 * Author: Gary R Hook <gary.hook@amd.com>9 */10 11#include <linux/module.h>12#include <linux/kernel.h>13#include <linux/kthread.h>14#include <linux/sched.h>15#include <linux/interrupt.h>16#include <linux/spinlock.h>17#include <linux/spinlock_types.h>18#include <linux/types.h>19#include <linux/mutex.h>20#include <linux/delay.h>21#include <linux/hw_random.h>22#include <linux/cpu.h>23#include <linux/atomic.h>24#ifdef CONFIG_X8625#include <asm/cpu_device_id.h>26#endif27#include <linux/ccp.h>28 29#include "ccp-dev.h"30 31#define MAX_CCPS 3232 33/* Limit CCP use to a specifed number of queues per device */34static unsigned int nqueues;35module_param(nqueues, uint, 0444);36MODULE_PARM_DESC(nqueues, "Number of queues per CCP (minimum 1; default: all available)");37 38/* Limit the maximum number of configured CCPs */39static atomic_t dev_count = ATOMIC_INIT(0);40static unsigned int max_devs = MAX_CCPS;41module_param(max_devs, uint, 0444);42MODULE_PARM_DESC(max_devs, "Maximum number of CCPs to enable (default: all; 0 disables all CCPs)");43 44struct ccp_tasklet_data {45 struct completion completion;46 struct ccp_cmd *cmd;47};48 49/* Human-readable error strings */50#define CCP_MAX_ERROR_CODE 6451static char *ccp_error_codes[] = {52 "",53 "ILLEGAL_ENGINE",54 "ILLEGAL_KEY_ID",55 "ILLEGAL_FUNCTION_TYPE",56 "ILLEGAL_FUNCTION_MODE",57 "ILLEGAL_FUNCTION_ENCRYPT",58 "ILLEGAL_FUNCTION_SIZE",59 "Zlib_MISSING_INIT_EOM",60 "ILLEGAL_FUNCTION_RSVD",61 "ILLEGAL_BUFFER_LENGTH",62 "VLSB_FAULT",63 "ILLEGAL_MEM_ADDR",64 "ILLEGAL_MEM_SEL",65 "ILLEGAL_CONTEXT_ID",66 "ILLEGAL_KEY_ADDR",67 "0xF Reserved",68 "Zlib_ILLEGAL_MULTI_QUEUE",69 "Zlib_ILLEGAL_JOBID_CHANGE",70 "CMD_TIMEOUT",71 "IDMA0_AXI_SLVERR",72 "IDMA0_AXI_DECERR",73 "0x15 Reserved",74 "IDMA1_AXI_SLAVE_FAULT",75 "IDMA1_AIXI_DECERR",76 "0x18 Reserved",77 "ZLIBVHB_AXI_SLVERR",78 "ZLIBVHB_AXI_DECERR",79 "0x1B Reserved",80 "ZLIB_UNEXPECTED_EOM",81 "ZLIB_EXTRA_DATA",82 "ZLIB_BTYPE",83 "ZLIB_UNDEFINED_SYMBOL",84 "ZLIB_UNDEFINED_DISTANCE_S",85 "ZLIB_CODE_LENGTH_SYMBOL",86 "ZLIB _VHB_ILLEGAL_FETCH",87 "ZLIB_UNCOMPRESSED_LEN",88 "ZLIB_LIMIT_REACHED",89 "ZLIB_CHECKSUM_MISMATCH0",90 "ODMA0_AXI_SLVERR",91 "ODMA0_AXI_DECERR",92 "0x28 Reserved",93 "ODMA1_AXI_SLVERR",94 "ODMA1_AXI_DECERR",95};96 97void ccp_log_error(struct ccp_device *d, unsigned int e)98{99 if (WARN_ON(e >= CCP_MAX_ERROR_CODE))100 return;101 102 if (e < ARRAY_SIZE(ccp_error_codes))103 dev_err(d->dev, "CCP error %d: %s\n", e, ccp_error_codes[e]);104 else105 dev_err(d->dev, "CCP error %d: Unknown Error\n", e);106}107 108/* List of CCPs, CCP count, read-write access lock, and access functions109 *110 * Lock structure: get ccp_unit_lock for reading whenever we need to111 * examine the CCP list. While holding it for reading we can acquire112 * the RR lock to update the round-robin next-CCP pointer. The unit lock113 * must be acquired before the RR lock.114 *115 * If the unit-lock is acquired for writing, we have total control over116 * the list, so there's no value in getting the RR lock.117 */118static DEFINE_RWLOCK(ccp_unit_lock);119static LIST_HEAD(ccp_units);120 121/* Round-robin counter */122static DEFINE_SPINLOCK(ccp_rr_lock);123static struct ccp_device *ccp_rr;124 125/**126 * ccp_add_device - add a CCP device to the list127 *128 * @ccp: ccp_device struct pointer129 *130 * Put this CCP on the unit list, which makes it available131 * for use.132 *133 * Returns zero if a CCP device is present, -ENODEV otherwise.134 */135void ccp_add_device(struct ccp_device *ccp)136{137 unsigned long flags;138 139 write_lock_irqsave(&ccp_unit_lock, flags);140 list_add_tail(&ccp->entry, &ccp_units);141 if (!ccp_rr)142 /* We already have the list lock (we're first) so this143 * pointer can't change on us. Set its initial value.144 */145 ccp_rr = ccp;146 write_unlock_irqrestore(&ccp_unit_lock, flags);147}148 149/**150 * ccp_del_device - remove a CCP device from the list151 *152 * @ccp: ccp_device struct pointer153 *154 * Remove this unit from the list of devices. If the next device155 * up for use is this one, adjust the pointer. If this is the last156 * device, NULL the pointer.157 */158void ccp_del_device(struct ccp_device *ccp)159{160 unsigned long flags;161 162 write_lock_irqsave(&ccp_unit_lock, flags);163 if (ccp_rr == ccp) {164 /* ccp_unit_lock is read/write; any read access165 * will be suspended while we make changes to the166 * list and RR pointer.167 */168 if (list_is_last(&ccp_rr->entry, &ccp_units))169 ccp_rr = list_first_entry(&ccp_units, struct ccp_device,170 entry);171 else172 ccp_rr = list_next_entry(ccp_rr, entry);173 }174 list_del(&ccp->entry);175 if (list_empty(&ccp_units))176 ccp_rr = NULL;177 write_unlock_irqrestore(&ccp_unit_lock, flags);178}179 180 181 182int ccp_register_rng(struct ccp_device *ccp)183{184 int ret = 0;185 186 dev_dbg(ccp->dev, "Registering RNG...\n");187 /* Register an RNG */188 ccp->hwrng.name = ccp->rngname;189 ccp->hwrng.read = ccp_trng_read;190 ret = hwrng_register(&ccp->hwrng);191 if (ret)192 dev_err(ccp->dev, "error registering hwrng (%d)\n", ret);193 194 return ret;195}196 197void ccp_unregister_rng(struct ccp_device *ccp)198{199 if (ccp->hwrng.name)200 hwrng_unregister(&ccp->hwrng);201}202 203static struct ccp_device *ccp_get_device(void)204{205 unsigned long flags;206 struct ccp_device *dp = NULL;207 208 /* We round-robin through the unit list.209 * The (ccp_rr) pointer refers to the next unit to use.210 */211 read_lock_irqsave(&ccp_unit_lock, flags);212 if (!list_empty(&ccp_units)) {213 spin_lock(&ccp_rr_lock);214 dp = ccp_rr;215 if (list_is_last(&ccp_rr->entry, &ccp_units))216 ccp_rr = list_first_entry(&ccp_units, struct ccp_device,217 entry);218 else219 ccp_rr = list_next_entry(ccp_rr, entry);220 spin_unlock(&ccp_rr_lock);221 }222 read_unlock_irqrestore(&ccp_unit_lock, flags);223 224 return dp;225}226 227/**228 * ccp_present - check if a CCP device is present229 *230 * Returns zero if a CCP device is present, -ENODEV otherwise.231 */232int ccp_present(void)233{234 unsigned long flags;235 int ret;236 237 read_lock_irqsave(&ccp_unit_lock, flags);238 ret = list_empty(&ccp_units);239 read_unlock_irqrestore(&ccp_unit_lock, flags);240 241 return ret ? -ENODEV : 0;242}243EXPORT_SYMBOL_GPL(ccp_present);244 245/**246 * ccp_version - get the version of the CCP device247 *248 * Returns the version from the first unit on the list;249 * otherwise a zero if no CCP device is present250 */251unsigned int ccp_version(void)252{253 struct ccp_device *dp;254 unsigned long flags;255 int ret = 0;256 257 read_lock_irqsave(&ccp_unit_lock, flags);258 if (!list_empty(&ccp_units)) {259 dp = list_first_entry(&ccp_units, struct ccp_device, entry);260 ret = dp->vdata->version;261 }262 read_unlock_irqrestore(&ccp_unit_lock, flags);263 264 return ret;265}266EXPORT_SYMBOL_GPL(ccp_version);267 268/**269 * ccp_enqueue_cmd - queue an operation for processing by the CCP270 *271 * @cmd: ccp_cmd struct to be processed272 *273 * Queue a cmd to be processed by the CCP. If queueing the cmd274 * would exceed the defined length of the cmd queue the cmd will275 * only be queued if the CCP_CMD_MAY_BACKLOG flag is set and will276 * result in a return code of -EBUSY.277 *278 * The callback routine specified in the ccp_cmd struct will be279 * called to notify the caller of completion (if the cmd was not280 * backlogged) or advancement out of the backlog. If the cmd has281 * advanced out of the backlog the "err" value of the callback282 * will be -EINPROGRESS. Any other "err" value during callback is283 * the result of the operation.284 *285 * The cmd has been successfully queued if:286 * the return code is -EINPROGRESS or287 * the return code is -EBUSY and CCP_CMD_MAY_BACKLOG flag is set288 */289int ccp_enqueue_cmd(struct ccp_cmd *cmd)290{291 struct ccp_device *ccp;292 unsigned long flags;293 unsigned int i;294 int ret;295 296 /* Some commands might need to be sent to a specific device */297 ccp = cmd->ccp ? cmd->ccp : ccp_get_device();298 299 if (!ccp)300 return -ENODEV;301 302 /* Caller must supply a callback routine */303 if (!cmd->callback)304 return -EINVAL;305 306 cmd->ccp = ccp;307 308 spin_lock_irqsave(&ccp->cmd_lock, flags);309 310 i = ccp->cmd_q_count;311 312 if (ccp->cmd_count >= MAX_CMD_QLEN) {313 if (cmd->flags & CCP_CMD_MAY_BACKLOG) {314 ret = -EBUSY;315 list_add_tail(&cmd->entry, &ccp->backlog);316 } else {317 ret = -ENOSPC;318 }319 } else {320 ret = -EINPROGRESS;321 ccp->cmd_count++;322 list_add_tail(&cmd->entry, &ccp->cmd);323 324 /* Find an idle queue */325 if (!ccp->suspending) {326 for (i = 0; i < ccp->cmd_q_count; i++) {327 if (ccp->cmd_q[i].active)328 continue;329 330 break;331 }332 }333 }334 335 spin_unlock_irqrestore(&ccp->cmd_lock, flags);336 337 /* If we found an idle queue, wake it up */338 if (i < ccp->cmd_q_count)339 wake_up_process(ccp->cmd_q[i].kthread);340 341 return ret;342}343EXPORT_SYMBOL_GPL(ccp_enqueue_cmd);344 345static void ccp_do_cmd_backlog(struct work_struct *work)346{347 struct ccp_cmd *cmd = container_of(work, struct ccp_cmd, work);348 struct ccp_device *ccp = cmd->ccp;349 unsigned long flags;350 unsigned int i;351 352 cmd->callback(cmd->data, -EINPROGRESS);353 354 spin_lock_irqsave(&ccp->cmd_lock, flags);355 356 ccp->cmd_count++;357 list_add_tail(&cmd->entry, &ccp->cmd);358 359 /* Find an idle queue */360 for (i = 0; i < ccp->cmd_q_count; i++) {361 if (ccp->cmd_q[i].active)362 continue;363 364 break;365 }366 367 spin_unlock_irqrestore(&ccp->cmd_lock, flags);368 369 /* If we found an idle queue, wake it up */370 if (i < ccp->cmd_q_count)371 wake_up_process(ccp->cmd_q[i].kthread);372}373 374static struct ccp_cmd *ccp_dequeue_cmd(struct ccp_cmd_queue *cmd_q)375{376 struct ccp_device *ccp = cmd_q->ccp;377 struct ccp_cmd *cmd = NULL;378 struct ccp_cmd *backlog = NULL;379 unsigned long flags;380 381 spin_lock_irqsave(&ccp->cmd_lock, flags);382 383 cmd_q->active = 0;384 385 if (ccp->suspending) {386 cmd_q->suspended = 1;387 388 spin_unlock_irqrestore(&ccp->cmd_lock, flags);389 wake_up_interruptible(&ccp->suspend_queue);390 391 return NULL;392 }393 394 if (ccp->cmd_count) {395 cmd_q->active = 1;396 397 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);398 list_del(&cmd->entry);399 400 ccp->cmd_count--;401 }402 403 if (!list_empty(&ccp->backlog)) {404 backlog = list_first_entry(&ccp->backlog, struct ccp_cmd,405 entry);406 list_del(&backlog->entry);407 }408 409 spin_unlock_irqrestore(&ccp->cmd_lock, flags);410 411 if (backlog) {412 INIT_WORK(&backlog->work, ccp_do_cmd_backlog);413 schedule_work(&backlog->work);414 }415 416 return cmd;417}418 419static void ccp_do_cmd_complete(unsigned long data)420{421 struct ccp_tasklet_data *tdata = (struct ccp_tasklet_data *)data;422 struct ccp_cmd *cmd = tdata->cmd;423 424 cmd->callback(cmd->data, cmd->ret);425 426 complete(&tdata->completion);427}428 429/**430 * ccp_cmd_queue_thread - create a kernel thread to manage a CCP queue431 *432 * @data: thread-specific data433 */434int ccp_cmd_queue_thread(void *data)435{436 struct ccp_cmd_queue *cmd_q = (struct ccp_cmd_queue *)data;437 struct ccp_cmd *cmd;438 struct ccp_tasklet_data tdata;439 struct tasklet_struct tasklet;440 441 tasklet_init(&tasklet, ccp_do_cmd_complete, (unsigned long)&tdata);442 443 set_current_state(TASK_INTERRUPTIBLE);444 while (!kthread_should_stop()) {445 schedule();446 447 set_current_state(TASK_INTERRUPTIBLE);448 449 cmd = ccp_dequeue_cmd(cmd_q);450 if (!cmd)451 continue;452 453 __set_current_state(TASK_RUNNING);454 455 /* Execute the command */456 cmd->ret = ccp_run_cmd(cmd_q, cmd);457 458 /* Schedule the completion callback */459 tdata.cmd = cmd;460 init_completion(&tdata.completion);461 tasklet_schedule(&tasklet);462 wait_for_completion(&tdata.completion);463 }464 465 __set_current_state(TASK_RUNNING);466 467 return 0;468}469 470/**471 * ccp_alloc_struct - allocate and initialize the ccp_device struct472 *473 * @sp: sp_device struct of the CCP474 */475struct ccp_device *ccp_alloc_struct(struct sp_device *sp)476{477 struct device *dev = sp->dev;478 struct ccp_device *ccp;479 480 ccp = devm_kzalloc(dev, sizeof(*ccp), GFP_KERNEL);481 if (!ccp)482 return NULL;483 ccp->dev = dev;484 ccp->sp = sp;485 ccp->axcache = sp->axcache;486 487 INIT_LIST_HEAD(&ccp->cmd);488 INIT_LIST_HEAD(&ccp->backlog);489 490 spin_lock_init(&ccp->cmd_lock);491 mutex_init(&ccp->req_mutex);492 mutex_init(&ccp->sb_mutex);493 ccp->sb_count = KSB_COUNT;494 ccp->sb_start = 0;495 496 /* Initialize the wait queues */497 init_waitqueue_head(&ccp->sb_queue);498 init_waitqueue_head(&ccp->suspend_queue);499 500 snprintf(ccp->name, MAX_CCP_NAME_LEN, "ccp-%u", sp->ord);501 snprintf(ccp->rngname, MAX_CCP_NAME_LEN, "ccp-%u-rng", sp->ord);502 503 return ccp;504}505 506int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait)507{508 struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng);509 u32 trng_value;510 int len = min_t(int, sizeof(trng_value), max);511 512 /* Locking is provided by the caller so we can update device513 * hwrng-related fields safely514 */515 trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG);516 if (!trng_value) {517 /* Zero is returned if not data is available or if a518 * bad-entropy error is present. Assume an error if519 * we exceed TRNG_RETRIES reads of zero.520 */521 if (ccp->hwrng_retries++ > TRNG_RETRIES)522 return -EIO;523 524 return 0;525 }526 527 /* Reset the counter and save the rng value */528 ccp->hwrng_retries = 0;529 memcpy(data, &trng_value, len);530 531 return len;532}533 534bool ccp_queues_suspended(struct ccp_device *ccp)535{536 unsigned int suspended = 0;537 unsigned long flags;538 unsigned int i;539 540 spin_lock_irqsave(&ccp->cmd_lock, flags);541 542 for (i = 0; i < ccp->cmd_q_count; i++)543 if (ccp->cmd_q[i].suspended)544 suspended++;545 546 spin_unlock_irqrestore(&ccp->cmd_lock, flags);547 548 return ccp->cmd_q_count == suspended;549}550 551void ccp_dev_suspend(struct sp_device *sp)552{553 struct ccp_device *ccp = sp->ccp_data;554 unsigned long flags;555 unsigned int i;556 557 /* If there's no device there's nothing to do */558 if (!ccp)559 return;560 561 spin_lock_irqsave(&ccp->cmd_lock, flags);562 563 ccp->suspending = 1;564 565 /* Wake all the queue kthreads to prepare for suspend */566 for (i = 0; i < ccp->cmd_q_count; i++)567 wake_up_process(ccp->cmd_q[i].kthread);568 569 spin_unlock_irqrestore(&ccp->cmd_lock, flags);570 571 /* Wait for all queue kthreads to say they're done */572 while (!ccp_queues_suspended(ccp))573 wait_event_interruptible(ccp->suspend_queue,574 ccp_queues_suspended(ccp));575}576 577void ccp_dev_resume(struct sp_device *sp)578{579 struct ccp_device *ccp = sp->ccp_data;580 unsigned long flags;581 unsigned int i;582 583 /* If there's no device there's nothing to do */584 if (!ccp)585 return;586 587 spin_lock_irqsave(&ccp->cmd_lock, flags);588 589 ccp->suspending = 0;590 591 /* Wake up all the kthreads */592 for (i = 0; i < ccp->cmd_q_count; i++) {593 ccp->cmd_q[i].suspended = 0;594 wake_up_process(ccp->cmd_q[i].kthread);595 }596 597 spin_unlock_irqrestore(&ccp->cmd_lock, flags);598}599 600int ccp_dev_init(struct sp_device *sp)601{602 struct device *dev = sp->dev;603 struct ccp_device *ccp;604 int ret;605 606 /*607 * Check how many we have so far, and stop after reaching608 * that number609 */610 if (atomic_inc_return(&dev_count) > max_devs)611 return 0; /* don't fail the load */612 613 ret = -ENOMEM;614 ccp = ccp_alloc_struct(sp);615 if (!ccp)616 goto e_err;617 sp->ccp_data = ccp;618 619 if (!nqueues || (nqueues > MAX_HW_QUEUES))620 ccp->max_q_count = MAX_HW_QUEUES;621 else622 ccp->max_q_count = nqueues;623 624 ccp->vdata = (struct ccp_vdata *)sp->dev_vdata->ccp_vdata;625 if (!ccp->vdata || !ccp->vdata->version) {626 ret = -ENODEV;627 dev_err(dev, "missing driver data\n");628 goto e_err;629 }630 631 ccp->use_tasklet = sp->use_tasklet;632 633 ccp->io_regs = sp->io_map + ccp->vdata->offset;634 if (ccp->vdata->setup)635 ccp->vdata->setup(ccp);636 637 ret = ccp->vdata->perform->init(ccp);638 if (ret) {639 /* A positive number means that the device cannot be initialized,640 * but no additional message is required.641 */642 if (ret > 0)643 goto e_quiet;644 645 /* An unexpected problem occurred, and should be reported in the log */646 goto e_err;647 }648 649 dev_notice(dev, "ccp enabled\n");650 651 return 0;652 653e_err:654 dev_notice(dev, "ccp initialization failed\n");655 656e_quiet:657 sp->ccp_data = NULL;658 659 return ret;660}661 662void ccp_dev_destroy(struct sp_device *sp)663{664 struct ccp_device *ccp = sp->ccp_data;665 666 if (!ccp)667 return;668 669 ccp->vdata->perform->destroy(ccp);670}671