310 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * AMD Passthru DMA device driver4 * -- Based on the CCP driver5 *6 * Copyright (C) 2016,2021 Advanced Micro Devices, Inc.7 *8 * Author: Sanjay R Mehta <sanju.mehta@amd.com>9 * Author: Gary R Hook <gary.hook@amd.com>10 */11 12#include <linux/bitfield.h>13#include <linux/dma-mapping.h>14#include <linux/debugfs.h>15#include <linux/interrupt.h>16#include <linux/kernel.h>17#include <linux/module.h>18#include <linux/pci.h>19 20#include "ptdma.h"21 22/* Human-readable error strings */23static char *pt_error_codes[] = {24 "",25 "ERR 01: ILLEGAL_ENGINE",26 "ERR 03: ILLEGAL_FUNCTION_TYPE",27 "ERR 04: ILLEGAL_FUNCTION_MODE",28 "ERR 06: ILLEGAL_FUNCTION_SIZE",29 "ERR 08: ILLEGAL_FUNCTION_RSVD",30 "ERR 09: ILLEGAL_BUFFER_LENGTH",31 "ERR 10: VLSB_FAULT",32 "ERR 11: ILLEGAL_MEM_ADDR",33 "ERR 12: ILLEGAL_MEM_SEL",34 "ERR 13: ILLEGAL_CONTEXT_ID",35 "ERR 15: 0xF Reserved",36 "ERR 18: CMD_TIMEOUT",37 "ERR 19: IDMA0_AXI_SLVERR",38 "ERR 20: IDMA0_AXI_DECERR",39 "ERR 21: 0x15 Reserved",40 "ERR 22: IDMA1_AXI_SLAVE_FAULT",41 "ERR 23: IDMA1_AIXI_DECERR",42 "ERR 24: 0x18 Reserved",43 "ERR 27: 0x1B Reserved",44 "ERR 38: ODMA0_AXI_SLVERR",45 "ERR 39: ODMA0_AXI_DECERR",46 "ERR 40: 0x28 Reserved",47 "ERR 41: ODMA1_AXI_SLVERR",48 "ERR 42: ODMA1_AXI_DECERR",49 "ERR 43: LSB_PARITY_ERR",50};51 52static void pt_log_error(struct pt_device *d, int e)53{54 dev_err(d->dev, "PTDMA error: %s (0x%x)\n", pt_error_codes[e], e);55}56 57void pt_start_queue(struct pt_cmd_queue *cmd_q)58{59 /* Turn on the run bit */60 iowrite32(cmd_q->qcontrol | CMD_Q_RUN, cmd_q->reg_control);61}62 63void pt_stop_queue(struct pt_cmd_queue *cmd_q)64{65 /* Turn off the run bit */66 iowrite32(cmd_q->qcontrol & ~CMD_Q_RUN, cmd_q->reg_control);67}68 69static int pt_core_execute_cmd(struct ptdma_desc *desc, struct pt_cmd_queue *cmd_q)70{71 bool soc = FIELD_GET(DWORD0_SOC, desc->dw0);72 u8 *q_desc = (u8 *)&cmd_q->qbase[cmd_q->qidx];73 u32 tail;74 unsigned long flags;75 76 if (soc) {77 desc->dw0 |= FIELD_PREP(DWORD0_IOC, desc->dw0);78 desc->dw0 &= ~DWORD0_SOC;79 }80 spin_lock_irqsave(&cmd_q->q_lock, flags);81 82 /* Copy 32-byte command descriptor to hw queue. */83 memcpy(q_desc, desc, 32);84 cmd_q->qidx = (cmd_q->qidx + 1) % CMD_Q_LEN;85 86 /* The data used by this command must be flushed to memory */87 wmb();88 89 /* Write the new tail address back to the queue register */90 tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);91 iowrite32(tail, cmd_q->reg_control + 0x0004);92 93 /* Turn the queue back on using our cached control register */94 pt_start_queue(cmd_q);95 spin_unlock_irqrestore(&cmd_q->q_lock, flags);96 97 return 0;98}99 100int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q,101 struct pt_passthru_engine *pt_engine)102{103 struct ptdma_desc desc;104 struct pt_device *pt = container_of(cmd_q, struct pt_device, cmd_q);105 106 cmd_q->cmd_error = 0;107 cmd_q->total_pt_ops++;108 memset(&desc, 0, sizeof(desc));109 desc.dw0 = CMD_DESC_DW0_VAL;110 desc.length = pt_engine->src_len;111 desc.src_lo = lower_32_bits(pt_engine->src_dma);112 desc.dw3.src_hi = upper_32_bits(pt_engine->src_dma);113 desc.dst_lo = lower_32_bits(pt_engine->dst_dma);114 desc.dw5.dst_hi = upper_32_bits(pt_engine->dst_dma);115 116 if (cmd_q->int_en)117 pt_core_enable_queue_interrupts(pt);118 else119 pt_core_disable_queue_interrupts(pt);120 121 return pt_core_execute_cmd(&desc, cmd_q);122}123 124static void pt_do_cmd_complete(unsigned long data)125{126 struct pt_tasklet_data *tdata = (struct pt_tasklet_data *)data;127 struct pt_cmd *cmd = tdata->cmd;128 struct pt_cmd_queue *cmd_q = &cmd->pt->cmd_q;129 u32 tail;130 131 if (cmd_q->cmd_error) {132 /*133 * Log the error and flush the queue by134 * moving the head pointer135 */136 tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);137 pt_log_error(cmd_q->pt, cmd_q->cmd_error);138 iowrite32(tail, cmd_q->reg_control + 0x0008);139 }140 141 cmd->pt_cmd_callback(cmd->data, cmd->ret);142}143 144void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q)145{146 u32 status;147 148 status = ioread32(cmd_q->reg_control + 0x0010);149 if (status) {150 cmd_q->int_status = status;151 cmd_q->q_status = ioread32(cmd_q->reg_control + 0x0100);152 cmd_q->q_int_status = ioread32(cmd_q->reg_control + 0x0104);153 154 /* On error, only save the first error value */155 if ((status & INT_ERROR) && !cmd_q->cmd_error)156 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);157 158 /* Acknowledge the completion */159 iowrite32(status, cmd_q->reg_control + 0x0010);160 pt_do_cmd_complete((ulong)&pt->tdata);161 }162}163 164static irqreturn_t pt_core_irq_handler(int irq, void *data)165{166 struct pt_device *pt = data;167 struct pt_cmd_queue *cmd_q = &pt->cmd_q;168 169 pt_core_disable_queue_interrupts(pt);170 pt->total_interrupts++;171 pt_check_status_trans(pt, cmd_q);172 pt_core_enable_queue_interrupts(pt);173 return IRQ_HANDLED;174}175 176int pt_core_init(struct pt_device *pt)177{178 char dma_pool_name[MAX_DMAPOOL_NAME_LEN];179 struct pt_cmd_queue *cmd_q = &pt->cmd_q;180 u32 dma_addr_lo, dma_addr_hi;181 struct device *dev = pt->dev;182 struct dma_pool *dma_pool;183 int ret;184 185 /* Allocate a dma pool for the queue */186 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q", dev_name(pt->dev));187 188 dma_pool = dma_pool_create(dma_pool_name, dev,189 PT_DMAPOOL_MAX_SIZE,190 PT_DMAPOOL_ALIGN, 0);191 if (!dma_pool)192 return -ENOMEM;193 194 /* ptdma core initialisation */195 iowrite32(CMD_CONFIG_VHB_EN, pt->io_regs + CMD_CONFIG_OFFSET);196 iowrite32(CMD_QUEUE_PRIO, pt->io_regs + CMD_QUEUE_PRIO_OFFSET);197 iowrite32(CMD_TIMEOUT_DISABLE, pt->io_regs + CMD_TIMEOUT_OFFSET);198 iowrite32(CMD_CLK_GATE_CONFIG, pt->io_regs + CMD_CLK_GATE_CTL_OFFSET);199 iowrite32(CMD_CONFIG_REQID, pt->io_regs + CMD_REQID_CONFIG_OFFSET);200 201 cmd_q->pt = pt;202 cmd_q->dma_pool = dma_pool;203 spin_lock_init(&cmd_q->q_lock);204 205 /* Page alignment satisfies our needs for N <= 128 */206 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);207 cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize,208 &cmd_q->qbase_dma,209 GFP_KERNEL);210 if (!cmd_q->qbase) {211 dev_err(dev, "unable to allocate command queue\n");212 ret = -ENOMEM;213 goto e_destroy_pool;214 }215 216 cmd_q->qidx = 0;217 218 /* Preset some register values */219 cmd_q->reg_control = pt->io_regs + CMD_Q_STATUS_INCR;220 221 /* Turn off the queues and disable interrupts until ready */222 pt_core_disable_queue_interrupts(pt);223 224 cmd_q->qcontrol = 0; /* Start with nothing */225 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);226 227 ioread32(cmd_q->reg_control + 0x0104);228 ioread32(cmd_q->reg_control + 0x0100);229 230 /* Clear the interrupt status */231 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);232 233 /* Request an irq */234 ret = request_irq(pt->pt_irq, pt_core_irq_handler, 0, dev_name(pt->dev), pt);235 if (ret) {236 dev_err(dev, "unable to allocate an IRQ\n");237 goto e_free_dma;238 }239 240 /* Update the device registers with queue information. */241 cmd_q->qcontrol &= ~CMD_Q_SIZE;242 cmd_q->qcontrol |= FIELD_PREP(CMD_Q_SIZE, QUEUE_SIZE_VAL);243 244 cmd_q->qdma_tail = cmd_q->qbase_dma;245 dma_addr_lo = lower_32_bits(cmd_q->qdma_tail);246 iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0004);247 iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0008);248 249 dma_addr_hi = upper_32_bits(cmd_q->qdma_tail);250 cmd_q->qcontrol |= (dma_addr_hi << 16);251 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);252 253 pt_core_enable_queue_interrupts(pt);254 255 /* Register the DMA engine support */256 ret = pt_dmaengine_register(pt);257 if (ret)258 goto e_free_irq;259 260 /* Set up debugfs entries */261 ptdma_debugfs_setup(pt);262 263 return 0;264 265e_free_irq:266 free_irq(pt->pt_irq, pt);267 268e_free_dma:269 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma);270 271e_destroy_pool:272 dma_pool_destroy(pt->cmd_q.dma_pool);273 274 return ret;275}276 277void pt_core_destroy(struct pt_device *pt)278{279 struct device *dev = pt->dev;280 struct pt_cmd_queue *cmd_q = &pt->cmd_q;281 struct pt_cmd *cmd;282 283 /* Unregister the DMA engine */284 pt_dmaengine_unregister(pt);285 286 /* Disable and clear interrupts */287 pt_core_disable_queue_interrupts(pt);288 289 /* Turn off the run bit */290 pt_stop_queue(cmd_q);291 292 /* Clear the interrupt status */293 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);294 ioread32(cmd_q->reg_control + 0x0104);295 ioread32(cmd_q->reg_control + 0x0100);296 297 free_irq(pt->pt_irq, pt);298 299 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,300 cmd_q->qbase_dma);301 302 /* Flush the cmd queue */303 while (!list_empty(&pt->cmd)) {304 /* Invoke the callback directly with an error code */305 cmd = list_first_entry(&pt->cmd, struct pt_cmd, entry);306 list_del(&cmd->entry);307 cmd->pt_cmd_callback(cmd->data, -ENODEV);308 }309}310