338 lines · c
1/* SPDX-License-Identifier: GPL-2.0-only */2/*3 * AMD Passthru DMA device driver4 * -- Based on the CCP driver5 *6 * Copyright (C) 2016,2021 Advanced Micro Devices, Inc.7 *8 * Author: Sanjay R Mehta <sanju.mehta@amd.com>9 * Author: Tom Lendacky <thomas.lendacky@amd.com>10 * Author: Gary R Hook <gary.hook@amd.com>11 */12 13#ifndef __PT_DEV_H__14#define __PT_DEV_H__15 16#include <linux/device.h>17#include <linux/dmaengine.h>18#include <linux/pci.h>19#include <linux/spinlock.h>20#include <linux/mutex.h>21#include <linux/list.h>22#include <linux/wait.h>23#include <linux/dmapool.h>24 25#include "../virt-dma.h"26 27#define MAX_PT_NAME_LEN 1628#define MAX_DMAPOOL_NAME_LEN 3229 30#define MAX_HW_QUEUES 131#define MAX_CMD_QLEN 10032 33#define PT_ENGINE_PASSTHRU 534 35/* Register Mappings */36#define IRQ_MASK_REG 0x04037#define IRQ_STATUS_REG 0x20038 39#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f)40 41#define CMD_QUEUE_PRIO_OFFSET 0x0042#define CMD_REQID_CONFIG_OFFSET 0x0443#define CMD_TIMEOUT_OFFSET 0x0844#define CMD_PT_VERSION 0x1045 46#define CMD_Q_CONTROL_BASE 0x000047#define CMD_Q_TAIL_LO_BASE 0x000448#define CMD_Q_HEAD_LO_BASE 0x000849#define CMD_Q_INT_ENABLE_BASE 0x000C50#define CMD_Q_INTERRUPT_STATUS_BASE 0x001051 52#define CMD_Q_STATUS_BASE 0x010053#define CMD_Q_INT_STATUS_BASE 0x010454#define CMD_Q_DMA_STATUS_BASE 0x010855#define CMD_Q_DMA_READ_STATUS_BASE 0x010C56#define CMD_Q_DMA_WRITE_STATUS_BASE 0x011057#define CMD_Q_ABORT_BASE 0x011458#define CMD_Q_AX_CACHE_BASE 0x011859 60#define CMD_CONFIG_OFFSET 0x112061#define CMD_CLK_GATE_CTL_OFFSET 0x600462 63#define CMD_DESC_DW0_VAL 0x50001264 65/* Address offset for virtual queue registers */66#define CMD_Q_STATUS_INCR 0x100067 68/* Bit masks */69#define CMD_CONFIG_REQID 070#define CMD_TIMEOUT_DISABLE 071#define CMD_CLK_DYN_GATING_DIS 072#define CMD_CLK_SW_GATE_MODE 073#define CMD_CLK_GATE_CTL 074#define CMD_QUEUE_PRIO GENMASK(2, 1)75#define CMD_CONFIG_VHB_EN BIT(0)76#define CMD_CLK_DYN_GATING_EN BIT(0)77#define CMD_CLK_HW_GATE_MODE BIT(0)78#define CMD_CLK_GATE_ON_DELAY BIT(12)79#define CMD_CLK_GATE_OFF_DELAY BIT(12)80 81#define CMD_CLK_GATE_CONFIG (CMD_CLK_GATE_CTL | \82 CMD_CLK_HW_GATE_MODE | \83 CMD_CLK_GATE_ON_DELAY | \84 CMD_CLK_DYN_GATING_EN | \85 CMD_CLK_GATE_OFF_DELAY)86 87#define CMD_Q_LEN 3288#define CMD_Q_RUN BIT(0)89#define CMD_Q_HALT BIT(1)90#define CMD_Q_MEM_LOCATION BIT(2)91#define CMD_Q_SIZE_MASK GENMASK(4, 0)92#define CMD_Q_SIZE GENMASK(7, 3)93#define CMD_Q_SHIFT GENMASK(1, 0)94#define QUEUE_SIZE_VAL ((ffs(CMD_Q_LEN) - 2) & \95 CMD_Q_SIZE_MASK)96#define Q_PTR_MASK (2 << (QUEUE_SIZE_VAL + 5) - 1)97#define Q_DESC_SIZE sizeof(struct ptdma_desc)98#define Q_SIZE(n) (CMD_Q_LEN * (n))99 100#define INT_COMPLETION BIT(0)101#define INT_ERROR BIT(1)102#define INT_QUEUE_STOPPED BIT(2)103#define INT_EMPTY_QUEUE BIT(3)104#define SUPPORTED_INTERRUPTS (INT_COMPLETION | INT_ERROR)105 106/****** Local Storage Block ******/107#define LSB_START 0108#define LSB_END 127109#define LSB_COUNT (LSB_END - LSB_START + 1)110 111#define PT_DMAPOOL_MAX_SIZE 64112#define PT_DMAPOOL_ALIGN BIT(5)113 114#define PT_PASSTHRU_BLOCKSIZE 512115 116struct pt_device;117 118struct pt_tasklet_data {119 struct completion completion;120 struct pt_cmd *cmd;121};122 123/*124 * struct pt_passthru_engine - pass-through operation125 * without performing DMA mapping126 * @mask: mask to be applied to data127 * @mask_len: length in bytes of mask128 * @src_dma: data to be used for this operation129 * @dst_dma: data produced by this operation130 * @src_len: length in bytes of data used for this operation131 *132 * Variables required to be set when calling pt_enqueue_cmd():133 * - bit_mod, byte_swap, src, dst, src_len134 * - mask, mask_len if bit_mod is not PT_PASSTHRU_BITWISE_NOOP135 */136struct pt_passthru_engine {137 dma_addr_t mask;138 u32 mask_len; /* In bytes */139 140 dma_addr_t src_dma, dst_dma;141 u64 src_len; /* In bytes */142};143 144/*145 * struct pt_cmd - PTDMA operation request146 * @entry: list element147 * @work: work element used for callbacks148 * @pt: PT device to be run on149 * @ret: operation return code150 * @flags: cmd processing flags151 * @engine: PTDMA operation to perform (passthru)152 * @engine_error: PT engine return code153 * @passthru: engine specific structures, refer to specific engine struct below154 * @callback: operation completion callback function155 * @data: parameter value to be supplied to the callback function156 *157 * Variables required to be set when calling pt_enqueue_cmd():158 * - engine, callback159 * - See the operation structures below for what is required for each160 * operation.161 */162struct pt_cmd {163 struct list_head entry;164 struct work_struct work;165 struct pt_device *pt;166 int ret;167 u32 engine;168 u32 engine_error;169 struct pt_passthru_engine passthru;170 /* Completion callback support */171 void (*pt_cmd_callback)(void *data, int err);172 void *data;173};174 175struct pt_dma_desc {176 struct virt_dma_desc vd;177 struct pt_device *pt;178 enum dma_status status;179 size_t len;180 bool issued_to_hw;181 struct pt_cmd pt_cmd;182};183 184struct pt_dma_chan {185 struct virt_dma_chan vc;186 struct pt_device *pt;187};188 189struct pt_cmd_queue {190 struct pt_device *pt;191 192 /* Queue dma pool */193 struct dma_pool *dma_pool;194 195 /* Queue base address (not necessarily aligned)*/196 struct ptdma_desc *qbase;197 198 /* Aligned queue start address (per requirement) */199 spinlock_t q_lock ____cacheline_aligned;200 unsigned int qidx;201 202 unsigned int qsize;203 dma_addr_t qbase_dma;204 dma_addr_t qdma_tail;205 206 unsigned int active;207 unsigned int suspended;208 209 /* Interrupt flag */210 bool int_en;211 212 /* Register addresses for queue */213 void __iomem *reg_control;214 u32 qcontrol; /* Cached control register */215 216 /* Status values from job */217 u32 int_status;218 u32 q_status;219 u32 q_int_status;220 u32 cmd_error;221 /* Queue Statistics */222 unsigned long total_pt_ops;223} ____cacheline_aligned;224 225struct pt_device {226 struct list_head entry;227 228 unsigned int ord;229 char name[MAX_PT_NAME_LEN];230 231 struct device *dev;232 233 /* Bus specific device information */234 struct pt_msix *pt_msix;235 236 struct pt_dev_vdata *dev_vdata;237 238 unsigned int pt_irq;239 240 /* I/O area used for device communication */241 void __iomem *io_regs;242 243 spinlock_t cmd_lock ____cacheline_aligned;244 unsigned int cmd_count;245 struct list_head cmd;246 247 /*248 * The command queue. This represent the queue available on the249 * PTDMA that are available for processing cmds250 */251 struct pt_cmd_queue cmd_q;252 253 /* Support for the DMA Engine capabilities */254 struct dma_device dma_dev;255 struct pt_dma_chan *pt_dma_chan;256 struct kmem_cache *dma_cmd_cache;257 struct kmem_cache *dma_desc_cache;258 259 wait_queue_head_t lsb_queue;260 261 /* Device Statistics */262 unsigned long total_interrupts;263 264 struct pt_tasklet_data tdata;265};266 267/*268 * descriptor for PTDMA commands269 * 8 32-bit words:270 * word 0: function; engine; control bits271 * word 1: length of source data272 * word 2: low 32 bits of source pointer273 * word 3: upper 16 bits of source pointer; source memory type274 * word 4: low 32 bits of destination pointer275 * word 5: upper 16 bits of destination pointer; destination memory type276 * word 6: reserved 32 bits277 * word 7: reserved 32 bits278 */279 280#define DWORD0_SOC BIT(0)281#define DWORD0_IOC BIT(1)282 283struct dword3 {284 unsigned int src_hi:16;285 unsigned int src_mem:2;286 unsigned int lsb_cxt_id:8;287 unsigned int rsvd1:5;288 unsigned int fixed:1;289};290 291struct dword5 {292 unsigned int dst_hi:16;293 unsigned int dst_mem:2;294 unsigned int rsvd1:13;295 unsigned int fixed:1;296};297 298struct ptdma_desc {299 u32 dw0;300 u32 length;301 u32 src_lo;302 struct dword3 dw3;303 u32 dst_lo;304 struct dword5 dw5;305 __le32 rsvd1;306 __le32 rsvd2;307};308 309/* Structure to hold PT device data */310struct pt_dev_vdata {311 const unsigned int bar;312};313 314int pt_dmaengine_register(struct pt_device *pt);315void pt_dmaengine_unregister(struct pt_device *pt);316 317void ptdma_debugfs_setup(struct pt_device *pt);318int pt_core_init(struct pt_device *pt);319void pt_core_destroy(struct pt_device *pt);320 321int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q,322 struct pt_passthru_engine *pt_engine);323 324void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q);325void pt_start_queue(struct pt_cmd_queue *cmd_q);326void pt_stop_queue(struct pt_cmd_queue *cmd_q);327 328static inline void pt_core_disable_queue_interrupts(struct pt_device *pt)329{330 iowrite32(0, pt->cmd_q.reg_control + 0x000C);331}332 333static inline void pt_core_enable_queue_interrupts(struct pt_device *pt)334{335 iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C);336}337#endif338