1326 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * DMA driver for Xilinx DMA/Bridge Subsystem4 *5 * Copyright (C) 2017-2020 Xilinx, Inc. All rights reserved.6 * Copyright (C) 2022, Advanced Micro Devices, Inc.7 */8 9/*10 * The DMA/Bridge Subsystem for PCI Express allows for the movement of data11 * between Host memory and the DMA subsystem. It does this by operating on12 * 'descriptors' that contain information about the source, destination and13 * amount of data to transfer. These direct memory transfers can be both in14 * the Host to Card (H2C) and Card to Host (C2H) transfers. The DMA can be15 * configured to have a single AXI4 Master interface shared by all channels16 * or one AXI4-Stream interface for each channel enabled. Memory transfers are17 * specified on a per-channel basis in descriptor linked lists, which the DMA18 * fetches from host memory and processes. Events such as descriptor completion19 * and errors are signaled using interrupts. The core also provides up to 1620 * user interrupt wires that generate interrupts to the host.21 */22 23#include <linux/mod_devicetable.h>24#include <linux/bitfield.h>25#include <linux/dmapool.h>26#include <linux/regmap.h>27#include <linux/dmaengine.h>28#include <linux/dma/amd_xdma.h>29#include <linux/platform_device.h>30#include <linux/platform_data/amd_xdma.h>31#include <linux/dma-mapping.h>32#include <linux/pci.h>33#include "../virt-dma.h"34#include "xdma-regs.h"35 36/* mmio regmap config for all XDMA registers */37static const struct regmap_config xdma_regmap_config = {38 .reg_bits = 32,39 .val_bits = 32,40 .reg_stride = 4,41 .max_register = XDMA_REG_SPACE_LEN,42};43 44/**45 * struct xdma_desc_block - Descriptor block46 * @virt_addr: Virtual address of block start47 * @dma_addr: DMA address of block start48 */49struct xdma_desc_block {50 void *virt_addr;51 dma_addr_t dma_addr;52};53 54/**55 * struct xdma_chan - Driver specific DMA channel structure56 * @vchan: Virtual channel57 * @xdev_hdl: Pointer to DMA device structure58 * @base: Offset of channel registers59 * @desc_pool: Descriptor pool60 * @busy: Busy flag of the channel61 * @dir: Transferring direction of the channel62 * @cfg: Transferring config of the channel63 * @irq: IRQ assigned to the channel64 */65struct xdma_chan {66 struct virt_dma_chan vchan;67 void *xdev_hdl;68 u32 base;69 struct dma_pool *desc_pool;70 bool busy;71 enum dma_transfer_direction dir;72 struct dma_slave_config cfg;73 u32 irq;74 struct completion last_interrupt;75 bool stop_requested;76};77 78/**79 * struct xdma_desc - DMA desc structure80 * @vdesc: Virtual DMA descriptor81 * @chan: DMA channel pointer82 * @dir: Transferring direction of the request83 * @desc_blocks: Hardware descriptor blocks84 * @dblk_num: Number of hardware descriptor blocks85 * @desc_num: Number of hardware descriptors86 * @completed_desc_num: Completed hardware descriptors87 * @cyclic: Cyclic transfer vs. scatter-gather88 * @interleaved_dma: Interleaved DMA transfer89 * @periods: Number of periods in the cyclic transfer90 * @period_size: Size of a period in bytes in cyclic transfers91 * @frames_left: Number of frames left in interleaved DMA transfer92 * @error: tx error flag93 */94struct xdma_desc {95 struct virt_dma_desc vdesc;96 struct xdma_chan *chan;97 enum dma_transfer_direction dir;98 struct xdma_desc_block *desc_blocks;99 u32 dblk_num;100 u32 desc_num;101 u32 completed_desc_num;102 bool cyclic;103 bool interleaved_dma;104 u32 periods;105 u32 period_size;106 u32 frames_left;107 bool error;108};109 110#define XDMA_DEV_STATUS_REG_DMA BIT(0)111#define XDMA_DEV_STATUS_INIT_MSIX BIT(1)112 113/**114 * struct xdma_device - DMA device structure115 * @pdev: Platform device pointer116 * @dma_dev: DMA device structure117 * @rmap: MMIO regmap for DMA registers118 * @h2c_chans: Host to Card channels119 * @c2h_chans: Card to Host channels120 * @h2c_chan_num: Number of H2C channels121 * @c2h_chan_num: Number of C2H channels122 * @irq_start: Start IRQ assigned to device123 * @irq_num: Number of IRQ assigned to device124 * @status: Initialization status125 */126struct xdma_device {127 struct platform_device *pdev;128 struct dma_device dma_dev;129 struct regmap *rmap;130 struct xdma_chan *h2c_chans;131 struct xdma_chan *c2h_chans;132 u32 h2c_chan_num;133 u32 c2h_chan_num;134 u32 irq_start;135 u32 irq_num;136 u32 status;137};138 139#define xdma_err(xdev, fmt, args...) \140 dev_err(&(xdev)->pdev->dev, fmt, ##args)141#define XDMA_CHAN_NUM(_xd) ({ \142 typeof(_xd) (xd) = (_xd); \143 ((xd)->h2c_chan_num + (xd)->c2h_chan_num); })144 145/* Get the last desc in a desc block */146static inline void *xdma_blk_last_desc(struct xdma_desc_block *block)147{148 return block->virt_addr + (XDMA_DESC_ADJACENT - 1) * XDMA_DESC_SIZE;149}150 151/**152 * xdma_link_sg_desc_blocks - Link SG descriptor blocks for DMA transfer153 * @sw_desc: Tx descriptor pointer154 */155static void xdma_link_sg_desc_blocks(struct xdma_desc *sw_desc)156{157 struct xdma_desc_block *block;158 u32 last_blk_desc, desc_control;159 struct xdma_hw_desc *desc;160 int i;161 162 desc_control = XDMA_DESC_CONTROL(XDMA_DESC_ADJACENT, 0);163 for (i = 1; i < sw_desc->dblk_num; i++) {164 block = &sw_desc->desc_blocks[i - 1];165 desc = xdma_blk_last_desc(block);166 167 if (!(i & XDMA_DESC_BLOCK_MASK)) {168 desc->control = cpu_to_le32(XDMA_DESC_CONTROL_LAST);169 continue;170 }171 desc->control = cpu_to_le32(desc_control);172 desc->next_desc = cpu_to_le64(block[1].dma_addr);173 }174 175 /* update the last block */176 last_blk_desc = (sw_desc->desc_num - 1) & XDMA_DESC_ADJACENT_MASK;177 if (((sw_desc->dblk_num - 1) & XDMA_DESC_BLOCK_MASK) > 0) {178 block = &sw_desc->desc_blocks[sw_desc->dblk_num - 2];179 desc = xdma_blk_last_desc(block);180 desc_control = XDMA_DESC_CONTROL(last_blk_desc + 1, 0);181 desc->control = cpu_to_le32(desc_control);182 }183 184 block = &sw_desc->desc_blocks[sw_desc->dblk_num - 1];185 desc = block->virt_addr + last_blk_desc * XDMA_DESC_SIZE;186 desc->control = cpu_to_le32(XDMA_DESC_CONTROL_LAST);187}188 189/**190 * xdma_link_cyclic_desc_blocks - Link cyclic descriptor blocks for DMA transfer191 * @sw_desc: Tx descriptor pointer192 */193static void xdma_link_cyclic_desc_blocks(struct xdma_desc *sw_desc)194{195 struct xdma_desc_block *block;196 struct xdma_hw_desc *desc;197 int i;198 199 block = sw_desc->desc_blocks;200 for (i = 0; i < sw_desc->desc_num - 1; i++) {201 desc = block->virt_addr + i * XDMA_DESC_SIZE;202 desc->next_desc = cpu_to_le64(block->dma_addr + ((i + 1) * XDMA_DESC_SIZE));203 }204 desc = block->virt_addr + i * XDMA_DESC_SIZE;205 desc->next_desc = cpu_to_le64(block->dma_addr);206}207 208static inline struct xdma_chan *to_xdma_chan(struct dma_chan *chan)209{210 return container_of(chan, struct xdma_chan, vchan.chan);211}212 213static inline struct xdma_desc *to_xdma_desc(struct virt_dma_desc *vdesc)214{215 return container_of(vdesc, struct xdma_desc, vdesc);216}217 218/**219 * xdma_channel_init - Initialize DMA channel registers220 * @chan: DMA channel pointer221 */222static int xdma_channel_init(struct xdma_chan *chan)223{224 struct xdma_device *xdev = chan->xdev_hdl;225 int ret;226 227 ret = regmap_write(xdev->rmap, chan->base + XDMA_CHAN_CONTROL_W1C,228 CHAN_CTRL_NON_INCR_ADDR);229 if (ret)230 return ret;231 232 ret = regmap_write(xdev->rmap, chan->base + XDMA_CHAN_INTR_ENABLE,233 CHAN_IM_ALL);234 if (ret)235 return ret;236 237 return 0;238}239 240/**241 * xdma_free_desc - Free descriptor242 * @vdesc: Virtual DMA descriptor243 */244static void xdma_free_desc(struct virt_dma_desc *vdesc)245{246 struct xdma_desc *sw_desc;247 int i;248 249 sw_desc = to_xdma_desc(vdesc);250 for (i = 0; i < sw_desc->dblk_num; i++) {251 if (!sw_desc->desc_blocks[i].virt_addr)252 break;253 dma_pool_free(sw_desc->chan->desc_pool,254 sw_desc->desc_blocks[i].virt_addr,255 sw_desc->desc_blocks[i].dma_addr);256 }257 kfree(sw_desc->desc_blocks);258 kfree(sw_desc);259}260 261/**262 * xdma_alloc_desc - Allocate descriptor263 * @chan: DMA channel pointer264 * @desc_num: Number of hardware descriptors265 * @cyclic: Whether this is a cyclic transfer266 */267static struct xdma_desc *268xdma_alloc_desc(struct xdma_chan *chan, u32 desc_num, bool cyclic)269{270 struct xdma_desc *sw_desc;271 struct xdma_hw_desc *desc;272 dma_addr_t dma_addr;273 u32 dblk_num;274 u32 control;275 void *addr;276 int i, j;277 278 sw_desc = kzalloc(sizeof(*sw_desc), GFP_NOWAIT);279 if (!sw_desc)280 return NULL;281 282 sw_desc->chan = chan;283 sw_desc->desc_num = desc_num;284 sw_desc->cyclic = cyclic;285 sw_desc->error = false;286 dblk_num = DIV_ROUND_UP(desc_num, XDMA_DESC_ADJACENT);287 sw_desc->desc_blocks = kcalloc(dblk_num, sizeof(*sw_desc->desc_blocks),288 GFP_NOWAIT);289 if (!sw_desc->desc_blocks)290 goto failed;291 292 if (cyclic)293 control = XDMA_DESC_CONTROL_CYCLIC;294 else295 control = XDMA_DESC_CONTROL(1, 0);296 297 sw_desc->dblk_num = dblk_num;298 for (i = 0; i < sw_desc->dblk_num; i++) {299 addr = dma_pool_alloc(chan->desc_pool, GFP_NOWAIT, &dma_addr);300 if (!addr)301 goto failed;302 303 sw_desc->desc_blocks[i].virt_addr = addr;304 sw_desc->desc_blocks[i].dma_addr = dma_addr;305 for (j = 0, desc = addr; j < XDMA_DESC_ADJACENT; j++)306 desc[j].control = cpu_to_le32(control);307 }308 309 if (cyclic)310 xdma_link_cyclic_desc_blocks(sw_desc);311 else312 xdma_link_sg_desc_blocks(sw_desc);313 314 return sw_desc;315 316failed:317 xdma_free_desc(&sw_desc->vdesc);318 return NULL;319}320 321/**322 * xdma_xfer_start - Start DMA transfer323 * @xchan: DMA channel pointer324 */325static int xdma_xfer_start(struct xdma_chan *xchan)326{327 struct virt_dma_desc *vd = vchan_next_desc(&xchan->vchan);328 struct xdma_device *xdev = xchan->xdev_hdl;329 struct xdma_desc_block *block;330 u32 val, completed_blocks;331 struct xdma_desc *desc;332 int ret;333 334 /*335 * check if there is not any submitted descriptor or channel is busy.336 * vchan lock should be held where this function is called.337 */338 if (!vd || xchan->busy)339 return -EINVAL;340 341 /* clear run stop bit to get ready for transfer */342 ret = regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_CONTROL_W1C,343 CHAN_CTRL_RUN_STOP);344 if (ret)345 return ret;346 347 desc = to_xdma_desc(vd);348 if (desc->dir != xchan->dir) {349 xdma_err(xdev, "incorrect request direction");350 return -EINVAL;351 }352 353 /* set DMA engine to the first descriptor block */354 completed_blocks = desc->completed_desc_num / XDMA_DESC_ADJACENT;355 block = &desc->desc_blocks[completed_blocks];356 val = lower_32_bits(block->dma_addr);357 ret = regmap_write(xdev->rmap, xchan->base + XDMA_SGDMA_DESC_LO, val);358 if (ret)359 return ret;360 361 val = upper_32_bits(block->dma_addr);362 ret = regmap_write(xdev->rmap, xchan->base + XDMA_SGDMA_DESC_HI, val);363 if (ret)364 return ret;365 366 if (completed_blocks + 1 == desc->dblk_num)367 val = (desc->desc_num - 1) & XDMA_DESC_ADJACENT_MASK;368 else369 val = XDMA_DESC_ADJACENT - 1;370 ret = regmap_write(xdev->rmap, xchan->base + XDMA_SGDMA_DESC_ADJ, val);371 if (ret)372 return ret;373 374 /* kick off DMA transfer */375 ret = regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_CONTROL,376 CHAN_CTRL_START);377 if (ret)378 return ret;379 380 xchan->busy = true;381 xchan->stop_requested = false;382 reinit_completion(&xchan->last_interrupt);383 384 return 0;385}386 387/**388 * xdma_xfer_stop - Stop DMA transfer389 * @xchan: DMA channel pointer390 */391static int xdma_xfer_stop(struct xdma_chan *xchan)392{393 int ret;394 struct xdma_device *xdev = xchan->xdev_hdl;395 396 /* clear run stop bit to prevent any further auto-triggering */397 ret = regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_CONTROL_W1C,398 CHAN_CTRL_RUN_STOP);399 if (ret)400 return ret;401 return ret;402}403 404/**405 * xdma_alloc_channels - Detect and allocate DMA channels406 * @xdev: DMA device pointer407 * @dir: Channel direction408 */409static int xdma_alloc_channels(struct xdma_device *xdev,410 enum dma_transfer_direction dir)411{412 struct xdma_platdata *pdata = dev_get_platdata(&xdev->pdev->dev);413 struct xdma_chan **chans, *xchan;414 u32 base, identifier, target;415 u32 *chan_num;416 int i, j, ret;417 418 if (dir == DMA_MEM_TO_DEV) {419 base = XDMA_CHAN_H2C_OFFSET;420 target = XDMA_CHAN_H2C_TARGET;421 chans = &xdev->h2c_chans;422 chan_num = &xdev->h2c_chan_num;423 } else if (dir == DMA_DEV_TO_MEM) {424 base = XDMA_CHAN_C2H_OFFSET;425 target = XDMA_CHAN_C2H_TARGET;426 chans = &xdev->c2h_chans;427 chan_num = &xdev->c2h_chan_num;428 } else {429 xdma_err(xdev, "invalid direction specified");430 return -EINVAL;431 }432 433 /* detect number of available DMA channels */434 for (i = 0, *chan_num = 0; i < pdata->max_dma_channels; i++) {435 ret = regmap_read(xdev->rmap, base + i * XDMA_CHAN_STRIDE,436 &identifier);437 if (ret)438 return ret;439 440 /* check if it is available DMA channel */441 if (XDMA_CHAN_CHECK_TARGET(identifier, target))442 (*chan_num)++;443 }444 445 if (!*chan_num) {446 xdma_err(xdev, "does not probe any channel");447 return -EINVAL;448 }449 450 *chans = devm_kcalloc(&xdev->pdev->dev, *chan_num, sizeof(**chans),451 GFP_KERNEL);452 if (!*chans)453 return -ENOMEM;454 455 for (i = 0, j = 0; i < pdata->max_dma_channels; i++) {456 ret = regmap_read(xdev->rmap, base + i * XDMA_CHAN_STRIDE,457 &identifier);458 if (ret)459 return ret;460 461 if (!XDMA_CHAN_CHECK_TARGET(identifier, target))462 continue;463 464 if (j == *chan_num) {465 xdma_err(xdev, "invalid channel number");466 return -EIO;467 }468 469 /* init channel structure and hardware */470 xchan = &(*chans)[j];471 xchan->xdev_hdl = xdev;472 xchan->base = base + i * XDMA_CHAN_STRIDE;473 xchan->dir = dir;474 xchan->stop_requested = false;475 init_completion(&xchan->last_interrupt);476 477 ret = xdma_channel_init(xchan);478 if (ret)479 return ret;480 xchan->vchan.desc_free = xdma_free_desc;481 vchan_init(&xchan->vchan, &xdev->dma_dev);482 483 j++;484 }485 486 dev_info(&xdev->pdev->dev, "configured %d %s channels", j,487 (dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H");488 489 return 0;490}491 492/**493 * xdma_issue_pending - Issue pending transactions494 * @chan: DMA channel pointer495 */496static void xdma_issue_pending(struct dma_chan *chan)497{498 struct xdma_chan *xdma_chan = to_xdma_chan(chan);499 unsigned long flags;500 501 spin_lock_irqsave(&xdma_chan->vchan.lock, flags);502 if (vchan_issue_pending(&xdma_chan->vchan))503 xdma_xfer_start(xdma_chan);504 spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags);505}506 507/**508 * xdma_terminate_all - Terminate all transactions509 * @chan: DMA channel pointer510 */511static int xdma_terminate_all(struct dma_chan *chan)512{513 struct xdma_chan *xdma_chan = to_xdma_chan(chan);514 struct virt_dma_desc *vd;515 unsigned long flags;516 LIST_HEAD(head);517 518 xdma_xfer_stop(xdma_chan);519 520 spin_lock_irqsave(&xdma_chan->vchan.lock, flags);521 522 xdma_chan->busy = false;523 xdma_chan->stop_requested = true;524 vd = vchan_next_desc(&xdma_chan->vchan);525 if (vd) {526 list_del(&vd->node);527 dma_cookie_complete(&vd->tx);528 vchan_terminate_vdesc(vd);529 }530 vchan_get_all_descriptors(&xdma_chan->vchan, &head);531 list_splice_tail(&head, &xdma_chan->vchan.desc_terminated);532 533 spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags);534 535 return 0;536}537 538/**539 * xdma_synchronize - Synchronize terminated transactions540 * @chan: DMA channel pointer541 */542static void xdma_synchronize(struct dma_chan *chan)543{544 struct xdma_chan *xdma_chan = to_xdma_chan(chan);545 struct xdma_device *xdev = xdma_chan->xdev_hdl;546 int st = 0;547 548 /* If the engine continues running, wait for the last interrupt */549 regmap_read(xdev->rmap, xdma_chan->base + XDMA_CHAN_STATUS, &st);550 if (st & XDMA_CHAN_STATUS_BUSY)551 wait_for_completion_timeout(&xdma_chan->last_interrupt, msecs_to_jiffies(1000));552 553 vchan_synchronize(&xdma_chan->vchan);554}555 556/**557 * xdma_fill_descs() - Fill hardware descriptors for one contiguous memory chunk.558 * More than one descriptor will be used if the size is bigger559 * than XDMA_DESC_BLEN_MAX.560 * @sw_desc: Descriptor container561 * @src_addr: First value for the ->src_addr field562 * @dst_addr: First value for the ->dst_addr field563 * @size: Size of the contiguous memory block564 * @filled_descs_num: Index of the first descriptor to take care of in @sw_desc565 */566static inline u32 xdma_fill_descs(struct xdma_desc *sw_desc, u64 src_addr,567 u64 dst_addr, u32 size, u32 filled_descs_num)568{569 u32 left = size, len, desc_num = filled_descs_num;570 struct xdma_desc_block *dblk;571 struct xdma_hw_desc *desc;572 573 dblk = sw_desc->desc_blocks + (desc_num / XDMA_DESC_ADJACENT);574 desc = dblk->virt_addr;575 desc += desc_num & XDMA_DESC_ADJACENT_MASK;576 do {577 len = min_t(u32, left, XDMA_DESC_BLEN_MAX);578 /* set hardware descriptor */579 desc->bytes = cpu_to_le32(len);580 desc->src_addr = cpu_to_le64(src_addr);581 desc->dst_addr = cpu_to_le64(dst_addr);582 if (!(++desc_num & XDMA_DESC_ADJACENT_MASK))583 desc = (++dblk)->virt_addr;584 else585 desc++;586 587 src_addr += len;588 dst_addr += len;589 left -= len;590 } while (left);591 592 return desc_num - filled_descs_num;593}594 595/**596 * xdma_prep_device_sg - prepare a descriptor for a DMA transaction597 * @chan: DMA channel pointer598 * @sgl: Transfer scatter gather list599 * @sg_len: Length of scatter gather list600 * @dir: Transfer direction601 * @flags: transfer ack flags602 * @context: APP words of the descriptor603 */604static struct dma_async_tx_descriptor *605xdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl,606 unsigned int sg_len, enum dma_transfer_direction dir,607 unsigned long flags, void *context)608{609 struct xdma_chan *xdma_chan = to_xdma_chan(chan);610 struct dma_async_tx_descriptor *tx_desc;611 struct xdma_desc *sw_desc;612 u32 desc_num = 0, i;613 u64 addr, dev_addr, *src, *dst;614 struct scatterlist *sg;615 616 for_each_sg(sgl, sg, sg_len, i)617 desc_num += DIV_ROUND_UP(sg_dma_len(sg), XDMA_DESC_BLEN_MAX);618 619 sw_desc = xdma_alloc_desc(xdma_chan, desc_num, false);620 if (!sw_desc)621 return NULL;622 sw_desc->dir = dir;623 sw_desc->cyclic = false;624 sw_desc->interleaved_dma = false;625 626 if (dir == DMA_MEM_TO_DEV) {627 dev_addr = xdma_chan->cfg.dst_addr;628 src = &addr;629 dst = &dev_addr;630 } else {631 dev_addr = xdma_chan->cfg.src_addr;632 src = &dev_addr;633 dst = &addr;634 }635 636 desc_num = 0;637 for_each_sg(sgl, sg, sg_len, i) {638 addr = sg_dma_address(sg);639 desc_num += xdma_fill_descs(sw_desc, *src, *dst, sg_dma_len(sg), desc_num);640 dev_addr += sg_dma_len(sg);641 }642 643 tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags);644 if (!tx_desc)645 goto failed;646 647 return tx_desc;648 649failed:650 xdma_free_desc(&sw_desc->vdesc);651 652 return NULL;653}654 655/**656 * xdma_prep_dma_cyclic - prepare for cyclic DMA transactions657 * @chan: DMA channel pointer658 * @address: Device DMA address to access659 * @size: Total length to transfer660 * @period_size: Period size to use for each transfer661 * @dir: Transfer direction662 * @flags: Transfer ack flags663 */664static struct dma_async_tx_descriptor *665xdma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t address,666 size_t size, size_t period_size,667 enum dma_transfer_direction dir,668 unsigned long flags)669{670 struct xdma_chan *xdma_chan = to_xdma_chan(chan);671 struct xdma_device *xdev = xdma_chan->xdev_hdl;672 unsigned int periods = size / period_size;673 struct dma_async_tx_descriptor *tx_desc;674 struct xdma_desc *sw_desc;675 u64 addr, dev_addr, *src, *dst;676 u32 desc_num;677 unsigned int i;678 679 /*680 * Simplify the whole logic by preventing an abnormally high number of681 * periods and periods size.682 */683 if (period_size > XDMA_DESC_BLEN_MAX) {684 xdma_err(xdev, "period size limited to %lu bytes\n", XDMA_DESC_BLEN_MAX);685 return NULL;686 }687 688 if (periods > XDMA_DESC_ADJACENT) {689 xdma_err(xdev, "number of periods limited to %u\n", XDMA_DESC_ADJACENT);690 return NULL;691 }692 693 sw_desc = xdma_alloc_desc(xdma_chan, periods, true);694 if (!sw_desc)695 return NULL;696 697 sw_desc->periods = periods;698 sw_desc->period_size = period_size;699 sw_desc->dir = dir;700 sw_desc->interleaved_dma = false;701 702 addr = address;703 if (dir == DMA_MEM_TO_DEV) {704 dev_addr = xdma_chan->cfg.dst_addr;705 src = &addr;706 dst = &dev_addr;707 } else {708 dev_addr = xdma_chan->cfg.src_addr;709 src = &dev_addr;710 dst = &addr;711 }712 713 desc_num = 0;714 for (i = 0; i < periods; i++) {715 desc_num += xdma_fill_descs(sw_desc, *src, *dst, period_size, desc_num);716 addr += period_size;717 }718 719 tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags);720 if (!tx_desc)721 goto failed;722 723 return tx_desc;724 725failed:726 xdma_free_desc(&sw_desc->vdesc);727 728 return NULL;729}730 731/**732 * xdma_prep_interleaved_dma - Prepare virtual descriptor for interleaved DMA transfers733 * @chan: DMA channel734 * @xt: DMA transfer template735 * @flags: tx flags736 */737static struct dma_async_tx_descriptor *738xdma_prep_interleaved_dma(struct dma_chan *chan,739 struct dma_interleaved_template *xt,740 unsigned long flags)741{742 int i;743 u32 desc_num = 0, period_size = 0;744 struct dma_async_tx_descriptor *tx_desc;745 struct xdma_chan *xchan = to_xdma_chan(chan);746 struct xdma_desc *sw_desc;747 u64 src_addr, dst_addr;748 749 for (i = 0; i < xt->frame_size; ++i)750 desc_num += DIV_ROUND_UP(xt->sgl[i].size, XDMA_DESC_BLEN_MAX);751 752 sw_desc = xdma_alloc_desc(xchan, desc_num, false);753 if (!sw_desc)754 return NULL;755 sw_desc->dir = xt->dir;756 sw_desc->interleaved_dma = true;757 sw_desc->cyclic = flags & DMA_PREP_REPEAT;758 sw_desc->frames_left = xt->numf;759 sw_desc->periods = xt->numf;760 761 desc_num = 0;762 src_addr = xt->src_start;763 dst_addr = xt->dst_start;764 for (i = 0; i < xt->frame_size; ++i) {765 desc_num += xdma_fill_descs(sw_desc, src_addr, dst_addr, xt->sgl[i].size, desc_num);766 src_addr += dmaengine_get_src_icg(xt, &xt->sgl[i]) + (xt->src_inc ?767 xt->sgl[i].size : 0);768 dst_addr += dmaengine_get_dst_icg(xt, &xt->sgl[i]) + (xt->dst_inc ?769 xt->sgl[i].size : 0);770 period_size += xt->sgl[i].size;771 }772 sw_desc->period_size = period_size;773 774 tx_desc = vchan_tx_prep(&xchan->vchan, &sw_desc->vdesc, flags);775 if (tx_desc)776 return tx_desc;777 778 xdma_free_desc(&sw_desc->vdesc);779 return NULL;780}781 782/**783 * xdma_device_config - Configure the DMA channel784 * @chan: DMA channel785 * @cfg: channel configuration786 */787static int xdma_device_config(struct dma_chan *chan,788 struct dma_slave_config *cfg)789{790 struct xdma_chan *xdma_chan = to_xdma_chan(chan);791 792 memcpy(&xdma_chan->cfg, cfg, sizeof(*cfg));793 794 return 0;795}796 797/**798 * xdma_free_chan_resources - Free channel resources799 * @chan: DMA channel800 */801static void xdma_free_chan_resources(struct dma_chan *chan)802{803 struct xdma_chan *xdma_chan = to_xdma_chan(chan);804 805 vchan_free_chan_resources(&xdma_chan->vchan);806 dma_pool_destroy(xdma_chan->desc_pool);807 xdma_chan->desc_pool = NULL;808}809 810/**811 * xdma_alloc_chan_resources - Allocate channel resources812 * @chan: DMA channel813 */814static int xdma_alloc_chan_resources(struct dma_chan *chan)815{816 struct xdma_chan *xdma_chan = to_xdma_chan(chan);817 struct xdma_device *xdev = xdma_chan->xdev_hdl;818 struct device *dev = xdev->dma_dev.dev;819 820 while (dev && !dev_is_pci(dev))821 dev = dev->parent;822 if (!dev) {823 xdma_err(xdev, "unable to find pci device");824 return -EINVAL;825 }826 827 xdma_chan->desc_pool = dma_pool_create(dma_chan_name(chan), dev, XDMA_DESC_BLOCK_SIZE,828 XDMA_DESC_BLOCK_ALIGN, XDMA_DESC_BLOCK_BOUNDARY);829 if (!xdma_chan->desc_pool) {830 xdma_err(xdev, "unable to allocate descriptor pool");831 return -ENOMEM;832 }833 834 return 0;835}836 837static enum dma_status xdma_tx_status(struct dma_chan *chan, dma_cookie_t cookie,838 struct dma_tx_state *state)839{840 struct xdma_chan *xdma_chan = to_xdma_chan(chan);841 struct xdma_desc *desc = NULL;842 struct virt_dma_desc *vd;843 enum dma_status ret;844 unsigned long flags;845 unsigned int period_idx;846 u32 residue = 0;847 848 ret = dma_cookie_status(chan, cookie, state);849 if (ret == DMA_COMPLETE)850 return ret;851 852 spin_lock_irqsave(&xdma_chan->vchan.lock, flags);853 854 vd = vchan_find_desc(&xdma_chan->vchan, cookie);855 if (!vd)856 goto out;857 858 desc = to_xdma_desc(vd);859 if (desc->error) {860 ret = DMA_ERROR;861 } else if (desc->cyclic) {862 period_idx = desc->completed_desc_num % desc->periods;863 residue = (desc->periods - period_idx) * desc->period_size;864 dma_set_residue(state, residue);865 }866out:867 spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags);868 869 return ret;870}871 872/**873 * xdma_channel_isr - XDMA channel interrupt handler874 * @irq: IRQ number875 * @dev_id: Pointer to the DMA channel structure876 */877static irqreturn_t xdma_channel_isr(int irq, void *dev_id)878{879 struct xdma_chan *xchan = dev_id;880 u32 complete_desc_num = 0;881 struct xdma_device *xdev = xchan->xdev_hdl;882 struct virt_dma_desc *vd, *next_vd;883 struct xdma_desc *desc;884 int ret;885 u32 st;886 bool repeat_tx;887 888 spin_lock(&xchan->vchan.lock);889 890 if (xchan->stop_requested)891 complete(&xchan->last_interrupt);892 893 /* get submitted request */894 vd = vchan_next_desc(&xchan->vchan);895 if (!vd)896 goto out;897 898 /* Clear-on-read the status register */899 ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_STATUS_RC, &st);900 if (ret)901 goto out;902 903 desc = to_xdma_desc(vd);904 905 st &= XDMA_CHAN_STATUS_MASK;906 if ((st & XDMA_CHAN_ERROR_MASK) ||907 !(st & (CHAN_CTRL_IE_DESC_COMPLETED | CHAN_CTRL_IE_DESC_STOPPED))) {908 desc->error = true;909 xdma_err(xdev, "channel error, status register value: 0x%x", st);910 goto out;911 }912 913 ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_COMPLETED_DESC,914 &complete_desc_num);915 if (ret)916 goto out;917 918 if (desc->interleaved_dma) {919 xchan->busy = false;920 desc->completed_desc_num += complete_desc_num;921 if (complete_desc_num == XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT) {922 xdma_xfer_start(xchan);923 goto out;924 }925 926 /* last desc of any frame */927 desc->frames_left--;928 if (desc->frames_left)929 goto out;930 931 /* last desc of the last frame */932 repeat_tx = vd->tx.flags & DMA_PREP_REPEAT;933 next_vd = list_first_entry_or_null(&vd->node, struct virt_dma_desc, node);934 if (next_vd)935 repeat_tx = repeat_tx && !(next_vd->tx.flags & DMA_PREP_LOAD_EOT);936 if (repeat_tx) {937 desc->frames_left = desc->periods;938 desc->completed_desc_num = 0;939 vchan_cyclic_callback(vd);940 } else {941 list_del(&vd->node);942 vchan_cookie_complete(vd);943 }944 /* start (or continue) the tx of a first desc on the vc.desc_issued list, if any */945 xdma_xfer_start(xchan);946 } else if (!desc->cyclic) {947 xchan->busy = false;948 desc->completed_desc_num += complete_desc_num;949 950 /* if all data blocks are transferred, remove and complete the request */951 if (desc->completed_desc_num == desc->desc_num) {952 list_del(&vd->node);953 vchan_cookie_complete(vd);954 goto out;955 }956 957 if (desc->completed_desc_num > desc->desc_num ||958 complete_desc_num != XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT)959 goto out;960 961 /* transfer the rest of data */962 xdma_xfer_start(xchan);963 } else {964 desc->completed_desc_num = complete_desc_num;965 vchan_cyclic_callback(vd);966 }967 968out:969 spin_unlock(&xchan->vchan.lock);970 return IRQ_HANDLED;971}972 973/**974 * xdma_irq_fini - Uninitialize IRQ975 * @xdev: DMA device pointer976 */977static void xdma_irq_fini(struct xdma_device *xdev)978{979 int i;980 981 /* disable interrupt */982 regmap_write(xdev->rmap, XDMA_IRQ_CHAN_INT_EN_W1C, ~0);983 984 /* free irq handler */985 for (i = 0; i < xdev->h2c_chan_num; i++)986 free_irq(xdev->h2c_chans[i].irq, &xdev->h2c_chans[i]);987 988 for (i = 0; i < xdev->c2h_chan_num; i++)989 free_irq(xdev->c2h_chans[i].irq, &xdev->c2h_chans[i]);990}991 992/**993 * xdma_set_vector_reg - configure hardware IRQ registers994 * @xdev: DMA device pointer995 * @vec_tbl_start: Start of IRQ registers996 * @irq_start: Start of IRQ997 * @irq_num: Number of IRQ998 */999static int xdma_set_vector_reg(struct xdma_device *xdev, u32 vec_tbl_start,1000 u32 irq_start, u32 irq_num)1001{1002 u32 shift, i, val = 0;1003 int ret;1004 1005 /* Each IRQ register is 32 bit and contains 4 IRQs */1006 while (irq_num > 0) {1007 for (i = 0; i < 4; i++) {1008 shift = XDMA_IRQ_VEC_SHIFT * i;1009 val |= irq_start << shift;1010 irq_start++;1011 irq_num--;1012 if (!irq_num)1013 break;1014 }1015 1016 /* write IRQ register */1017 ret = regmap_write(xdev->rmap, vec_tbl_start, val);1018 if (ret)1019 return ret;1020 vec_tbl_start += sizeof(u32);1021 val = 0;1022 }1023 1024 return 0;1025}1026 1027/**1028 * xdma_irq_init - initialize IRQs1029 * @xdev: DMA device pointer1030 */1031static int xdma_irq_init(struct xdma_device *xdev)1032{1033 u32 irq = xdev->irq_start;1034 u32 user_irq_start;1035 int i, j, ret;1036 1037 /* return failure if there are not enough IRQs */1038 if (xdev->irq_num < XDMA_CHAN_NUM(xdev)) {1039 xdma_err(xdev, "not enough irq");1040 return -EINVAL;1041 }1042 1043 /* setup H2C interrupt handler */1044 for (i = 0; i < xdev->h2c_chan_num; i++) {1045 ret = request_irq(irq, xdma_channel_isr, 0,1046 "xdma-h2c-channel", &xdev->h2c_chans[i]);1047 if (ret) {1048 xdma_err(xdev, "H2C channel%d request irq%d failed: %d",1049 i, irq, ret);1050 goto failed_init_h2c;1051 }1052 xdev->h2c_chans[i].irq = irq;1053 irq++;1054 }1055 1056 /* setup C2H interrupt handler */1057 for (j = 0; j < xdev->c2h_chan_num; j++) {1058 ret = request_irq(irq, xdma_channel_isr, 0,1059 "xdma-c2h-channel", &xdev->c2h_chans[j]);1060 if (ret) {1061 xdma_err(xdev, "C2H channel%d request irq%d failed: %d",1062 j, irq, ret);1063 goto failed_init_c2h;1064 }1065 xdev->c2h_chans[j].irq = irq;1066 irq++;1067 }1068 1069 /* config hardware IRQ registers */1070 ret = xdma_set_vector_reg(xdev, XDMA_IRQ_CHAN_VEC_NUM, 0,1071 XDMA_CHAN_NUM(xdev));1072 if (ret) {1073 xdma_err(xdev, "failed to set channel vectors: %d", ret);1074 goto failed_init_c2h;1075 }1076 1077 /* config user IRQ registers if needed */1078 user_irq_start = XDMA_CHAN_NUM(xdev);1079 if (xdev->irq_num > user_irq_start) {1080 ret = xdma_set_vector_reg(xdev, XDMA_IRQ_USER_VEC_NUM,1081 user_irq_start,1082 xdev->irq_num - user_irq_start);1083 if (ret) {1084 xdma_err(xdev, "failed to set user vectors: %d", ret);1085 goto failed_init_c2h;1086 }1087 }1088 1089 /* enable interrupt */1090 ret = regmap_write(xdev->rmap, XDMA_IRQ_CHAN_INT_EN_W1S, ~0);1091 if (ret)1092 goto failed_init_c2h;1093 1094 return 0;1095 1096failed_init_c2h:1097 while (j--)1098 free_irq(xdev->c2h_chans[j].irq, &xdev->c2h_chans[j]);1099failed_init_h2c:1100 while (i--)1101 free_irq(xdev->h2c_chans[i].irq, &xdev->h2c_chans[i]);1102 1103 return ret;1104}1105 1106static bool xdma_filter_fn(struct dma_chan *chan, void *param)1107{1108 struct xdma_chan *xdma_chan = to_xdma_chan(chan);1109 struct xdma_chan_info *chan_info = param;1110 1111 return chan_info->dir == xdma_chan->dir;1112}1113 1114/**1115 * xdma_disable_user_irq - Disable user interrupt1116 * @pdev: Pointer to the platform_device structure1117 * @irq_num: System IRQ number1118 */1119void xdma_disable_user_irq(struct platform_device *pdev, u32 irq_num)1120{1121 struct xdma_device *xdev = platform_get_drvdata(pdev);1122 u32 index;1123 1124 index = irq_num - xdev->irq_start;1125 if (index < XDMA_CHAN_NUM(xdev) || index >= xdev->irq_num) {1126 xdma_err(xdev, "invalid user irq number");1127 return;1128 }1129 index -= XDMA_CHAN_NUM(xdev);1130 1131 regmap_write(xdev->rmap, XDMA_IRQ_USER_INT_EN_W1C, 1 << index);1132}1133EXPORT_SYMBOL(xdma_disable_user_irq);1134 1135/**1136 * xdma_enable_user_irq - Enable user logic interrupt1137 * @pdev: Pointer to the platform_device structure1138 * @irq_num: System IRQ number1139 */1140int xdma_enable_user_irq(struct platform_device *pdev, u32 irq_num)1141{1142 struct xdma_device *xdev = platform_get_drvdata(pdev);1143 u32 index;1144 int ret;1145 1146 index = irq_num - xdev->irq_start;1147 if (index < XDMA_CHAN_NUM(xdev) || index >= xdev->irq_num) {1148 xdma_err(xdev, "invalid user irq number");1149 return -EINVAL;1150 }1151 index -= XDMA_CHAN_NUM(xdev);1152 1153 ret = regmap_write(xdev->rmap, XDMA_IRQ_USER_INT_EN_W1S, 1 << index);1154 if (ret)1155 return ret;1156 1157 return 0;1158}1159EXPORT_SYMBOL(xdma_enable_user_irq);1160 1161/**1162 * xdma_get_user_irq - Get system IRQ number1163 * @pdev: Pointer to the platform_device structure1164 * @user_irq_index: User logic IRQ wire index1165 *1166 * Return: The system IRQ number allocated for the given wire index.1167 */1168int xdma_get_user_irq(struct platform_device *pdev, u32 user_irq_index)1169{1170 struct xdma_device *xdev = platform_get_drvdata(pdev);1171 1172 if (XDMA_CHAN_NUM(xdev) + user_irq_index >= xdev->irq_num) {1173 xdma_err(xdev, "invalid user irq index");1174 return -EINVAL;1175 }1176 1177 return xdev->irq_start + XDMA_CHAN_NUM(xdev) + user_irq_index;1178}1179EXPORT_SYMBOL(xdma_get_user_irq);1180 1181/**1182 * xdma_remove - Driver remove function1183 * @pdev: Pointer to the platform_device structure1184 */1185static void xdma_remove(struct platform_device *pdev)1186{1187 struct xdma_device *xdev = platform_get_drvdata(pdev);1188 1189 if (xdev->status & XDMA_DEV_STATUS_INIT_MSIX)1190 xdma_irq_fini(xdev);1191 1192 if (xdev->status & XDMA_DEV_STATUS_REG_DMA)1193 dma_async_device_unregister(&xdev->dma_dev);1194}1195 1196/**1197 * xdma_probe - Driver probe function1198 * @pdev: Pointer to the platform_device structure1199 */1200static int xdma_probe(struct platform_device *pdev)1201{1202 struct xdma_platdata *pdata = dev_get_platdata(&pdev->dev);1203 struct xdma_device *xdev;1204 void __iomem *reg_base;1205 struct resource *res;1206 int ret = -ENODEV;1207 1208 if (pdata->max_dma_channels > XDMA_MAX_CHANNELS) {1209 dev_err(&pdev->dev, "invalid max dma channels %d",1210 pdata->max_dma_channels);1211 return -EINVAL;1212 }1213 1214 xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL);1215 if (!xdev)1216 return -ENOMEM;1217 1218 platform_set_drvdata(pdev, xdev);1219 xdev->pdev = pdev;1220 1221 res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);1222 if (!res) {1223 xdma_err(xdev, "failed to get irq resource");1224 goto failed;1225 }1226 xdev->irq_start = res->start;1227 xdev->irq_num = resource_size(res);1228 1229 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);1230 if (!res) {1231 xdma_err(xdev, "failed to get io resource");1232 goto failed;1233 }1234 1235 reg_base = devm_ioremap_resource(&pdev->dev, res);1236 if (IS_ERR(reg_base)) {1237 xdma_err(xdev, "ioremap failed");1238 goto failed;1239 }1240 1241 xdev->rmap = devm_regmap_init_mmio(&pdev->dev, reg_base,1242 &xdma_regmap_config);1243 if (!xdev->rmap) {1244 xdma_err(xdev, "config regmap failed: %d", ret);1245 goto failed;1246 }1247 INIT_LIST_HEAD(&xdev->dma_dev.channels);1248 1249 ret = xdma_alloc_channels(xdev, DMA_MEM_TO_DEV);1250 if (ret) {1251 xdma_err(xdev, "config H2C channels failed: %d", ret);1252 goto failed;1253 }1254 1255 ret = xdma_alloc_channels(xdev, DMA_DEV_TO_MEM);1256 if (ret) {1257 xdma_err(xdev, "config C2H channels failed: %d", ret);1258 goto failed;1259 }1260 1261 dma_cap_set(DMA_SLAVE, xdev->dma_dev.cap_mask);1262 dma_cap_set(DMA_PRIVATE, xdev->dma_dev.cap_mask);1263 dma_cap_set(DMA_CYCLIC, xdev->dma_dev.cap_mask);1264 dma_cap_set(DMA_INTERLEAVE, xdev->dma_dev.cap_mask);1265 dma_cap_set(DMA_REPEAT, xdev->dma_dev.cap_mask);1266 dma_cap_set(DMA_LOAD_EOT, xdev->dma_dev.cap_mask);1267 1268 xdev->dma_dev.dev = &pdev->dev;1269 xdev->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;1270 xdev->dma_dev.device_free_chan_resources = xdma_free_chan_resources;1271 xdev->dma_dev.device_alloc_chan_resources = xdma_alloc_chan_resources;1272 xdev->dma_dev.device_tx_status = xdma_tx_status;1273 xdev->dma_dev.device_prep_slave_sg = xdma_prep_device_sg;1274 xdev->dma_dev.device_config = xdma_device_config;1275 xdev->dma_dev.device_issue_pending = xdma_issue_pending;1276 xdev->dma_dev.device_terminate_all = xdma_terminate_all;1277 xdev->dma_dev.device_synchronize = xdma_synchronize;1278 xdev->dma_dev.filter.map = pdata->device_map;1279 xdev->dma_dev.filter.mapcnt = pdata->device_map_cnt;1280 xdev->dma_dev.filter.fn = xdma_filter_fn;1281 xdev->dma_dev.device_prep_dma_cyclic = xdma_prep_dma_cyclic;1282 xdev->dma_dev.device_prep_interleaved_dma = xdma_prep_interleaved_dma;1283 1284 ret = dma_async_device_register(&xdev->dma_dev);1285 if (ret) {1286 xdma_err(xdev, "failed to register Xilinx XDMA: %d", ret);1287 goto failed;1288 }1289 xdev->status |= XDMA_DEV_STATUS_REG_DMA;1290 1291 ret = xdma_irq_init(xdev);1292 if (ret) {1293 xdma_err(xdev, "failed to init msix: %d", ret);1294 goto failed;1295 }1296 xdev->status |= XDMA_DEV_STATUS_INIT_MSIX;1297 1298 return 0;1299 1300failed:1301 xdma_remove(pdev);1302 1303 return ret;1304}1305 1306static const struct platform_device_id xdma_id_table[] = {1307 { "xdma", 0},1308 { },1309};1310MODULE_DEVICE_TABLE(platform, xdma_id_table);1311 1312static struct platform_driver xdma_driver = {1313 .driver = {1314 .name = "xdma",1315 },1316 .id_table = xdma_id_table,1317 .probe = xdma_probe,1318 .remove_new = xdma_remove,1319};1320 1321module_platform_driver(xdma_driver);1322 1323MODULE_DESCRIPTION("AMD XDMA driver");1324MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>");1325MODULE_LICENSE("GPL");1326