1017 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.4 * Synopsys DesignWare eDMA core driver5 *6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>7 */8 9#include <linux/module.h>10#include <linux/device.h>11#include <linux/kernel.h>12#include <linux/dmaengine.h>13#include <linux/err.h>14#include <linux/interrupt.h>15#include <linux/irq.h>16#include <linux/dma/edma.h>17#include <linux/dma-mapping.h>18 19#include "dw-edma-core.h"20#include "dw-edma-v0-core.h"21#include "dw-hdma-v0-core.h"22#include "../dmaengine.h"23#include "../virt-dma.h"24 25static inline26struct device *dchan2dev(struct dma_chan *dchan)27{28 return &dchan->dev->device;29}30 31static inline32struct device *chan2dev(struct dw_edma_chan *chan)33{34 return &chan->vc.chan.dev->device;35}36 37static inline38struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)39{40 return container_of(vd, struct dw_edma_desc, vd);41}42 43static inline44u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr)45{46 struct dw_edma_chip *chip = chan->dw->chip;47 48 if (chip->ops->pci_address)49 return chip->ops->pci_address(chip->dev, cpu_addr);50 51 return cpu_addr;52}53 54static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk)55{56 struct dw_edma_burst *burst;57 58 burst = kzalloc(sizeof(*burst), GFP_NOWAIT);59 if (unlikely(!burst))60 return NULL;61 62 INIT_LIST_HEAD(&burst->list);63 if (chunk->burst) {64 /* Create and add new element into the linked list */65 chunk->bursts_alloc++;66 list_add_tail(&burst->list, &chunk->burst->list);67 } else {68 /* List head */69 chunk->bursts_alloc = 0;70 chunk->burst = burst;71 }72 73 return burst;74}75 76static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc)77{78 struct dw_edma_chip *chip = desc->chan->dw->chip;79 struct dw_edma_chan *chan = desc->chan;80 struct dw_edma_chunk *chunk;81 82 chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT);83 if (unlikely(!chunk))84 return NULL;85 86 INIT_LIST_HEAD(&chunk->list);87 chunk->chan = chan;88 /* Toggling change bit (CB) in each chunk, this is a mechanism to89 * inform the eDMA HW block that this is a new linked list ready90 * to be consumed.91 * - Odd chunks originate CB equal to 092 * - Even chunks originate CB equal to 193 */94 chunk->cb = !(desc->chunks_alloc % 2);95 if (chan->dir == EDMA_DIR_WRITE) {96 chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr;97 chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr;98 } else {99 chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr;100 chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr;101 }102 103 if (desc->chunk) {104 /* Create and add new element into the linked list */105 if (!dw_edma_alloc_burst(chunk)) {106 kfree(chunk);107 return NULL;108 }109 desc->chunks_alloc++;110 list_add_tail(&chunk->list, &desc->chunk->list);111 } else {112 /* List head */113 chunk->burst = NULL;114 desc->chunks_alloc = 0;115 desc->chunk = chunk;116 }117 118 return chunk;119}120 121static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan)122{123 struct dw_edma_desc *desc;124 125 desc = kzalloc(sizeof(*desc), GFP_NOWAIT);126 if (unlikely(!desc))127 return NULL;128 129 desc->chan = chan;130 if (!dw_edma_alloc_chunk(desc)) {131 kfree(desc);132 return NULL;133 }134 135 return desc;136}137 138static void dw_edma_free_burst(struct dw_edma_chunk *chunk)139{140 struct dw_edma_burst *child, *_next;141 142 /* Remove all the list elements */143 list_for_each_entry_safe(child, _next, &chunk->burst->list, list) {144 list_del(&child->list);145 kfree(child);146 chunk->bursts_alloc--;147 }148 149 /* Remove the list head */150 kfree(child);151 chunk->burst = NULL;152}153 154static void dw_edma_free_chunk(struct dw_edma_desc *desc)155{156 struct dw_edma_chunk *child, *_next;157 158 if (!desc->chunk)159 return;160 161 /* Remove all the list elements */162 list_for_each_entry_safe(child, _next, &desc->chunk->list, list) {163 dw_edma_free_burst(child);164 list_del(&child->list);165 kfree(child);166 desc->chunks_alloc--;167 }168 169 /* Remove the list head */170 kfree(child);171 desc->chunk = NULL;172}173 174static void dw_edma_free_desc(struct dw_edma_desc *desc)175{176 dw_edma_free_chunk(desc);177 kfree(desc);178}179 180static void vchan_free_desc(struct virt_dma_desc *vdesc)181{182 dw_edma_free_desc(vd2dw_edma_desc(vdesc));183}184 185static int dw_edma_start_transfer(struct dw_edma_chan *chan)186{187 struct dw_edma *dw = chan->dw;188 struct dw_edma_chunk *child;189 struct dw_edma_desc *desc;190 struct virt_dma_desc *vd;191 192 vd = vchan_next_desc(&chan->vc);193 if (!vd)194 return 0;195 196 desc = vd2dw_edma_desc(vd);197 if (!desc)198 return 0;199 200 child = list_first_entry_or_null(&desc->chunk->list,201 struct dw_edma_chunk, list);202 if (!child)203 return 0;204 205 dw_edma_core_start(dw, child, !desc->xfer_sz);206 desc->xfer_sz += child->ll_region.sz;207 dw_edma_free_burst(child);208 list_del(&child->list);209 kfree(child);210 desc->chunks_alloc--;211 212 return 1;213}214 215static void dw_edma_device_caps(struct dma_chan *dchan,216 struct dma_slave_caps *caps)217{218 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);219 220 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {221 if (chan->dir == EDMA_DIR_READ)222 caps->directions = BIT(DMA_DEV_TO_MEM);223 else224 caps->directions = BIT(DMA_MEM_TO_DEV);225 } else {226 if (chan->dir == EDMA_DIR_WRITE)227 caps->directions = BIT(DMA_DEV_TO_MEM);228 else229 caps->directions = BIT(DMA_MEM_TO_DEV);230 }231}232 233static int dw_edma_device_config(struct dma_chan *dchan,234 struct dma_slave_config *config)235{236 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);237 238 memcpy(&chan->config, config, sizeof(*config));239 chan->configured = true;240 241 return 0;242}243 244static int dw_edma_device_pause(struct dma_chan *dchan)245{246 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);247 int err = 0;248 249 if (!chan->configured)250 err = -EPERM;251 else if (chan->status != EDMA_ST_BUSY)252 err = -EPERM;253 else if (chan->request != EDMA_REQ_NONE)254 err = -EPERM;255 else256 chan->request = EDMA_REQ_PAUSE;257 258 return err;259}260 261static int dw_edma_device_resume(struct dma_chan *dchan)262{263 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);264 int err = 0;265 266 if (!chan->configured) {267 err = -EPERM;268 } else if (chan->status != EDMA_ST_PAUSE) {269 err = -EPERM;270 } else if (chan->request != EDMA_REQ_NONE) {271 err = -EPERM;272 } else {273 chan->status = EDMA_ST_BUSY;274 dw_edma_start_transfer(chan);275 }276 277 return err;278}279 280static int dw_edma_device_terminate_all(struct dma_chan *dchan)281{282 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);283 int err = 0;284 285 if (!chan->configured) {286 /* Do nothing */287 } else if (chan->status == EDMA_ST_PAUSE) {288 chan->status = EDMA_ST_IDLE;289 chan->configured = false;290 } else if (chan->status == EDMA_ST_IDLE) {291 chan->configured = false;292 } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) {293 /*294 * The channel is in a false BUSY state, probably didn't295 * receive or lost an interrupt296 */297 chan->status = EDMA_ST_IDLE;298 chan->configured = false;299 } else if (chan->request > EDMA_REQ_PAUSE) {300 err = -EPERM;301 } else {302 chan->request = EDMA_REQ_STOP;303 }304 305 return err;306}307 308static void dw_edma_device_issue_pending(struct dma_chan *dchan)309{310 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);311 unsigned long flags;312 313 if (!chan->configured)314 return;315 316 spin_lock_irqsave(&chan->vc.lock, flags);317 if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE &&318 chan->status == EDMA_ST_IDLE) {319 chan->status = EDMA_ST_BUSY;320 dw_edma_start_transfer(chan);321 }322 spin_unlock_irqrestore(&chan->vc.lock, flags);323}324 325static enum dma_status326dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,327 struct dma_tx_state *txstate)328{329 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);330 struct dw_edma_desc *desc;331 struct virt_dma_desc *vd;332 unsigned long flags;333 enum dma_status ret;334 u32 residue = 0;335 336 ret = dma_cookie_status(dchan, cookie, txstate);337 if (ret == DMA_COMPLETE)338 return ret;339 340 if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE)341 ret = DMA_PAUSED;342 343 if (!txstate)344 goto ret_residue;345 346 spin_lock_irqsave(&chan->vc.lock, flags);347 vd = vchan_find_desc(&chan->vc, cookie);348 if (vd) {349 desc = vd2dw_edma_desc(vd);350 if (desc)351 residue = desc->alloc_sz - desc->xfer_sz;352 }353 spin_unlock_irqrestore(&chan->vc.lock, flags);354 355ret_residue:356 dma_set_residue(txstate, residue);357 358 return ret;359}360 361static struct dma_async_tx_descriptor *362dw_edma_device_transfer(struct dw_edma_transfer *xfer)363{364 struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);365 enum dma_transfer_direction dir = xfer->direction;366 struct scatterlist *sg = NULL;367 struct dw_edma_chunk *chunk;368 struct dw_edma_burst *burst;369 struct dw_edma_desc *desc;370 u64 src_addr, dst_addr;371 size_t fsz = 0;372 u32 cnt = 0;373 int i;374 375 if (!chan->configured)376 return NULL;377 378 /*379 * Local Root Port/End-point Remote End-point380 * +-----------------------+ PCIe bus +----------------------+381 * | | +-+ | |382 * | DEV_TO_MEM Rx Ch <----+ +---+ Tx Ch DEV_TO_MEM |383 * | | | | | |384 * | MEM_TO_DEV Tx Ch +----+ +---> Rx Ch MEM_TO_DEV |385 * | | +-+ | |386 * +-----------------------+ +----------------------+387 *388 * 1. Normal logic:389 * If eDMA is embedded into the DW PCIe RP/EP and controlled from the390 * CPU/Application side, the Rx channel (EDMA_DIR_READ) will be used391 * for the device read operations (DEV_TO_MEM) and the Tx channel392 * (EDMA_DIR_WRITE) - for the write operations (MEM_TO_DEV).393 *394 * 2. Inverted logic:395 * If eDMA is embedded into a Remote PCIe EP and is controlled by the396 * MWr/MRd TLPs sent from the CPU's PCIe host controller, the Tx397 * channel (EDMA_DIR_WRITE) will be used for the device read operations398 * (DEV_TO_MEM) and the Rx channel (EDMA_DIR_READ) - for the write399 * operations (MEM_TO_DEV).400 *401 * It is the client driver responsibility to choose a proper channel402 * for the DMA transfers.403 */404 if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {405 if ((chan->dir == EDMA_DIR_READ && dir != DMA_DEV_TO_MEM) ||406 (chan->dir == EDMA_DIR_WRITE && dir != DMA_MEM_TO_DEV))407 return NULL;408 } else {409 if ((chan->dir == EDMA_DIR_WRITE && dir != DMA_DEV_TO_MEM) ||410 (chan->dir == EDMA_DIR_READ && dir != DMA_MEM_TO_DEV))411 return NULL;412 }413 414 if (xfer->type == EDMA_XFER_CYCLIC) {415 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt)416 return NULL;417 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {418 if (xfer->xfer.sg.len < 1)419 return NULL;420 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {421 if (!xfer->xfer.il->numf || xfer->xfer.il->frame_size < 1)422 return NULL;423 if (!xfer->xfer.il->src_inc || !xfer->xfer.il->dst_inc)424 return NULL;425 } else {426 return NULL;427 }428 429 desc = dw_edma_alloc_desc(chan);430 if (unlikely(!desc))431 goto err_alloc;432 433 chunk = dw_edma_alloc_chunk(desc);434 if (unlikely(!chunk))435 goto err_alloc;436 437 if (xfer->type == EDMA_XFER_INTERLEAVED) {438 src_addr = xfer->xfer.il->src_start;439 dst_addr = xfer->xfer.il->dst_start;440 } else {441 src_addr = chan->config.src_addr;442 dst_addr = chan->config.dst_addr;443 }444 445 if (dir == DMA_DEV_TO_MEM)446 src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr);447 else448 dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr);449 450 if (xfer->type == EDMA_XFER_CYCLIC) {451 cnt = xfer->xfer.cyclic.cnt;452 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {453 cnt = xfer->xfer.sg.len;454 sg = xfer->xfer.sg.sgl;455 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {456 cnt = xfer->xfer.il->numf * xfer->xfer.il->frame_size;457 fsz = xfer->xfer.il->frame_size;458 }459 460 for (i = 0; i < cnt; i++) {461 if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg)462 break;463 464 if (chunk->bursts_alloc == chan->ll_max) {465 chunk = dw_edma_alloc_chunk(desc);466 if (unlikely(!chunk))467 goto err_alloc;468 }469 470 burst = dw_edma_alloc_burst(chunk);471 if (unlikely(!burst))472 goto err_alloc;473 474 if (xfer->type == EDMA_XFER_CYCLIC)475 burst->sz = xfer->xfer.cyclic.len;476 else if (xfer->type == EDMA_XFER_SCATTER_GATHER)477 burst->sz = sg_dma_len(sg);478 else if (xfer->type == EDMA_XFER_INTERLEAVED)479 burst->sz = xfer->xfer.il->sgl[i % fsz].size;480 481 chunk->ll_region.sz += burst->sz;482 desc->alloc_sz += burst->sz;483 484 if (dir == DMA_DEV_TO_MEM) {485 burst->sar = src_addr;486 if (xfer->type == EDMA_XFER_CYCLIC) {487 burst->dar = xfer->xfer.cyclic.paddr;488 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {489 src_addr += sg_dma_len(sg);490 burst->dar = sg_dma_address(sg);491 /* Unlike the typical assumption by other492 * drivers/IPs the peripheral memory isn't493 * a FIFO memory, in this case, it's a494 * linear memory and that why the source495 * and destination addresses are increased496 * by the same portion (data length)497 */498 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {499 burst->dar = dst_addr;500 }501 } else {502 burst->dar = dst_addr;503 if (xfer->type == EDMA_XFER_CYCLIC) {504 burst->sar = xfer->xfer.cyclic.paddr;505 } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {506 dst_addr += sg_dma_len(sg);507 burst->sar = sg_dma_address(sg);508 /* Unlike the typical assumption by other509 * drivers/IPs the peripheral memory isn't510 * a FIFO memory, in this case, it's a511 * linear memory and that why the source512 * and destination addresses are increased513 * by the same portion (data length)514 */515 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {516 burst->sar = src_addr;517 }518 }519 520 if (xfer->type == EDMA_XFER_SCATTER_GATHER) {521 sg = sg_next(sg);522 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {523 struct dma_interleaved_template *il = xfer->xfer.il;524 struct data_chunk *dc = &il->sgl[i % fsz];525 526 src_addr += burst->sz;527 if (il->src_sgl)528 src_addr += dmaengine_get_src_icg(il, dc);529 530 dst_addr += burst->sz;531 if (il->dst_sgl)532 dst_addr += dmaengine_get_dst_icg(il, dc);533 }534 }535 536 return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags);537 538err_alloc:539 if (desc)540 dw_edma_free_desc(desc);541 542 return NULL;543}544 545static struct dma_async_tx_descriptor *546dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,547 unsigned int len,548 enum dma_transfer_direction direction,549 unsigned long flags, void *context)550{551 struct dw_edma_transfer xfer;552 553 xfer.dchan = dchan;554 xfer.direction = direction;555 xfer.xfer.sg.sgl = sgl;556 xfer.xfer.sg.len = len;557 xfer.flags = flags;558 xfer.type = EDMA_XFER_SCATTER_GATHER;559 560 return dw_edma_device_transfer(&xfer);561}562 563static struct dma_async_tx_descriptor *564dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr,565 size_t len, size_t count,566 enum dma_transfer_direction direction,567 unsigned long flags)568{569 struct dw_edma_transfer xfer;570 571 xfer.dchan = dchan;572 xfer.direction = direction;573 xfer.xfer.cyclic.paddr = paddr;574 xfer.xfer.cyclic.len = len;575 xfer.xfer.cyclic.cnt = count;576 xfer.flags = flags;577 xfer.type = EDMA_XFER_CYCLIC;578 579 return dw_edma_device_transfer(&xfer);580}581 582static struct dma_async_tx_descriptor *583dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,584 struct dma_interleaved_template *ilt,585 unsigned long flags)586{587 struct dw_edma_transfer xfer;588 589 xfer.dchan = dchan;590 xfer.direction = ilt->dir;591 xfer.xfer.il = ilt;592 xfer.flags = flags;593 xfer.type = EDMA_XFER_INTERLEAVED;594 595 return dw_edma_device_transfer(&xfer);596}597 598static void dw_edma_done_interrupt(struct dw_edma_chan *chan)599{600 struct dw_edma_desc *desc;601 struct virt_dma_desc *vd;602 unsigned long flags;603 604 spin_lock_irqsave(&chan->vc.lock, flags);605 vd = vchan_next_desc(&chan->vc);606 if (vd) {607 switch (chan->request) {608 case EDMA_REQ_NONE:609 desc = vd2dw_edma_desc(vd);610 if (!desc->chunks_alloc) {611 list_del(&vd->node);612 vchan_cookie_complete(vd);613 }614 615 /* Continue transferring if there are remaining chunks or issued requests.616 */617 chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE;618 break;619 620 case EDMA_REQ_STOP:621 list_del(&vd->node);622 vchan_cookie_complete(vd);623 chan->request = EDMA_REQ_NONE;624 chan->status = EDMA_ST_IDLE;625 break;626 627 case EDMA_REQ_PAUSE:628 chan->request = EDMA_REQ_NONE;629 chan->status = EDMA_ST_PAUSE;630 break;631 632 default:633 break;634 }635 }636 spin_unlock_irqrestore(&chan->vc.lock, flags);637}638 639static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)640{641 struct virt_dma_desc *vd;642 unsigned long flags;643 644 spin_lock_irqsave(&chan->vc.lock, flags);645 vd = vchan_next_desc(&chan->vc);646 if (vd) {647 list_del(&vd->node);648 vchan_cookie_complete(vd);649 }650 spin_unlock_irqrestore(&chan->vc.lock, flags);651 chan->request = EDMA_REQ_NONE;652 chan->status = EDMA_ST_IDLE;653}654 655static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)656{657 struct dw_edma_irq *dw_irq = data;658 659 return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE,660 dw_edma_done_interrupt,661 dw_edma_abort_interrupt);662}663 664static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data)665{666 struct dw_edma_irq *dw_irq = data;667 668 return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ,669 dw_edma_done_interrupt,670 dw_edma_abort_interrupt);671}672 673static irqreturn_t dw_edma_interrupt_common(int irq, void *data)674{675 irqreturn_t ret = IRQ_NONE;676 677 ret |= dw_edma_interrupt_write(irq, data);678 ret |= dw_edma_interrupt_read(irq, data);679 680 return ret;681}682 683static int dw_edma_alloc_chan_resources(struct dma_chan *dchan)684{685 struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);686 687 if (chan->status != EDMA_ST_IDLE)688 return -EBUSY;689 690 return 0;691}692 693static void dw_edma_free_chan_resources(struct dma_chan *dchan)694{695 unsigned long timeout = jiffies + msecs_to_jiffies(5000);696 int ret;697 698 while (time_before(jiffies, timeout)) {699 ret = dw_edma_device_terminate_all(dchan);700 if (!ret)701 break;702 703 if (time_after_eq(jiffies, timeout))704 return;705 706 cpu_relax();707 }708}709 710static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)711{712 struct dw_edma_chip *chip = dw->chip;713 struct device *dev = chip->dev;714 struct dw_edma_chan *chan;715 struct dw_edma_irq *irq;716 struct dma_device *dma;717 u32 i, ch_cnt;718 u32 pos;719 720 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;721 dma = &dw->dma;722 723 INIT_LIST_HEAD(&dma->channels);724 725 for (i = 0; i < ch_cnt; i++) {726 chan = &dw->chan[i];727 728 chan->dw = dw;729 730 if (i < dw->wr_ch_cnt) {731 chan->id = i;732 chan->dir = EDMA_DIR_WRITE;733 } else {734 chan->id = i - dw->wr_ch_cnt;735 chan->dir = EDMA_DIR_READ;736 }737 738 chan->configured = false;739 chan->request = EDMA_REQ_NONE;740 chan->status = EDMA_ST_IDLE;741 742 if (chan->dir == EDMA_DIR_WRITE)743 chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ);744 else745 chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ);746 chan->ll_max -= 1;747 748 dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n",749 chan->dir == EDMA_DIR_WRITE ? "write" : "read",750 chan->id, chan->ll_max);751 752 if (dw->nr_irqs == 1)753 pos = 0;754 else if (chan->dir == EDMA_DIR_WRITE)755 pos = chan->id % wr_alloc;756 else757 pos = wr_alloc + chan->id % rd_alloc;758 759 irq = &dw->irq[pos];760 761 if (chan->dir == EDMA_DIR_WRITE)762 irq->wr_mask |= BIT(chan->id);763 else764 irq->rd_mask |= BIT(chan->id);765 766 irq->dw = dw;767 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi));768 769 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n",770 chan->dir == EDMA_DIR_WRITE ? "write" : "read", chan->id,771 chan->msi.address_hi, chan->msi.address_lo,772 chan->msi.data);773 774 chan->vc.desc_free = vchan_free_desc;775 chan->vc.chan.private = chan->dir == EDMA_DIR_WRITE ?776 &dw->chip->dt_region_wr[chan->id] :777 &dw->chip->dt_region_rd[chan->id];778 779 vchan_init(&chan->vc, dma);780 781 dw_edma_core_ch_config(chan);782 }783 784 /* Set DMA channel capabilities */785 dma_cap_zero(dma->cap_mask);786 dma_cap_set(DMA_SLAVE, dma->cap_mask);787 dma_cap_set(DMA_CYCLIC, dma->cap_mask);788 dma_cap_set(DMA_PRIVATE, dma->cap_mask);789 dma_cap_set(DMA_INTERLEAVE, dma->cap_mask);790 dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);791 dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);792 dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);793 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;794 795 /* Set DMA channel callbacks */796 dma->dev = chip->dev;797 dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources;798 dma->device_free_chan_resources = dw_edma_free_chan_resources;799 dma->device_caps = dw_edma_device_caps;800 dma->device_config = dw_edma_device_config;801 dma->device_pause = dw_edma_device_pause;802 dma->device_resume = dw_edma_device_resume;803 dma->device_terminate_all = dw_edma_device_terminate_all;804 dma->device_issue_pending = dw_edma_device_issue_pending;805 dma->device_tx_status = dw_edma_device_tx_status;806 dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg;807 dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic;808 dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma;809 810 dma_set_max_seg_size(dma->dev, U32_MAX);811 812 /* Register DMA device */813 return dma_async_device_register(dma);814}815 816static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt)817{818 if (*nr_irqs && *alloc < cnt) {819 (*alloc)++;820 (*nr_irqs)--;821 }822}823 824static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt)825{826 while (*mask * alloc < cnt)827 (*mask)++;828}829 830static int dw_edma_irq_request(struct dw_edma *dw,831 u32 *wr_alloc, u32 *rd_alloc)832{833 struct dw_edma_chip *chip = dw->chip;834 struct device *dev = dw->chip->dev;835 u32 wr_mask = 1;836 u32 rd_mask = 1;837 int i, err = 0;838 u32 ch_cnt;839 int irq;840 841 ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;842 843 if (chip->nr_irqs < 1 || !chip->ops->irq_vector)844 return -EINVAL;845 846 dw->irq = devm_kcalloc(dev, chip->nr_irqs, sizeof(*dw->irq), GFP_KERNEL);847 if (!dw->irq)848 return -ENOMEM;849 850 if (chip->nr_irqs == 1) {851 /* Common IRQ shared among all channels */852 irq = chip->ops->irq_vector(dev, 0);853 err = request_irq(irq, dw_edma_interrupt_common,854 IRQF_SHARED, dw->name, &dw->irq[0]);855 if (err) {856 dw->nr_irqs = 0;857 return err;858 }859 860 if (irq_get_msi_desc(irq))861 get_cached_msi_msg(irq, &dw->irq[0].msi);862 863 dw->nr_irqs = 1;864 } else {865 /* Distribute IRQs equally among all channels */866 int tmp = chip->nr_irqs;867 868 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) {869 dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt);870 dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt);871 }872 873 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt);874 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt);875 876 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) {877 irq = chip->ops->irq_vector(dev, i);878 err = request_irq(irq,879 i < *wr_alloc ?880 dw_edma_interrupt_write :881 dw_edma_interrupt_read,882 IRQF_SHARED, dw->name,883 &dw->irq[i]);884 if (err)885 goto err_irq_free;886 887 if (irq_get_msi_desc(irq))888 get_cached_msi_msg(irq, &dw->irq[i].msi);889 }890 891 dw->nr_irqs = i;892 }893 894 return 0;895 896err_irq_free:897 for (i--; i >= 0; i--) {898 irq = chip->ops->irq_vector(dev, i);899 free_irq(irq, &dw->irq[i]);900 }901 902 return err;903}904 905int dw_edma_probe(struct dw_edma_chip *chip)906{907 struct device *dev;908 struct dw_edma *dw;909 u32 wr_alloc = 0;910 u32 rd_alloc = 0;911 int i, err;912 913 if (!chip)914 return -EINVAL;915 916 dev = chip->dev;917 if (!dev || !chip->ops)918 return -EINVAL;919 920 dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL);921 if (!dw)922 return -ENOMEM;923 924 dw->chip = chip;925 926 if (dw->chip->mf == EDMA_MF_HDMA_NATIVE)927 dw_hdma_v0_core_register(dw);928 else929 dw_edma_v0_core_register(dw);930 931 raw_spin_lock_init(&dw->lock);932 933 dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt,934 dw_edma_core_ch_count(dw, EDMA_DIR_WRITE));935 dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH);936 937 dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt,938 dw_edma_core_ch_count(dw, EDMA_DIR_READ));939 dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH);940 941 if (!dw->wr_ch_cnt && !dw->rd_ch_cnt)942 return -EINVAL;943 944 dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n",945 dw->wr_ch_cnt, dw->rd_ch_cnt);946 947 /* Allocate channels */948 dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt,949 sizeof(*dw->chan), GFP_KERNEL);950 if (!dw->chan)951 return -ENOMEM;952 953 snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s",954 dev_name(chip->dev));955 956 /* Disable eDMA, only to establish the ideal initial conditions */957 dw_edma_core_off(dw);958 959 /* Request IRQs */960 err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc);961 if (err)962 return err;963 964 /* Setup write/read channels */965 err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc);966 if (err)967 goto err_irq_free;968 969 /* Turn debugfs on */970 dw_edma_core_debugfs_on(dw);971 972 chip->dw = dw;973 974 return 0;975 976err_irq_free:977 for (i = (dw->nr_irqs - 1); i >= 0; i--)978 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);979 980 return err;981}982EXPORT_SYMBOL_GPL(dw_edma_probe);983 984int dw_edma_remove(struct dw_edma_chip *chip)985{986 struct dw_edma_chan *chan, *_chan;987 struct device *dev = chip->dev;988 struct dw_edma *dw = chip->dw;989 int i;990 991 /* Skip removal if no private data found */992 if (!dw)993 return -ENODEV;994 995 /* Disable eDMA */996 dw_edma_core_off(dw);997 998 /* Free irqs */999 for (i = (dw->nr_irqs - 1); i >= 0; i--)1000 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);1001 1002 /* Deregister eDMA device */1003 dma_async_device_unregister(&dw->dma);1004 list_for_each_entry_safe(chan, _chan, &dw->dma.channels,1005 vc.chan.device_node) {1006 tasklet_kill(&chan->vc.task);1007 list_del(&chan->vc.chan.device_node);1008 }1009 1010 return 0;1011}1012EXPORT_SYMBOL_GPL(dw_edma_remove);1013 1014MODULE_LICENSE("GPL v2");1015MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver");1016MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>");1017