1407 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Core driver for the Synopsys DesignWare DMA Controller4 *5 * Copyright (C) 2007-2008 Atmel Corporation6 * Copyright (C) 2010-2011 ST Microelectronics7 * Copyright (C) 2013 Intel Corporation8 */9 10#include <linux/bitops.h>11#include <linux/delay.h>12#include <linux/dmaengine.h>13#include <linux/dma-mapping.h>14#include <linux/dmapool.h>15#include <linux/err.h>16#include <linux/init.h>17#include <linux/interrupt.h>18#include <linux/io.h>19#include <linux/log2.h>20#include <linux/mm.h>21#include <linux/module.h>22#include <linux/slab.h>23#include <linux/pm_runtime.h>24 25#include "../dmaengine.h"26#include "internal.h"27 28/*29 * This supports the Synopsys "DesignWare AHB Central DMA Controller",30 * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all31 * of which use ARM any more). See the "Databook" from Synopsys for32 * information beyond what licensees probably provide.33 */34 35/* The set of bus widths supported by the DMA controller */36#define DW_DMA_BUSWIDTHS \37 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \38 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \39 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \40 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)41 42/*----------------------------------------------------------------------*/43 44static struct device *chan2dev(struct dma_chan *chan)45{46 return &chan->dev->device;47}48 49static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)50{51 return to_dw_desc(dwc->active_list.next);52}53 54static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx)55{56 struct dw_desc *desc = txd_to_dw_desc(tx);57 struct dw_dma_chan *dwc = to_dw_dma_chan(tx->chan);58 dma_cookie_t cookie;59 unsigned long flags;60 61 spin_lock_irqsave(&dwc->lock, flags);62 cookie = dma_cookie_assign(tx);63 64 /*65 * REVISIT: We should attempt to chain as many descriptors as66 * possible, perhaps even appending to those already submitted67 * for DMA. But this is hard to do in a race-free manner.68 */69 70 list_add_tail(&desc->desc_node, &dwc->queue);71 spin_unlock_irqrestore(&dwc->lock, flags);72 dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n",73 __func__, desc->txd.cookie);74 75 return cookie;76}77 78static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc)79{80 struct dw_dma *dw = to_dw_dma(dwc->chan.device);81 struct dw_desc *desc;82 dma_addr_t phys;83 84 desc = dma_pool_zalloc(dw->desc_pool, GFP_ATOMIC, &phys);85 if (!desc)86 return NULL;87 88 dwc->descs_allocated++;89 INIT_LIST_HEAD(&desc->tx_list);90 dma_async_tx_descriptor_init(&desc->txd, &dwc->chan);91 desc->txd.tx_submit = dwc_tx_submit;92 desc->txd.flags = DMA_CTRL_ACK;93 desc->txd.phys = phys;94 return desc;95}96 97static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)98{99 struct dw_dma *dw = to_dw_dma(dwc->chan.device);100 struct dw_desc *child, *_next;101 102 if (unlikely(!desc))103 return;104 105 list_for_each_entry_safe(child, _next, &desc->tx_list, desc_node) {106 list_del(&child->desc_node);107 dma_pool_free(dw->desc_pool, child, child->txd.phys);108 dwc->descs_allocated--;109 }110 111 dma_pool_free(dw->desc_pool, desc, desc->txd.phys);112 dwc->descs_allocated--;113}114 115static void dwc_initialize(struct dw_dma_chan *dwc)116{117 struct dw_dma *dw = to_dw_dma(dwc->chan.device);118 119 dw->initialize_chan(dwc);120 121 /* Enable interrupts */122 channel_set_bit(dw, MASK.XFER, dwc->mask);123 channel_set_bit(dw, MASK.ERROR, dwc->mask);124}125 126/*----------------------------------------------------------------------*/127 128static inline void dwc_dump_chan_regs(struct dw_dma_chan *dwc)129{130 dev_err(chan2dev(&dwc->chan),131 " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",132 channel_readl(dwc, SAR),133 channel_readl(dwc, DAR),134 channel_readl(dwc, LLP),135 channel_readl(dwc, CTL_HI),136 channel_readl(dwc, CTL_LO));137}138 139static inline void dwc_chan_disable(struct dw_dma *dw, struct dw_dma_chan *dwc)140{141 channel_clear_bit(dw, CH_EN, dwc->mask);142 while (dma_readl(dw, CH_EN) & dwc->mask)143 cpu_relax();144}145 146/*----------------------------------------------------------------------*/147 148/* Perform single block transfer */149static inline void dwc_do_single_block(struct dw_dma_chan *dwc,150 struct dw_desc *desc)151{152 struct dw_dma *dw = to_dw_dma(dwc->chan.device);153 u32 ctllo;154 155 /*156 * Software emulation of LLP mode relies on interrupts to continue157 * multi block transfer.158 */159 ctllo = lli_read(desc, ctllo) | DWC_CTLL_INT_EN;160 161 channel_writel(dwc, SAR, lli_read(desc, sar));162 channel_writel(dwc, DAR, lli_read(desc, dar));163 channel_writel(dwc, CTL_LO, ctllo);164 channel_writel(dwc, CTL_HI, lli_read(desc, ctlhi));165 channel_set_bit(dw, CH_EN, dwc->mask);166 167 /* Move pointer to next descriptor */168 dwc->tx_node_active = dwc->tx_node_active->next;169}170 171/* Called with dwc->lock held and bh disabled */172static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)173{174 struct dw_dma *dw = to_dw_dma(dwc->chan.device);175 u8 lms = DWC_LLP_LMS(dwc->dws.m_master);176 unsigned long was_soft_llp;177 178 /* ASSERT: channel is idle */179 if (dma_readl(dw, CH_EN) & dwc->mask) {180 dev_err(chan2dev(&dwc->chan),181 "%s: BUG: Attempted to start non-idle channel\n",182 __func__);183 dwc_dump_chan_regs(dwc);184 185 /* The tasklet will hopefully advance the queue... */186 return;187 }188 189 if (dwc->nollp) {190 was_soft_llp = test_and_set_bit(DW_DMA_IS_SOFT_LLP,191 &dwc->flags);192 if (was_soft_llp) {193 dev_err(chan2dev(&dwc->chan),194 "BUG: Attempted to start new LLP transfer inside ongoing one\n");195 return;196 }197 198 dwc_initialize(dwc);199 200 first->residue = first->total_len;201 dwc->tx_node_active = &first->tx_list;202 203 /* Submit first block */204 dwc_do_single_block(dwc, first);205 206 return;207 }208 209 dwc_initialize(dwc);210 211 channel_writel(dwc, LLP, first->txd.phys | lms);212 channel_writel(dwc, CTL_LO, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);213 channel_writel(dwc, CTL_HI, 0);214 channel_set_bit(dw, CH_EN, dwc->mask);215}216 217static void dwc_dostart_first_queued(struct dw_dma_chan *dwc)218{219 struct dw_desc *desc;220 221 if (list_empty(&dwc->queue))222 return;223 224 list_move(dwc->queue.next, &dwc->active_list);225 desc = dwc_first_active(dwc);226 dev_vdbg(chan2dev(&dwc->chan), "%s: started %u\n", __func__, desc->txd.cookie);227 dwc_dostart(dwc, desc);228}229 230/*----------------------------------------------------------------------*/231 232static void233dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,234 bool callback_required)235{236 struct dma_async_tx_descriptor *txd = &desc->txd;237 struct dw_desc *child;238 unsigned long flags;239 struct dmaengine_desc_callback cb;240 241 dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);242 243 spin_lock_irqsave(&dwc->lock, flags);244 dma_cookie_complete(txd);245 if (callback_required)246 dmaengine_desc_get_callback(txd, &cb);247 else248 memset(&cb, 0, sizeof(cb));249 250 /* async_tx_ack */251 list_for_each_entry(child, &desc->tx_list, desc_node)252 async_tx_ack(&child->txd);253 async_tx_ack(&desc->txd);254 dwc_desc_put(dwc, desc);255 spin_unlock_irqrestore(&dwc->lock, flags);256 257 dmaengine_desc_callback_invoke(&cb, NULL);258}259 260static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)261{262 struct dw_desc *desc, *_desc;263 LIST_HEAD(list);264 unsigned long flags;265 266 spin_lock_irqsave(&dwc->lock, flags);267 if (dma_readl(dw, CH_EN) & dwc->mask) {268 dev_err(chan2dev(&dwc->chan),269 "BUG: XFER bit set, but channel not idle!\n");270 271 /* Try to continue after resetting the channel... */272 dwc_chan_disable(dw, dwc);273 }274 275 /*276 * Submit queued descriptors ASAP, i.e. before we go through277 * the completed ones.278 */279 list_splice_init(&dwc->active_list, &list);280 dwc_dostart_first_queued(dwc);281 282 spin_unlock_irqrestore(&dwc->lock, flags);283 284 list_for_each_entry_safe(desc, _desc, &list, desc_node)285 dwc_descriptor_complete(dwc, desc, true);286}287 288/* Returns how many bytes were already received from source */289static inline u32 dwc_get_sent(struct dw_dma_chan *dwc)290{291 struct dw_dma *dw = to_dw_dma(dwc->chan.device);292 u32 ctlhi = channel_readl(dwc, CTL_HI);293 u32 ctllo = channel_readl(dwc, CTL_LO);294 295 return dw->block2bytes(dwc, ctlhi, ctllo >> 4 & 7);296}297 298static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)299{300 dma_addr_t llp;301 struct dw_desc *desc, *_desc;302 struct dw_desc *child;303 u32 status_xfer;304 unsigned long flags;305 306 spin_lock_irqsave(&dwc->lock, flags);307 llp = channel_readl(dwc, LLP);308 status_xfer = dma_readl(dw, RAW.XFER);309 310 if (status_xfer & dwc->mask) {311 /* Everything we've submitted is done */312 dma_writel(dw, CLEAR.XFER, dwc->mask);313 314 if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) {315 struct list_head *head, *active = dwc->tx_node_active;316 317 /*318 * We are inside first active descriptor.319 * Otherwise something is really wrong.320 */321 desc = dwc_first_active(dwc);322 323 head = &desc->tx_list;324 if (active != head) {325 /* Update residue to reflect last sent descriptor */326 if (active == head->next)327 desc->residue -= desc->len;328 else329 desc->residue -= to_dw_desc(active->prev)->len;330 331 child = to_dw_desc(active);332 333 /* Submit next block */334 dwc_do_single_block(dwc, child);335 336 spin_unlock_irqrestore(&dwc->lock, flags);337 return;338 }339 340 /* We are done here */341 clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);342 }343 344 spin_unlock_irqrestore(&dwc->lock, flags);345 346 dwc_complete_all(dw, dwc);347 return;348 }349 350 if (list_empty(&dwc->active_list)) {351 spin_unlock_irqrestore(&dwc->lock, flags);352 return;353 }354 355 if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) {356 dev_vdbg(chan2dev(&dwc->chan), "%s: soft LLP mode\n", __func__);357 spin_unlock_irqrestore(&dwc->lock, flags);358 return;359 }360 361 dev_vdbg(chan2dev(&dwc->chan), "%s: llp=%pad\n", __func__, &llp);362 363 list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {364 /* Initial residue value */365 desc->residue = desc->total_len;366 367 /* Check first descriptors addr */368 if (desc->txd.phys == DWC_LLP_LOC(llp)) {369 spin_unlock_irqrestore(&dwc->lock, flags);370 return;371 }372 373 /* Check first descriptors llp */374 if (lli_read(desc, llp) == llp) {375 /* This one is currently in progress */376 desc->residue -= dwc_get_sent(dwc);377 spin_unlock_irqrestore(&dwc->lock, flags);378 return;379 }380 381 desc->residue -= desc->len;382 list_for_each_entry(child, &desc->tx_list, desc_node) {383 if (lli_read(child, llp) == llp) {384 /* Currently in progress */385 desc->residue -= dwc_get_sent(dwc);386 spin_unlock_irqrestore(&dwc->lock, flags);387 return;388 }389 desc->residue -= child->len;390 }391 392 /*393 * No descriptors so far seem to be in progress, i.e.394 * this one must be done.395 */396 spin_unlock_irqrestore(&dwc->lock, flags);397 dwc_descriptor_complete(dwc, desc, true);398 spin_lock_irqsave(&dwc->lock, flags);399 }400 401 dev_err(chan2dev(&dwc->chan),402 "BUG: All descriptors done, but channel not idle!\n");403 404 /* Try to continue after resetting the channel... */405 dwc_chan_disable(dw, dwc);406 407 dwc_dostart_first_queued(dwc);408 spin_unlock_irqrestore(&dwc->lock, flags);409}410 411static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_desc *desc)412{413 dev_crit(chan2dev(&dwc->chan), " desc: s0x%x d0x%x l0x%x c0x%x:%x\n",414 lli_read(desc, sar),415 lli_read(desc, dar),416 lli_read(desc, llp),417 lli_read(desc, ctlhi),418 lli_read(desc, ctllo));419}420 421static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)422{423 struct dw_desc *bad_desc;424 struct dw_desc *child;425 unsigned long flags;426 427 dwc_scan_descriptors(dw, dwc);428 429 spin_lock_irqsave(&dwc->lock, flags);430 431 /*432 * The descriptor currently at the head of the active list is433 * borked. Since we don't have any way to report errors, we'll434 * just have to scream loudly and try to carry on.435 */436 bad_desc = dwc_first_active(dwc);437 list_del_init(&bad_desc->desc_node);438 list_move(dwc->queue.next, dwc->active_list.prev);439 440 /* Clear the error flag and try to restart the controller */441 dma_writel(dw, CLEAR.ERROR, dwc->mask);442 if (!list_empty(&dwc->active_list))443 dwc_dostart(dwc, dwc_first_active(dwc));444 445 /*446 * WARN may seem harsh, but since this only happens447 * when someone submits a bad physical address in a448 * descriptor, we should consider ourselves lucky that the449 * controller flagged an error instead of scribbling over450 * random memory locations.451 */452 dev_WARN(chan2dev(&dwc->chan), "Bad descriptor submitted for DMA!\n"453 " cookie: %d\n", bad_desc->txd.cookie);454 dwc_dump_lli(dwc, bad_desc);455 list_for_each_entry(child, &bad_desc->tx_list, desc_node)456 dwc_dump_lli(dwc, child);457 458 spin_unlock_irqrestore(&dwc->lock, flags);459 460 /* Pretend the descriptor completed successfully */461 dwc_descriptor_complete(dwc, bad_desc, true);462}463 464static void dw_dma_tasklet(struct tasklet_struct *t)465{466 struct dw_dma *dw = from_tasklet(dw, t, tasklet);467 struct dw_dma_chan *dwc;468 u32 status_xfer;469 u32 status_err;470 unsigned int i;471 472 status_xfer = dma_readl(dw, RAW.XFER);473 status_err = dma_readl(dw, RAW.ERROR);474 475 dev_vdbg(dw->dma.dev, "%s: status_err=%x\n", __func__, status_err);476 477 for (i = 0; i < dw->dma.chancnt; i++) {478 dwc = &dw->chan[i];479 if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags))480 dev_vdbg(dw->dma.dev, "Cyclic xfer is not implemented\n");481 else if (status_err & (1 << i))482 dwc_handle_error(dw, dwc);483 else if (status_xfer & (1 << i))484 dwc_scan_descriptors(dw, dwc);485 }486 487 /* Re-enable interrupts */488 channel_set_bit(dw, MASK.XFER, dw->all_chan_mask);489 channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask);490}491 492static irqreturn_t dw_dma_interrupt(int irq, void *dev_id)493{494 struct dw_dma *dw = dev_id;495 u32 status;496 497 /* Check if we have any interrupt from the DMAC which is not in use */498 if (!dw->in_use)499 return IRQ_NONE;500 501 status = dma_readl(dw, STATUS_INT);502 dev_vdbg(dw->dma.dev, "%s: status=0x%x\n", __func__, status);503 504 /* Check if we have any interrupt from the DMAC */505 if (!status)506 return IRQ_NONE;507 508 /*509 * Just disable the interrupts. We'll turn them back on in the510 * softirq handler.511 */512 channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);513 channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);514 channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);515 516 status = dma_readl(dw, STATUS_INT);517 if (status) {518 dev_err(dw->dma.dev,519 "BUG: Unexpected interrupts pending: 0x%x\n",520 status);521 522 /* Try to recover */523 channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1);524 channel_clear_bit(dw, MASK.BLOCK, (1 << 8) - 1);525 channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1);526 channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1);527 channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1);528 }529 530 tasklet_schedule(&dw->tasklet);531 532 return IRQ_HANDLED;533}534 535/*----------------------------------------------------------------------*/536 537static struct dma_async_tx_descriptor *538dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,539 size_t len, unsigned long flags)540{541 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);542 struct dw_dma *dw = to_dw_dma(chan->device);543 struct dw_desc *desc;544 struct dw_desc *first;545 struct dw_desc *prev;546 size_t xfer_count;547 size_t offset;548 u8 m_master = dwc->dws.m_master;549 unsigned int src_width;550 unsigned int dst_width;551 unsigned int data_width = dw->pdata->data_width[m_master];552 u32 ctllo, ctlhi;553 u8 lms = DWC_LLP_LMS(m_master);554 555 dev_vdbg(chan2dev(chan),556 "%s: d%pad s%pad l0x%zx f0x%lx\n", __func__,557 &dest, &src, len, flags);558 559 if (unlikely(!len)) {560 dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__);561 return NULL;562 }563 564 dwc->direction = DMA_MEM_TO_MEM;565 566 src_width = dst_width = __ffs(data_width | src | dest | len);567 568 ctllo = dw->prepare_ctllo(dwc)569 | DWC_CTLL_DST_WIDTH(dst_width)570 | DWC_CTLL_SRC_WIDTH(src_width)571 | DWC_CTLL_DST_INC572 | DWC_CTLL_SRC_INC573 | DWC_CTLL_FC_M2M;574 prev = first = NULL;575 576 for (offset = 0; offset < len; offset += xfer_count) {577 desc = dwc_desc_get(dwc);578 if (!desc)579 goto err_desc_get;580 581 ctlhi = dw->bytes2block(dwc, len - offset, src_width, &xfer_count);582 583 lli_write(desc, sar, src + offset);584 lli_write(desc, dar, dest + offset);585 lli_write(desc, ctllo, ctllo);586 lli_write(desc, ctlhi, ctlhi);587 desc->len = xfer_count;588 589 if (!first) {590 first = desc;591 } else {592 lli_write(prev, llp, desc->txd.phys | lms);593 list_add_tail(&desc->desc_node, &first->tx_list);594 }595 prev = desc;596 }597 598 if (flags & DMA_PREP_INTERRUPT)599 /* Trigger interrupt after last block */600 lli_set(prev, ctllo, DWC_CTLL_INT_EN);601 602 prev->lli.llp = 0;603 lli_clear(prev, ctllo, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);604 first->txd.flags = flags;605 first->total_len = len;606 607 return &first->txd;608 609err_desc_get:610 dwc_desc_put(dwc, first);611 return NULL;612}613 614static struct dma_async_tx_descriptor *615dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,616 unsigned int sg_len, enum dma_transfer_direction direction,617 unsigned long flags, void *context)618{619 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);620 struct dw_dma *dw = to_dw_dma(chan->device);621 struct dma_slave_config *sconfig = &dwc->dma_sconfig;622 struct dw_desc *prev;623 struct dw_desc *first;624 u32 ctllo, ctlhi;625 u8 lms = DWC_LLP_LMS(dwc->dws.m_master);626 dma_addr_t reg;627 unsigned int reg_width;628 unsigned int mem_width;629 unsigned int i;630 struct scatterlist *sg;631 size_t total_len = 0;632 633 dev_vdbg(chan2dev(chan), "%s\n", __func__);634 635 if (unlikely(!is_slave_direction(direction) || !sg_len))636 return NULL;637 638 dwc->direction = direction;639 640 prev = first = NULL;641 642 switch (direction) {643 case DMA_MEM_TO_DEV:644 reg_width = __ffs(sconfig->dst_addr_width);645 reg = sconfig->dst_addr;646 ctllo = dw->prepare_ctllo(dwc)647 | DWC_CTLL_DST_WIDTH(reg_width)648 | DWC_CTLL_DST_FIX649 | DWC_CTLL_SRC_INC;650 651 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_M2P) :652 DWC_CTLL_FC(DW_DMA_FC_D_M2P);653 654 for_each_sg(sgl, sg, sg_len, i) {655 struct dw_desc *desc;656 u32 len, mem;657 size_t dlen;658 659 mem = sg_dma_address(sg);660 len = sg_dma_len(sg);661 662 mem_width = __ffs(sconfig->src_addr_width | mem | len);663 664slave_sg_todev_fill_desc:665 desc = dwc_desc_get(dwc);666 if (!desc)667 goto err_desc_get;668 669 ctlhi = dw->bytes2block(dwc, len, mem_width, &dlen);670 671 lli_write(desc, sar, mem);672 lli_write(desc, dar, reg);673 lli_write(desc, ctlhi, ctlhi);674 lli_write(desc, ctllo, ctllo | DWC_CTLL_SRC_WIDTH(mem_width));675 desc->len = dlen;676 677 if (!first) {678 first = desc;679 } else {680 lli_write(prev, llp, desc->txd.phys | lms);681 list_add_tail(&desc->desc_node, &first->tx_list);682 }683 prev = desc;684 685 mem += dlen;686 len -= dlen;687 total_len += dlen;688 689 if (len)690 goto slave_sg_todev_fill_desc;691 }692 break;693 case DMA_DEV_TO_MEM:694 reg_width = __ffs(sconfig->src_addr_width);695 reg = sconfig->src_addr;696 ctllo = dw->prepare_ctllo(dwc)697 | DWC_CTLL_SRC_WIDTH(reg_width)698 | DWC_CTLL_DST_INC699 | DWC_CTLL_SRC_FIX;700 701 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_P2M) :702 DWC_CTLL_FC(DW_DMA_FC_D_P2M);703 704 for_each_sg(sgl, sg, sg_len, i) {705 struct dw_desc *desc;706 u32 len, mem;707 size_t dlen;708 709 mem = sg_dma_address(sg);710 len = sg_dma_len(sg);711 712slave_sg_fromdev_fill_desc:713 desc = dwc_desc_get(dwc);714 if (!desc)715 goto err_desc_get;716 717 ctlhi = dw->bytes2block(dwc, len, reg_width, &dlen);718 719 lli_write(desc, sar, reg);720 lli_write(desc, dar, mem);721 lli_write(desc, ctlhi, ctlhi);722 mem_width = __ffs(sconfig->dst_addr_width | mem);723 lli_write(desc, ctllo, ctllo | DWC_CTLL_DST_WIDTH(mem_width));724 desc->len = dlen;725 726 if (!first) {727 first = desc;728 } else {729 lli_write(prev, llp, desc->txd.phys | lms);730 list_add_tail(&desc->desc_node, &first->tx_list);731 }732 prev = desc;733 734 mem += dlen;735 len -= dlen;736 total_len += dlen;737 738 if (len)739 goto slave_sg_fromdev_fill_desc;740 }741 break;742 default:743 return NULL;744 }745 746 if (flags & DMA_PREP_INTERRUPT)747 /* Trigger interrupt after last block */748 lli_set(prev, ctllo, DWC_CTLL_INT_EN);749 750 prev->lli.llp = 0;751 lli_clear(prev, ctllo, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);752 first->total_len = total_len;753 754 return &first->txd;755 756err_desc_get:757 dev_err(chan2dev(chan),758 "not enough descriptors available. Direction %d\n", direction);759 dwc_desc_put(dwc, first);760 return NULL;761}762 763bool dw_dma_filter(struct dma_chan *chan, void *param)764{765 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);766 struct dw_dma_slave *dws = param;767 768 if (dws->dma_dev != chan->device->dev)769 return false;770 771 /* permit channels in accordance with the channels mask */772 if (dws->channels && !(dws->channels & dwc->mask))773 return false;774 775 /* We have to copy data since dws can be temporary storage */776 memcpy(&dwc->dws, dws, sizeof(struct dw_dma_slave));777 778 return true;779}780EXPORT_SYMBOL_GPL(dw_dma_filter);781 782static int dwc_verify_maxburst(struct dma_chan *chan)783{784 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);785 786 dwc->dma_sconfig.src_maxburst =787 clamp(dwc->dma_sconfig.src_maxburst, 1U, dwc->max_burst);788 dwc->dma_sconfig.dst_maxburst =789 clamp(dwc->dma_sconfig.dst_maxburst, 1U, dwc->max_burst);790 791 dwc->dma_sconfig.src_maxburst =792 rounddown_pow_of_two(dwc->dma_sconfig.src_maxburst);793 dwc->dma_sconfig.dst_maxburst =794 rounddown_pow_of_two(dwc->dma_sconfig.dst_maxburst);795 796 return 0;797}798 799static int dwc_verify_p_buswidth(struct dma_chan *chan)800{801 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);802 struct dw_dma *dw = to_dw_dma(chan->device);803 u32 reg_width, max_width;804 805 if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV)806 reg_width = dwc->dma_sconfig.dst_addr_width;807 else if (dwc->dma_sconfig.direction == DMA_DEV_TO_MEM)808 reg_width = dwc->dma_sconfig.src_addr_width;809 else /* DMA_MEM_TO_MEM */810 return 0;811 812 max_width = dw->pdata->data_width[dwc->dws.p_master];813 814 /* Fall-back to 1-byte transfer width if undefined */815 if (reg_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)816 reg_width = DMA_SLAVE_BUSWIDTH_1_BYTE;817 else if (!is_power_of_2(reg_width) || reg_width > max_width)818 return -EINVAL;819 else /* bus width is valid */820 return 0;821 822 /* Update undefined addr width value */823 if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV)824 dwc->dma_sconfig.dst_addr_width = reg_width;825 else /* DMA_DEV_TO_MEM */826 dwc->dma_sconfig.src_addr_width = reg_width;827 828 return 0;829}830 831static int dwc_verify_m_buswidth(struct dma_chan *chan)832{833 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);834 struct dw_dma *dw = to_dw_dma(chan->device);835 u32 reg_width, reg_burst, mem_width;836 837 mem_width = dw->pdata->data_width[dwc->dws.m_master];838 839 /*840 * It's possible to have a data portion locked in the DMA FIFO in case841 * of the channel suspension. Subsequent channel disabling will cause842 * that data silent loss. In order to prevent that maintain the src and843 * dst transfer widths coherency by means of the relation:844 * (CTLx.SRC_TR_WIDTH * CTLx.SRC_MSIZE >= CTLx.DST_TR_WIDTH)845 * Look for the details in the commit message that brings this change.846 *847 * Note the DMA configs utilized in the calculations below must have848 * been verified to have correct values by this method call.849 */850 if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) {851 reg_width = dwc->dma_sconfig.dst_addr_width;852 if (mem_width < reg_width)853 return -EINVAL;854 855 dwc->dma_sconfig.src_addr_width = mem_width;856 } else if (dwc->dma_sconfig.direction == DMA_DEV_TO_MEM) {857 reg_width = dwc->dma_sconfig.src_addr_width;858 reg_burst = dwc->dma_sconfig.src_maxburst;859 860 dwc->dma_sconfig.dst_addr_width = min(mem_width, reg_width * reg_burst);861 }862 863 return 0;864}865 866static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig)867{868 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);869 int ret;870 871 memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig));872 873 ret = dwc_verify_maxburst(chan);874 if (ret)875 return ret;876 877 ret = dwc_verify_p_buswidth(chan);878 if (ret)879 return ret;880 881 ret = dwc_verify_m_buswidth(chan);882 if (ret)883 return ret;884 885 return 0;886}887 888static void dwc_chan_pause(struct dw_dma_chan *dwc, bool drain)889{890 struct dw_dma *dw = to_dw_dma(dwc->chan.device);891 unsigned int count = 20; /* timeout iterations */892 893 dw->suspend_chan(dwc, drain);894 895 while (!(channel_readl(dwc, CFG_LO) & DWC_CFGL_FIFO_EMPTY) && count--)896 udelay(2);897 898 set_bit(DW_DMA_IS_PAUSED, &dwc->flags);899}900 901static int dwc_pause(struct dma_chan *chan)902{903 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);904 unsigned long flags;905 906 spin_lock_irqsave(&dwc->lock, flags);907 dwc_chan_pause(dwc, false);908 spin_unlock_irqrestore(&dwc->lock, flags);909 910 return 0;911}912 913static inline void dwc_chan_resume(struct dw_dma_chan *dwc, bool drain)914{915 struct dw_dma *dw = to_dw_dma(dwc->chan.device);916 917 dw->resume_chan(dwc, drain);918 919 clear_bit(DW_DMA_IS_PAUSED, &dwc->flags);920}921 922static int dwc_resume(struct dma_chan *chan)923{924 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);925 unsigned long flags;926 927 spin_lock_irqsave(&dwc->lock, flags);928 929 if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags))930 dwc_chan_resume(dwc, false);931 932 spin_unlock_irqrestore(&dwc->lock, flags);933 934 return 0;935}936 937static int dwc_terminate_all(struct dma_chan *chan)938{939 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);940 struct dw_dma *dw = to_dw_dma(chan->device);941 struct dw_desc *desc, *_desc;942 unsigned long flags;943 LIST_HEAD(list);944 945 spin_lock_irqsave(&dwc->lock, flags);946 947 clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);948 949 dwc_chan_pause(dwc, true);950 951 dwc_chan_disable(dw, dwc);952 953 dwc_chan_resume(dwc, true);954 955 /* active_list entries will end up before queued entries */956 list_splice_init(&dwc->queue, &list);957 list_splice_init(&dwc->active_list, &list);958 959 spin_unlock_irqrestore(&dwc->lock, flags);960 961 /* Flush all pending and queued descriptors */962 list_for_each_entry_safe(desc, _desc, &list, desc_node)963 dwc_descriptor_complete(dwc, desc, false);964 965 return 0;966}967 968static struct dw_desc *dwc_find_desc(struct dw_dma_chan *dwc, dma_cookie_t c)969{970 struct dw_desc *desc;971 972 list_for_each_entry(desc, &dwc->active_list, desc_node)973 if (desc->txd.cookie == c)974 return desc;975 976 return NULL;977}978 979static u32 dwc_get_residue_and_status(struct dw_dma_chan *dwc, dma_cookie_t cookie,980 enum dma_status *status)981{982 struct dw_desc *desc;983 unsigned long flags;984 u32 residue;985 986 spin_lock_irqsave(&dwc->lock, flags);987 988 desc = dwc_find_desc(dwc, cookie);989 if (desc) {990 if (desc == dwc_first_active(dwc)) {991 residue = desc->residue;992 if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags) && residue)993 residue -= dwc_get_sent(dwc);994 if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags))995 *status = DMA_PAUSED;996 } else {997 residue = desc->total_len;998 }999 } else {1000 residue = 0;1001 }1002 1003 spin_unlock_irqrestore(&dwc->lock, flags);1004 return residue;1005}1006 1007static enum dma_status1008dwc_tx_status(struct dma_chan *chan,1009 dma_cookie_t cookie,1010 struct dma_tx_state *txstate)1011{1012 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);1013 enum dma_status ret;1014 1015 ret = dma_cookie_status(chan, cookie, txstate);1016 if (ret == DMA_COMPLETE)1017 return ret;1018 1019 dwc_scan_descriptors(to_dw_dma(chan->device), dwc);1020 1021 ret = dma_cookie_status(chan, cookie, txstate);1022 if (ret == DMA_COMPLETE)1023 return ret;1024 1025 dma_set_residue(txstate, dwc_get_residue_and_status(dwc, cookie, &ret));1026 return ret;1027}1028 1029static void dwc_issue_pending(struct dma_chan *chan)1030{1031 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);1032 unsigned long flags;1033 1034 spin_lock_irqsave(&dwc->lock, flags);1035 if (list_empty(&dwc->active_list))1036 dwc_dostart_first_queued(dwc);1037 spin_unlock_irqrestore(&dwc->lock, flags);1038}1039 1040/*----------------------------------------------------------------------*/1041 1042void do_dw_dma_off(struct dw_dma *dw)1043{1044 dma_writel(dw, CFG, 0);1045 1046 channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);1047 channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);1048 channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);1049 channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);1050 channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);1051 1052 while (dma_readl(dw, CFG) & DW_CFG_DMA_EN)1053 cpu_relax();1054}1055 1056void do_dw_dma_on(struct dw_dma *dw)1057{1058 dma_writel(dw, CFG, DW_CFG_DMA_EN);1059}1060 1061static int dwc_alloc_chan_resources(struct dma_chan *chan)1062{1063 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);1064 struct dw_dma *dw = to_dw_dma(chan->device);1065 1066 dev_vdbg(chan2dev(chan), "%s\n", __func__);1067 1068 /* ASSERT: channel is idle */1069 if (dma_readl(dw, CH_EN) & dwc->mask) {1070 dev_dbg(chan2dev(chan), "DMA channel not idle?\n");1071 return -EIO;1072 }1073 1074 dma_cookie_init(chan);1075 1076 /*1077 * NOTE: some controllers may have additional features that we1078 * need to initialize here, like "scatter-gather" (which1079 * doesn't mean what you think it means), and status writeback.1080 */1081 1082 /*1083 * We need controller-specific data to set up slave transfers.1084 */1085 if (chan->private && !dw_dma_filter(chan, chan->private)) {1086 dev_warn(chan2dev(chan), "Wrong controller-specific data\n");1087 return -EINVAL;1088 }1089 1090 /* Enable controller here if needed */1091 if (!dw->in_use)1092 do_dw_dma_on(dw);1093 dw->in_use |= dwc->mask;1094 1095 return 0;1096}1097 1098static void dwc_free_chan_resources(struct dma_chan *chan)1099{1100 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);1101 struct dw_dma *dw = to_dw_dma(chan->device);1102 unsigned long flags;1103 1104 dev_dbg(chan2dev(chan), "%s: descs allocated=%u\n", __func__,1105 dwc->descs_allocated);1106 1107 /* ASSERT: channel is idle */1108 BUG_ON(!list_empty(&dwc->active_list));1109 BUG_ON(!list_empty(&dwc->queue));1110 BUG_ON(dma_readl(to_dw_dma(chan->device), CH_EN) & dwc->mask);1111 1112 spin_lock_irqsave(&dwc->lock, flags);1113 1114 /* Clear custom channel configuration */1115 memset(&dwc->dws, 0, sizeof(struct dw_dma_slave));1116 1117 /* Disable interrupts */1118 channel_clear_bit(dw, MASK.XFER, dwc->mask);1119 channel_clear_bit(dw, MASK.BLOCK, dwc->mask);1120 channel_clear_bit(dw, MASK.ERROR, dwc->mask);1121 1122 spin_unlock_irqrestore(&dwc->lock, flags);1123 1124 /* Disable controller in case it was a last user */1125 dw->in_use &= ~dwc->mask;1126 if (!dw->in_use)1127 do_dw_dma_off(dw);1128 1129 dev_vdbg(chan2dev(chan), "%s: done\n", __func__);1130}1131 1132static void dwc_caps(struct dma_chan *chan, struct dma_slave_caps *caps)1133{1134 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);1135 1136 caps->max_burst = dwc->max_burst;1137 1138 /*1139 * It might be crucial for some devices to have the hardware1140 * accelerated multi-block transfers supported, aka LLPs in DW DMAC1141 * notation. So if LLPs are supported then max_sg_burst is set to1142 * zero which means unlimited number of SG entries can be handled in a1143 * single DMA transaction, otherwise it's just one SG entry.1144 */1145 if (dwc->nollp)1146 caps->max_sg_burst = 1;1147 else1148 caps->max_sg_burst = 0;1149}1150 1151int do_dma_probe(struct dw_dma_chip *chip)1152{1153 struct dw_dma *dw = chip->dw;1154 struct dw_dma_platform_data *pdata;1155 bool autocfg = false;1156 unsigned int dw_params;1157 unsigned int i;1158 int ret;1159 1160 dw->pdata = devm_kzalloc(chip->dev, sizeof(*dw->pdata), GFP_KERNEL);1161 if (!dw->pdata)1162 return -ENOMEM;1163 1164 dw->regs = chip->regs;1165 1166 pm_runtime_get_sync(chip->dev);1167 1168 if (!chip->pdata) {1169 dw_params = dma_readl(dw, DW_PARAMS);1170 dev_dbg(chip->dev, "DW_PARAMS: 0x%08x\n", dw_params);1171 1172 autocfg = dw_params >> DW_PARAMS_EN & 1;1173 if (!autocfg) {1174 ret = -EINVAL;1175 goto err_pdata;1176 }1177 1178 /* Reassign the platform data pointer */1179 pdata = dw->pdata;1180 1181 /* Get hardware configuration parameters */1182 pdata->nr_channels = (dw_params >> DW_PARAMS_NR_CHAN & 7) + 1;1183 pdata->nr_masters = (dw_params >> DW_PARAMS_NR_MASTER & 3) + 1;1184 for (i = 0; i < pdata->nr_masters; i++) {1185 pdata->data_width[i] =1186 4 << (dw_params >> DW_PARAMS_DATA_WIDTH(i) & 3);1187 }1188 pdata->block_size = dma_readl(dw, MAX_BLK_SIZE);1189 1190 /* Fill platform data with the default values */1191 pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING;1192 pdata->chan_priority = CHAN_PRIORITY_ASCENDING;1193 } else if (chip->pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS) {1194 ret = -EINVAL;1195 goto err_pdata;1196 } else {1197 memcpy(dw->pdata, chip->pdata, sizeof(*dw->pdata));1198 1199 /* Reassign the platform data pointer */1200 pdata = dw->pdata;1201 }1202 1203 dw->chan = devm_kcalloc(chip->dev, pdata->nr_channels, sizeof(*dw->chan),1204 GFP_KERNEL);1205 if (!dw->chan) {1206 ret = -ENOMEM;1207 goto err_pdata;1208 }1209 1210 /* Calculate all channel mask before DMA setup */1211 dw->all_chan_mask = (1 << pdata->nr_channels) - 1;1212 1213 /* Force dma off, just in case */1214 dw->disable(dw);1215 1216 /* Device and instance ID for IRQ and DMA pool */1217 dw->set_device_name(dw, chip->id);1218 1219 /* Create a pool of consistent memory blocks for hardware descriptors */1220 dw->desc_pool = dmam_pool_create(dw->name, chip->dev,1221 sizeof(struct dw_desc), 4, 0);1222 if (!dw->desc_pool) {1223 dev_err(chip->dev, "No memory for descriptors dma pool\n");1224 ret = -ENOMEM;1225 goto err_pdata;1226 }1227 1228 tasklet_setup(&dw->tasklet, dw_dma_tasklet);1229 1230 ret = request_irq(chip->irq, dw_dma_interrupt, IRQF_SHARED,1231 dw->name, dw);1232 if (ret)1233 goto err_pdata;1234 1235 INIT_LIST_HEAD(&dw->dma.channels);1236 for (i = 0; i < pdata->nr_channels; i++) {1237 struct dw_dma_chan *dwc = &dw->chan[i];1238 1239 dwc->chan.device = &dw->dma;1240 dma_cookie_init(&dwc->chan);1241 if (pdata->chan_allocation_order == CHAN_ALLOCATION_ASCENDING)1242 list_add_tail(&dwc->chan.device_node,1243 &dw->dma.channels);1244 else1245 list_add(&dwc->chan.device_node, &dw->dma.channels);1246 1247 /* 7 is highest priority & 0 is lowest. */1248 if (pdata->chan_priority == CHAN_PRIORITY_ASCENDING)1249 dwc->priority = pdata->nr_channels - i - 1;1250 else1251 dwc->priority = i;1252 1253 dwc->ch_regs = &__dw_regs(dw)->CHAN[i];1254 spin_lock_init(&dwc->lock);1255 dwc->mask = 1 << i;1256 1257 INIT_LIST_HEAD(&dwc->active_list);1258 INIT_LIST_HEAD(&dwc->queue);1259 1260 channel_clear_bit(dw, CH_EN, dwc->mask);1261 1262 dwc->direction = DMA_TRANS_NONE;1263 1264 /* Hardware configuration */1265 if (autocfg) {1266 unsigned int r = DW_DMA_MAX_NR_CHANNELS - i - 1;1267 void __iomem *addr = &__dw_regs(dw)->DWC_PARAMS[r];1268 unsigned int dwc_params = readl(addr);1269 1270 dev_dbg(chip->dev, "DWC_PARAMS[%d]: 0x%08x\n", i,1271 dwc_params);1272 1273 /*1274 * Decode maximum block size for given channel. The1275 * stored 4 bit value represents blocks from 0x00 for 31276 * up to 0x0a for 4095.1277 */1278 dwc->block_size =1279 (4 << ((pdata->block_size >> 4 * i) & 0xf)) - 1;1280 1281 /*1282 * According to the DW DMA databook the true scatter-1283 * gether LLPs aren't available if either multi-block1284 * config is disabled (CHx_MULTI_BLK_EN == 0) or the1285 * LLP register is hard-coded to zeros1286 * (CHx_HC_LLP == 1).1287 */1288 dwc->nollp =1289 (dwc_params >> DWC_PARAMS_MBLK_EN & 0x1) == 0 ||1290 (dwc_params >> DWC_PARAMS_HC_LLP & 0x1) == 1;1291 dwc->max_burst =1292 (0x4 << (dwc_params >> DWC_PARAMS_MSIZE & 0x7));1293 } else {1294 dwc->block_size = pdata->block_size;1295 dwc->nollp = !pdata->multi_block[i];1296 dwc->max_burst = pdata->max_burst[i] ?: DW_DMA_MAX_BURST;1297 }1298 }1299 1300 /* Clear all interrupts on all channels. */1301 dma_writel(dw, CLEAR.XFER, dw->all_chan_mask);1302 dma_writel(dw, CLEAR.BLOCK, dw->all_chan_mask);1303 dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask);1304 dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask);1305 dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask);1306 1307 /* Set capabilities */1308 dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);1309 dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask);1310 dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask);1311 1312 dw->dma.dev = chip->dev;1313 dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources;1314 dw->dma.device_free_chan_resources = dwc_free_chan_resources;1315 1316 dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy;1317 dw->dma.device_prep_slave_sg = dwc_prep_slave_sg;1318 1319 dw->dma.device_caps = dwc_caps;1320 dw->dma.device_config = dwc_config;1321 dw->dma.device_pause = dwc_pause;1322 dw->dma.device_resume = dwc_resume;1323 dw->dma.device_terminate_all = dwc_terminate_all;1324 1325 dw->dma.device_tx_status = dwc_tx_status;1326 dw->dma.device_issue_pending = dwc_issue_pending;1327 1328 /* DMA capabilities */1329 dw->dma.min_burst = DW_DMA_MIN_BURST;1330 dw->dma.max_burst = DW_DMA_MAX_BURST;1331 dw->dma.src_addr_widths = DW_DMA_BUSWIDTHS;1332 dw->dma.dst_addr_widths = DW_DMA_BUSWIDTHS;1333 dw->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |1334 BIT(DMA_MEM_TO_MEM);1335 dw->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;1336 1337 /*1338 * For now there is no hardware with non uniform maximum block size1339 * across all of the device channels, so we set the maximum segment1340 * size as the block size found for the very first channel.1341 */1342 dma_set_max_seg_size(dw->dma.dev, dw->chan[0].block_size);1343 1344 ret = dma_async_device_register(&dw->dma);1345 if (ret)1346 goto err_dma_register;1347 1348 dev_info(chip->dev, "DesignWare DMA Controller, %d channels\n",1349 pdata->nr_channels);1350 1351 pm_runtime_put_sync_suspend(chip->dev);1352 1353 return 0;1354 1355err_dma_register:1356 free_irq(chip->irq, dw);1357err_pdata:1358 pm_runtime_put_sync_suspend(chip->dev);1359 return ret;1360}1361 1362int do_dma_remove(struct dw_dma_chip *chip)1363{1364 struct dw_dma *dw = chip->dw;1365 struct dw_dma_chan *dwc, *_dwc;1366 1367 pm_runtime_get_sync(chip->dev);1368 1369 do_dw_dma_off(dw);1370 dma_async_device_unregister(&dw->dma);1371 1372 free_irq(chip->irq, dw);1373 tasklet_kill(&dw->tasklet);1374 1375 list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels,1376 chan.device_node) {1377 list_del(&dwc->chan.device_node);1378 channel_clear_bit(dw, CH_EN, dwc->mask);1379 }1380 1381 pm_runtime_put_sync_suspend(chip->dev);1382 return 0;1383}1384 1385int do_dw_dma_disable(struct dw_dma_chip *chip)1386{1387 struct dw_dma *dw = chip->dw;1388 1389 dw->disable(dw);1390 return 0;1391}1392EXPORT_SYMBOL_GPL(do_dw_dma_disable);1393 1394int do_dw_dma_enable(struct dw_dma_chip *chip)1395{1396 struct dw_dma *dw = chip->dw;1397 1398 dw->enable(dw);1399 return 0;1400}1401EXPORT_SYMBOL_GPL(do_dw_dma_enable);1402 1403MODULE_LICENSE("GPL v2");1404MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller core driver");1405MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");1406MODULE_AUTHOR("Viresh Kumar <vireshk@kernel.org>");1407