1430 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Freescale MPC85xx, MPC83xx DMA Engine support4 *5 * Copyright (C) 2007-2010 Freescale Semiconductor, Inc. All rights reserved.6 *7 * Author:8 * Zhang Wei <wei.zhang@freescale.com>, Jul 20079 * Ebony Zhu <ebony.zhu@freescale.com>, May 200710 *11 * Description:12 * DMA engine driver for Freescale MPC8540 DMA controller, which is13 * also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc.14 * The support for MPC8349 DMA controller is also added.15 *16 * This driver instructs the DMA controller to issue the PCI Read Multiple17 * command for PCI read operations, instead of using the default PCI Read Line18 * command. Please be aware that this setting may result in read pre-fetching19 * on some platforms.20 */21 22#include <linux/init.h>23#include <linux/module.h>24#include <linux/pci.h>25#include <linux/slab.h>26#include <linux/interrupt.h>27#include <linux/dmaengine.h>28#include <linux/delay.h>29#include <linux/dma-mapping.h>30#include <linux/dmapool.h>31#include <linux/of.h>32#include <linux/of_address.h>33#include <linux/of_irq.h>34#include <linux/platform_device.h>35#include <linux/fsldma.h>36#include "dmaengine.h"37#include "fsldma.h"38 39#define chan_dbg(chan, fmt, arg...) \40 dev_dbg(chan->dev, "%s: " fmt, chan->name, ##arg)41#define chan_err(chan, fmt, arg...) \42 dev_err(chan->dev, "%s: " fmt, chan->name, ##arg)43 44static const char msg_ld_oom[] = "No free memory for link descriptor";45 46/*47 * Register Helpers48 */49 50static void set_sr(struct fsldma_chan *chan, u32 val)51{52 FSL_DMA_OUT(chan, &chan->regs->sr, val, 32);53}54 55static u32 get_sr(struct fsldma_chan *chan)56{57 return FSL_DMA_IN(chan, &chan->regs->sr, 32);58}59 60static void set_mr(struct fsldma_chan *chan, u32 val)61{62 FSL_DMA_OUT(chan, &chan->regs->mr, val, 32);63}64 65static u32 get_mr(struct fsldma_chan *chan)66{67 return FSL_DMA_IN(chan, &chan->regs->mr, 32);68}69 70static void set_cdar(struct fsldma_chan *chan, dma_addr_t addr)71{72 FSL_DMA_OUT(chan, &chan->regs->cdar, addr | FSL_DMA_SNEN, 64);73}74 75static dma_addr_t get_cdar(struct fsldma_chan *chan)76{77 return FSL_DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN;78}79 80static void set_bcr(struct fsldma_chan *chan, u32 val)81{82 FSL_DMA_OUT(chan, &chan->regs->bcr, val, 32);83}84 85static u32 get_bcr(struct fsldma_chan *chan)86{87 return FSL_DMA_IN(chan, &chan->regs->bcr, 32);88}89 90/*91 * Descriptor Helpers92 */93 94static void set_desc_cnt(struct fsldma_chan *chan,95 struct fsl_dma_ld_hw *hw, u32 count)96{97 hw->count = CPU_TO_DMA(chan, count, 32);98}99 100static void set_desc_src(struct fsldma_chan *chan,101 struct fsl_dma_ld_hw *hw, dma_addr_t src)102{103 u64 snoop_bits;104 105 snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)106 ? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0;107 hw->src_addr = CPU_TO_DMA(chan, snoop_bits | src, 64);108}109 110static void set_desc_dst(struct fsldma_chan *chan,111 struct fsl_dma_ld_hw *hw, dma_addr_t dst)112{113 u64 snoop_bits;114 115 snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)116 ? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0;117 hw->dst_addr = CPU_TO_DMA(chan, snoop_bits | dst, 64);118}119 120static void set_desc_next(struct fsldma_chan *chan,121 struct fsl_dma_ld_hw *hw, dma_addr_t next)122{123 u64 snoop_bits;124 125 snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)126 ? FSL_DMA_SNEN : 0;127 hw->next_ln_addr = CPU_TO_DMA(chan, snoop_bits | next, 64);128}129 130static void set_ld_eol(struct fsldma_chan *chan, struct fsl_desc_sw *desc)131{132 u64 snoop_bits;133 134 snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)135 ? FSL_DMA_SNEN : 0;136 137 desc->hw.next_ln_addr = CPU_TO_DMA(chan,138 DMA_TO_CPU(chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL139 | snoop_bits, 64);140}141 142/*143 * DMA Engine Hardware Control Helpers144 */145 146static void dma_init(struct fsldma_chan *chan)147{148 /* Reset the channel */149 set_mr(chan, 0);150 151 switch (chan->feature & FSL_DMA_IP_MASK) {152 case FSL_DMA_IP_85XX:153 /* Set the channel to below modes:154 * EIE - Error interrupt enable155 * EOLNIE - End of links interrupt enable156 * BWC - Bandwidth sharing among channels157 */158 set_mr(chan, FSL_DMA_MR_BWC | FSL_DMA_MR_EIE159 | FSL_DMA_MR_EOLNIE);160 break;161 case FSL_DMA_IP_83XX:162 /* Set the channel to below modes:163 * EOTIE - End-of-transfer interrupt enable164 * PRC_RM - PCI read multiple165 */166 set_mr(chan, FSL_DMA_MR_EOTIE | FSL_DMA_MR_PRC_RM);167 break;168 }169}170 171static int dma_is_idle(struct fsldma_chan *chan)172{173 u32 sr = get_sr(chan);174 return (!(sr & FSL_DMA_SR_CB)) || (sr & FSL_DMA_SR_CH);175}176 177/*178 * Start the DMA controller179 *180 * Preconditions:181 * - the CDAR register must point to the start descriptor182 * - the MRn[CS] bit must be cleared183 */184static void dma_start(struct fsldma_chan *chan)185{186 u32 mode;187 188 mode = get_mr(chan);189 190 if (chan->feature & FSL_DMA_CHAN_PAUSE_EXT) {191 set_bcr(chan, 0);192 mode |= FSL_DMA_MR_EMP_EN;193 } else {194 mode &= ~FSL_DMA_MR_EMP_EN;195 }196 197 if (chan->feature & FSL_DMA_CHAN_START_EXT) {198 mode |= FSL_DMA_MR_EMS_EN;199 } else {200 mode &= ~FSL_DMA_MR_EMS_EN;201 mode |= FSL_DMA_MR_CS;202 }203 204 set_mr(chan, mode);205}206 207static void dma_halt(struct fsldma_chan *chan)208{209 u32 mode;210 int i;211 212 /* read the mode register */213 mode = get_mr(chan);214 215 /*216 * The 85xx controller supports channel abort, which will stop217 * the current transfer. On 83xx, this bit is the transfer error218 * mask bit, which should not be changed.219 */220 if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {221 mode |= FSL_DMA_MR_CA;222 set_mr(chan, mode);223 224 mode &= ~FSL_DMA_MR_CA;225 }226 227 /* stop the DMA controller */228 mode &= ~(FSL_DMA_MR_CS | FSL_DMA_MR_EMS_EN);229 set_mr(chan, mode);230 231 /* wait for the DMA controller to become idle */232 for (i = 0; i < 100; i++) {233 if (dma_is_idle(chan))234 return;235 236 udelay(10);237 }238 239 if (!dma_is_idle(chan))240 chan_err(chan, "DMA halt timeout!\n");241}242 243/**244 * fsl_chan_set_src_loop_size - Set source address hold transfer size245 * @chan : Freescale DMA channel246 * @size : Address loop size, 0 for disable loop247 *248 * The set source address hold transfer size. The source249 * address hold or loop transfer size is when the DMA transfer250 * data from source address (SA), if the loop size is 4, the DMA will251 * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA,252 * SA + 1 ... and so on.253 */254static void fsl_chan_set_src_loop_size(struct fsldma_chan *chan, int size)255{256 u32 mode;257 258 mode = get_mr(chan);259 260 switch (size) {261 case 0:262 mode &= ~FSL_DMA_MR_SAHE;263 break;264 case 1:265 case 2:266 case 4:267 case 8:268 mode &= ~FSL_DMA_MR_SAHTS_MASK;269 mode |= FSL_DMA_MR_SAHE | (__ilog2(size) << 14);270 break;271 }272 273 set_mr(chan, mode);274}275 276/**277 * fsl_chan_set_dst_loop_size - Set destination address hold transfer size278 * @chan : Freescale DMA channel279 * @size : Address loop size, 0 for disable loop280 *281 * The set destination address hold transfer size. The destination282 * address hold or loop transfer size is when the DMA transfer283 * data to destination address (TA), if the loop size is 4, the DMA will284 * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA,285 * TA + 1 ... and so on.286 */287static void fsl_chan_set_dst_loop_size(struct fsldma_chan *chan, int size)288{289 u32 mode;290 291 mode = get_mr(chan);292 293 switch (size) {294 case 0:295 mode &= ~FSL_DMA_MR_DAHE;296 break;297 case 1:298 case 2:299 case 4:300 case 8:301 mode &= ~FSL_DMA_MR_DAHTS_MASK;302 mode |= FSL_DMA_MR_DAHE | (__ilog2(size) << 16);303 break;304 }305 306 set_mr(chan, mode);307}308 309/**310 * fsl_chan_set_request_count - Set DMA Request Count for external control311 * @chan : Freescale DMA channel312 * @size : Number of bytes to transfer in a single request313 *314 * The Freescale DMA channel can be controlled by the external signal DREQ#.315 * The DMA request count is how many bytes are allowed to transfer before316 * pausing the channel, after which a new assertion of DREQ# resumes channel317 * operation.318 *319 * A size of 0 disables external pause control. The maximum size is 1024.320 */321static void fsl_chan_set_request_count(struct fsldma_chan *chan, int size)322{323 u32 mode;324 325 BUG_ON(size > 1024);326 327 mode = get_mr(chan);328 mode &= ~FSL_DMA_MR_BWC_MASK;329 mode |= (__ilog2(size) << 24) & FSL_DMA_MR_BWC_MASK;330 331 set_mr(chan, mode);332}333 334/**335 * fsl_chan_toggle_ext_pause - Toggle channel external pause status336 * @chan : Freescale DMA channel337 * @enable : 0 is disabled, 1 is enabled.338 *339 * The Freescale DMA channel can be controlled by the external signal DREQ#.340 * The DMA Request Count feature should be used in addition to this feature341 * to set the number of bytes to transfer before pausing the channel.342 */343static void fsl_chan_toggle_ext_pause(struct fsldma_chan *chan, int enable)344{345 if (enable)346 chan->feature |= FSL_DMA_CHAN_PAUSE_EXT;347 else348 chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT;349}350 351/**352 * fsl_chan_toggle_ext_start - Toggle channel external start status353 * @chan : Freescale DMA channel354 * @enable : 0 is disabled, 1 is enabled.355 *356 * If enable the external start, the channel can be started by an357 * external DMA start pin. So the dma_start() does not start the358 * transfer immediately. The DMA channel will wait for the359 * control pin asserted.360 */361static void fsl_chan_toggle_ext_start(struct fsldma_chan *chan, int enable)362{363 if (enable)364 chan->feature |= FSL_DMA_CHAN_START_EXT;365 else366 chan->feature &= ~FSL_DMA_CHAN_START_EXT;367}368 369int fsl_dma_external_start(struct dma_chan *dchan, int enable)370{371 struct fsldma_chan *chan;372 373 if (!dchan)374 return -EINVAL;375 376 chan = to_fsl_chan(dchan);377 378 fsl_chan_toggle_ext_start(chan, enable);379 return 0;380}381EXPORT_SYMBOL_GPL(fsl_dma_external_start);382 383static void append_ld_queue(struct fsldma_chan *chan, struct fsl_desc_sw *desc)384{385 struct fsl_desc_sw *tail = to_fsl_desc(chan->ld_pending.prev);386 387 if (list_empty(&chan->ld_pending))388 goto out_splice;389 390 /*391 * Add the hardware descriptor to the chain of hardware descriptors392 * that already exists in memory.393 *394 * This will un-set the EOL bit of the existing transaction, and the395 * last link in this transaction will become the EOL descriptor.396 */397 set_desc_next(chan, &tail->hw, desc->async_tx.phys);398 399 /*400 * Add the software descriptor and all children to the list401 * of pending transactions402 */403out_splice:404 list_splice_tail_init(&desc->tx_list, &chan->ld_pending);405}406 407static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)408{409 struct fsldma_chan *chan = to_fsl_chan(tx->chan);410 struct fsl_desc_sw *desc = tx_to_fsl_desc(tx);411 struct fsl_desc_sw *child;412 dma_cookie_t cookie = -EINVAL;413 414 spin_lock_bh(&chan->desc_lock);415 416#ifdef CONFIG_PM417 if (unlikely(chan->pm_state != RUNNING)) {418 chan_dbg(chan, "cannot submit due to suspend\n");419 spin_unlock_bh(&chan->desc_lock);420 return -1;421 }422#endif423 424 /*425 * assign cookies to all of the software descriptors426 * that make up this transaction427 */428 list_for_each_entry(child, &desc->tx_list, node) {429 cookie = dma_cookie_assign(&child->async_tx);430 }431 432 /* put this transaction onto the tail of the pending queue */433 append_ld_queue(chan, desc);434 435 spin_unlock_bh(&chan->desc_lock);436 437 return cookie;438}439 440/**441 * fsl_dma_free_descriptor - Free descriptor from channel's DMA pool.442 * @chan : Freescale DMA channel443 * @desc: descriptor to be freed444 */445static void fsl_dma_free_descriptor(struct fsldma_chan *chan,446 struct fsl_desc_sw *desc)447{448 list_del(&desc->node);449 chan_dbg(chan, "LD %p free\n", desc);450 dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);451}452 453/**454 * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool.455 * @chan : Freescale DMA channel456 *457 * Return - The descriptor allocated. NULL for failed.458 */459static struct fsl_desc_sw *fsl_dma_alloc_descriptor(struct fsldma_chan *chan)460{461 struct fsl_desc_sw *desc;462 dma_addr_t pdesc;463 464 desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &pdesc);465 if (!desc) {466 chan_dbg(chan, "out of memory for link descriptor\n");467 return NULL;468 }469 470 INIT_LIST_HEAD(&desc->tx_list);471 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);472 desc->async_tx.tx_submit = fsl_dma_tx_submit;473 desc->async_tx.phys = pdesc;474 475 chan_dbg(chan, "LD %p allocated\n", desc);476 477 return desc;478}479 480/**481 * fsldma_clean_completed_descriptor - free all descriptors which482 * has been completed and acked483 * @chan: Freescale DMA channel484 *485 * This function is used on all completed and acked descriptors.486 * All descriptors should only be freed in this function.487 */488static void fsldma_clean_completed_descriptor(struct fsldma_chan *chan)489{490 struct fsl_desc_sw *desc, *_desc;491 492 /* Run the callback for each descriptor, in order */493 list_for_each_entry_safe(desc, _desc, &chan->ld_completed, node)494 if (async_tx_test_ack(&desc->async_tx))495 fsl_dma_free_descriptor(chan, desc);496}497 498/**499 * fsldma_run_tx_complete_actions - cleanup a single link descriptor500 * @chan: Freescale DMA channel501 * @desc: descriptor to cleanup and free502 * @cookie: Freescale DMA transaction identifier503 *504 * This function is used on a descriptor which has been executed by the DMA505 * controller. It will run any callbacks, submit any dependencies.506 */507static dma_cookie_t fsldma_run_tx_complete_actions(struct fsldma_chan *chan,508 struct fsl_desc_sw *desc, dma_cookie_t cookie)509{510 struct dma_async_tx_descriptor *txd = &desc->async_tx;511 dma_cookie_t ret = cookie;512 513 BUG_ON(txd->cookie < 0);514 515 if (txd->cookie > 0) {516 ret = txd->cookie;517 518 dma_descriptor_unmap(txd);519 /* Run the link descriptor callback function */520 dmaengine_desc_get_callback_invoke(txd, NULL);521 }522 523 /* Run any dependencies */524 dma_run_dependencies(txd);525 526 return ret;527}528 529/**530 * fsldma_clean_running_descriptor - move the completed descriptor from531 * ld_running to ld_completed532 * @chan: Freescale DMA channel533 * @desc: the descriptor which is completed534 *535 * Free the descriptor directly if acked by async_tx api, or move it to536 * queue ld_completed.537 */538static void fsldma_clean_running_descriptor(struct fsldma_chan *chan,539 struct fsl_desc_sw *desc)540{541 /* Remove from the list of transactions */542 list_del(&desc->node);543 544 /*545 * the client is allowed to attach dependent operations546 * until 'ack' is set547 */548 if (!async_tx_test_ack(&desc->async_tx)) {549 /*550 * Move this descriptor to the list of descriptors which is551 * completed, but still awaiting the 'ack' bit to be set.552 */553 list_add_tail(&desc->node, &chan->ld_completed);554 return;555 }556 557 dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);558}559 560/**561 * fsl_chan_xfer_ld_queue - transfer any pending transactions562 * @chan : Freescale DMA channel563 *564 * HARDWARE STATE: idle565 * LOCKING: must hold chan->desc_lock566 */567static void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan)568{569 struct fsl_desc_sw *desc;570 571 /*572 * If the list of pending descriptors is empty, then we573 * don't need to do any work at all574 */575 if (list_empty(&chan->ld_pending)) {576 chan_dbg(chan, "no pending LDs\n");577 return;578 }579 580 /*581 * The DMA controller is not idle, which means that the interrupt582 * handler will start any queued transactions when it runs after583 * this transaction finishes584 */585 if (!chan->idle) {586 chan_dbg(chan, "DMA controller still busy\n");587 return;588 }589 590 /*591 * If there are some link descriptors which have not been592 * transferred, we need to start the controller593 */594 595 /*596 * Move all elements from the queue of pending transactions597 * onto the list of running transactions598 */599 chan_dbg(chan, "idle, starting controller\n");600 desc = list_first_entry(&chan->ld_pending, struct fsl_desc_sw, node);601 list_splice_tail_init(&chan->ld_pending, &chan->ld_running);602 603 /*604 * The 85xx DMA controller doesn't clear the channel start bit605 * automatically at the end of a transfer. Therefore we must clear606 * it in software before starting the transfer.607 */608 if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {609 u32 mode;610 611 mode = get_mr(chan);612 mode &= ~FSL_DMA_MR_CS;613 set_mr(chan, mode);614 }615 616 /*617 * Program the descriptor's address into the DMA controller,618 * then start the DMA transaction619 */620 set_cdar(chan, desc->async_tx.phys);621 get_cdar(chan);622 623 dma_start(chan);624 chan->idle = false;625}626 627/**628 * fsldma_cleanup_descriptors - cleanup link descriptors which are completed629 * and move them to ld_completed to free until flag 'ack' is set630 * @chan: Freescale DMA channel631 *632 * This function is used on descriptors which have been executed by the DMA633 * controller. It will run any callbacks, submit any dependencies, then634 * free these descriptors if flag 'ack' is set.635 */636static void fsldma_cleanup_descriptors(struct fsldma_chan *chan)637{638 struct fsl_desc_sw *desc, *_desc;639 dma_cookie_t cookie = 0;640 dma_addr_t curr_phys = get_cdar(chan);641 int seen_current = 0;642 643 fsldma_clean_completed_descriptor(chan);644 645 /* Run the callback for each descriptor, in order */646 list_for_each_entry_safe(desc, _desc, &chan->ld_running, node) {647 /*648 * do not advance past the current descriptor loaded into the649 * hardware channel, subsequent descriptors are either in650 * process or have not been submitted651 */652 if (seen_current)653 break;654 655 /*656 * stop the search if we reach the current descriptor and the657 * channel is busy658 */659 if (desc->async_tx.phys == curr_phys) {660 seen_current = 1;661 if (!dma_is_idle(chan))662 break;663 }664 665 cookie = fsldma_run_tx_complete_actions(chan, desc, cookie);666 667 fsldma_clean_running_descriptor(chan, desc);668 }669 670 /*671 * Start any pending transactions automatically672 *673 * In the ideal case, we keep the DMA controller busy while we go674 * ahead and free the descriptors below.675 */676 fsl_chan_xfer_ld_queue(chan);677 678 if (cookie > 0)679 chan->common.completed_cookie = cookie;680}681 682/**683 * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel.684 * @chan : Freescale DMA channel685 *686 * This function will create a dma pool for descriptor allocation.687 *688 * Return - The number of descriptors allocated.689 */690static int fsl_dma_alloc_chan_resources(struct dma_chan *dchan)691{692 struct fsldma_chan *chan = to_fsl_chan(dchan);693 694 /* Has this channel already been allocated? */695 if (chan->desc_pool)696 return 1;697 698 /*699 * We need the descriptor to be aligned to 32bytes700 * for meeting FSL DMA specification requirement.701 */702 chan->desc_pool = dma_pool_create(chan->name, chan->dev,703 sizeof(struct fsl_desc_sw),704 __alignof__(struct fsl_desc_sw), 0);705 if (!chan->desc_pool) {706 chan_err(chan, "unable to allocate descriptor pool\n");707 return -ENOMEM;708 }709 710 /* there is at least one descriptor free to be allocated */711 return 1;712}713 714/**715 * fsldma_free_desc_list - Free all descriptors in a queue716 * @chan: Freescae DMA channel717 * @list: the list to free718 *719 * LOCKING: must hold chan->desc_lock720 */721static void fsldma_free_desc_list(struct fsldma_chan *chan,722 struct list_head *list)723{724 struct fsl_desc_sw *desc, *_desc;725 726 list_for_each_entry_safe(desc, _desc, list, node)727 fsl_dma_free_descriptor(chan, desc);728}729 730static void fsldma_free_desc_list_reverse(struct fsldma_chan *chan,731 struct list_head *list)732{733 struct fsl_desc_sw *desc, *_desc;734 735 list_for_each_entry_safe_reverse(desc, _desc, list, node)736 fsl_dma_free_descriptor(chan, desc);737}738 739/**740 * fsl_dma_free_chan_resources - Free all resources of the channel.741 * @chan : Freescale DMA channel742 */743static void fsl_dma_free_chan_resources(struct dma_chan *dchan)744{745 struct fsldma_chan *chan = to_fsl_chan(dchan);746 747 chan_dbg(chan, "free all channel resources\n");748 spin_lock_bh(&chan->desc_lock);749 fsldma_cleanup_descriptors(chan);750 fsldma_free_desc_list(chan, &chan->ld_pending);751 fsldma_free_desc_list(chan, &chan->ld_running);752 fsldma_free_desc_list(chan, &chan->ld_completed);753 spin_unlock_bh(&chan->desc_lock);754 755 dma_pool_destroy(chan->desc_pool);756 chan->desc_pool = NULL;757}758 759static struct dma_async_tx_descriptor *760fsl_dma_prep_memcpy(struct dma_chan *dchan,761 dma_addr_t dma_dst, dma_addr_t dma_src,762 size_t len, unsigned long flags)763{764 struct fsldma_chan *chan;765 struct fsl_desc_sw *first = NULL, *prev = NULL, *new;766 size_t copy;767 768 if (!dchan)769 return NULL;770 771 if (!len)772 return NULL;773 774 chan = to_fsl_chan(dchan);775 776 do {777 778 /* Allocate the link descriptor from DMA pool */779 new = fsl_dma_alloc_descriptor(chan);780 if (!new) {781 chan_err(chan, "%s\n", msg_ld_oom);782 goto fail;783 }784 785 copy = min(len, (size_t)FSL_DMA_BCR_MAX_CNT);786 787 set_desc_cnt(chan, &new->hw, copy);788 set_desc_src(chan, &new->hw, dma_src);789 set_desc_dst(chan, &new->hw, dma_dst);790 791 if (!first)792 first = new;793 else794 set_desc_next(chan, &prev->hw, new->async_tx.phys);795 796 new->async_tx.cookie = 0;797 async_tx_ack(&new->async_tx);798 799 prev = new;800 len -= copy;801 dma_src += copy;802 dma_dst += copy;803 804 /* Insert the link descriptor to the LD ring */805 list_add_tail(&new->node, &first->tx_list);806 } while (len);807 808 new->async_tx.flags = flags; /* client is in control of this ack */809 new->async_tx.cookie = -EBUSY;810 811 /* Set End-of-link to the last link descriptor of new list */812 set_ld_eol(chan, new);813 814 return &first->async_tx;815 816fail:817 if (!first)818 return NULL;819 820 fsldma_free_desc_list_reverse(chan, &first->tx_list);821 return NULL;822}823 824static int fsl_dma_device_terminate_all(struct dma_chan *dchan)825{826 struct fsldma_chan *chan;827 828 if (!dchan)829 return -EINVAL;830 831 chan = to_fsl_chan(dchan);832 833 spin_lock_bh(&chan->desc_lock);834 835 /* Halt the DMA engine */836 dma_halt(chan);837 838 /* Remove and free all of the descriptors in the LD queue */839 fsldma_free_desc_list(chan, &chan->ld_pending);840 fsldma_free_desc_list(chan, &chan->ld_running);841 fsldma_free_desc_list(chan, &chan->ld_completed);842 chan->idle = true;843 844 spin_unlock_bh(&chan->desc_lock);845 return 0;846}847 848static int fsl_dma_device_config(struct dma_chan *dchan,849 struct dma_slave_config *config)850{851 struct fsldma_chan *chan;852 int size;853 854 if (!dchan)855 return -EINVAL;856 857 chan = to_fsl_chan(dchan);858 859 /* make sure the channel supports setting burst size */860 if (!chan->set_request_count)861 return -ENXIO;862 863 /* we set the controller burst size depending on direction */864 if (config->direction == DMA_MEM_TO_DEV)865 size = config->dst_addr_width * config->dst_maxburst;866 else867 size = config->src_addr_width * config->src_maxburst;868 869 chan->set_request_count(chan, size);870 return 0;871}872 873 874/**875 * fsl_dma_memcpy_issue_pending - Issue the DMA start command876 * @chan : Freescale DMA channel877 */878static void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan)879{880 struct fsldma_chan *chan = to_fsl_chan(dchan);881 882 spin_lock_bh(&chan->desc_lock);883 fsl_chan_xfer_ld_queue(chan);884 spin_unlock_bh(&chan->desc_lock);885}886 887/**888 * fsl_tx_status - Determine the DMA status889 * @chan : Freescale DMA channel890 */891static enum dma_status fsl_tx_status(struct dma_chan *dchan,892 dma_cookie_t cookie,893 struct dma_tx_state *txstate)894{895 struct fsldma_chan *chan = to_fsl_chan(dchan);896 enum dma_status ret;897 898 ret = dma_cookie_status(dchan, cookie, txstate);899 if (ret == DMA_COMPLETE)900 return ret;901 902 spin_lock_bh(&chan->desc_lock);903 fsldma_cleanup_descriptors(chan);904 spin_unlock_bh(&chan->desc_lock);905 906 return dma_cookie_status(dchan, cookie, txstate);907}908 909/*----------------------------------------------------------------------------*/910/* Interrupt Handling */911/*----------------------------------------------------------------------------*/912 913static irqreturn_t fsldma_chan_irq(int irq, void *data)914{915 struct fsldma_chan *chan = data;916 u32 stat;917 918 /* save and clear the status register */919 stat = get_sr(chan);920 set_sr(chan, stat);921 chan_dbg(chan, "irq: stat = 0x%x\n", stat);922 923 /* check that this was really our device */924 stat &= ~(FSL_DMA_SR_CB | FSL_DMA_SR_CH);925 if (!stat)926 return IRQ_NONE;927 928 if (stat & FSL_DMA_SR_TE)929 chan_err(chan, "Transfer Error!\n");930 931 /*932 * Programming Error933 * The DMA_INTERRUPT async_tx is a NULL transfer, which will934 * trigger a PE interrupt.935 */936 if (stat & FSL_DMA_SR_PE) {937 chan_dbg(chan, "irq: Programming Error INT\n");938 stat &= ~FSL_DMA_SR_PE;939 if (get_bcr(chan) != 0)940 chan_err(chan, "Programming Error!\n");941 }942 943 /*944 * For MPC8349, EOCDI event need to update cookie945 * and start the next transfer if it exist.946 */947 if (stat & FSL_DMA_SR_EOCDI) {948 chan_dbg(chan, "irq: End-of-Chain link INT\n");949 stat &= ~FSL_DMA_SR_EOCDI;950 }951 952 /*953 * If it current transfer is the end-of-transfer,954 * we should clear the Channel Start bit for955 * prepare next transfer.956 */957 if (stat & FSL_DMA_SR_EOLNI) {958 chan_dbg(chan, "irq: End-of-link INT\n");959 stat &= ~FSL_DMA_SR_EOLNI;960 }961 962 /* check that the DMA controller is really idle */963 if (!dma_is_idle(chan))964 chan_err(chan, "irq: controller not idle!\n");965 966 /* check that we handled all of the bits */967 if (stat)968 chan_err(chan, "irq: unhandled sr 0x%08x\n", stat);969 970 /*971 * Schedule the tasklet to handle all cleanup of the current972 * transaction. It will start a new transaction if there is973 * one pending.974 */975 tasklet_schedule(&chan->tasklet);976 chan_dbg(chan, "irq: Exit\n");977 return IRQ_HANDLED;978}979 980static void dma_do_tasklet(struct tasklet_struct *t)981{982 struct fsldma_chan *chan = from_tasklet(chan, t, tasklet);983 984 chan_dbg(chan, "tasklet entry\n");985 986 spin_lock(&chan->desc_lock);987 988 /* the hardware is now idle and ready for more */989 chan->idle = true;990 991 /* Run all cleanup for descriptors which have been completed */992 fsldma_cleanup_descriptors(chan);993 994 spin_unlock(&chan->desc_lock);995 996 chan_dbg(chan, "tasklet exit\n");997}998 999static irqreturn_t fsldma_ctrl_irq(int irq, void *data)1000{1001 struct fsldma_device *fdev = data;1002 struct fsldma_chan *chan;1003 unsigned int handled = 0;1004 u32 gsr, mask;1005 int i;1006 1007 gsr = (fdev->feature & FSL_DMA_BIG_ENDIAN) ? in_be32(fdev->regs)1008 : in_le32(fdev->regs);1009 mask = 0xff000000;1010 dev_dbg(fdev->dev, "IRQ: gsr 0x%.8x\n", gsr);1011 1012 for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {1013 chan = fdev->chan[i];1014 if (!chan)1015 continue;1016 1017 if (gsr & mask) {1018 dev_dbg(fdev->dev, "IRQ: chan %d\n", chan->id);1019 fsldma_chan_irq(irq, chan);1020 handled++;1021 }1022 1023 gsr &= ~mask;1024 mask >>= 8;1025 }1026 1027 return IRQ_RETVAL(handled);1028}1029 1030static void fsldma_free_irqs(struct fsldma_device *fdev)1031{1032 struct fsldma_chan *chan;1033 int i;1034 1035 if (fdev->irq) {1036 dev_dbg(fdev->dev, "free per-controller IRQ\n");1037 free_irq(fdev->irq, fdev);1038 return;1039 }1040 1041 for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {1042 chan = fdev->chan[i];1043 if (chan && chan->irq) {1044 chan_dbg(chan, "free per-channel IRQ\n");1045 free_irq(chan->irq, chan);1046 }1047 }1048}1049 1050static int fsldma_request_irqs(struct fsldma_device *fdev)1051{1052 struct fsldma_chan *chan;1053 int ret;1054 int i;1055 1056 /* if we have a per-controller IRQ, use that */1057 if (fdev->irq) {1058 dev_dbg(fdev->dev, "request per-controller IRQ\n");1059 ret = request_irq(fdev->irq, fsldma_ctrl_irq, IRQF_SHARED,1060 "fsldma-controller", fdev);1061 return ret;1062 }1063 1064 /* no per-controller IRQ, use the per-channel IRQs */1065 for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {1066 chan = fdev->chan[i];1067 if (!chan)1068 continue;1069 1070 if (!chan->irq) {1071 chan_err(chan, "interrupts property missing in device tree\n");1072 ret = -ENODEV;1073 goto out_unwind;1074 }1075 1076 chan_dbg(chan, "request per-channel IRQ\n");1077 ret = request_irq(chan->irq, fsldma_chan_irq, IRQF_SHARED,1078 "fsldma-chan", chan);1079 if (ret) {1080 chan_err(chan, "unable to request per-channel IRQ\n");1081 goto out_unwind;1082 }1083 }1084 1085 return 0;1086 1087out_unwind:1088 for (/* none */; i >= 0; i--) {1089 chan = fdev->chan[i];1090 if (!chan)1091 continue;1092 1093 if (!chan->irq)1094 continue;1095 1096 free_irq(chan->irq, chan);1097 }1098 1099 return ret;1100}1101 1102/*----------------------------------------------------------------------------*/1103/* OpenFirmware Subsystem */1104/*----------------------------------------------------------------------------*/1105 1106static int fsl_dma_chan_probe(struct fsldma_device *fdev,1107 struct device_node *node, u32 feature, const char *compatible)1108{1109 struct fsldma_chan *chan;1110 struct resource res;1111 int err;1112 1113 /* alloc channel */1114 chan = kzalloc(sizeof(*chan), GFP_KERNEL);1115 if (!chan) {1116 err = -ENOMEM;1117 goto out_return;1118 }1119 1120 /* ioremap registers for use */1121 chan->regs = of_iomap(node, 0);1122 if (!chan->regs) {1123 dev_err(fdev->dev, "unable to ioremap registers\n");1124 err = -ENOMEM;1125 goto out_free_chan;1126 }1127 1128 err = of_address_to_resource(node, 0, &res);1129 if (err) {1130 dev_err(fdev->dev, "unable to find 'reg' property\n");1131 goto out_iounmap_regs;1132 }1133 1134 chan->feature = feature;1135 if (!fdev->feature)1136 fdev->feature = chan->feature;1137 1138 /*1139 * If the DMA device's feature is different than the feature1140 * of its channels, report the bug1141 */1142 WARN_ON(fdev->feature != chan->feature);1143 1144 chan->dev = fdev->dev;1145 chan->id = (res.start & 0xfff) < 0x300 ?1146 ((res.start - 0x100) & 0xfff) >> 7 :1147 ((res.start - 0x200) & 0xfff) >> 7;1148 if (chan->id >= FSL_DMA_MAX_CHANS_PER_DEVICE) {1149 dev_err(fdev->dev, "too many channels for device\n");1150 err = -EINVAL;1151 goto out_iounmap_regs;1152 }1153 1154 fdev->chan[chan->id] = chan;1155 tasklet_setup(&chan->tasklet, dma_do_tasklet);1156 snprintf(chan->name, sizeof(chan->name), "chan%d", chan->id);1157 1158 /* Initialize the channel */1159 dma_init(chan);1160 1161 /* Clear cdar registers */1162 set_cdar(chan, 0);1163 1164 switch (chan->feature & FSL_DMA_IP_MASK) {1165 case FSL_DMA_IP_85XX:1166 chan->toggle_ext_pause = fsl_chan_toggle_ext_pause;1167 fallthrough;1168 case FSL_DMA_IP_83XX:1169 chan->toggle_ext_start = fsl_chan_toggle_ext_start;1170 chan->set_src_loop_size = fsl_chan_set_src_loop_size;1171 chan->set_dst_loop_size = fsl_chan_set_dst_loop_size;1172 chan->set_request_count = fsl_chan_set_request_count;1173 }1174 1175 spin_lock_init(&chan->desc_lock);1176 INIT_LIST_HEAD(&chan->ld_pending);1177 INIT_LIST_HEAD(&chan->ld_running);1178 INIT_LIST_HEAD(&chan->ld_completed);1179 chan->idle = true;1180#ifdef CONFIG_PM1181 chan->pm_state = RUNNING;1182#endif1183 1184 chan->common.device = &fdev->common;1185 dma_cookie_init(&chan->common);1186 1187 /* find the IRQ line, if it exists in the device tree */1188 chan->irq = irq_of_parse_and_map(node, 0);1189 1190 /* Add the channel to DMA device channel list */1191 list_add_tail(&chan->common.device_node, &fdev->common.channels);1192 1193 dev_info(fdev->dev, "#%d (%s), irq %d\n", chan->id, compatible,1194 chan->irq ? chan->irq : fdev->irq);1195 1196 return 0;1197 1198out_iounmap_regs:1199 iounmap(chan->regs);1200out_free_chan:1201 kfree(chan);1202out_return:1203 return err;1204}1205 1206static void fsl_dma_chan_remove(struct fsldma_chan *chan)1207{1208 irq_dispose_mapping(chan->irq);1209 list_del(&chan->common.device_node);1210 iounmap(chan->regs);1211 kfree(chan);1212}1213 1214static int fsldma_of_probe(struct platform_device *op)1215{1216 struct fsldma_device *fdev;1217 struct device_node *child;1218 unsigned int i;1219 int err;1220 1221 fdev = kzalloc(sizeof(*fdev), GFP_KERNEL);1222 if (!fdev) {1223 err = -ENOMEM;1224 goto out_return;1225 }1226 1227 fdev->dev = &op->dev;1228 INIT_LIST_HEAD(&fdev->common.channels);1229 1230 /* ioremap the registers for use */1231 fdev->regs = of_iomap(op->dev.of_node, 0);1232 if (!fdev->regs) {1233 dev_err(&op->dev, "unable to ioremap registers\n");1234 err = -ENOMEM;1235 goto out_free;1236 }1237 1238 /* map the channel IRQ if it exists, but don't hookup the handler yet */1239 fdev->irq = irq_of_parse_and_map(op->dev.of_node, 0);1240 1241 dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);1242 dma_cap_set(DMA_SLAVE, fdev->common.cap_mask);1243 fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources;1244 fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources;1245 fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy;1246 fdev->common.device_tx_status = fsl_tx_status;1247 fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending;1248 fdev->common.device_config = fsl_dma_device_config;1249 fdev->common.device_terminate_all = fsl_dma_device_terminate_all;1250 fdev->common.dev = &op->dev;1251 1252 fdev->common.src_addr_widths = FSL_DMA_BUSWIDTHS;1253 fdev->common.dst_addr_widths = FSL_DMA_BUSWIDTHS;1254 fdev->common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);1255 fdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;1256 1257 dma_set_mask(&(op->dev), DMA_BIT_MASK(36));1258 1259 platform_set_drvdata(op, fdev);1260 1261 /*1262 * We cannot use of_platform_bus_probe() because there is no1263 * of_platform_bus_remove(). Instead, we manually instantiate every DMA1264 * channel object.1265 */1266 for_each_child_of_node(op->dev.of_node, child) {1267 if (of_device_is_compatible(child, "fsl,eloplus-dma-channel")) {1268 fsl_dma_chan_probe(fdev, child,1269 FSL_DMA_IP_85XX | FSL_DMA_BIG_ENDIAN,1270 "fsl,eloplus-dma-channel");1271 }1272 1273 if (of_device_is_compatible(child, "fsl,elo-dma-channel")) {1274 fsl_dma_chan_probe(fdev, child,1275 FSL_DMA_IP_83XX | FSL_DMA_LITTLE_ENDIAN,1276 "fsl,elo-dma-channel");1277 }1278 }1279 1280 /*1281 * Hookup the IRQ handler(s)1282 *1283 * If we have a per-controller interrupt, we prefer that to the1284 * per-channel interrupts to reduce the number of shared interrupt1285 * handlers on the same IRQ line1286 */1287 err = fsldma_request_irqs(fdev);1288 if (err) {1289 dev_err(fdev->dev, "unable to request IRQs\n");1290 goto out_free_fdev;1291 }1292 1293 dma_async_device_register(&fdev->common);1294 return 0;1295 1296out_free_fdev:1297 for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {1298 if (fdev->chan[i])1299 fsl_dma_chan_remove(fdev->chan[i]);1300 }1301 irq_dispose_mapping(fdev->irq);1302 iounmap(fdev->regs);1303out_free:1304 kfree(fdev);1305out_return:1306 return err;1307}1308 1309static void fsldma_of_remove(struct platform_device *op)1310{1311 struct fsldma_device *fdev;1312 unsigned int i;1313 1314 fdev = platform_get_drvdata(op);1315 dma_async_device_unregister(&fdev->common);1316 1317 fsldma_free_irqs(fdev);1318 1319 for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {1320 if (fdev->chan[i])1321 fsl_dma_chan_remove(fdev->chan[i]);1322 }1323 irq_dispose_mapping(fdev->irq);1324 1325 iounmap(fdev->regs);1326 kfree(fdev);1327}1328 1329#ifdef CONFIG_PM1330static int fsldma_suspend_late(struct device *dev)1331{1332 struct fsldma_device *fdev = dev_get_drvdata(dev);1333 struct fsldma_chan *chan;1334 int i;1335 1336 for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {1337 chan = fdev->chan[i];1338 if (!chan)1339 continue;1340 1341 spin_lock_bh(&chan->desc_lock);1342 if (unlikely(!chan->idle))1343 goto out;1344 chan->regs_save.mr = get_mr(chan);1345 chan->pm_state = SUSPENDED;1346 spin_unlock_bh(&chan->desc_lock);1347 }1348 return 0;1349 1350out:1351 for (; i >= 0; i--) {1352 chan = fdev->chan[i];1353 if (!chan)1354 continue;1355 chan->pm_state = RUNNING;1356 spin_unlock_bh(&chan->desc_lock);1357 }1358 return -EBUSY;1359}1360 1361static int fsldma_resume_early(struct device *dev)1362{1363 struct fsldma_device *fdev = dev_get_drvdata(dev);1364 struct fsldma_chan *chan;1365 u32 mode;1366 int i;1367 1368 for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {1369 chan = fdev->chan[i];1370 if (!chan)1371 continue;1372 1373 spin_lock_bh(&chan->desc_lock);1374 mode = chan->regs_save.mr1375 & ~FSL_DMA_MR_CS & ~FSL_DMA_MR_CC & ~FSL_DMA_MR_CA;1376 set_mr(chan, mode);1377 chan->pm_state = RUNNING;1378 spin_unlock_bh(&chan->desc_lock);1379 }1380 1381 return 0;1382}1383 1384static const struct dev_pm_ops fsldma_pm_ops = {1385 .suspend_late = fsldma_suspend_late,1386 .resume_early = fsldma_resume_early,1387};1388#endif1389 1390static const struct of_device_id fsldma_of_ids[] = {1391 { .compatible = "fsl,elo3-dma", },1392 { .compatible = "fsl,eloplus-dma", },1393 { .compatible = "fsl,elo-dma", },1394 {}1395};1396MODULE_DEVICE_TABLE(of, fsldma_of_ids);1397 1398static struct platform_driver fsldma_of_driver = {1399 .driver = {1400 .name = "fsl-elo-dma",1401 .of_match_table = fsldma_of_ids,1402#ifdef CONFIG_PM1403 .pm = &fsldma_pm_ops,1404#endif1405 },1406 .probe = fsldma_of_probe,1407 .remove_new = fsldma_of_remove,1408};1409 1410/*----------------------------------------------------------------------------*/1411/* Module Init / Exit */1412/*----------------------------------------------------------------------------*/1413 1414static __init int fsldma_init(void)1415{1416 pr_info("Freescale Elo series DMA driver\n");1417 return platform_driver_register(&fsldma_of_driver);1418}1419 1420static void __exit fsldma_exit(void)1421{1422 platform_driver_unregister(&fsldma_of_driver);1423}1424 1425subsys_initcall(fsldma_init);1426module_exit(fsldma_exit);1427 1428MODULE_DESCRIPTION("Freescale Elo series DMA driver");1429MODULE_LICENSE("GPL");1430