1124 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Copyright (C) Freescale Semicondutor, Inc. 2007, 2008.4 * Copyright (C) Semihalf 20095 * Copyright (C) Ilya Yanok, Emcraft Systems 20106 * Copyright (C) Alexander Popov, Promcontroller 20147 * Copyright (C) Mario Six, Guntermann & Drunck GmbH, 20168 *9 * Written by Piotr Ziecik <kosmo@semihalf.com>. Hardware description10 * (defines, structures and comments) was taken from MPC5121 DMA driver11 * written by Hongjun Chen <hong-jun.chen@freescale.com>.12 *13 * Approved as OSADL project by a majority of OSADL members and funded14 * by OSADL membership fees in 2009; for details see www.osadl.org.15 */16 17/*18 * MPC512x and MPC8308 DMA driver. It supports memory to memory data transfers19 * (tested using dmatest module) and data transfers between memory and20 * peripheral I/O memory by means of slave scatter/gather with these21 * limitations:22 * - chunked transfers (described by s/g lists with more than one item) are23 * refused as long as proper support for scatter/gather is missing24 * - transfers on MPC8308 always start from software as this SoC does not have25 * external request lines for peripheral flow control26 * - memory <-> I/O memory transfer chunks of sizes of 1, 2, 4, 16 (for27 * MPC512x), and 32 bytes are supported, and, consequently, source28 * addresses and destination addresses must be aligned accordingly;29 * furthermore, for MPC512x SoCs, the transfer size must be aligned on30 * (chunk size * maxburst)31 */32 33#include <linux/module.h>34#include <linux/dmaengine.h>35#include <linux/dma-mapping.h>36#include <linux/interrupt.h>37#include <linux/io.h>38#include <linux/slab.h>39#include <linux/of.h>40#include <linux/of_address.h>41#include <linux/of_irq.h>42#include <linux/of_dma.h>43#include <linux/platform_device.h>44 45#include <linux/random.h>46 47#include "dmaengine.h"48 49/* Number of DMA Transfer descriptors allocated per channel */50#define MPC_DMA_DESCRIPTORS 6451 52/* Macro definitions */53#define MPC_DMA_TCD_OFFSET 0x100054 55/*56 * Maximum channel counts for individual hardware variants57 * and the maximum channel count over all supported controllers,58 * used for data structure size59 */60#define MPC8308_DMACHAN_MAX 1661#define MPC512x_DMACHAN_MAX 6462#define MPC_DMA_CHANNELS 6463 64/* Arbitration mode of group and channel */65#define MPC_DMA_DMACR_EDCG (1 << 31)66#define MPC_DMA_DMACR_ERGA (1 << 3)67#define MPC_DMA_DMACR_ERCA (1 << 2)68 69/* Error codes */70#define MPC_DMA_DMAES_VLD (1 << 31)71#define MPC_DMA_DMAES_GPE (1 << 15)72#define MPC_DMA_DMAES_CPE (1 << 14)73#define MPC_DMA_DMAES_ERRCHN(err) \74 (((err) >> 8) & 0x3f)75#define MPC_DMA_DMAES_SAE (1 << 7)76#define MPC_DMA_DMAES_SOE (1 << 6)77#define MPC_DMA_DMAES_DAE (1 << 5)78#define MPC_DMA_DMAES_DOE (1 << 4)79#define MPC_DMA_DMAES_NCE (1 << 3)80#define MPC_DMA_DMAES_SGE (1 << 2)81#define MPC_DMA_DMAES_SBE (1 << 1)82#define MPC_DMA_DMAES_DBE (1 << 0)83 84#define MPC_DMA_DMAGPOR_SNOOP_ENABLE (1 << 6)85 86#define MPC_DMA_TSIZE_1 0x0087#define MPC_DMA_TSIZE_2 0x0188#define MPC_DMA_TSIZE_4 0x0289#define MPC_DMA_TSIZE_16 0x0490#define MPC_DMA_TSIZE_32 0x0591 92/* MPC5121 DMA engine registers */93struct __attribute__ ((__packed__)) mpc_dma_regs {94 /* 0x00 */95 u32 dmacr; /* DMA control register */96 u32 dmaes; /* DMA error status */97 /* 0x08 */98 u32 dmaerqh; /* DMA enable request high(channels 63~32) */99 u32 dmaerql; /* DMA enable request low(channels 31~0) */100 u32 dmaeeih; /* DMA enable error interrupt high(ch63~32) */101 u32 dmaeeil; /* DMA enable error interrupt low(ch31~0) */102 /* 0x18 */103 u8 dmaserq; /* DMA set enable request */104 u8 dmacerq; /* DMA clear enable request */105 u8 dmaseei; /* DMA set enable error interrupt */106 u8 dmaceei; /* DMA clear enable error interrupt */107 /* 0x1c */108 u8 dmacint; /* DMA clear interrupt request */109 u8 dmacerr; /* DMA clear error */110 u8 dmassrt; /* DMA set start bit */111 u8 dmacdne; /* DMA clear DONE status bit */112 /* 0x20 */113 u32 dmainth; /* DMA interrupt request high(ch63~32) */114 u32 dmaintl; /* DMA interrupt request low(ch31~0) */115 u32 dmaerrh; /* DMA error high(ch63~32) */116 u32 dmaerrl; /* DMA error low(ch31~0) */117 /* 0x30 */118 u32 dmahrsh; /* DMA hw request status high(ch63~32) */119 u32 dmahrsl; /* DMA hardware request status low(ch31~0) */120 union {121 u32 dmaihsa; /* DMA interrupt high select AXE(ch63~32) */122 u32 dmagpor; /* (General purpose register on MPC8308) */123 };124 u32 dmailsa; /* DMA interrupt low select AXE(ch31~0) */125 /* 0x40 ~ 0xff */126 u32 reserve0[48]; /* Reserved */127 /* 0x100 */128 u8 dchpri[MPC_DMA_CHANNELS];129 /* DMA channels(0~63) priority */130};131 132struct __attribute__ ((__packed__)) mpc_dma_tcd {133 /* 0x00 */134 u32 saddr; /* Source address */135 136 u32 smod:5; /* Source address modulo */137 u32 ssize:3; /* Source data transfer size */138 u32 dmod:5; /* Destination address modulo */139 u32 dsize:3; /* Destination data transfer size */140 u32 soff:16; /* Signed source address offset */141 142 /* 0x08 */143 u32 nbytes; /* Inner "minor" byte count */144 u32 slast; /* Last source address adjustment */145 u32 daddr; /* Destination address */146 147 /* 0x14 */148 u32 citer_elink:1; /* Enable channel-to-channel linking on149 * minor loop complete150 */151 u32 citer_linkch:6; /* Link channel for minor loop complete */152 u32 citer:9; /* Current "major" iteration count */153 u32 doff:16; /* Signed destination address offset */154 155 /* 0x18 */156 u32 dlast_sga; /* Last Destination address adjustment/scatter157 * gather address158 */159 160 /* 0x1c */161 u32 biter_elink:1; /* Enable channel-to-channel linking on major162 * loop complete163 */164 u32 biter_linkch:6;165 u32 biter:9; /* Beginning "major" iteration count */166 u32 bwc:2; /* Bandwidth control */167 u32 major_linkch:6; /* Link channel number */168 u32 done:1; /* Channel done */169 u32 active:1; /* Channel active */170 u32 major_elink:1; /* Enable channel-to-channel linking on major171 * loop complete172 */173 u32 e_sg:1; /* Enable scatter/gather processing */174 u32 d_req:1; /* Disable request */175 u32 int_half:1; /* Enable an interrupt when major counter is176 * half complete177 */178 u32 int_maj:1; /* Enable an interrupt when major iteration179 * count completes180 */181 u32 start:1; /* Channel start */182};183 184struct mpc_dma_desc {185 struct dma_async_tx_descriptor desc;186 struct mpc_dma_tcd *tcd;187 dma_addr_t tcd_paddr;188 int error;189 struct list_head node;190 int will_access_peripheral;191};192 193struct mpc_dma_chan {194 struct dma_chan chan;195 struct list_head free;196 struct list_head prepared;197 struct list_head queued;198 struct list_head active;199 struct list_head completed;200 struct mpc_dma_tcd *tcd;201 dma_addr_t tcd_paddr;202 203 /* Settings for access to peripheral FIFO */204 dma_addr_t src_per_paddr;205 u32 src_tcd_nunits;206 u8 swidth;207 dma_addr_t dst_per_paddr;208 u32 dst_tcd_nunits;209 u8 dwidth;210 211 /* Lock for this structure */212 spinlock_t lock;213};214 215struct mpc_dma {216 struct dma_device dma;217 struct tasklet_struct tasklet;218 struct mpc_dma_chan channels[MPC_DMA_CHANNELS];219 struct mpc_dma_regs __iomem *regs;220 struct mpc_dma_tcd __iomem *tcd;221 int irq;222 int irq2;223 uint error_status;224 int is_mpc8308;225 226 /* Lock for error_status field in this structure */227 spinlock_t error_status_lock;228};229 230#define DRV_NAME "mpc512x_dma"231 232/* Convert struct dma_chan to struct mpc_dma_chan */233static inline struct mpc_dma_chan *dma_chan_to_mpc_dma_chan(struct dma_chan *c)234{235 return container_of(c, struct mpc_dma_chan, chan);236}237 238/* Convert struct dma_chan to struct mpc_dma */239static inline struct mpc_dma *dma_chan_to_mpc_dma(struct dma_chan *c)240{241 struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(c);242 243 return container_of(mchan, struct mpc_dma, channels[c->chan_id]);244}245 246/*247 * Execute all queued DMA descriptors.248 *249 * Following requirements must be met while calling mpc_dma_execute():250 * a) mchan->lock is acquired,251 * b) mchan->active list is empty,252 * c) mchan->queued list contains at least one entry.253 */254static void mpc_dma_execute(struct mpc_dma_chan *mchan)255{256 struct mpc_dma *mdma = dma_chan_to_mpc_dma(&mchan->chan);257 struct mpc_dma_desc *first = NULL;258 struct mpc_dma_desc *prev = NULL;259 struct mpc_dma_desc *mdesc;260 int cid = mchan->chan.chan_id;261 262 while (!list_empty(&mchan->queued)) {263 mdesc = list_first_entry(&mchan->queued,264 struct mpc_dma_desc, node);265 /*266 * Grab either several mem-to-mem transfer descriptors267 * or one peripheral transfer descriptor,268 * don't mix mem-to-mem and peripheral transfer descriptors269 * within the same 'active' list.270 */271 if (mdesc->will_access_peripheral) {272 if (list_empty(&mchan->active))273 list_move_tail(&mdesc->node, &mchan->active);274 break;275 } else {276 list_move_tail(&mdesc->node, &mchan->active);277 }278 }279 280 /* Chain descriptors into one transaction */281 list_for_each_entry(mdesc, &mchan->active, node) {282 if (!first)283 first = mdesc;284 285 if (!prev) {286 prev = mdesc;287 continue;288 }289 290 prev->tcd->dlast_sga = mdesc->tcd_paddr;291 prev->tcd->e_sg = 1;292 mdesc->tcd->start = 1;293 294 prev = mdesc;295 }296 297 prev->tcd->int_maj = 1;298 299 /* Send first descriptor in chain into hardware */300 memcpy_toio(&mdma->tcd[cid], first->tcd, sizeof(struct mpc_dma_tcd));301 302 if (first != prev)303 mdma->tcd[cid].e_sg = 1;304 305 if (mdma->is_mpc8308) {306 /* MPC8308, no request lines, software initiated start */307 out_8(&mdma->regs->dmassrt, cid);308 } else if (first->will_access_peripheral) {309 /* Peripherals involved, start by external request signal */310 out_8(&mdma->regs->dmaserq, cid);311 } else {312 /* Memory to memory transfer, software initiated start */313 out_8(&mdma->regs->dmassrt, cid);314 }315}316 317/* Handle interrupt on one half of DMA controller (32 channels) */318static void mpc_dma_irq_process(struct mpc_dma *mdma, u32 is, u32 es, int off)319{320 struct mpc_dma_chan *mchan;321 struct mpc_dma_desc *mdesc;322 u32 status = is | es;323 int ch;324 325 while ((ch = fls(status) - 1) >= 0) {326 status &= ~(1 << ch);327 mchan = &mdma->channels[ch + off];328 329 spin_lock(&mchan->lock);330 331 out_8(&mdma->regs->dmacint, ch + off);332 out_8(&mdma->regs->dmacerr, ch + off);333 334 /* Check error status */335 if (es & (1 << ch))336 list_for_each_entry(mdesc, &mchan->active, node)337 mdesc->error = -EIO;338 339 /* Execute queued descriptors */340 list_splice_tail_init(&mchan->active, &mchan->completed);341 if (!list_empty(&mchan->queued))342 mpc_dma_execute(mchan);343 344 spin_unlock(&mchan->lock);345 }346}347 348/* Interrupt handler */349static irqreturn_t mpc_dma_irq(int irq, void *data)350{351 struct mpc_dma *mdma = data;352 uint es;353 354 /* Save error status register */355 es = in_be32(&mdma->regs->dmaes);356 spin_lock(&mdma->error_status_lock);357 if ((es & MPC_DMA_DMAES_VLD) && mdma->error_status == 0)358 mdma->error_status = es;359 spin_unlock(&mdma->error_status_lock);360 361 /* Handle interrupt on each channel */362 if (mdma->dma.chancnt > 32) {363 mpc_dma_irq_process(mdma, in_be32(&mdma->regs->dmainth),364 in_be32(&mdma->regs->dmaerrh), 32);365 }366 mpc_dma_irq_process(mdma, in_be32(&mdma->regs->dmaintl),367 in_be32(&mdma->regs->dmaerrl), 0);368 369 /* Schedule tasklet */370 tasklet_schedule(&mdma->tasklet);371 372 return IRQ_HANDLED;373}374 375/* process completed descriptors */376static void mpc_dma_process_completed(struct mpc_dma *mdma)377{378 dma_cookie_t last_cookie = 0;379 struct mpc_dma_chan *mchan;380 struct mpc_dma_desc *mdesc;381 struct dma_async_tx_descriptor *desc;382 unsigned long flags;383 LIST_HEAD(list);384 int i;385 386 for (i = 0; i < mdma->dma.chancnt; i++) {387 mchan = &mdma->channels[i];388 389 /* Get all completed descriptors */390 spin_lock_irqsave(&mchan->lock, flags);391 if (!list_empty(&mchan->completed))392 list_splice_tail_init(&mchan->completed, &list);393 spin_unlock_irqrestore(&mchan->lock, flags);394 395 if (list_empty(&list))396 continue;397 398 /* Execute callbacks and run dependencies */399 list_for_each_entry(mdesc, &list, node) {400 desc = &mdesc->desc;401 402 dmaengine_desc_get_callback_invoke(desc, NULL);403 404 last_cookie = desc->cookie;405 dma_run_dependencies(desc);406 }407 408 /* Free descriptors */409 spin_lock_irqsave(&mchan->lock, flags);410 list_splice_tail_init(&list, &mchan->free);411 mchan->chan.completed_cookie = last_cookie;412 spin_unlock_irqrestore(&mchan->lock, flags);413 }414}415 416/* DMA Tasklet */417static void mpc_dma_tasklet(struct tasklet_struct *t)418{419 struct mpc_dma *mdma = from_tasklet(mdma, t, tasklet);420 unsigned long flags;421 uint es;422 423 spin_lock_irqsave(&mdma->error_status_lock, flags);424 es = mdma->error_status;425 mdma->error_status = 0;426 spin_unlock_irqrestore(&mdma->error_status_lock, flags);427 428 /* Print nice error report */429 if (es) {430 dev_err(mdma->dma.dev,431 "Hardware reported following error(s) on channel %u:\n",432 MPC_DMA_DMAES_ERRCHN(es));433 434 if (es & MPC_DMA_DMAES_GPE)435 dev_err(mdma->dma.dev, "- Group Priority Error\n");436 if (es & MPC_DMA_DMAES_CPE)437 dev_err(mdma->dma.dev, "- Channel Priority Error\n");438 if (es & MPC_DMA_DMAES_SAE)439 dev_err(mdma->dma.dev, "- Source Address Error\n");440 if (es & MPC_DMA_DMAES_SOE)441 dev_err(mdma->dma.dev, "- Source Offset Configuration Error\n");442 if (es & MPC_DMA_DMAES_DAE)443 dev_err(mdma->dma.dev, "- Destination Address Error\n");444 if (es & MPC_DMA_DMAES_DOE)445 dev_err(mdma->dma.dev, "- Destination Offset Configuration Error\n");446 if (es & MPC_DMA_DMAES_NCE)447 dev_err(mdma->dma.dev, "- NBytes/Citter Configuration Error\n");448 if (es & MPC_DMA_DMAES_SGE)449 dev_err(mdma->dma.dev, "- Scatter/Gather Configuration Error\n");450 if (es & MPC_DMA_DMAES_SBE)451 dev_err(mdma->dma.dev, "- Source Bus Error\n");452 if (es & MPC_DMA_DMAES_DBE)453 dev_err(mdma->dma.dev, "- Destination Bus Error\n");454 }455 456 mpc_dma_process_completed(mdma);457}458 459/* Submit descriptor to hardware */460static dma_cookie_t mpc_dma_tx_submit(struct dma_async_tx_descriptor *txd)461{462 struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(txd->chan);463 struct mpc_dma_desc *mdesc;464 unsigned long flags;465 dma_cookie_t cookie;466 467 mdesc = container_of(txd, struct mpc_dma_desc, desc);468 469 spin_lock_irqsave(&mchan->lock, flags);470 471 /* Move descriptor to queue */472 list_move_tail(&mdesc->node, &mchan->queued);473 474 /* If channel is idle, execute all queued descriptors */475 if (list_empty(&mchan->active))476 mpc_dma_execute(mchan);477 478 /* Update cookie */479 cookie = dma_cookie_assign(txd);480 spin_unlock_irqrestore(&mchan->lock, flags);481 482 return cookie;483}484 485/* Alloc channel resources */486static int mpc_dma_alloc_chan_resources(struct dma_chan *chan)487{488 struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);489 struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);490 struct mpc_dma_desc *mdesc;491 struct mpc_dma_tcd *tcd;492 dma_addr_t tcd_paddr;493 unsigned long flags;494 LIST_HEAD(descs);495 int i;496 497 /* Alloc DMA memory for Transfer Control Descriptors */498 tcd = dma_alloc_coherent(mdma->dma.dev,499 MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd),500 &tcd_paddr, GFP_KERNEL);501 if (!tcd)502 return -ENOMEM;503 504 /* Alloc descriptors for this channel */505 for (i = 0; i < MPC_DMA_DESCRIPTORS; i++) {506 mdesc = kzalloc(sizeof(struct mpc_dma_desc), GFP_KERNEL);507 if (!mdesc) {508 dev_notice(mdma->dma.dev,509 "Memory allocation error. Allocated only %u descriptors\n", i);510 break;511 }512 513 dma_async_tx_descriptor_init(&mdesc->desc, chan);514 mdesc->desc.flags = DMA_CTRL_ACK;515 mdesc->desc.tx_submit = mpc_dma_tx_submit;516 517 mdesc->tcd = &tcd[i];518 mdesc->tcd_paddr = tcd_paddr + (i * sizeof(struct mpc_dma_tcd));519 520 list_add_tail(&mdesc->node, &descs);521 }522 523 /* Return error only if no descriptors were allocated */524 if (i == 0) {525 dma_free_coherent(mdma->dma.dev,526 MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd),527 tcd, tcd_paddr);528 return -ENOMEM;529 }530 531 spin_lock_irqsave(&mchan->lock, flags);532 mchan->tcd = tcd;533 mchan->tcd_paddr = tcd_paddr;534 list_splice_tail_init(&descs, &mchan->free);535 spin_unlock_irqrestore(&mchan->lock, flags);536 537 /* Enable Error Interrupt */538 out_8(&mdma->regs->dmaseei, chan->chan_id);539 540 return 0;541}542 543/* Free channel resources */544static void mpc_dma_free_chan_resources(struct dma_chan *chan)545{546 struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);547 struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);548 struct mpc_dma_desc *mdesc, *tmp;549 struct mpc_dma_tcd *tcd;550 dma_addr_t tcd_paddr;551 unsigned long flags;552 LIST_HEAD(descs);553 554 spin_lock_irqsave(&mchan->lock, flags);555 556 /* Channel must be idle */557 BUG_ON(!list_empty(&mchan->prepared));558 BUG_ON(!list_empty(&mchan->queued));559 BUG_ON(!list_empty(&mchan->active));560 BUG_ON(!list_empty(&mchan->completed));561 562 /* Move data */563 list_splice_tail_init(&mchan->free, &descs);564 tcd = mchan->tcd;565 tcd_paddr = mchan->tcd_paddr;566 567 spin_unlock_irqrestore(&mchan->lock, flags);568 569 /* Free DMA memory used by descriptors */570 dma_free_coherent(mdma->dma.dev,571 MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd),572 tcd, tcd_paddr);573 574 /* Free descriptors */575 list_for_each_entry_safe(mdesc, tmp, &descs, node)576 kfree(mdesc);577 578 /* Disable Error Interrupt */579 out_8(&mdma->regs->dmaceei, chan->chan_id);580}581 582/* Send all pending descriptor to hardware */583static void mpc_dma_issue_pending(struct dma_chan *chan)584{585 /*586 * We are posting descriptors to the hardware as soon as587 * they are ready, so this function does nothing.588 */589}590 591/* Check request completion status */592static enum dma_status593mpc_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie,594 struct dma_tx_state *txstate)595{596 return dma_cookie_status(chan, cookie, txstate);597}598 599/* Prepare descriptor for memory to memory copy */600static struct dma_async_tx_descriptor *601mpc_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,602 size_t len, unsigned long flags)603{604 struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);605 struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);606 struct mpc_dma_desc *mdesc = NULL;607 struct mpc_dma_tcd *tcd;608 unsigned long iflags;609 610 /* Get free descriptor */611 spin_lock_irqsave(&mchan->lock, iflags);612 if (!list_empty(&mchan->free)) {613 mdesc = list_first_entry(&mchan->free, struct mpc_dma_desc,614 node);615 list_del(&mdesc->node);616 }617 spin_unlock_irqrestore(&mchan->lock, iflags);618 619 if (!mdesc) {620 /* try to free completed descriptors */621 mpc_dma_process_completed(mdma);622 return NULL;623 }624 625 mdesc->error = 0;626 mdesc->will_access_peripheral = 0;627 tcd = mdesc->tcd;628 629 /* Prepare Transfer Control Descriptor for this transaction */630 memset(tcd, 0, sizeof(struct mpc_dma_tcd));631 632 if (IS_ALIGNED(src | dst | len, 32)) {633 tcd->ssize = MPC_DMA_TSIZE_32;634 tcd->dsize = MPC_DMA_TSIZE_32;635 tcd->soff = 32;636 tcd->doff = 32;637 } else if (!mdma->is_mpc8308 && IS_ALIGNED(src | dst | len, 16)) {638 /* MPC8308 doesn't support 16 byte transfers */639 tcd->ssize = MPC_DMA_TSIZE_16;640 tcd->dsize = MPC_DMA_TSIZE_16;641 tcd->soff = 16;642 tcd->doff = 16;643 } else if (IS_ALIGNED(src | dst | len, 4)) {644 tcd->ssize = MPC_DMA_TSIZE_4;645 tcd->dsize = MPC_DMA_TSIZE_4;646 tcd->soff = 4;647 tcd->doff = 4;648 } else if (IS_ALIGNED(src | dst | len, 2)) {649 tcd->ssize = MPC_DMA_TSIZE_2;650 tcd->dsize = MPC_DMA_TSIZE_2;651 tcd->soff = 2;652 tcd->doff = 2;653 } else {654 tcd->ssize = MPC_DMA_TSIZE_1;655 tcd->dsize = MPC_DMA_TSIZE_1;656 tcd->soff = 1;657 tcd->doff = 1;658 }659 660 tcd->saddr = src;661 tcd->daddr = dst;662 tcd->nbytes = len;663 tcd->biter = 1;664 tcd->citer = 1;665 666 /* Place descriptor in prepared list */667 spin_lock_irqsave(&mchan->lock, iflags);668 list_add_tail(&mdesc->node, &mchan->prepared);669 spin_unlock_irqrestore(&mchan->lock, iflags);670 671 return &mdesc->desc;672}673 674inline u8 buswidth_to_dmatsize(u8 buswidth)675{676 u8 res;677 678 for (res = 0; buswidth > 1; buswidth /= 2)679 res++;680 return res;681}682 683static struct dma_async_tx_descriptor *684mpc_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,685 unsigned int sg_len, enum dma_transfer_direction direction,686 unsigned long flags, void *context)687{688 struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);689 struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);690 struct mpc_dma_desc *mdesc = NULL;691 dma_addr_t per_paddr;692 u32 tcd_nunits;693 struct mpc_dma_tcd *tcd;694 unsigned long iflags;695 struct scatterlist *sg;696 size_t len;697 int iter, i;698 699 /* Currently there is no proper support for scatter/gather */700 if (sg_len != 1)701 return NULL;702 703 if (!is_slave_direction(direction))704 return NULL;705 706 for_each_sg(sgl, sg, sg_len, i) {707 spin_lock_irqsave(&mchan->lock, iflags);708 709 mdesc = list_first_entry(&mchan->free,710 struct mpc_dma_desc, node);711 if (!mdesc) {712 spin_unlock_irqrestore(&mchan->lock, iflags);713 /* Try to free completed descriptors */714 mpc_dma_process_completed(mdma);715 return NULL;716 }717 718 list_del(&mdesc->node);719 720 if (direction == DMA_DEV_TO_MEM) {721 per_paddr = mchan->src_per_paddr;722 tcd_nunits = mchan->src_tcd_nunits;723 } else {724 per_paddr = mchan->dst_per_paddr;725 tcd_nunits = mchan->dst_tcd_nunits;726 }727 728 spin_unlock_irqrestore(&mchan->lock, iflags);729 730 if (per_paddr == 0 || tcd_nunits == 0)731 goto err_prep;732 733 mdesc->error = 0;734 mdesc->will_access_peripheral = 1;735 736 /* Prepare Transfer Control Descriptor for this transaction */737 tcd = mdesc->tcd;738 739 memset(tcd, 0, sizeof(struct mpc_dma_tcd));740 741 if (direction == DMA_DEV_TO_MEM) {742 tcd->saddr = per_paddr;743 tcd->daddr = sg_dma_address(sg);744 745 if (!IS_ALIGNED(sg_dma_address(sg), mchan->dwidth))746 goto err_prep;747 748 tcd->soff = 0;749 tcd->doff = mchan->dwidth;750 } else {751 tcd->saddr = sg_dma_address(sg);752 tcd->daddr = per_paddr;753 754 if (!IS_ALIGNED(sg_dma_address(sg), mchan->swidth))755 goto err_prep;756 757 tcd->soff = mchan->swidth;758 tcd->doff = 0;759 }760 761 tcd->ssize = buswidth_to_dmatsize(mchan->swidth);762 tcd->dsize = buswidth_to_dmatsize(mchan->dwidth);763 764 if (mdma->is_mpc8308) {765 tcd->nbytes = sg_dma_len(sg);766 if (!IS_ALIGNED(tcd->nbytes, mchan->swidth))767 goto err_prep;768 769 /* No major loops for MPC8303 */770 tcd->biter = 1;771 tcd->citer = 1;772 } else {773 len = sg_dma_len(sg);774 tcd->nbytes = tcd_nunits * tcd->ssize;775 if (!IS_ALIGNED(len, tcd->nbytes))776 goto err_prep;777 778 iter = len / tcd->nbytes;779 if (iter >= 1 << 15) {780 /* len is too big */781 goto err_prep;782 }783 /* citer_linkch contains the high bits of iter */784 tcd->biter = iter & 0x1ff;785 tcd->biter_linkch = iter >> 9;786 tcd->citer = tcd->biter;787 tcd->citer_linkch = tcd->biter_linkch;788 }789 790 tcd->e_sg = 0;791 tcd->d_req = 1;792 793 /* Place descriptor in prepared list */794 spin_lock_irqsave(&mchan->lock, iflags);795 list_add_tail(&mdesc->node, &mchan->prepared);796 spin_unlock_irqrestore(&mchan->lock, iflags);797 }798 799 return &mdesc->desc;800 801err_prep:802 /* Put the descriptor back */803 spin_lock_irqsave(&mchan->lock, iflags);804 list_add_tail(&mdesc->node, &mchan->free);805 spin_unlock_irqrestore(&mchan->lock, iflags);806 807 return NULL;808}809 810inline bool is_buswidth_valid(u8 buswidth, bool is_mpc8308)811{812 switch (buswidth) {813 case 16:814 if (is_mpc8308)815 return false;816 break;817 case 1:818 case 2:819 case 4:820 case 32:821 break;822 default:823 return false;824 }825 826 return true;827}828 829static int mpc_dma_device_config(struct dma_chan *chan,830 struct dma_slave_config *cfg)831{832 struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);833 struct mpc_dma *mdma = dma_chan_to_mpc_dma(&mchan->chan);834 unsigned long flags;835 836 /*837 * Software constraints:838 * - only transfers between a peripheral device and memory are839 * supported840 * - transfer chunk sizes of 1, 2, 4, 16 (for MPC512x), and 32 bytes841 * are supported, and, consequently, source addresses and842 * destination addresses; must be aligned accordingly; furthermore,843 * for MPC512x SoCs, the transfer size must be aligned on (chunk844 * size * maxburst)845 * - during the transfer, the RAM address is incremented by the size846 * of transfer chunk847 * - the peripheral port's address is constant during the transfer.848 */849 850 if (!IS_ALIGNED(cfg->src_addr, cfg->src_addr_width) ||851 !IS_ALIGNED(cfg->dst_addr, cfg->dst_addr_width)) {852 return -EINVAL;853 }854 855 if (!is_buswidth_valid(cfg->src_addr_width, mdma->is_mpc8308) ||856 !is_buswidth_valid(cfg->dst_addr_width, mdma->is_mpc8308))857 return -EINVAL;858 859 spin_lock_irqsave(&mchan->lock, flags);860 861 mchan->src_per_paddr = cfg->src_addr;862 mchan->src_tcd_nunits = cfg->src_maxburst;863 mchan->swidth = cfg->src_addr_width;864 mchan->dst_per_paddr = cfg->dst_addr;865 mchan->dst_tcd_nunits = cfg->dst_maxburst;866 mchan->dwidth = cfg->dst_addr_width;867 868 /* Apply defaults */869 if (mchan->src_tcd_nunits == 0)870 mchan->src_tcd_nunits = 1;871 if (mchan->dst_tcd_nunits == 0)872 mchan->dst_tcd_nunits = 1;873 874 spin_unlock_irqrestore(&mchan->lock, flags);875 876 return 0;877}878 879static int mpc_dma_device_terminate_all(struct dma_chan *chan)880{881 struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);882 struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);883 unsigned long flags;884 885 /* Disable channel requests */886 spin_lock_irqsave(&mchan->lock, flags);887 888 out_8(&mdma->regs->dmacerq, chan->chan_id);889 list_splice_tail_init(&mchan->prepared, &mchan->free);890 list_splice_tail_init(&mchan->queued, &mchan->free);891 list_splice_tail_init(&mchan->active, &mchan->free);892 893 spin_unlock_irqrestore(&mchan->lock, flags);894 895 return 0;896}897 898static int mpc_dma_probe(struct platform_device *op)899{900 struct device_node *dn = op->dev.of_node;901 struct device *dev = &op->dev;902 struct dma_device *dma;903 struct mpc_dma *mdma;904 struct mpc_dma_chan *mchan;905 struct resource res;906 ulong regs_start, regs_size;907 int retval, i;908 u8 chancnt;909 910 mdma = devm_kzalloc(dev, sizeof(struct mpc_dma), GFP_KERNEL);911 if (!mdma) {912 retval = -ENOMEM;913 goto err;914 }915 916 mdma->irq = irq_of_parse_and_map(dn, 0);917 if (!mdma->irq) {918 dev_err(dev, "Error mapping IRQ!\n");919 retval = -EINVAL;920 goto err;921 }922 923 if (of_device_is_compatible(dn, "fsl,mpc8308-dma")) {924 mdma->is_mpc8308 = 1;925 mdma->irq2 = irq_of_parse_and_map(dn, 1);926 if (!mdma->irq2) {927 dev_err(dev, "Error mapping IRQ!\n");928 retval = -EINVAL;929 goto err_dispose1;930 }931 }932 933 retval = of_address_to_resource(dn, 0, &res);934 if (retval) {935 dev_err(dev, "Error parsing memory region!\n");936 goto err_dispose2;937 }938 939 regs_start = res.start;940 regs_size = resource_size(&res);941 942 if (!devm_request_mem_region(dev, regs_start, regs_size, DRV_NAME)) {943 dev_err(dev, "Error requesting memory region!\n");944 retval = -EBUSY;945 goto err_dispose2;946 }947 948 mdma->regs = devm_ioremap(dev, regs_start, regs_size);949 if (!mdma->regs) {950 dev_err(dev, "Error mapping memory region!\n");951 retval = -ENOMEM;952 goto err_dispose2;953 }954 955 mdma->tcd = (struct mpc_dma_tcd *)((u8 *)(mdma->regs)956 + MPC_DMA_TCD_OFFSET);957 958 retval = request_irq(mdma->irq, &mpc_dma_irq, 0, DRV_NAME, mdma);959 if (retval) {960 dev_err(dev, "Error requesting IRQ!\n");961 retval = -EINVAL;962 goto err_dispose2;963 }964 965 if (mdma->is_mpc8308) {966 retval = request_irq(mdma->irq2, &mpc_dma_irq, 0,967 DRV_NAME, mdma);968 if (retval) {969 dev_err(dev, "Error requesting IRQ2!\n");970 retval = -EINVAL;971 goto err_free1;972 }973 }974 975 spin_lock_init(&mdma->error_status_lock);976 977 dma = &mdma->dma;978 dma->dev = dev;979 dma->device_alloc_chan_resources = mpc_dma_alloc_chan_resources;980 dma->device_free_chan_resources = mpc_dma_free_chan_resources;981 dma->device_issue_pending = mpc_dma_issue_pending;982 dma->device_tx_status = mpc_dma_tx_status;983 dma->device_prep_dma_memcpy = mpc_dma_prep_memcpy;984 dma->device_prep_slave_sg = mpc_dma_prep_slave_sg;985 dma->device_config = mpc_dma_device_config;986 dma->device_terminate_all = mpc_dma_device_terminate_all;987 988 INIT_LIST_HEAD(&dma->channels);989 dma_cap_set(DMA_MEMCPY, dma->cap_mask);990 dma_cap_set(DMA_SLAVE, dma->cap_mask);991 992 if (mdma->is_mpc8308)993 chancnt = MPC8308_DMACHAN_MAX;994 else995 chancnt = MPC512x_DMACHAN_MAX;996 997 for (i = 0; i < chancnt; i++) {998 mchan = &mdma->channels[i];999 1000 mchan->chan.device = dma;1001 dma_cookie_init(&mchan->chan);1002 1003 INIT_LIST_HEAD(&mchan->free);1004 INIT_LIST_HEAD(&mchan->prepared);1005 INIT_LIST_HEAD(&mchan->queued);1006 INIT_LIST_HEAD(&mchan->active);1007 INIT_LIST_HEAD(&mchan->completed);1008 1009 spin_lock_init(&mchan->lock);1010 list_add_tail(&mchan->chan.device_node, &dma->channels);1011 }1012 1013 tasklet_setup(&mdma->tasklet, mpc_dma_tasklet);1014 1015 /*1016 * Configure DMA Engine:1017 * - Dynamic clock,1018 * - Round-robin group arbitration,1019 * - Round-robin channel arbitration.1020 */1021 if (mdma->is_mpc8308) {1022 /* MPC8308 has 16 channels and lacks some registers */1023 out_be32(&mdma->regs->dmacr, MPC_DMA_DMACR_ERCA);1024 1025 /* enable snooping */1026 out_be32(&mdma->regs->dmagpor, MPC_DMA_DMAGPOR_SNOOP_ENABLE);1027 /* Disable error interrupts */1028 out_be32(&mdma->regs->dmaeeil, 0);1029 1030 /* Clear interrupts status */1031 out_be32(&mdma->regs->dmaintl, 0xFFFF);1032 out_be32(&mdma->regs->dmaerrl, 0xFFFF);1033 } else {1034 out_be32(&mdma->regs->dmacr, MPC_DMA_DMACR_EDCG |1035 MPC_DMA_DMACR_ERGA |1036 MPC_DMA_DMACR_ERCA);1037 1038 /* Disable hardware DMA requests */1039 out_be32(&mdma->regs->dmaerqh, 0);1040 out_be32(&mdma->regs->dmaerql, 0);1041 1042 /* Disable error interrupts */1043 out_be32(&mdma->regs->dmaeeih, 0);1044 out_be32(&mdma->regs->dmaeeil, 0);1045 1046 /* Clear interrupts status */1047 out_be32(&mdma->regs->dmainth, 0xFFFFFFFF);1048 out_be32(&mdma->regs->dmaintl, 0xFFFFFFFF);1049 out_be32(&mdma->regs->dmaerrh, 0xFFFFFFFF);1050 out_be32(&mdma->regs->dmaerrl, 0xFFFFFFFF);1051 1052 /* Route interrupts to IPIC */1053 out_be32(&mdma->regs->dmaihsa, 0);1054 out_be32(&mdma->regs->dmailsa, 0);1055 }1056 1057 /* Register DMA engine */1058 dev_set_drvdata(dev, mdma);1059 retval = dma_async_device_register(dma);1060 if (retval)1061 goto err_free2;1062 1063 /* Register with OF helpers for DMA lookups (nonfatal) */1064 if (dev->of_node) {1065 retval = of_dma_controller_register(dev->of_node,1066 of_dma_xlate_by_chan_id, mdma);1067 if (retval)1068 dev_warn(dev, "Could not register for OF lookup\n");1069 }1070 1071 return 0;1072 1073err_free2:1074 if (mdma->is_mpc8308)1075 free_irq(mdma->irq2, mdma);1076err_free1:1077 free_irq(mdma->irq, mdma);1078err_dispose2:1079 if (mdma->is_mpc8308)1080 irq_dispose_mapping(mdma->irq2);1081err_dispose1:1082 irq_dispose_mapping(mdma->irq);1083err:1084 return retval;1085}1086 1087static void mpc_dma_remove(struct platform_device *op)1088{1089 struct device *dev = &op->dev;1090 struct mpc_dma *mdma = dev_get_drvdata(dev);1091 1092 if (dev->of_node)1093 of_dma_controller_free(dev->of_node);1094 dma_async_device_unregister(&mdma->dma);1095 if (mdma->is_mpc8308) {1096 free_irq(mdma->irq2, mdma);1097 irq_dispose_mapping(mdma->irq2);1098 }1099 free_irq(mdma->irq, mdma);1100 irq_dispose_mapping(mdma->irq);1101 tasklet_kill(&mdma->tasklet);1102}1103 1104static const struct of_device_id mpc_dma_match[] = {1105 { .compatible = "fsl,mpc5121-dma", },1106 { .compatible = "fsl,mpc8308-dma", },1107 {},1108};1109MODULE_DEVICE_TABLE(of, mpc_dma_match);1110 1111static struct platform_driver mpc_dma_driver = {1112 .probe = mpc_dma_probe,1113 .remove_new = mpc_dma_remove,1114 .driver = {1115 .name = DRV_NAME,1116 .of_match_table = mpc_dma_match,1117 },1118};1119 1120module_platform_driver(mpc_dma_driver);1121 1122MODULE_LICENSE("GPL");1123MODULE_AUTHOR("Piotr Ziecik <kosmo@semihalf.com>");1124