1101 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * SA11x0 DMAengine support4 *5 * Copyright (C) 2012 Russell King6 * Derived in part from arch/arm/mach-sa1100/dma.c,7 * Copyright (C) 2000, 2001 by Nicolas Pitre8 */9#include <linux/sched.h>10#include <linux/device.h>11#include <linux/dmaengine.h>12#include <linux/init.h>13#include <linux/interrupt.h>14#include <linux/kernel.h>15#include <linux/module.h>16#include <linux/platform_device.h>17#include <linux/slab.h>18#include <linux/spinlock.h>19 20#include "virt-dma.h"21 22#define NR_PHY_CHAN 623#define DMA_ALIGN 324#define DMA_MAX_SIZE 0x1fff25#define DMA_CHUNK_SIZE 0x100026 27#define DMA_DDAR 0x0028#define DMA_DCSR_S 0x0429#define DMA_DCSR_C 0x0830#define DMA_DCSR_R 0x0c31#define DMA_DBSA 0x1032#define DMA_DBTA 0x1433#define DMA_DBSB 0x1834#define DMA_DBTB 0x1c35#define DMA_SIZE 0x2036 37#define DCSR_RUN (1 << 0)38#define DCSR_IE (1 << 1)39#define DCSR_ERROR (1 << 2)40#define DCSR_DONEA (1 << 3)41#define DCSR_STRTA (1 << 4)42#define DCSR_DONEB (1 << 5)43#define DCSR_STRTB (1 << 6)44#define DCSR_BIU (1 << 7)45 46#define DDAR_RW (1 << 0) /* 0 = W, 1 = R */47#define DDAR_E (1 << 1) /* 0 = LE, 1 = BE */48#define DDAR_BS (1 << 2) /* 0 = BS4, 1 = BS8 */49#define DDAR_DW (1 << 3) /* 0 = 8b, 1 = 16b */50#define DDAR_Ser0UDCTr (0x0 << 4)51#define DDAR_Ser0UDCRc (0x1 << 4)52#define DDAR_Ser1SDLCTr (0x2 << 4)53#define DDAR_Ser1SDLCRc (0x3 << 4)54#define DDAR_Ser1UARTTr (0x4 << 4)55#define DDAR_Ser1UARTRc (0x5 << 4)56#define DDAR_Ser2ICPTr (0x6 << 4)57#define DDAR_Ser2ICPRc (0x7 << 4)58#define DDAR_Ser3UARTTr (0x8 << 4)59#define DDAR_Ser3UARTRc (0x9 << 4)60#define DDAR_Ser4MCP0Tr (0xa << 4)61#define DDAR_Ser4MCP0Rc (0xb << 4)62#define DDAR_Ser4MCP1Tr (0xc << 4)63#define DDAR_Ser4MCP1Rc (0xd << 4)64#define DDAR_Ser4SSPTr (0xe << 4)65#define DDAR_Ser4SSPRc (0xf << 4)66 67struct sa11x0_dma_sg {68 u32 addr;69 u32 len;70};71 72struct sa11x0_dma_desc {73 struct virt_dma_desc vd;74 75 u32 ddar;76 size_t size;77 unsigned period;78 bool cyclic;79 80 unsigned sglen;81 struct sa11x0_dma_sg sg[] __counted_by(sglen);82};83 84struct sa11x0_dma_phy;85 86struct sa11x0_dma_chan {87 struct virt_dma_chan vc;88 89 /* protected by c->vc.lock */90 struct sa11x0_dma_phy *phy;91 enum dma_status status;92 93 /* protected by d->lock */94 struct list_head node;95 96 u32 ddar;97 const char *name;98};99 100struct sa11x0_dma_phy {101 void __iomem *base;102 struct sa11x0_dma_dev *dev;103 unsigned num;104 105 struct sa11x0_dma_chan *vchan;106 107 /* Protected by c->vc.lock */108 unsigned sg_load;109 struct sa11x0_dma_desc *txd_load;110 unsigned sg_done;111 struct sa11x0_dma_desc *txd_done;112 u32 dbs[2];113 u32 dbt[2];114 u32 dcsr;115};116 117struct sa11x0_dma_dev {118 struct dma_device slave;119 void __iomem *base;120 spinlock_t lock;121 struct tasklet_struct task;122 struct list_head chan_pending;123 struct sa11x0_dma_phy phy[NR_PHY_CHAN];124};125 126static struct sa11x0_dma_chan *to_sa11x0_dma_chan(struct dma_chan *chan)127{128 return container_of(chan, struct sa11x0_dma_chan, vc.chan);129}130 131static struct sa11x0_dma_dev *to_sa11x0_dma(struct dma_device *dmadev)132{133 return container_of(dmadev, struct sa11x0_dma_dev, slave);134}135 136static struct sa11x0_dma_desc *sa11x0_dma_next_desc(struct sa11x0_dma_chan *c)137{138 struct virt_dma_desc *vd = vchan_next_desc(&c->vc);139 140 return vd ? container_of(vd, struct sa11x0_dma_desc, vd) : NULL;141}142 143static void sa11x0_dma_free_desc(struct virt_dma_desc *vd)144{145 kfree(container_of(vd, struct sa11x0_dma_desc, vd));146}147 148static void sa11x0_dma_start_desc(struct sa11x0_dma_phy *p, struct sa11x0_dma_desc *txd)149{150 list_del(&txd->vd.node);151 p->txd_load = txd;152 p->sg_load = 0;153 154 dev_vdbg(p->dev->slave.dev, "pchan %u: txd %p[%x]: starting: DDAR:%x\n",155 p->num, &txd->vd, txd->vd.tx.cookie, txd->ddar);156}157 158static void noinline sa11x0_dma_start_sg(struct sa11x0_dma_phy *p,159 struct sa11x0_dma_chan *c)160{161 struct sa11x0_dma_desc *txd = p->txd_load;162 struct sa11x0_dma_sg *sg;163 void __iomem *base = p->base;164 unsigned dbsx, dbtx;165 u32 dcsr;166 167 if (!txd)168 return;169 170 dcsr = readl_relaxed(base + DMA_DCSR_R);171 172 /* Don't try to load the next transfer if both buffers are started */173 if ((dcsr & (DCSR_STRTA | DCSR_STRTB)) == (DCSR_STRTA | DCSR_STRTB))174 return;175 176 if (p->sg_load == txd->sglen) {177 if (!txd->cyclic) {178 struct sa11x0_dma_desc *txn = sa11x0_dma_next_desc(c);179 180 /*181 * We have reached the end of the current descriptor.182 * Peek at the next descriptor, and if compatible with183 * the current, start processing it.184 */185 if (txn && txn->ddar == txd->ddar) {186 txd = txn;187 sa11x0_dma_start_desc(p, txn);188 } else {189 p->txd_load = NULL;190 return;191 }192 } else {193 /* Cyclic: reset back to beginning */194 p->sg_load = 0;195 }196 }197 198 sg = &txd->sg[p->sg_load++];199 200 /* Select buffer to load according to channel status */201 if (((dcsr & (DCSR_BIU | DCSR_STRTB)) == (DCSR_BIU | DCSR_STRTB)) ||202 ((dcsr & (DCSR_BIU | DCSR_STRTA)) == 0)) {203 dbsx = DMA_DBSA;204 dbtx = DMA_DBTA;205 dcsr = DCSR_STRTA | DCSR_IE | DCSR_RUN;206 } else {207 dbsx = DMA_DBSB;208 dbtx = DMA_DBTB;209 dcsr = DCSR_STRTB | DCSR_IE | DCSR_RUN;210 }211 212 writel_relaxed(sg->addr, base + dbsx);213 writel_relaxed(sg->len, base + dbtx);214 writel(dcsr, base + DMA_DCSR_S);215 216 dev_dbg(p->dev->slave.dev, "pchan %u: load: DCSR:%02x DBS%c:%08x DBT%c:%08x\n",217 p->num, dcsr,218 'A' + (dbsx == DMA_DBSB), sg->addr,219 'A' + (dbtx == DMA_DBTB), sg->len);220}221 222static void noinline sa11x0_dma_complete(struct sa11x0_dma_phy *p,223 struct sa11x0_dma_chan *c)224{225 struct sa11x0_dma_desc *txd = p->txd_done;226 227 if (++p->sg_done == txd->sglen) {228 if (!txd->cyclic) {229 vchan_cookie_complete(&txd->vd);230 231 p->sg_done = 0;232 p->txd_done = p->txd_load;233 234 if (!p->txd_done)235 tasklet_schedule(&p->dev->task);236 } else {237 if ((p->sg_done % txd->period) == 0)238 vchan_cyclic_callback(&txd->vd);239 240 /* Cyclic: reset back to beginning */241 p->sg_done = 0;242 }243 }244 245 sa11x0_dma_start_sg(p, c);246}247 248static irqreturn_t sa11x0_dma_irq(int irq, void *dev_id)249{250 struct sa11x0_dma_phy *p = dev_id;251 struct sa11x0_dma_dev *d = p->dev;252 struct sa11x0_dma_chan *c;253 u32 dcsr;254 255 dcsr = readl_relaxed(p->base + DMA_DCSR_R);256 if (!(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB)))257 return IRQ_NONE;258 259 /* Clear reported status bits */260 writel_relaxed(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB),261 p->base + DMA_DCSR_C);262 263 dev_dbg(d->slave.dev, "pchan %u: irq: DCSR:%02x\n", p->num, dcsr);264 265 if (dcsr & DCSR_ERROR) {266 dev_err(d->slave.dev, "pchan %u: error. DCSR:%02x DDAR:%08x DBSA:%08x DBTA:%08x DBSB:%08x DBTB:%08x\n",267 p->num, dcsr,268 readl_relaxed(p->base + DMA_DDAR),269 readl_relaxed(p->base + DMA_DBSA),270 readl_relaxed(p->base + DMA_DBTA),271 readl_relaxed(p->base + DMA_DBSB),272 readl_relaxed(p->base + DMA_DBTB));273 }274 275 c = p->vchan;276 if (c) {277 unsigned long flags;278 279 spin_lock_irqsave(&c->vc.lock, flags);280 /*281 * Now that we're holding the lock, check that the vchan282 * really is associated with this pchan before touching the283 * hardware. This should always succeed, because we won't284 * change p->vchan or c->phy while the channel is actively285 * transferring.286 */287 if (c->phy == p) {288 if (dcsr & DCSR_DONEA)289 sa11x0_dma_complete(p, c);290 if (dcsr & DCSR_DONEB)291 sa11x0_dma_complete(p, c);292 }293 spin_unlock_irqrestore(&c->vc.lock, flags);294 }295 296 return IRQ_HANDLED;297}298 299static void sa11x0_dma_start_txd(struct sa11x0_dma_chan *c)300{301 struct sa11x0_dma_desc *txd = sa11x0_dma_next_desc(c);302 303 /* If the issued list is empty, we have no further txds to process */304 if (txd) {305 struct sa11x0_dma_phy *p = c->phy;306 307 sa11x0_dma_start_desc(p, txd);308 p->txd_done = txd;309 p->sg_done = 0;310 311 /* The channel should not have any transfers started */312 WARN_ON(readl_relaxed(p->base + DMA_DCSR_R) &313 (DCSR_STRTA | DCSR_STRTB));314 315 /* Clear the run and start bits before changing DDAR */316 writel_relaxed(DCSR_RUN | DCSR_STRTA | DCSR_STRTB,317 p->base + DMA_DCSR_C);318 writel_relaxed(txd->ddar, p->base + DMA_DDAR);319 320 /* Try to start both buffers */321 sa11x0_dma_start_sg(p, c);322 sa11x0_dma_start_sg(p, c);323 }324}325 326static void sa11x0_dma_tasklet(struct tasklet_struct *t)327{328 struct sa11x0_dma_dev *d = from_tasklet(d, t, task);329 struct sa11x0_dma_phy *p;330 struct sa11x0_dma_chan *c;331 unsigned pch, pch_alloc = 0;332 333 dev_dbg(d->slave.dev, "tasklet enter\n");334 335 list_for_each_entry(c, &d->slave.channels, vc.chan.device_node) {336 spin_lock_irq(&c->vc.lock);337 p = c->phy;338 if (p && !p->txd_done) {339 sa11x0_dma_start_txd(c);340 if (!p->txd_done) {341 /* No current txd associated with this channel */342 dev_dbg(d->slave.dev, "pchan %u: free\n", p->num);343 344 /* Mark this channel free */345 c->phy = NULL;346 p->vchan = NULL;347 }348 }349 spin_unlock_irq(&c->vc.lock);350 }351 352 spin_lock_irq(&d->lock);353 for (pch = 0; pch < NR_PHY_CHAN; pch++) {354 p = &d->phy[pch];355 356 if (p->vchan == NULL && !list_empty(&d->chan_pending)) {357 c = list_first_entry(&d->chan_pending,358 struct sa11x0_dma_chan, node);359 list_del_init(&c->node);360 361 pch_alloc |= 1 << pch;362 363 /* Mark this channel allocated */364 p->vchan = c;365 366 dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc);367 }368 }369 spin_unlock_irq(&d->lock);370 371 for (pch = 0; pch < NR_PHY_CHAN; pch++) {372 if (pch_alloc & (1 << pch)) {373 p = &d->phy[pch];374 c = p->vchan;375 376 spin_lock_irq(&c->vc.lock);377 c->phy = p;378 379 sa11x0_dma_start_txd(c);380 spin_unlock_irq(&c->vc.lock);381 }382 }383 384 dev_dbg(d->slave.dev, "tasklet exit\n");385}386 387 388static void sa11x0_dma_free_chan_resources(struct dma_chan *chan)389{390 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);391 struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);392 unsigned long flags;393 394 spin_lock_irqsave(&d->lock, flags);395 list_del_init(&c->node);396 spin_unlock_irqrestore(&d->lock, flags);397 398 vchan_free_chan_resources(&c->vc);399}400 401static dma_addr_t sa11x0_dma_pos(struct sa11x0_dma_phy *p)402{403 unsigned reg;404 u32 dcsr;405 406 dcsr = readl_relaxed(p->base + DMA_DCSR_R);407 408 if ((dcsr & (DCSR_BIU | DCSR_STRTA)) == DCSR_STRTA ||409 (dcsr & (DCSR_BIU | DCSR_STRTB)) == DCSR_BIU)410 reg = DMA_DBSA;411 else412 reg = DMA_DBSB;413 414 return readl_relaxed(p->base + reg);415}416 417static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,418 dma_cookie_t cookie, struct dma_tx_state *state)419{420 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);421 struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);422 struct sa11x0_dma_phy *p;423 struct virt_dma_desc *vd;424 unsigned long flags;425 enum dma_status ret;426 427 ret = dma_cookie_status(&c->vc.chan, cookie, state);428 if (ret == DMA_COMPLETE)429 return ret;430 431 if (!state)432 return c->status;433 434 spin_lock_irqsave(&c->vc.lock, flags);435 p = c->phy;436 437 /*438 * If the cookie is on our issue queue, then the residue is439 * its total size.440 */441 vd = vchan_find_desc(&c->vc, cookie);442 if (vd) {443 state->residue = container_of(vd, struct sa11x0_dma_desc, vd)->size;444 } else if (!p) {445 state->residue = 0;446 } else {447 struct sa11x0_dma_desc *txd;448 size_t bytes = 0;449 450 if (p->txd_done && p->txd_done->vd.tx.cookie == cookie)451 txd = p->txd_done;452 else if (p->txd_load && p->txd_load->vd.tx.cookie == cookie)453 txd = p->txd_load;454 else455 txd = NULL;456 457 ret = c->status;458 if (txd) {459 dma_addr_t addr = sa11x0_dma_pos(p);460 unsigned i;461 462 dev_vdbg(d->slave.dev, "tx_status: addr:%pad\n", &addr);463 464 for (i = 0; i < txd->sglen; i++) {465 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n",466 i, txd->sg[i].addr, txd->sg[i].len);467 if (addr >= txd->sg[i].addr &&468 addr < txd->sg[i].addr + txd->sg[i].len) {469 unsigned len;470 471 len = txd->sg[i].len -472 (addr - txd->sg[i].addr);473 dev_vdbg(d->slave.dev, "tx_status: [%u] +%x\n",474 i, len);475 bytes += len;476 i++;477 break;478 }479 }480 for (; i < txd->sglen; i++) {481 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x ++\n",482 i, txd->sg[i].addr, txd->sg[i].len);483 bytes += txd->sg[i].len;484 }485 }486 state->residue = bytes;487 }488 spin_unlock_irqrestore(&c->vc.lock, flags);489 490 dev_vdbg(d->slave.dev, "tx_status: bytes 0x%x\n", state->residue);491 492 return ret;493}494 495/*496 * Move pending txds to the issued list, and re-init pending list.497 * If not already pending, add this channel to the list of pending498 * channels and trigger the tasklet to run.499 */500static void sa11x0_dma_issue_pending(struct dma_chan *chan)501{502 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);503 struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);504 unsigned long flags;505 506 spin_lock_irqsave(&c->vc.lock, flags);507 if (vchan_issue_pending(&c->vc)) {508 if (!c->phy) {509 spin_lock(&d->lock);510 if (list_empty(&c->node)) {511 list_add_tail(&c->node, &d->chan_pending);512 tasklet_schedule(&d->task);513 dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);514 }515 spin_unlock(&d->lock);516 }517 } else518 dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);519 spin_unlock_irqrestore(&c->vc.lock, flags);520}521 522static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(523 struct dma_chan *chan, struct scatterlist *sg, unsigned int sglen,524 enum dma_transfer_direction dir, unsigned long flags, void *context)525{526 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);527 struct sa11x0_dma_desc *txd;528 struct scatterlist *sgent;529 unsigned i, j = sglen;530 size_t size = 0;531 532 /* SA11x0 channels can only operate in their native direction */533 if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {534 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",535 &c->vc, c->ddar, dir);536 return NULL;537 }538 539 /* Do not allow zero-sized txds */540 if (sglen == 0)541 return NULL;542 543 for_each_sg(sg, sgent, sglen, i) {544 dma_addr_t addr = sg_dma_address(sgent);545 unsigned int len = sg_dma_len(sgent);546 547 if (len > DMA_MAX_SIZE)548 j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1;549 if (addr & DMA_ALIGN) {550 dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %pad\n",551 &c->vc, &addr);552 return NULL;553 }554 }555 556 txd = kzalloc(struct_size(txd, sg, j), GFP_ATOMIC);557 if (!txd) {558 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);559 return NULL;560 }561 txd->sglen = j;562 563 j = 0;564 for_each_sg(sg, sgent, sglen, i) {565 dma_addr_t addr = sg_dma_address(sgent);566 unsigned len = sg_dma_len(sgent);567 568 size += len;569 570 do {571 unsigned tlen = len;572 573 /*574 * Check whether the transfer will fit. If not, try575 * to split the transfer up such that we end up with576 * equal chunks - but make sure that we preserve the577 * alignment. This avoids small segments.578 */579 if (tlen > DMA_MAX_SIZE) {580 unsigned mult = DIV_ROUND_UP(tlen,581 DMA_MAX_SIZE & ~DMA_ALIGN);582 583 tlen = (tlen / mult) & ~DMA_ALIGN;584 }585 586 txd->sg[j].addr = addr;587 txd->sg[j].len = tlen;588 589 addr += tlen;590 len -= tlen;591 j++;592 } while (len);593 }594 595 txd->ddar = c->ddar;596 txd->size = size;597 598 dev_dbg(chan->device->dev, "vchan %p: txd %p: size %zu nr %u\n",599 &c->vc, &txd->vd, txd->size, txd->sglen);600 601 return vchan_tx_prep(&c->vc, &txd->vd, flags);602}603 604static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic(605 struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,606 enum dma_transfer_direction dir, unsigned long flags)607{608 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);609 struct sa11x0_dma_desc *txd;610 unsigned i, j, k, sglen, sgperiod;611 612 /* SA11x0 channels can only operate in their native direction */613 if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {614 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",615 &c->vc, c->ddar, dir);616 return NULL;617 }618 619 sgperiod = DIV_ROUND_UP(period, DMA_MAX_SIZE & ~DMA_ALIGN);620 sglen = size * sgperiod / period;621 622 /* Do not allow zero-sized txds */623 if (sglen == 0)624 return NULL;625 626 txd = kzalloc(struct_size(txd, sg, sglen), GFP_ATOMIC);627 if (!txd) {628 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);629 return NULL;630 }631 txd->sglen = sglen;632 633 for (i = k = 0; i < size / period; i++) {634 size_t tlen, len = period;635 636 for (j = 0; j < sgperiod; j++, k++) {637 tlen = len;638 639 if (tlen > DMA_MAX_SIZE) {640 unsigned mult = DIV_ROUND_UP(tlen, DMA_MAX_SIZE & ~DMA_ALIGN);641 tlen = (tlen / mult) & ~DMA_ALIGN;642 }643 644 txd->sg[k].addr = addr;645 txd->sg[k].len = tlen;646 addr += tlen;647 len -= tlen;648 }649 650 WARN_ON(len != 0);651 }652 653 WARN_ON(k != sglen);654 655 txd->ddar = c->ddar;656 txd->size = size;657 txd->cyclic = 1;658 txd->period = sgperiod;659 660 return vchan_tx_prep(&c->vc, &txd->vd, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);661}662 663static int sa11x0_dma_device_config(struct dma_chan *chan,664 struct dma_slave_config *cfg)665{666 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);667 u32 ddar = c->ddar & ((0xf << 4) | DDAR_RW);668 dma_addr_t addr;669 enum dma_slave_buswidth width;670 u32 maxburst;671 672 if (ddar & DDAR_RW) {673 addr = cfg->src_addr;674 width = cfg->src_addr_width;675 maxburst = cfg->src_maxburst;676 } else {677 addr = cfg->dst_addr;678 width = cfg->dst_addr_width;679 maxburst = cfg->dst_maxburst;680 }681 682 if ((width != DMA_SLAVE_BUSWIDTH_1_BYTE &&683 width != DMA_SLAVE_BUSWIDTH_2_BYTES) ||684 (maxburst != 4 && maxburst != 8))685 return -EINVAL;686 687 if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)688 ddar |= DDAR_DW;689 if (maxburst == 8)690 ddar |= DDAR_BS;691 692 dev_dbg(c->vc.chan.device->dev, "vchan %p: dma_slave_config addr %pad width %u burst %u\n",693 &c->vc, &addr, width, maxburst);694 695 c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6;696 697 return 0;698}699 700static int sa11x0_dma_device_pause(struct dma_chan *chan)701{702 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);703 struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);704 struct sa11x0_dma_phy *p;705 unsigned long flags;706 707 dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc);708 spin_lock_irqsave(&c->vc.lock, flags);709 if (c->status == DMA_IN_PROGRESS) {710 c->status = DMA_PAUSED;711 712 p = c->phy;713 if (p) {714 writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);715 } else {716 spin_lock(&d->lock);717 list_del_init(&c->node);718 spin_unlock(&d->lock);719 }720 }721 spin_unlock_irqrestore(&c->vc.lock, flags);722 723 return 0;724}725 726static int sa11x0_dma_device_resume(struct dma_chan *chan)727{728 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);729 struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);730 struct sa11x0_dma_phy *p;731 unsigned long flags;732 733 dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc);734 spin_lock_irqsave(&c->vc.lock, flags);735 if (c->status == DMA_PAUSED) {736 c->status = DMA_IN_PROGRESS;737 738 p = c->phy;739 if (p) {740 writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_S);741 } else if (!list_empty(&c->vc.desc_issued)) {742 spin_lock(&d->lock);743 list_add_tail(&c->node, &d->chan_pending);744 spin_unlock(&d->lock);745 }746 }747 spin_unlock_irqrestore(&c->vc.lock, flags);748 749 return 0;750}751 752static int sa11x0_dma_device_terminate_all(struct dma_chan *chan)753{754 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);755 struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);756 struct sa11x0_dma_phy *p;757 LIST_HEAD(head);758 unsigned long flags;759 760 dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);761 /* Clear the tx descriptor lists */762 spin_lock_irqsave(&c->vc.lock, flags);763 vchan_get_all_descriptors(&c->vc, &head);764 765 p = c->phy;766 if (p) {767 dev_dbg(d->slave.dev, "pchan %u: terminating\n", p->num);768 /* vchan is assigned to a pchan - stop the channel */769 writel(DCSR_RUN | DCSR_IE |770 DCSR_STRTA | DCSR_DONEA |771 DCSR_STRTB | DCSR_DONEB,772 p->base + DMA_DCSR_C);773 774 if (p->txd_load) {775 if (p->txd_load != p->txd_done)776 list_add_tail(&p->txd_load->vd.node, &head);777 p->txd_load = NULL;778 }779 if (p->txd_done) {780 list_add_tail(&p->txd_done->vd.node, &head);781 p->txd_done = NULL;782 }783 c->phy = NULL;784 spin_lock(&d->lock);785 p->vchan = NULL;786 spin_unlock(&d->lock);787 tasklet_schedule(&d->task);788 }789 spin_unlock_irqrestore(&c->vc.lock, flags);790 vchan_dma_desc_free_list(&c->vc, &head);791 792 return 0;793}794 795struct sa11x0_dma_channel_desc {796 u32 ddar;797 const char *name;798};799 800#define CD(d1, d2) { .ddar = DDAR_##d1 | d2, .name = #d1 }801static const struct sa11x0_dma_channel_desc chan_desc[] = {802 CD(Ser0UDCTr, 0),803 CD(Ser0UDCRc, DDAR_RW),804 CD(Ser1SDLCTr, 0),805 CD(Ser1SDLCRc, DDAR_RW),806 CD(Ser1UARTTr, 0),807 CD(Ser1UARTRc, DDAR_RW),808 CD(Ser2ICPTr, 0),809 CD(Ser2ICPRc, DDAR_RW),810 CD(Ser3UARTTr, 0),811 CD(Ser3UARTRc, DDAR_RW),812 CD(Ser4MCP0Tr, 0),813 CD(Ser4MCP0Rc, DDAR_RW),814 CD(Ser4MCP1Tr, 0),815 CD(Ser4MCP1Rc, DDAR_RW),816 CD(Ser4SSPTr, 0),817 CD(Ser4SSPRc, DDAR_RW),818};819 820static const struct dma_slave_map sa11x0_dma_map[] = {821 { "sa11x0-ir", "tx", "Ser2ICPTr" },822 { "sa11x0-ir", "rx", "Ser2ICPRc" },823 { "sa11x0-ssp", "tx", "Ser4SSPTr" },824 { "sa11x0-ssp", "rx", "Ser4SSPRc" },825};826 827static bool sa11x0_dma_filter_fn(struct dma_chan *chan, void *param)828{829 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);830 const char *p = param;831 832 return !strcmp(c->name, p);833}834 835static int sa11x0_dma_init_dmadev(struct dma_device *dmadev,836 struct device *dev)837{838 unsigned i;839 840 INIT_LIST_HEAD(&dmadev->channels);841 dmadev->dev = dev;842 dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources;843 dmadev->device_config = sa11x0_dma_device_config;844 dmadev->device_pause = sa11x0_dma_device_pause;845 dmadev->device_resume = sa11x0_dma_device_resume;846 dmadev->device_terminate_all = sa11x0_dma_device_terminate_all;847 dmadev->device_tx_status = sa11x0_dma_tx_status;848 dmadev->device_issue_pending = sa11x0_dma_issue_pending;849 850 for (i = 0; i < ARRAY_SIZE(chan_desc); i++) {851 struct sa11x0_dma_chan *c;852 853 c = kzalloc(sizeof(*c), GFP_KERNEL);854 if (!c) {855 dev_err(dev, "no memory for channel %u\n", i);856 return -ENOMEM;857 }858 859 c->status = DMA_IN_PROGRESS;860 c->ddar = chan_desc[i].ddar;861 c->name = chan_desc[i].name;862 INIT_LIST_HEAD(&c->node);863 864 c->vc.desc_free = sa11x0_dma_free_desc;865 vchan_init(&c->vc, dmadev);866 }867 868 return dma_async_device_register(dmadev);869}870 871static int sa11x0_dma_request_irq(struct platform_device *pdev, int nr,872 void *data)873{874 int irq = platform_get_irq(pdev, nr);875 876 if (irq <= 0)877 return -ENXIO;878 879 return request_irq(irq, sa11x0_dma_irq, 0, dev_name(&pdev->dev), data);880}881 882static void sa11x0_dma_free_irq(struct platform_device *pdev, int nr,883 void *data)884{885 int irq = platform_get_irq(pdev, nr);886 if (irq > 0)887 free_irq(irq, data);888}889 890static void sa11x0_dma_free_channels(struct dma_device *dmadev)891{892 struct sa11x0_dma_chan *c, *cn;893 894 list_for_each_entry_safe(c, cn, &dmadev->channels, vc.chan.device_node) {895 list_del(&c->vc.chan.device_node);896 tasklet_kill(&c->vc.task);897 kfree(c);898 }899}900 901static int sa11x0_dma_probe(struct platform_device *pdev)902{903 struct sa11x0_dma_dev *d;904 struct resource *res;905 unsigned i;906 int ret;907 908 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);909 if (!res)910 return -ENXIO;911 912 d = kzalloc(sizeof(*d), GFP_KERNEL);913 if (!d) {914 ret = -ENOMEM;915 goto err_alloc;916 }917 918 spin_lock_init(&d->lock);919 INIT_LIST_HEAD(&d->chan_pending);920 921 d->slave.filter.fn = sa11x0_dma_filter_fn;922 d->slave.filter.mapcnt = ARRAY_SIZE(sa11x0_dma_map);923 d->slave.filter.map = sa11x0_dma_map;924 925 d->base = ioremap(res->start, resource_size(res));926 if (!d->base) {927 ret = -ENOMEM;928 goto err_ioremap;929 }930 931 tasklet_setup(&d->task, sa11x0_dma_tasklet);932 933 for (i = 0; i < NR_PHY_CHAN; i++) {934 struct sa11x0_dma_phy *p = &d->phy[i];935 936 p->dev = d;937 p->num = i;938 p->base = d->base + i * DMA_SIZE;939 writel_relaxed(DCSR_RUN | DCSR_IE | DCSR_ERROR |940 DCSR_DONEA | DCSR_STRTA | DCSR_DONEB | DCSR_STRTB,941 p->base + DMA_DCSR_C);942 writel_relaxed(0, p->base + DMA_DDAR);943 944 ret = sa11x0_dma_request_irq(pdev, i, p);945 if (ret) {946 while (i) {947 i--;948 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);949 }950 goto err_irq;951 }952 }953 954 dma_cap_set(DMA_SLAVE, d->slave.cap_mask);955 dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);956 d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg;957 d->slave.device_prep_dma_cyclic = sa11x0_dma_prep_dma_cyclic;958 d->slave.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);959 d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;960 d->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |961 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);962 d->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |963 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);964 ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev);965 if (ret) {966 dev_warn(d->slave.dev, "failed to register slave async device: %d\n",967 ret);968 goto err_slave_reg;969 }970 971 platform_set_drvdata(pdev, d);972 return 0;973 974 err_slave_reg:975 sa11x0_dma_free_channels(&d->slave);976 for (i = 0; i < NR_PHY_CHAN; i++)977 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);978 err_irq:979 tasklet_kill(&d->task);980 iounmap(d->base);981 err_ioremap:982 kfree(d);983 err_alloc:984 return ret;985}986 987static void sa11x0_dma_remove(struct platform_device *pdev)988{989 struct sa11x0_dma_dev *d = platform_get_drvdata(pdev);990 unsigned pch;991 992 dma_async_device_unregister(&d->slave);993 994 sa11x0_dma_free_channels(&d->slave);995 for (pch = 0; pch < NR_PHY_CHAN; pch++)996 sa11x0_dma_free_irq(pdev, pch, &d->phy[pch]);997 tasklet_kill(&d->task);998 iounmap(d->base);999 kfree(d);1000}1001 1002static __maybe_unused int sa11x0_dma_suspend(struct device *dev)1003{1004 struct sa11x0_dma_dev *d = dev_get_drvdata(dev);1005 unsigned pch;1006 1007 for (pch = 0; pch < NR_PHY_CHAN; pch++) {1008 struct sa11x0_dma_phy *p = &d->phy[pch];1009 u32 dcsr, saved_dcsr;1010 1011 dcsr = saved_dcsr = readl_relaxed(p->base + DMA_DCSR_R);1012 if (dcsr & DCSR_RUN) {1013 writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);1014 dcsr = readl_relaxed(p->base + DMA_DCSR_R);1015 }1016 1017 saved_dcsr &= DCSR_RUN | DCSR_IE;1018 if (dcsr & DCSR_BIU) {1019 p->dbs[0] = readl_relaxed(p->base + DMA_DBSB);1020 p->dbt[0] = readl_relaxed(p->base + DMA_DBTB);1021 p->dbs[1] = readl_relaxed(p->base + DMA_DBSA);1022 p->dbt[1] = readl_relaxed(p->base + DMA_DBTA);1023 saved_dcsr |= (dcsr & DCSR_STRTA ? DCSR_STRTB : 0) |1024 (dcsr & DCSR_STRTB ? DCSR_STRTA : 0);1025 } else {1026 p->dbs[0] = readl_relaxed(p->base + DMA_DBSA);1027 p->dbt[0] = readl_relaxed(p->base + DMA_DBTA);1028 p->dbs[1] = readl_relaxed(p->base + DMA_DBSB);1029 p->dbt[1] = readl_relaxed(p->base + DMA_DBTB);1030 saved_dcsr |= dcsr & (DCSR_STRTA | DCSR_STRTB);1031 }1032 p->dcsr = saved_dcsr;1033 1034 writel(DCSR_STRTA | DCSR_STRTB, p->base + DMA_DCSR_C);1035 }1036 1037 return 0;1038}1039 1040static __maybe_unused int sa11x0_dma_resume(struct device *dev)1041{1042 struct sa11x0_dma_dev *d = dev_get_drvdata(dev);1043 unsigned pch;1044 1045 for (pch = 0; pch < NR_PHY_CHAN; pch++) {1046 struct sa11x0_dma_phy *p = &d->phy[pch];1047 struct sa11x0_dma_desc *txd = NULL;1048 u32 dcsr = readl_relaxed(p->base + DMA_DCSR_R);1049 1050 WARN_ON(dcsr & (DCSR_BIU | DCSR_STRTA | DCSR_STRTB | DCSR_RUN));1051 1052 if (p->txd_done)1053 txd = p->txd_done;1054 else if (p->txd_load)1055 txd = p->txd_load;1056 1057 if (!txd)1058 continue;1059 1060 writel_relaxed(txd->ddar, p->base + DMA_DDAR);1061 1062 writel_relaxed(p->dbs[0], p->base + DMA_DBSA);1063 writel_relaxed(p->dbt[0], p->base + DMA_DBTA);1064 writel_relaxed(p->dbs[1], p->base + DMA_DBSB);1065 writel_relaxed(p->dbt[1], p->base + DMA_DBTB);1066 writel_relaxed(p->dcsr, p->base + DMA_DCSR_S);1067 }1068 1069 return 0;1070}1071 1072static const struct dev_pm_ops sa11x0_dma_pm_ops = {1073 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(sa11x0_dma_suspend, sa11x0_dma_resume)1074};1075 1076static struct platform_driver sa11x0_dma_driver = {1077 .driver = {1078 .name = "sa11x0-dma",1079 .pm = &sa11x0_dma_pm_ops,1080 },1081 .probe = sa11x0_dma_probe,1082 .remove_new = sa11x0_dma_remove,1083};1084 1085static int __init sa11x0_dma_init(void)1086{1087 return platform_driver_register(&sa11x0_dma_driver);1088}1089subsys_initcall(sa11x0_dma_init);1090 1091static void __exit sa11x0_dma_exit(void)1092{1093 platform_driver_unregister(&sa11x0_dma_driver);1094}1095module_exit(sa11x0_dma_exit);1096 1097MODULE_AUTHOR("Russell King");1098MODULE_DESCRIPTION("SA-11x0 DMA driver");1099MODULE_LICENSE("GPL v2");1100MODULE_ALIAS("platform:sa11x0-dma");1101