1045 lines · c
1// SPDX-License-Identifier: GPL-2.0+2/*3 * BCM2835 DMA engine support4 *5 * Author: Florian Meier <florian.meier@koalo.de>6 * Copyright 20137 *8 * Based on9 * OMAP DMAengine support by Russell King10 *11 * BCM2708 DMA Driver12 * Copyright (C) 2010 Broadcom13 *14 * Raspberry Pi PCM I2S ALSA Driver15 * Copyright (c) by Phil Poole 201316 *17 * MARVELL MMP Peripheral DMA Driver18 * Copyright 2012 Marvell International Ltd.19 */20#include <linux/dmaengine.h>21#include <linux/dma-mapping.h>22#include <linux/dmapool.h>23#include <linux/err.h>24#include <linux/init.h>25#include <linux/interrupt.h>26#include <linux/list.h>27#include <linux/module.h>28#include <linux/platform_device.h>29#include <linux/slab.h>30#include <linux/io.h>31#include <linux/spinlock.h>32#include <linux/of.h>33#include <linux/of_dma.h>34 35#include "virt-dma.h"36 37#define BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED 1438#define BCM2835_DMA_CHAN_NAME_SIZE 839 40/**41 * struct bcm2835_dmadev - BCM2835 DMA controller42 * @ddev: DMA device43 * @base: base address of register map44 * @zero_page: bus address of zero page (to detect transactions copying from45 * zero page and avoid accessing memory if so)46 */47struct bcm2835_dmadev {48 struct dma_device ddev;49 void __iomem *base;50 dma_addr_t zero_page;51};52 53struct bcm2835_dma_cb {54 uint32_t info;55 uint32_t src;56 uint32_t dst;57 uint32_t length;58 uint32_t stride;59 uint32_t next;60 uint32_t pad[2];61};62 63struct bcm2835_cb_entry {64 struct bcm2835_dma_cb *cb;65 dma_addr_t paddr;66};67 68struct bcm2835_chan {69 struct virt_dma_chan vc;70 71 struct dma_slave_config cfg;72 unsigned int dreq;73 74 int ch;75 struct bcm2835_desc *desc;76 struct dma_pool *cb_pool;77 78 void __iomem *chan_base;79 int irq_number;80 unsigned int irq_flags;81 82 bool is_lite_channel;83};84 85struct bcm2835_desc {86 struct bcm2835_chan *c;87 struct virt_dma_desc vd;88 enum dma_transfer_direction dir;89 90 unsigned int frames;91 size_t size;92 93 bool cyclic;94 95 struct bcm2835_cb_entry cb_list[];96};97 98#define BCM2835_DMA_CS 0x0099#define BCM2835_DMA_ADDR 0x04100#define BCM2835_DMA_TI 0x08101#define BCM2835_DMA_SOURCE_AD 0x0c102#define BCM2835_DMA_DEST_AD 0x10103#define BCM2835_DMA_LEN 0x14104#define BCM2835_DMA_STRIDE 0x18105#define BCM2835_DMA_NEXTCB 0x1c106#define BCM2835_DMA_DEBUG 0x20107 108/* DMA CS Control and Status bits */109#define BCM2835_DMA_ACTIVE BIT(0) /* activate the DMA */110#define BCM2835_DMA_END BIT(1) /* current CB has ended */111#define BCM2835_DMA_INT BIT(2) /* interrupt status */112#define BCM2835_DMA_DREQ BIT(3) /* DREQ state */113#define BCM2835_DMA_ISPAUSED BIT(4) /* Pause requested or not active */114#define BCM2835_DMA_ISHELD BIT(5) /* Is held by DREQ flow control */115#define BCM2835_DMA_WAITING_FOR_WRITES BIT(6) /* waiting for last116 * AXI-write to ack117 */118#define BCM2835_DMA_ERR BIT(8)119#define BCM2835_DMA_PRIORITY(x) ((x & 15) << 16) /* AXI priority */120#define BCM2835_DMA_PANIC_PRIORITY(x) ((x & 15) << 20) /* panic priority */121/* current value of TI.BCM2835_DMA_WAIT_RESP */122#define BCM2835_DMA_WAIT_FOR_WRITES BIT(28)123#define BCM2835_DMA_DIS_DEBUG BIT(29) /* disable debug pause signal */124#define BCM2835_DMA_ABORT BIT(30) /* Stop current CB, go to next, WO */125#define BCM2835_DMA_RESET BIT(31) /* WO, self clearing */126 127/* Transfer information bits - also bcm2835_cb.info field */128#define BCM2835_DMA_INT_EN BIT(0)129#define BCM2835_DMA_TDMODE BIT(1) /* 2D-Mode */130#define BCM2835_DMA_WAIT_RESP BIT(3) /* wait for AXI-write to be acked */131#define BCM2835_DMA_D_INC BIT(4)132#define BCM2835_DMA_D_WIDTH BIT(5) /* 128bit writes if set */133#define BCM2835_DMA_D_DREQ BIT(6) /* enable DREQ for destination */134#define BCM2835_DMA_D_IGNORE BIT(7) /* ignore destination writes */135#define BCM2835_DMA_S_INC BIT(8)136#define BCM2835_DMA_S_WIDTH BIT(9) /* 128bit writes if set */137#define BCM2835_DMA_S_DREQ BIT(10) /* enable SREQ for source */138#define BCM2835_DMA_S_IGNORE BIT(11) /* ignore source reads - read 0 */139#define BCM2835_DMA_BURST_LENGTH(x) ((x & 15) << 12)140#define BCM2835_DMA_PER_MAP(x) ((x & 31) << 16) /* REQ source */141#define BCM2835_DMA_WAIT(x) ((x & 31) << 21) /* add DMA-wait cycles */142#define BCM2835_DMA_NO_WIDE_BURSTS BIT(26) /* no 2 beat write bursts */143 144/* debug register bits */145#define BCM2835_DMA_DEBUG_LAST_NOT_SET_ERR BIT(0)146#define BCM2835_DMA_DEBUG_FIFO_ERR BIT(1)147#define BCM2835_DMA_DEBUG_READ_ERR BIT(2)148#define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_SHIFT 4149#define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_BITS 4150#define BCM2835_DMA_DEBUG_ID_SHIFT 16151#define BCM2835_DMA_DEBUG_ID_BITS 9152#define BCM2835_DMA_DEBUG_STATE_SHIFT 16153#define BCM2835_DMA_DEBUG_STATE_BITS 9154#define BCM2835_DMA_DEBUG_VERSION_SHIFT 25155#define BCM2835_DMA_DEBUG_VERSION_BITS 3156#define BCM2835_DMA_DEBUG_LITE BIT(28)157 158/* shared registers for all dma channels */159#define BCM2835_DMA_INT_STATUS 0xfe0160#define BCM2835_DMA_ENABLE 0xff0161 162#define BCM2835_DMA_DATA_TYPE_S8 1163#define BCM2835_DMA_DATA_TYPE_S16 2164#define BCM2835_DMA_DATA_TYPE_S32 4165#define BCM2835_DMA_DATA_TYPE_S128 16166 167/* Valid only for channels 0 - 14, 15 has its own base address */168#define BCM2835_DMA_CHAN(n) ((n) << 8) /* Base address */169#define BCM2835_DMA_CHANIO(base, n) ((base) + BCM2835_DMA_CHAN(n))170 171/* the max dma length for different channels */172#define MAX_DMA_LEN SZ_1G173#define MAX_LITE_DMA_LEN (SZ_64K - 4)174 175static inline size_t bcm2835_dma_max_frame_length(struct bcm2835_chan *c)176{177 /* lite and normal channels have different max frame length */178 return c->is_lite_channel ? MAX_LITE_DMA_LEN : MAX_DMA_LEN;179}180 181/* how many frames of max_len size do we need to transfer len bytes */182static inline size_t bcm2835_dma_frames_for_length(size_t len,183 size_t max_len)184{185 return DIV_ROUND_UP(len, max_len);186}187 188static inline struct bcm2835_dmadev *to_bcm2835_dma_dev(struct dma_device *d)189{190 return container_of(d, struct bcm2835_dmadev, ddev);191}192 193static inline struct bcm2835_chan *to_bcm2835_dma_chan(struct dma_chan *c)194{195 return container_of(c, struct bcm2835_chan, vc.chan);196}197 198static inline struct bcm2835_desc *to_bcm2835_dma_desc(199 struct dma_async_tx_descriptor *t)200{201 return container_of(t, struct bcm2835_desc, vd.tx);202}203 204static void bcm2835_dma_free_cb_chain(struct bcm2835_desc *desc)205{206 size_t i;207 208 for (i = 0; i < desc->frames; i++)209 dma_pool_free(desc->c->cb_pool, desc->cb_list[i].cb,210 desc->cb_list[i].paddr);211 212 kfree(desc);213}214 215static void bcm2835_dma_desc_free(struct virt_dma_desc *vd)216{217 bcm2835_dma_free_cb_chain(218 container_of(vd, struct bcm2835_desc, vd));219}220 221static void bcm2835_dma_create_cb_set_length(222 struct bcm2835_chan *chan,223 struct bcm2835_dma_cb *control_block,224 size_t len,225 size_t period_len,226 size_t *total_len,227 u32 finalextrainfo)228{229 size_t max_len = bcm2835_dma_max_frame_length(chan);230 231 /* set the length taking lite-channel limitations into account */232 control_block->length = min_t(u32, len, max_len);233 234 /* finished if we have no period_length */235 if (!period_len)236 return;237 238 /*239 * period_len means: that we need to generate240 * transfers that are terminating at every241 * multiple of period_len - this is typically242 * used to set the interrupt flag in info243 * which is required during cyclic transfers244 */245 246 /* have we filled in period_length yet? */247 if (*total_len + control_block->length < period_len) {248 /* update number of bytes in this period so far */249 *total_len += control_block->length;250 return;251 }252 253 /* calculate the length that remains to reach period_length */254 control_block->length = period_len - *total_len;255 256 /* reset total_length for next period */257 *total_len = 0;258 259 /* add extrainfo bits in info */260 control_block->info |= finalextrainfo;261}262 263static inline size_t bcm2835_dma_count_frames_for_sg(264 struct bcm2835_chan *c,265 struct scatterlist *sgl,266 unsigned int sg_len)267{268 size_t frames = 0;269 struct scatterlist *sgent;270 unsigned int i;271 size_t plength = bcm2835_dma_max_frame_length(c);272 273 for_each_sg(sgl, sgent, sg_len, i)274 frames += bcm2835_dma_frames_for_length(275 sg_dma_len(sgent), plength);276 277 return frames;278}279 280/**281 * bcm2835_dma_create_cb_chain - create a control block and fills data in282 *283 * @chan: the @dma_chan for which we run this284 * @direction: the direction in which we transfer285 * @cyclic: it is a cyclic transfer286 * @info: the default info bits to apply per controlblock287 * @frames: number of controlblocks to allocate288 * @src: the src address to assign (if the S_INC bit is set289 * in @info, then it gets incremented)290 * @dst: the dst address to assign (if the D_INC bit is set291 * in @info, then it gets incremented)292 * @buf_len: the full buffer length (may also be 0)293 * @period_len: the period length when to apply @finalextrainfo294 * in addition to the last transfer295 * this will also break some control-blocks early296 * @finalextrainfo: additional bits in last controlblock297 * (or when period_len is reached in case of cyclic)298 * @gfp: the GFP flag to use for allocation299 */300static struct bcm2835_desc *bcm2835_dma_create_cb_chain(301 struct dma_chan *chan, enum dma_transfer_direction direction,302 bool cyclic, u32 info, u32 finalextrainfo, size_t frames,303 dma_addr_t src, dma_addr_t dst, size_t buf_len,304 size_t period_len, gfp_t gfp)305{306 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);307 size_t len = buf_len, total_len;308 size_t frame;309 struct bcm2835_desc *d;310 struct bcm2835_cb_entry *cb_entry;311 struct bcm2835_dma_cb *control_block;312 313 if (!frames)314 return NULL;315 316 /* allocate and setup the descriptor. */317 d = kzalloc(struct_size(d, cb_list, frames), gfp);318 if (!d)319 return NULL;320 321 d->c = c;322 d->dir = direction;323 d->cyclic = cyclic;324 325 /*326 * Iterate over all frames, create a control block327 * for each frame and link them together.328 */329 for (frame = 0, total_len = 0; frame < frames; d->frames++, frame++) {330 cb_entry = &d->cb_list[frame];331 cb_entry->cb = dma_pool_alloc(c->cb_pool, gfp,332 &cb_entry->paddr);333 if (!cb_entry->cb)334 goto error_cb;335 336 /* fill in the control block */337 control_block = cb_entry->cb;338 control_block->info = info;339 control_block->src = src;340 control_block->dst = dst;341 control_block->stride = 0;342 control_block->next = 0;343 /* set up length in control_block if requested */344 if (buf_len) {345 /* calculate length honoring period_length */346 bcm2835_dma_create_cb_set_length(347 c, control_block,348 len, period_len, &total_len,349 cyclic ? finalextrainfo : 0);350 351 /* calculate new remaining length */352 len -= control_block->length;353 }354 355 /* link this the last controlblock */356 if (frame)357 d->cb_list[frame - 1].cb->next = cb_entry->paddr;358 359 /* update src and dst and length */360 if (src && (info & BCM2835_DMA_S_INC))361 src += control_block->length;362 if (dst && (info & BCM2835_DMA_D_INC))363 dst += control_block->length;364 365 /* Length of total transfer */366 d->size += control_block->length;367 }368 369 /* the last frame requires extra flags */370 d->cb_list[d->frames - 1].cb->info |= finalextrainfo;371 372 /* detect a size mismatch */373 if (buf_len && (d->size != buf_len))374 goto error_cb;375 376 return d;377error_cb:378 bcm2835_dma_free_cb_chain(d);379 380 return NULL;381}382 383static void bcm2835_dma_fill_cb_chain_with_sg(384 struct dma_chan *chan,385 enum dma_transfer_direction direction,386 struct bcm2835_cb_entry *cb,387 struct scatterlist *sgl,388 unsigned int sg_len)389{390 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);391 size_t len, max_len;392 unsigned int i;393 dma_addr_t addr;394 struct scatterlist *sgent;395 396 max_len = bcm2835_dma_max_frame_length(c);397 for_each_sg(sgl, sgent, sg_len, i) {398 for (addr = sg_dma_address(sgent), len = sg_dma_len(sgent);399 len > 0;400 addr += cb->cb->length, len -= cb->cb->length, cb++) {401 if (direction == DMA_DEV_TO_MEM)402 cb->cb->dst = addr;403 else404 cb->cb->src = addr;405 cb->cb->length = min(len, max_len);406 }407 }408}409 410static void bcm2835_dma_abort(struct bcm2835_chan *c)411{412 void __iomem *chan_base = c->chan_base;413 long int timeout = 10000;414 415 /*416 * A zero control block address means the channel is idle.417 * (The ACTIVE flag in the CS register is not a reliable indicator.)418 */419 if (!readl(chan_base + BCM2835_DMA_ADDR))420 return;421 422 /* Write 0 to the active bit - Pause the DMA */423 writel(0, chan_base + BCM2835_DMA_CS);424 425 /* Wait for any current AXI transfer to complete */426 while ((readl(chan_base + BCM2835_DMA_CS) &427 BCM2835_DMA_WAITING_FOR_WRITES) && --timeout)428 cpu_relax();429 430 /* Peripheral might be stuck and fail to signal AXI write responses */431 if (!timeout)432 dev_err(c->vc.chan.device->dev,433 "failed to complete outstanding writes\n");434 435 writel(BCM2835_DMA_RESET, chan_base + BCM2835_DMA_CS);436}437 438static void bcm2835_dma_start_desc(struct bcm2835_chan *c)439{440 struct virt_dma_desc *vd = vchan_next_desc(&c->vc);441 struct bcm2835_desc *d;442 443 if (!vd) {444 c->desc = NULL;445 return;446 }447 448 list_del(&vd->node);449 450 c->desc = d = to_bcm2835_dma_desc(&vd->tx);451 452 writel(d->cb_list[0].paddr, c->chan_base + BCM2835_DMA_ADDR);453 writel(BCM2835_DMA_ACTIVE, c->chan_base + BCM2835_DMA_CS);454}455 456static irqreturn_t bcm2835_dma_callback(int irq, void *data)457{458 struct bcm2835_chan *c = data;459 struct bcm2835_desc *d;460 unsigned long flags;461 462 /* check the shared interrupt */463 if (c->irq_flags & IRQF_SHARED) {464 /* check if the interrupt is enabled */465 flags = readl(c->chan_base + BCM2835_DMA_CS);466 /* if not set then we are not the reason for the irq */467 if (!(flags & BCM2835_DMA_INT))468 return IRQ_NONE;469 }470 471 spin_lock_irqsave(&c->vc.lock, flags);472 473 /*474 * Clear the INT flag to receive further interrupts. Keep the channel475 * active in case the descriptor is cyclic or in case the client has476 * already terminated the descriptor and issued a new one. (May happen477 * if this IRQ handler is threaded.) If the channel is finished, it478 * will remain idle despite the ACTIVE flag being set.479 */480 writel(BCM2835_DMA_INT | BCM2835_DMA_ACTIVE,481 c->chan_base + BCM2835_DMA_CS);482 483 d = c->desc;484 485 if (d) {486 if (d->cyclic) {487 /* call the cyclic callback */488 vchan_cyclic_callback(&d->vd);489 } else if (!readl(c->chan_base + BCM2835_DMA_ADDR)) {490 vchan_cookie_complete(&c->desc->vd);491 bcm2835_dma_start_desc(c);492 }493 }494 495 spin_unlock_irqrestore(&c->vc.lock, flags);496 497 return IRQ_HANDLED;498}499 500static int bcm2835_dma_alloc_chan_resources(struct dma_chan *chan)501{502 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);503 struct device *dev = c->vc.chan.device->dev;504 505 dev_dbg(dev, "Allocating DMA channel %d\n", c->ch);506 507 /*508 * Control blocks are 256 bit in length and must start at a 256 bit509 * (32 byte) aligned address (BCM2835 ARM Peripherals, sec. 4.2.1.1).510 */511 c->cb_pool = dma_pool_create(dev_name(dev), dev,512 sizeof(struct bcm2835_dma_cb), 32, 0);513 if (!c->cb_pool) {514 dev_err(dev, "unable to allocate descriptor pool\n");515 return -ENOMEM;516 }517 518 return request_irq(c->irq_number, bcm2835_dma_callback,519 c->irq_flags, "DMA IRQ", c);520}521 522static void bcm2835_dma_free_chan_resources(struct dma_chan *chan)523{524 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);525 526 vchan_free_chan_resources(&c->vc);527 free_irq(c->irq_number, c);528 dma_pool_destroy(c->cb_pool);529 530 dev_dbg(c->vc.chan.device->dev, "Freeing DMA channel %u\n", c->ch);531}532 533static size_t bcm2835_dma_desc_size(struct bcm2835_desc *d)534{535 return d->size;536}537 538static size_t bcm2835_dma_desc_size_pos(struct bcm2835_desc *d, dma_addr_t addr)539{540 unsigned int i;541 size_t size;542 543 for (size = i = 0; i < d->frames; i++) {544 struct bcm2835_dma_cb *control_block = d->cb_list[i].cb;545 size_t this_size = control_block->length;546 dma_addr_t dma;547 548 if (d->dir == DMA_DEV_TO_MEM)549 dma = control_block->dst;550 else551 dma = control_block->src;552 553 if (size)554 size += this_size;555 else if (addr >= dma && addr < dma + this_size)556 size += dma + this_size - addr;557 }558 559 return size;560}561 562static enum dma_status bcm2835_dma_tx_status(struct dma_chan *chan,563 dma_cookie_t cookie, struct dma_tx_state *txstate)564{565 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);566 struct virt_dma_desc *vd;567 enum dma_status ret;568 unsigned long flags;569 570 ret = dma_cookie_status(chan, cookie, txstate);571 if (ret == DMA_COMPLETE || !txstate)572 return ret;573 574 spin_lock_irqsave(&c->vc.lock, flags);575 vd = vchan_find_desc(&c->vc, cookie);576 if (vd) {577 txstate->residue =578 bcm2835_dma_desc_size(to_bcm2835_dma_desc(&vd->tx));579 } else if (c->desc && c->desc->vd.tx.cookie == cookie) {580 struct bcm2835_desc *d = c->desc;581 dma_addr_t pos;582 583 if (d->dir == DMA_MEM_TO_DEV)584 pos = readl(c->chan_base + BCM2835_DMA_SOURCE_AD);585 else if (d->dir == DMA_DEV_TO_MEM)586 pos = readl(c->chan_base + BCM2835_DMA_DEST_AD);587 else588 pos = 0;589 590 txstate->residue = bcm2835_dma_desc_size_pos(d, pos);591 } else {592 txstate->residue = 0;593 }594 595 spin_unlock_irqrestore(&c->vc.lock, flags);596 597 return ret;598}599 600static void bcm2835_dma_issue_pending(struct dma_chan *chan)601{602 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);603 unsigned long flags;604 605 spin_lock_irqsave(&c->vc.lock, flags);606 if (vchan_issue_pending(&c->vc) && !c->desc)607 bcm2835_dma_start_desc(c);608 609 spin_unlock_irqrestore(&c->vc.lock, flags);610}611 612static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_memcpy(613 struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,614 size_t len, unsigned long flags)615{616 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);617 struct bcm2835_desc *d;618 u32 info = BCM2835_DMA_D_INC | BCM2835_DMA_S_INC;619 u32 extra = BCM2835_DMA_INT_EN | BCM2835_DMA_WAIT_RESP;620 size_t max_len = bcm2835_dma_max_frame_length(c);621 size_t frames;622 623 /* if src, dst or len is not given return with an error */624 if (!src || !dst || !len)625 return NULL;626 627 /* calculate number of frames */628 frames = bcm2835_dma_frames_for_length(len, max_len);629 630 /* allocate the CB chain - this also fills in the pointers */631 d = bcm2835_dma_create_cb_chain(chan, DMA_MEM_TO_MEM, false,632 info, extra, frames,633 src, dst, len, 0, GFP_KERNEL);634 if (!d)635 return NULL;636 637 return vchan_tx_prep(&c->vc, &d->vd, flags);638}639 640static struct dma_async_tx_descriptor *bcm2835_dma_prep_slave_sg(641 struct dma_chan *chan,642 struct scatterlist *sgl, unsigned int sg_len,643 enum dma_transfer_direction direction,644 unsigned long flags, void *context)645{646 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);647 struct bcm2835_desc *d;648 dma_addr_t src = 0, dst = 0;649 u32 info = BCM2835_DMA_WAIT_RESP;650 u32 extra = BCM2835_DMA_INT_EN;651 size_t frames;652 653 if (!is_slave_direction(direction)) {654 dev_err(chan->device->dev,655 "%s: bad direction?\n", __func__);656 return NULL;657 }658 659 if (c->dreq != 0)660 info |= BCM2835_DMA_PER_MAP(c->dreq);661 662 if (direction == DMA_DEV_TO_MEM) {663 if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)664 return NULL;665 src = c->cfg.src_addr;666 info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC;667 } else {668 if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)669 return NULL;670 dst = c->cfg.dst_addr;671 info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC;672 }673 674 /* count frames in sg list */675 frames = bcm2835_dma_count_frames_for_sg(c, sgl, sg_len);676 677 /* allocate the CB chain */678 d = bcm2835_dma_create_cb_chain(chan, direction, false,679 info, extra,680 frames, src, dst, 0, 0,681 GFP_NOWAIT);682 if (!d)683 return NULL;684 685 /* fill in frames with scatterlist pointers */686 bcm2835_dma_fill_cb_chain_with_sg(chan, direction, d->cb_list,687 sgl, sg_len);688 689 return vchan_tx_prep(&c->vc, &d->vd, flags);690}691 692static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic(693 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,694 size_t period_len, enum dma_transfer_direction direction,695 unsigned long flags)696{697 struct bcm2835_dmadev *od = to_bcm2835_dma_dev(chan->device);698 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);699 struct bcm2835_desc *d;700 dma_addr_t src, dst;701 u32 info = BCM2835_DMA_WAIT_RESP;702 u32 extra = 0;703 size_t max_len = bcm2835_dma_max_frame_length(c);704 size_t frames;705 706 /* Grab configuration */707 if (!is_slave_direction(direction)) {708 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);709 return NULL;710 }711 712 if (!buf_len) {713 dev_err(chan->device->dev,714 "%s: bad buffer length (= 0)\n", __func__);715 return NULL;716 }717 718 if (flags & DMA_PREP_INTERRUPT)719 extra |= BCM2835_DMA_INT_EN;720 else721 period_len = buf_len;722 723 /*724 * warn if buf_len is not a multiple of period_len - this may leed725 * to unexpected latencies for interrupts and thus audiable clicks726 */727 if (buf_len % period_len)728 dev_warn_once(chan->device->dev,729 "%s: buffer_length (%zd) is not a multiple of period_len (%zd)\n",730 __func__, buf_len, period_len);731 732 /* Setup DREQ channel */733 if (c->dreq != 0)734 info |= BCM2835_DMA_PER_MAP(c->dreq);735 736 if (direction == DMA_DEV_TO_MEM) {737 if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)738 return NULL;739 src = c->cfg.src_addr;740 dst = buf_addr;741 info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC;742 } else {743 if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)744 return NULL;745 dst = c->cfg.dst_addr;746 src = buf_addr;747 info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC;748 749 /* non-lite channels can write zeroes w/o accessing memory */750 if (buf_addr == od->zero_page && !c->is_lite_channel)751 info |= BCM2835_DMA_S_IGNORE;752 }753 754 /* calculate number of frames */755 frames = /* number of periods */756 DIV_ROUND_UP(buf_len, period_len) *757 /* number of frames per period */758 bcm2835_dma_frames_for_length(period_len, max_len);759 760 /*761 * allocate the CB chain762 * note that we need to use GFP_NOWAIT, as the ALSA i2s dmaengine763 * implementation calls prep_dma_cyclic with interrupts disabled.764 */765 d = bcm2835_dma_create_cb_chain(chan, direction, true,766 info, extra,767 frames, src, dst, buf_len,768 period_len, GFP_NOWAIT);769 if (!d)770 return NULL;771 772 /* wrap around into a loop */773 d->cb_list[d->frames - 1].cb->next = d->cb_list[0].paddr;774 775 return vchan_tx_prep(&c->vc, &d->vd, flags);776}777 778static int bcm2835_dma_slave_config(struct dma_chan *chan,779 struct dma_slave_config *cfg)780{781 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);782 783 c->cfg = *cfg;784 785 return 0;786}787 788static int bcm2835_dma_terminate_all(struct dma_chan *chan)789{790 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);791 unsigned long flags;792 LIST_HEAD(head);793 794 spin_lock_irqsave(&c->vc.lock, flags);795 796 /* stop DMA activity */797 if (c->desc) {798 vchan_terminate_vdesc(&c->desc->vd);799 c->desc = NULL;800 bcm2835_dma_abort(c);801 }802 803 vchan_get_all_descriptors(&c->vc, &head);804 spin_unlock_irqrestore(&c->vc.lock, flags);805 vchan_dma_desc_free_list(&c->vc, &head);806 807 return 0;808}809 810static void bcm2835_dma_synchronize(struct dma_chan *chan)811{812 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);813 814 vchan_synchronize(&c->vc);815}816 817static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id,818 int irq, unsigned int irq_flags)819{820 struct bcm2835_chan *c;821 822 c = devm_kzalloc(d->ddev.dev, sizeof(*c), GFP_KERNEL);823 if (!c)824 return -ENOMEM;825 826 c->vc.desc_free = bcm2835_dma_desc_free;827 vchan_init(&c->vc, &d->ddev);828 829 c->chan_base = BCM2835_DMA_CHANIO(d->base, chan_id);830 c->ch = chan_id;831 c->irq_number = irq;832 c->irq_flags = irq_flags;833 834 /* check in DEBUG register if this is a LITE channel */835 if (readl(c->chan_base + BCM2835_DMA_DEBUG) &836 BCM2835_DMA_DEBUG_LITE)837 c->is_lite_channel = true;838 839 return 0;840}841 842static void bcm2835_dma_free(struct bcm2835_dmadev *od)843{844 struct bcm2835_chan *c, *next;845 846 list_for_each_entry_safe(c, next, &od->ddev.channels,847 vc.chan.device_node) {848 list_del(&c->vc.chan.device_node);849 tasklet_kill(&c->vc.task);850 }851 852 dma_unmap_page_attrs(od->ddev.dev, od->zero_page, PAGE_SIZE,853 DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);854}855 856static const struct of_device_id bcm2835_dma_of_match[] = {857 { .compatible = "brcm,bcm2835-dma", },858 {},859};860MODULE_DEVICE_TABLE(of, bcm2835_dma_of_match);861 862static struct dma_chan *bcm2835_dma_xlate(struct of_phandle_args *spec,863 struct of_dma *ofdma)864{865 struct bcm2835_dmadev *d = ofdma->of_dma_data;866 struct dma_chan *chan;867 868 chan = dma_get_any_slave_channel(&d->ddev);869 if (!chan)870 return NULL;871 872 /* Set DREQ from param */873 to_bcm2835_dma_chan(chan)->dreq = spec->args[0];874 875 return chan;876}877 878static int bcm2835_dma_probe(struct platform_device *pdev)879{880 struct bcm2835_dmadev *od;881 void __iomem *base;882 int rc;883 int i, j;884 int irq[BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED + 1];885 int irq_flags;886 uint32_t chans_available;887 char chan_name[BCM2835_DMA_CHAN_NAME_SIZE];888 889 if (!pdev->dev.dma_mask)890 pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask;891 892 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));893 if (rc) {894 dev_err(&pdev->dev, "Unable to set DMA mask\n");895 return rc;896 }897 898 od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);899 if (!od)900 return -ENOMEM;901 902 dma_set_max_seg_size(&pdev->dev, 0x3FFFFFFF);903 904 base = devm_platform_ioremap_resource(pdev, 0);905 if (IS_ERR(base))906 return PTR_ERR(base);907 908 od->base = base;909 910 dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);911 dma_cap_set(DMA_PRIVATE, od->ddev.cap_mask);912 dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);913 dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);914 od->ddev.device_alloc_chan_resources = bcm2835_dma_alloc_chan_resources;915 od->ddev.device_free_chan_resources = bcm2835_dma_free_chan_resources;916 od->ddev.device_tx_status = bcm2835_dma_tx_status;917 od->ddev.device_issue_pending = bcm2835_dma_issue_pending;918 od->ddev.device_prep_dma_cyclic = bcm2835_dma_prep_dma_cyclic;919 od->ddev.device_prep_slave_sg = bcm2835_dma_prep_slave_sg;920 od->ddev.device_prep_dma_memcpy = bcm2835_dma_prep_dma_memcpy;921 od->ddev.device_config = bcm2835_dma_slave_config;922 od->ddev.device_terminate_all = bcm2835_dma_terminate_all;923 od->ddev.device_synchronize = bcm2835_dma_synchronize;924 od->ddev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);925 od->ddev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);926 od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |927 BIT(DMA_MEM_TO_MEM);928 od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;929 od->ddev.descriptor_reuse = true;930 od->ddev.dev = &pdev->dev;931 INIT_LIST_HEAD(&od->ddev.channels);932 933 platform_set_drvdata(pdev, od);934 935 od->zero_page = dma_map_page_attrs(od->ddev.dev, ZERO_PAGE(0), 0,936 PAGE_SIZE, DMA_TO_DEVICE,937 DMA_ATTR_SKIP_CPU_SYNC);938 if (dma_mapping_error(od->ddev.dev, od->zero_page)) {939 dev_err(&pdev->dev, "Failed to map zero page\n");940 return -ENOMEM;941 }942 943 /* Request DMA channel mask from device tree */944 if (of_property_read_u32(pdev->dev.of_node,945 "brcm,dma-channel-mask",946 &chans_available)) {947 dev_err(&pdev->dev, "Failed to get channel mask\n");948 rc = -EINVAL;949 goto err_no_dma;950 }951 952 /* get irqs for each channel that we support */953 for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) {954 /* skip masked out channels */955 if (!(chans_available & (1 << i))) {956 irq[i] = -1;957 continue;958 }959 960 /* get the named irq */961 snprintf(chan_name, sizeof(chan_name), "dma%i", i);962 irq[i] = platform_get_irq_byname(pdev, chan_name);963 if (irq[i] >= 0)964 continue;965 966 /* legacy device tree case handling */967 dev_warn_once(&pdev->dev,968 "missing interrupt-names property in device tree - legacy interpretation is used\n");969 /*970 * in case of channel >= 11971 * use the 11th interrupt and that is shared972 */973 irq[i] = platform_get_irq(pdev, i < 11 ? i : 11);974 }975 976 /* get irqs for each channel */977 for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) {978 /* skip channels without irq */979 if (irq[i] < 0)980 continue;981 982 /* check if there are other channels that also use this irq */983 irq_flags = 0;984 for (j = 0; j <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; j++)985 if ((i != j) && (irq[j] == irq[i])) {986 irq_flags = IRQF_SHARED;987 break;988 }989 990 /* initialize the channel */991 rc = bcm2835_dma_chan_init(od, i, irq[i], irq_flags);992 if (rc)993 goto err_no_dma;994 }995 996 dev_dbg(&pdev->dev, "Initialized %i DMA channels\n", i);997 998 /* Device-tree DMA controller registration */999 rc = of_dma_controller_register(pdev->dev.of_node,1000 bcm2835_dma_xlate, od);1001 if (rc) {1002 dev_err(&pdev->dev, "Failed to register DMA controller\n");1003 goto err_no_dma;1004 }1005 1006 rc = dma_async_device_register(&od->ddev);1007 if (rc) {1008 dev_err(&pdev->dev,1009 "Failed to register slave DMA engine device: %d\n", rc);1010 goto err_no_dma;1011 }1012 1013 dev_dbg(&pdev->dev, "Load BCM2835 DMA engine driver\n");1014 1015 return 0;1016 1017err_no_dma:1018 bcm2835_dma_free(od);1019 return rc;1020}1021 1022static void bcm2835_dma_remove(struct platform_device *pdev)1023{1024 struct bcm2835_dmadev *od = platform_get_drvdata(pdev);1025 1026 dma_async_device_unregister(&od->ddev);1027 bcm2835_dma_free(od);1028}1029 1030static struct platform_driver bcm2835_dma_driver = {1031 .probe = bcm2835_dma_probe,1032 .remove_new = bcm2835_dma_remove,1033 .driver = {1034 .name = "bcm2835-dma",1035 .of_match_table = of_match_ptr(bcm2835_dma_of_match),1036 },1037};1038 1039module_platform_driver(bcm2835_dma_driver);1040 1041MODULE_ALIAS("platform:bcm2835-dma");1042MODULE_DESCRIPTION("BCM2835 DMA engine driver");1043MODULE_AUTHOR("Florian Meier <florian.meier@koalo.de>");1044MODULE_LICENSE("GPL");1045