669 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * MOXA ART SoCs DMA Engine support.4 *5 * Copyright (C) 2013 Jonas Jensen6 *7 * Jonas Jensen <jonas.jensen@gmail.com>8 */9 10#include <linux/dmaengine.h>11#include <linux/dma-mapping.h>12#include <linux/err.h>13#include <linux/init.h>14#include <linux/interrupt.h>15#include <linux/list.h>16#include <linux/module.h>17#include <linux/platform_device.h>18#include <linux/slab.h>19#include <linux/spinlock.h>20#include <linux/of_address.h>21#include <linux/of_irq.h>22#include <linux/of_dma.h>23#include <linux/bitops.h>24 25#include <asm/cacheflush.h>26 27#include "dmaengine.h"28#include "virt-dma.h"29 30#define APB_DMA_MAX_CHANNEL 431 32#define REG_OFF_ADDRESS_SOURCE 033#define REG_OFF_ADDRESS_DEST 434#define REG_OFF_CYCLES 835#define REG_OFF_CTRL 1236#define REG_OFF_CHAN_SIZE 1637 38#define APB_DMA_ENABLE BIT(0)39#define APB_DMA_FIN_INT_STS BIT(1)40#define APB_DMA_FIN_INT_EN BIT(2)41#define APB_DMA_BURST_MODE BIT(3)42#define APB_DMA_ERR_INT_STS BIT(4)43#define APB_DMA_ERR_INT_EN BIT(5)44 45/*46 * Unset: APB47 * Set: AHB48 */49#define APB_DMA_SOURCE_SELECT 0x4050#define APB_DMA_DEST_SELECT 0x8051 52#define APB_DMA_SOURCE 0x10053#define APB_DMA_DEST 0x100054 55#define APB_DMA_SOURCE_MASK 0x70056#define APB_DMA_DEST_MASK 0x700057 58/*59 * 000: No increment60 * 001: +1 (Burst=0), +4 (Burst=1)61 * 010: +2 (Burst=0), +8 (Burst=1)62 * 011: +4 (Burst=0), +16 (Burst=1)63 * 101: -1 (Burst=0), -4 (Burst=1)64 * 110: -2 (Burst=0), -8 (Burst=1)65 * 111: -4 (Burst=0), -16 (Burst=1)66 */67#define APB_DMA_SOURCE_INC_0 068#define APB_DMA_SOURCE_INC_1_4 0x10069#define APB_DMA_SOURCE_INC_2_8 0x20070#define APB_DMA_SOURCE_INC_4_16 0x30071#define APB_DMA_SOURCE_DEC_1_4 0x50072#define APB_DMA_SOURCE_DEC_2_8 0x60073#define APB_DMA_SOURCE_DEC_4_16 0x70074#define APB_DMA_DEST_INC_0 075#define APB_DMA_DEST_INC_1_4 0x100076#define APB_DMA_DEST_INC_2_8 0x200077#define APB_DMA_DEST_INC_4_16 0x300078#define APB_DMA_DEST_DEC_1_4 0x500079#define APB_DMA_DEST_DEC_2_8 0x600080#define APB_DMA_DEST_DEC_4_16 0x700081 82/*83 * Request signal select source/destination address for DMA hardware handshake.84 *85 * The request line number is a property of the DMA controller itself,86 * e.g. MMC must always request channels where dma_slave_config->slave_id is 5.87 *88 * 0: No request / Grant signal89 * 1-15: Request / Grant signal90 */91#define APB_DMA_SOURCE_REQ_NO 0x100000092#define APB_DMA_SOURCE_REQ_NO_MASK 0xf00000093#define APB_DMA_DEST_REQ_NO 0x1000094#define APB_DMA_DEST_REQ_NO_MASK 0xf000095 96#define APB_DMA_DATA_WIDTH 0x10000097#define APB_DMA_DATA_WIDTH_MASK 0x30000098/*99 * Data width of transfer:100 *101 * 00: Word102 * 01: Half103 * 10: Byte104 */105#define APB_DMA_DATA_WIDTH_4 0106#define APB_DMA_DATA_WIDTH_2 0x100000107#define APB_DMA_DATA_WIDTH_1 0x200000108 109#define APB_DMA_CYCLES_MASK 0x00ffffff110 111#define MOXART_DMA_DATA_TYPE_S8 0x00112#define MOXART_DMA_DATA_TYPE_S16 0x01113#define MOXART_DMA_DATA_TYPE_S32 0x02114 115struct moxart_sg {116 dma_addr_t addr;117 uint32_t len;118};119 120struct moxart_desc {121 enum dma_transfer_direction dma_dir;122 dma_addr_t dev_addr;123 unsigned int sglen;124 unsigned int dma_cycles;125 struct virt_dma_desc vd;126 uint8_t es;127 struct moxart_sg sg[] __counted_by(sglen);128};129 130struct moxart_chan {131 struct virt_dma_chan vc;132 133 void __iomem *base;134 struct moxart_desc *desc;135 136 struct dma_slave_config cfg;137 138 bool allocated;139 bool error;140 int ch_num;141 unsigned int line_reqno;142 unsigned int sgidx;143};144 145struct moxart_dmadev {146 struct dma_device dma_slave;147 struct moxart_chan slave_chans[APB_DMA_MAX_CHANNEL];148 unsigned int irq;149};150 151static const unsigned int es_bytes[] = {152 [MOXART_DMA_DATA_TYPE_S8] = 1,153 [MOXART_DMA_DATA_TYPE_S16] = 2,154 [MOXART_DMA_DATA_TYPE_S32] = 4,155};156 157static struct device *chan2dev(struct dma_chan *chan)158{159 return &chan->dev->device;160}161 162static inline struct moxart_chan *to_moxart_dma_chan(struct dma_chan *c)163{164 return container_of(c, struct moxart_chan, vc.chan);165}166 167static inline struct moxart_desc *to_moxart_dma_desc(168 struct dma_async_tx_descriptor *t)169{170 return container_of(t, struct moxart_desc, vd.tx);171}172 173static void moxart_dma_desc_free(struct virt_dma_desc *vd)174{175 kfree(container_of(vd, struct moxart_desc, vd));176}177 178static int moxart_terminate_all(struct dma_chan *chan)179{180 struct moxart_chan *ch = to_moxart_dma_chan(chan);181 unsigned long flags;182 LIST_HEAD(head);183 u32 ctrl;184 185 dev_dbg(chan2dev(chan), "%s: ch=%p\n", __func__, ch);186 187 spin_lock_irqsave(&ch->vc.lock, flags);188 189 if (ch->desc) {190 moxart_dma_desc_free(&ch->desc->vd);191 ch->desc = NULL;192 }193 194 ctrl = readl(ch->base + REG_OFF_CTRL);195 ctrl &= ~(APB_DMA_ENABLE | APB_DMA_FIN_INT_EN | APB_DMA_ERR_INT_EN);196 writel(ctrl, ch->base + REG_OFF_CTRL);197 198 vchan_get_all_descriptors(&ch->vc, &head);199 spin_unlock_irqrestore(&ch->vc.lock, flags);200 vchan_dma_desc_free_list(&ch->vc, &head);201 202 return 0;203}204 205static int moxart_slave_config(struct dma_chan *chan,206 struct dma_slave_config *cfg)207{208 struct moxart_chan *ch = to_moxart_dma_chan(chan);209 u32 ctrl;210 211 ch->cfg = *cfg;212 213 ctrl = readl(ch->base + REG_OFF_CTRL);214 ctrl |= APB_DMA_BURST_MODE;215 ctrl &= ~(APB_DMA_DEST_MASK | APB_DMA_SOURCE_MASK);216 ctrl &= ~(APB_DMA_DEST_REQ_NO_MASK | APB_DMA_SOURCE_REQ_NO_MASK);217 218 switch (ch->cfg.src_addr_width) {219 case DMA_SLAVE_BUSWIDTH_1_BYTE:220 ctrl |= APB_DMA_DATA_WIDTH_1;221 if (ch->cfg.direction != DMA_MEM_TO_DEV)222 ctrl |= APB_DMA_DEST_INC_1_4;223 else224 ctrl |= APB_DMA_SOURCE_INC_1_4;225 break;226 case DMA_SLAVE_BUSWIDTH_2_BYTES:227 ctrl |= APB_DMA_DATA_WIDTH_2;228 if (ch->cfg.direction != DMA_MEM_TO_DEV)229 ctrl |= APB_DMA_DEST_INC_2_8;230 else231 ctrl |= APB_DMA_SOURCE_INC_2_8;232 break;233 case DMA_SLAVE_BUSWIDTH_4_BYTES:234 ctrl &= ~APB_DMA_DATA_WIDTH;235 if (ch->cfg.direction != DMA_MEM_TO_DEV)236 ctrl |= APB_DMA_DEST_INC_4_16;237 else238 ctrl |= APB_DMA_SOURCE_INC_4_16;239 break;240 default:241 return -EINVAL;242 }243 244 if (ch->cfg.direction == DMA_MEM_TO_DEV) {245 ctrl &= ~APB_DMA_DEST_SELECT;246 ctrl |= APB_DMA_SOURCE_SELECT;247 ctrl |= (ch->line_reqno << 16 &248 APB_DMA_DEST_REQ_NO_MASK);249 } else {250 ctrl |= APB_DMA_DEST_SELECT;251 ctrl &= ~APB_DMA_SOURCE_SELECT;252 ctrl |= (ch->line_reqno << 24 &253 APB_DMA_SOURCE_REQ_NO_MASK);254 }255 256 writel(ctrl, ch->base + REG_OFF_CTRL);257 258 return 0;259}260 261static struct dma_async_tx_descriptor *moxart_prep_slave_sg(262 struct dma_chan *chan, struct scatterlist *sgl,263 unsigned int sg_len, enum dma_transfer_direction dir,264 unsigned long tx_flags, void *context)265{266 struct moxart_chan *ch = to_moxart_dma_chan(chan);267 struct moxart_desc *d;268 enum dma_slave_buswidth dev_width;269 dma_addr_t dev_addr;270 struct scatterlist *sgent;271 unsigned int es;272 unsigned int i;273 274 if (!is_slave_direction(dir)) {275 dev_err(chan2dev(chan), "%s: invalid DMA direction\n",276 __func__);277 return NULL;278 }279 280 if (dir == DMA_DEV_TO_MEM) {281 dev_addr = ch->cfg.src_addr;282 dev_width = ch->cfg.src_addr_width;283 } else {284 dev_addr = ch->cfg.dst_addr;285 dev_width = ch->cfg.dst_addr_width;286 }287 288 switch (dev_width) {289 case DMA_SLAVE_BUSWIDTH_1_BYTE:290 es = MOXART_DMA_DATA_TYPE_S8;291 break;292 case DMA_SLAVE_BUSWIDTH_2_BYTES:293 es = MOXART_DMA_DATA_TYPE_S16;294 break;295 case DMA_SLAVE_BUSWIDTH_4_BYTES:296 es = MOXART_DMA_DATA_TYPE_S32;297 break;298 default:299 dev_err(chan2dev(chan), "%s: unsupported data width (%u)\n",300 __func__, dev_width);301 return NULL;302 }303 304 d = kzalloc(struct_size(d, sg, sg_len), GFP_ATOMIC);305 if (!d)306 return NULL;307 d->sglen = sg_len;308 309 d->dma_dir = dir;310 d->dev_addr = dev_addr;311 d->es = es;312 313 for_each_sg(sgl, sgent, sg_len, i) {314 d->sg[i].addr = sg_dma_address(sgent);315 d->sg[i].len = sg_dma_len(sgent);316 }317 318 ch->error = 0;319 320 return vchan_tx_prep(&ch->vc, &d->vd, tx_flags);321}322 323static struct dma_chan *moxart_of_xlate(struct of_phandle_args *dma_spec,324 struct of_dma *ofdma)325{326 struct moxart_dmadev *mdc = ofdma->of_dma_data;327 struct dma_chan *chan;328 struct moxart_chan *ch;329 330 chan = dma_get_any_slave_channel(&mdc->dma_slave);331 if (!chan)332 return NULL;333 334 ch = to_moxart_dma_chan(chan);335 ch->line_reqno = dma_spec->args[0];336 337 return chan;338}339 340static int moxart_alloc_chan_resources(struct dma_chan *chan)341{342 struct moxart_chan *ch = to_moxart_dma_chan(chan);343 344 dev_dbg(chan2dev(chan), "%s: allocating channel #%u\n",345 __func__, ch->ch_num);346 ch->allocated = 1;347 348 return 0;349}350 351static void moxart_free_chan_resources(struct dma_chan *chan)352{353 struct moxart_chan *ch = to_moxart_dma_chan(chan);354 355 vchan_free_chan_resources(&ch->vc);356 357 dev_dbg(chan2dev(chan), "%s: freeing channel #%u\n",358 __func__, ch->ch_num);359 ch->allocated = 0;360}361 362static void moxart_dma_set_params(struct moxart_chan *ch, dma_addr_t src_addr,363 dma_addr_t dst_addr)364{365 writel(src_addr, ch->base + REG_OFF_ADDRESS_SOURCE);366 writel(dst_addr, ch->base + REG_OFF_ADDRESS_DEST);367}368 369static void moxart_set_transfer_params(struct moxart_chan *ch, unsigned int len)370{371 struct moxart_desc *d = ch->desc;372 unsigned int sglen_div = es_bytes[d->es];373 374 d->dma_cycles = len >> sglen_div;375 376 /*377 * There are 4 cycles on 64 bytes copied, i.e. one cycle copies 16378 * bytes ( when width is APB_DMAB_DATA_WIDTH_4 ).379 */380 writel(d->dma_cycles, ch->base + REG_OFF_CYCLES);381 382 dev_dbg(chan2dev(&ch->vc.chan), "%s: set %u DMA cycles (len=%u)\n",383 __func__, d->dma_cycles, len);384}385 386static void moxart_start_dma(struct moxart_chan *ch)387{388 u32 ctrl;389 390 ctrl = readl(ch->base + REG_OFF_CTRL);391 ctrl |= (APB_DMA_ENABLE | APB_DMA_FIN_INT_EN | APB_DMA_ERR_INT_EN);392 writel(ctrl, ch->base + REG_OFF_CTRL);393}394 395static void moxart_dma_start_sg(struct moxart_chan *ch, unsigned int idx)396{397 struct moxart_desc *d = ch->desc;398 struct moxart_sg *sg = ch->desc->sg + idx;399 400 if (ch->desc->dma_dir == DMA_MEM_TO_DEV)401 moxart_dma_set_params(ch, sg->addr, d->dev_addr);402 else if (ch->desc->dma_dir == DMA_DEV_TO_MEM)403 moxart_dma_set_params(ch, d->dev_addr, sg->addr);404 405 moxart_set_transfer_params(ch, sg->len);406 407 moxart_start_dma(ch);408}409 410static void moxart_dma_start_desc(struct dma_chan *chan)411{412 struct moxart_chan *ch = to_moxart_dma_chan(chan);413 struct virt_dma_desc *vd;414 415 vd = vchan_next_desc(&ch->vc);416 417 if (!vd) {418 ch->desc = NULL;419 return;420 }421 422 list_del(&vd->node);423 424 ch->desc = to_moxart_dma_desc(&vd->tx);425 ch->sgidx = 0;426 427 moxart_dma_start_sg(ch, 0);428}429 430static void moxart_issue_pending(struct dma_chan *chan)431{432 struct moxart_chan *ch = to_moxart_dma_chan(chan);433 unsigned long flags;434 435 spin_lock_irqsave(&ch->vc.lock, flags);436 if (vchan_issue_pending(&ch->vc) && !ch->desc)437 moxart_dma_start_desc(chan);438 spin_unlock_irqrestore(&ch->vc.lock, flags);439}440 441static size_t moxart_dma_desc_size(struct moxart_desc *d,442 unsigned int completed_sgs)443{444 unsigned int i;445 size_t size;446 447 for (size = i = completed_sgs; i < d->sglen; i++)448 size += d->sg[i].len;449 450 return size;451}452 453static size_t moxart_dma_desc_size_in_flight(struct moxart_chan *ch)454{455 size_t size;456 unsigned int completed_cycles, cycles;457 458 size = moxart_dma_desc_size(ch->desc, ch->sgidx);459 cycles = readl(ch->base + REG_OFF_CYCLES);460 completed_cycles = (ch->desc->dma_cycles - cycles);461 size -= completed_cycles << es_bytes[ch->desc->es];462 463 dev_dbg(chan2dev(&ch->vc.chan), "%s: size=%zu\n", __func__, size);464 465 return size;466}467 468static enum dma_status moxart_tx_status(struct dma_chan *chan,469 dma_cookie_t cookie,470 struct dma_tx_state *txstate)471{472 struct moxart_chan *ch = to_moxart_dma_chan(chan);473 struct virt_dma_desc *vd;474 struct moxart_desc *d;475 enum dma_status ret;476 unsigned long flags;477 478 /*479 * dma_cookie_status() assigns initial residue value.480 */481 ret = dma_cookie_status(chan, cookie, txstate);482 483 spin_lock_irqsave(&ch->vc.lock, flags);484 vd = vchan_find_desc(&ch->vc, cookie);485 if (vd) {486 d = to_moxart_dma_desc(&vd->tx);487 txstate->residue = moxart_dma_desc_size(d, 0);488 } else if (ch->desc && ch->desc->vd.tx.cookie == cookie) {489 txstate->residue = moxart_dma_desc_size_in_flight(ch);490 }491 spin_unlock_irqrestore(&ch->vc.lock, flags);492 493 if (ch->error)494 return DMA_ERROR;495 496 return ret;497}498 499static void moxart_dma_init(struct dma_device *dma, struct device *dev)500{501 dma->device_prep_slave_sg = moxart_prep_slave_sg;502 dma->device_alloc_chan_resources = moxart_alloc_chan_resources;503 dma->device_free_chan_resources = moxart_free_chan_resources;504 dma->device_issue_pending = moxart_issue_pending;505 dma->device_tx_status = moxart_tx_status;506 dma->device_config = moxart_slave_config;507 dma->device_terminate_all = moxart_terminate_all;508 dma->dev = dev;509 510 INIT_LIST_HEAD(&dma->channels);511}512 513static irqreturn_t moxart_dma_interrupt(int irq, void *devid)514{515 struct moxart_dmadev *mc = devid;516 struct moxart_chan *ch = &mc->slave_chans[0];517 unsigned int i;518 u32 ctrl;519 520 dev_dbg(chan2dev(&ch->vc.chan), "%s\n", __func__);521 522 for (i = 0; i < APB_DMA_MAX_CHANNEL; i++, ch++) {523 if (!ch->allocated)524 continue;525 526 ctrl = readl(ch->base + REG_OFF_CTRL);527 528 dev_dbg(chan2dev(&ch->vc.chan), "%s: ch=%p ch->base=%p ctrl=%x\n",529 __func__, ch, ch->base, ctrl);530 531 if (ctrl & APB_DMA_FIN_INT_STS) {532 ctrl &= ~APB_DMA_FIN_INT_STS;533 if (ch->desc) {534 spin_lock(&ch->vc.lock);535 if (++ch->sgidx < ch->desc->sglen) {536 moxart_dma_start_sg(ch, ch->sgidx);537 } else {538 vchan_cookie_complete(&ch->desc->vd);539 moxart_dma_start_desc(&ch->vc.chan);540 }541 spin_unlock(&ch->vc.lock);542 }543 }544 545 if (ctrl & APB_DMA_ERR_INT_STS) {546 ctrl &= ~APB_DMA_ERR_INT_STS;547 ch->error = 1;548 }549 550 writel(ctrl, ch->base + REG_OFF_CTRL);551 }552 553 return IRQ_HANDLED;554}555 556static int moxart_probe(struct platform_device *pdev)557{558 struct device *dev = &pdev->dev;559 struct device_node *node = dev->of_node;560 void __iomem *dma_base_addr;561 int ret, i;562 unsigned int irq;563 struct moxart_chan *ch;564 struct moxart_dmadev *mdc;565 566 mdc = devm_kzalloc(dev, sizeof(*mdc), GFP_KERNEL);567 if (!mdc)568 return -ENOMEM;569 570 irq = irq_of_parse_and_map(node, 0);571 if (!irq) {572 dev_err(dev, "no IRQ resource\n");573 return -EINVAL;574 }575 576 dma_base_addr = devm_platform_ioremap_resource(pdev, 0);577 if (IS_ERR(dma_base_addr))578 return PTR_ERR(dma_base_addr);579 580 dma_cap_zero(mdc->dma_slave.cap_mask);581 dma_cap_set(DMA_SLAVE, mdc->dma_slave.cap_mask);582 dma_cap_set(DMA_PRIVATE, mdc->dma_slave.cap_mask);583 584 moxart_dma_init(&mdc->dma_slave, dev);585 586 ch = &mdc->slave_chans[0];587 for (i = 0; i < APB_DMA_MAX_CHANNEL; i++, ch++) {588 ch->ch_num = i;589 ch->base = dma_base_addr + i * REG_OFF_CHAN_SIZE;590 ch->allocated = 0;591 592 ch->vc.desc_free = moxart_dma_desc_free;593 vchan_init(&ch->vc, &mdc->dma_slave);594 595 dev_dbg(dev, "%s: chs[%d]: ch->ch_num=%u ch->base=%p\n",596 __func__, i, ch->ch_num, ch->base);597 }598 599 platform_set_drvdata(pdev, mdc);600 601 ret = devm_request_irq(dev, irq, moxart_dma_interrupt, 0,602 "moxart-dma-engine", mdc);603 if (ret) {604 dev_err(dev, "devm_request_irq failed\n");605 return ret;606 }607 mdc->irq = irq;608 609 ret = dma_async_device_register(&mdc->dma_slave);610 if (ret) {611 dev_err(dev, "dma_async_device_register failed\n");612 return ret;613 }614 615 ret = of_dma_controller_register(node, moxart_of_xlate, mdc);616 if (ret) {617 dev_err(dev, "of_dma_controller_register failed\n");618 dma_async_device_unregister(&mdc->dma_slave);619 return ret;620 }621 622 dev_dbg(dev, "%s: IRQ=%u\n", __func__, irq);623 624 return 0;625}626 627static void moxart_remove(struct platform_device *pdev)628{629 struct moxart_dmadev *m = platform_get_drvdata(pdev);630 631 devm_free_irq(&pdev->dev, m->irq, m);632 633 dma_async_device_unregister(&m->dma_slave);634 635 if (pdev->dev.of_node)636 of_dma_controller_free(pdev->dev.of_node);637}638 639static const struct of_device_id moxart_dma_match[] = {640 { .compatible = "moxa,moxart-dma" },641 { }642};643MODULE_DEVICE_TABLE(of, moxart_dma_match);644 645static struct platform_driver moxart_driver = {646 .probe = moxart_probe,647 .remove_new = moxart_remove,648 .driver = {649 .name = "moxart-dma-engine",650 .of_match_table = moxart_dma_match,651 },652};653 654static int moxart_init(void)655{656 return platform_driver_register(&moxart_driver);657}658subsys_initcall(moxart_init);659 660static void __exit moxart_exit(void)661{662 platform_driver_unregister(&moxart_driver);663}664module_exit(moxart_exit);665 666MODULE_AUTHOR("Jonas Jensen <jonas.jensen@gmail.com>");667MODULE_DESCRIPTION("MOXART DMA engine driver");668MODULE_LICENSE("GPL v2");669