761 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * drivers/dma/fsl-edma.c4 *5 * Copyright 2013-2014 Freescale Semiconductor, Inc.6 *7 * Driver for the Freescale eDMA engine with flexible channel multiplexing8 * capability for DMA request sources. The eDMA block can be found on some9 * Vybrid and Layerscape SoCs.10 */11 12#include <dt-bindings/dma/fsl-edma.h>13#include <linux/bitfield.h>14#include <linux/module.h>15#include <linux/interrupt.h>16#include <linux/clk.h>17#include <linux/of.h>18#include <linux/of_dma.h>19#include <linux/dma-mapping.h>20#include <linux/pm_runtime.h>21#include <linux/pm_domain.h>22#include <linux/property.h>23 24#include "fsl-edma-common.h"25 26static void fsl_edma_synchronize(struct dma_chan *chan)27{28 struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);29 30 vchan_synchronize(&fsl_chan->vchan);31}32 33static irqreturn_t fsl_edma_tx_handler(int irq, void *dev_id)34{35 struct fsl_edma_engine *fsl_edma = dev_id;36 unsigned int intr, ch;37 struct edma_regs *regs = &fsl_edma->regs;38 39 intr = edma_readl(fsl_edma, regs->intl);40 if (!intr)41 return IRQ_NONE;42 43 for (ch = 0; ch < fsl_edma->n_chans; ch++) {44 if (intr & (0x1 << ch)) {45 edma_writeb(fsl_edma, EDMA_CINT_CINT(ch), regs->cint);46 fsl_edma_tx_chan_handler(&fsl_edma->chans[ch]);47 }48 }49 return IRQ_HANDLED;50}51 52static irqreturn_t fsl_edma3_tx_handler(int irq, void *dev_id)53{54 struct fsl_edma_chan *fsl_chan = dev_id;55 unsigned int intr;56 57 intr = edma_readl_chreg(fsl_chan, ch_int);58 if (!intr)59 return IRQ_HANDLED;60 61 edma_writel_chreg(fsl_chan, 1, ch_int);62 63 fsl_edma_tx_chan_handler(fsl_chan);64 65 return IRQ_HANDLED;66}67 68static irqreturn_t fsl_edma2_tx_handler(int irq, void *devi_id)69{70 struct fsl_edma_chan *fsl_chan = devi_id;71 72 return fsl_edma_tx_handler(irq, fsl_chan->edma);73}74 75static irqreturn_t fsl_edma_err_handler(int irq, void *dev_id)76{77 struct fsl_edma_engine *fsl_edma = dev_id;78 unsigned int err, ch;79 struct edma_regs *regs = &fsl_edma->regs;80 81 err = edma_readl(fsl_edma, regs->errl);82 if (!err)83 return IRQ_NONE;84 85 for (ch = 0; ch < fsl_edma->n_chans; ch++) {86 if (err & (0x1 << ch)) {87 fsl_edma_disable_request(&fsl_edma->chans[ch]);88 edma_writeb(fsl_edma, EDMA_CERR_CERR(ch), regs->cerr);89 fsl_edma_err_chan_handler(&fsl_edma->chans[ch]);90 }91 }92 return IRQ_HANDLED;93}94 95static irqreturn_t fsl_edma_irq_handler(int irq, void *dev_id)96{97 if (fsl_edma_tx_handler(irq, dev_id) == IRQ_HANDLED)98 return IRQ_HANDLED;99 100 return fsl_edma_err_handler(irq, dev_id);101}102 103static bool fsl_edma_srcid_in_use(struct fsl_edma_engine *fsl_edma, u32 srcid)104{105 struct fsl_edma_chan *fsl_chan;106 int i;107 108 for (i = 0; i < fsl_edma->n_chans; i++) {109 fsl_chan = &fsl_edma->chans[i];110 111 if (fsl_chan->srcid && srcid == fsl_chan->srcid) {112 dev_err(&fsl_chan->pdev->dev, "The srcid is in use, can't use!");113 return true;114 }115 }116 return false;117}118 119static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec,120 struct of_dma *ofdma)121{122 struct fsl_edma_engine *fsl_edma = ofdma->of_dma_data;123 struct dma_chan *chan, *_chan;124 struct fsl_edma_chan *fsl_chan;125 u32 dmamux_nr = fsl_edma->drvdata->dmamuxs;126 unsigned long chans_per_mux = fsl_edma->n_chans / dmamux_nr;127 128 if (dma_spec->args_count != 2)129 return NULL;130 131 guard(mutex)(&fsl_edma->fsl_edma_mutex);132 133 list_for_each_entry_safe(chan, _chan, &fsl_edma->dma_dev.channels, device_node) {134 if (chan->client_count)135 continue;136 137 if (fsl_edma_srcid_in_use(fsl_edma, dma_spec->args[1]))138 return NULL;139 140 if ((chan->chan_id / chans_per_mux) == dma_spec->args[0]) {141 chan = dma_get_slave_channel(chan);142 if (chan) {143 chan->device->privatecnt++;144 fsl_chan = to_fsl_edma_chan(chan);145 fsl_chan->srcid = dma_spec->args[1];146 147 if (!fsl_chan->srcid) {148 dev_err(&fsl_chan->pdev->dev, "Invalidate srcid %d\n",149 fsl_chan->srcid);150 return NULL;151 }152 153 fsl_edma_chan_mux(fsl_chan, fsl_chan->srcid,154 true);155 return chan;156 }157 }158 }159 return NULL;160}161 162static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec,163 struct of_dma *ofdma)164{165 struct fsl_edma_engine *fsl_edma = ofdma->of_dma_data;166 struct dma_chan *chan, *_chan;167 struct fsl_edma_chan *fsl_chan;168 bool b_chmux;169 int i;170 171 if (dma_spec->args_count != 3)172 return NULL;173 174 b_chmux = !!(fsl_edma->drvdata->flags & FSL_EDMA_DRV_HAS_CHMUX);175 176 guard(mutex)(&fsl_edma->fsl_edma_mutex);177 list_for_each_entry_safe(chan, _chan, &fsl_edma->dma_dev.channels,178 device_node) {179 180 if (chan->client_count)181 continue;182 183 fsl_chan = to_fsl_edma_chan(chan);184 if (fsl_edma_srcid_in_use(fsl_edma, dma_spec->args[0]))185 return NULL;186 i = fsl_chan - fsl_edma->chans;187 188 fsl_chan->priority = dma_spec->args[1];189 fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX;190 fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE;191 fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO;192 193 if ((dma_spec->args[2] & FSL_EDMA_EVEN_CH) && (i & 0x1))194 continue;195 196 if ((dma_spec->args[2] & FSL_EDMA_ODD_CH) && !(i & 0x1))197 continue;198 199 if (!b_chmux && i == dma_spec->args[0]) {200 chan = dma_get_slave_channel(chan);201 chan->device->privatecnt++;202 return chan;203 } else if (b_chmux && !fsl_chan->srcid) {204 /* if controller support channel mux, choose a free channel */205 chan = dma_get_slave_channel(chan);206 chan->device->privatecnt++;207 fsl_chan->srcid = dma_spec->args[0];208 return chan;209 }210 }211 return NULL;212}213 214static int215fsl_edma_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma)216{217 int ret;218 219 edma_writel(fsl_edma, ~0, fsl_edma->regs.intl);220 221 fsl_edma->txirq = platform_get_irq_byname(pdev, "edma-tx");222 if (fsl_edma->txirq < 0)223 return fsl_edma->txirq;224 225 fsl_edma->errirq = platform_get_irq_byname(pdev, "edma-err");226 if (fsl_edma->errirq < 0)227 return fsl_edma->errirq;228 229 if (fsl_edma->txirq == fsl_edma->errirq) {230 ret = devm_request_irq(&pdev->dev, fsl_edma->txirq,231 fsl_edma_irq_handler, 0, "eDMA", fsl_edma);232 if (ret) {233 dev_err(&pdev->dev, "Can't register eDMA IRQ.\n");234 return ret;235 }236 } else {237 ret = devm_request_irq(&pdev->dev, fsl_edma->txirq,238 fsl_edma_tx_handler, 0, "eDMA tx", fsl_edma);239 if (ret) {240 dev_err(&pdev->dev, "Can't register eDMA tx IRQ.\n");241 return ret;242 }243 244 ret = devm_request_irq(&pdev->dev, fsl_edma->errirq,245 fsl_edma_err_handler, 0, "eDMA err", fsl_edma);246 if (ret) {247 dev_err(&pdev->dev, "Can't register eDMA err IRQ.\n");248 return ret;249 }250 }251 252 return 0;253}254 255static int fsl_edma3_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma)256{257 int i;258 259 for (i = 0; i < fsl_edma->n_chans; i++) {260 261 struct fsl_edma_chan *fsl_chan = &fsl_edma->chans[i];262 263 if (fsl_edma->chan_masked & BIT(i))264 continue;265 266 /* request channel irq */267 fsl_chan->txirq = platform_get_irq(pdev, i);268 if (fsl_chan->txirq < 0)269 return -EINVAL;270 271 fsl_chan->irq_handler = fsl_edma3_tx_handler;272 }273 274 return 0;275}276 277static int278fsl_edma2_irq_init(struct platform_device *pdev,279 struct fsl_edma_engine *fsl_edma)280{281 int i, ret, irq;282 int count;283 284 edma_writel(fsl_edma, ~0, fsl_edma->regs.intl);285 286 count = platform_irq_count(pdev);287 dev_dbg(&pdev->dev, "%s Found %d interrupts\r\n", __func__, count);288 if (count <= 2) {289 dev_err(&pdev->dev, "Interrupts in DTS not correct.\n");290 return -EINVAL;291 }292 /*293 * 16 channel independent interrupts + 1 error interrupt on i.mx7ulp.294 * 2 channel share one interrupt, for example, ch0/ch16, ch1/ch17...295 * For now, just simply request irq without IRQF_SHARED flag, since 16296 * channels are enough on i.mx7ulp whose M4 domain own some peripherals.297 */298 for (i = 0; i < count; i++) {299 irq = platform_get_irq(pdev, i);300 ret = 0;301 if (irq < 0)302 return -ENXIO;303 304 /* The last IRQ is for eDMA err */305 if (i == count - 1) {306 ret = devm_request_irq(&pdev->dev, irq,307 fsl_edma_err_handler,308 0, "eDMA2-ERR", fsl_edma);309 } else {310 fsl_edma->chans[i].txirq = irq;311 fsl_edma->chans[i].irq_handler = fsl_edma2_tx_handler;312 }313 314 if (ret)315 return ret;316 }317 318 return 0;319}320 321static void fsl_edma_irq_exit(322 struct platform_device *pdev, struct fsl_edma_engine *fsl_edma)323{324 if (fsl_edma->txirq == fsl_edma->errirq) {325 devm_free_irq(&pdev->dev, fsl_edma->txirq, fsl_edma);326 } else {327 devm_free_irq(&pdev->dev, fsl_edma->txirq, fsl_edma);328 devm_free_irq(&pdev->dev, fsl_edma->errirq, fsl_edma);329 }330}331 332static void fsl_disable_clocks(struct fsl_edma_engine *fsl_edma, int nr_clocks)333{334 int i;335 336 for (i = 0; i < nr_clocks; i++)337 clk_disable_unprepare(fsl_edma->muxclk[i]);338}339 340static struct fsl_edma_drvdata vf610_data = {341 .dmamuxs = DMAMUX_NR,342 .flags = FSL_EDMA_DRV_WRAP_IO,343 .chreg_off = EDMA_TCD,344 .chreg_space_sz = sizeof(struct fsl_edma_hw_tcd),345 .setup_irq = fsl_edma_irq_init,346};347 348static struct fsl_edma_drvdata ls1028a_data = {349 .dmamuxs = DMAMUX_NR,350 .flags = FSL_EDMA_DRV_MUX_SWAP | FSL_EDMA_DRV_WRAP_IO,351 .chreg_off = EDMA_TCD,352 .chreg_space_sz = sizeof(struct fsl_edma_hw_tcd),353 .setup_irq = fsl_edma_irq_init,354};355 356static struct fsl_edma_drvdata imx7ulp_data = {357 .dmamuxs = 1,358 .chreg_off = EDMA_TCD,359 .chreg_space_sz = sizeof(struct fsl_edma_hw_tcd),360 .flags = FSL_EDMA_DRV_HAS_DMACLK | FSL_EDMA_DRV_CONFIG32,361 .setup_irq = fsl_edma2_irq_init,362};363 364static struct fsl_edma_drvdata imx8qm_data = {365 .flags = FSL_EDMA_DRV_HAS_PD | FSL_EDMA_DRV_EDMA3 | FSL_EDMA_DRV_MEM_REMOTE,366 .chreg_space_sz = 0x10000,367 .chreg_off = 0x10000,368 .setup_irq = fsl_edma3_irq_init,369};370 371static struct fsl_edma_drvdata imx8ulp_data = {372 .flags = FSL_EDMA_DRV_HAS_CHMUX | FSL_EDMA_DRV_HAS_CHCLK | FSL_EDMA_DRV_HAS_DMACLK |373 FSL_EDMA_DRV_EDMA3,374 .chreg_space_sz = 0x10000,375 .chreg_off = 0x10000,376 .mux_off = 0x10000 + offsetof(struct fsl_edma3_ch_reg, ch_mux),377 .mux_skip = 0x10000,378 .setup_irq = fsl_edma3_irq_init,379};380 381static struct fsl_edma_drvdata imx93_data3 = {382 .flags = FSL_EDMA_DRV_HAS_DMACLK | FSL_EDMA_DRV_EDMA3,383 .chreg_space_sz = 0x10000,384 .chreg_off = 0x10000,385 .setup_irq = fsl_edma3_irq_init,386};387 388static struct fsl_edma_drvdata imx93_data4 = {389 .flags = FSL_EDMA_DRV_HAS_CHMUX | FSL_EDMA_DRV_HAS_DMACLK | FSL_EDMA_DRV_EDMA4,390 .chreg_space_sz = 0x8000,391 .chreg_off = 0x10000,392 .mux_off = 0x10000 + offsetof(struct fsl_edma3_ch_reg, ch_mux),393 .mux_skip = 0x8000,394 .setup_irq = fsl_edma3_irq_init,395};396 397static struct fsl_edma_drvdata imx95_data5 = {398 .flags = FSL_EDMA_DRV_HAS_CHMUX | FSL_EDMA_DRV_HAS_DMACLK | FSL_EDMA_DRV_EDMA4 |399 FSL_EDMA_DRV_TCD64,400 .chreg_space_sz = 0x8000,401 .chreg_off = 0x10000,402 .mux_off = 0x200,403 .mux_skip = sizeof(u32),404 .setup_irq = fsl_edma3_irq_init,405};406 407static const struct of_device_id fsl_edma_dt_ids[] = {408 { .compatible = "fsl,vf610-edma", .data = &vf610_data},409 { .compatible = "fsl,ls1028a-edma", .data = &ls1028a_data},410 { .compatible = "fsl,imx7ulp-edma", .data = &imx7ulp_data},411 { .compatible = "fsl,imx8qm-edma", .data = &imx8qm_data},412 { .compatible = "fsl,imx8ulp-edma", .data = &imx8ulp_data},413 { .compatible = "fsl,imx93-edma3", .data = &imx93_data3},414 { .compatible = "fsl,imx93-edma4", .data = &imx93_data4},415 { .compatible = "fsl,imx95-edma5", .data = &imx95_data5},416 { /* sentinel */ }417};418MODULE_DEVICE_TABLE(of, fsl_edma_dt_ids);419 420static int fsl_edma3_attach_pd(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma)421{422 struct fsl_edma_chan *fsl_chan;423 struct device_link *link;424 struct device *pd_chan;425 struct device *dev;426 int i;427 428 dev = &pdev->dev;429 430 for (i = 0; i < fsl_edma->n_chans; i++) {431 if (fsl_edma->chan_masked & BIT(i))432 continue;433 434 fsl_chan = &fsl_edma->chans[i];435 436 pd_chan = dev_pm_domain_attach_by_id(dev, i);437 if (IS_ERR_OR_NULL(pd_chan)) {438 dev_err(dev, "Failed attach pd %d\n", i);439 return -EINVAL;440 }441 442 link = device_link_add(dev, pd_chan, DL_FLAG_STATELESS |443 DL_FLAG_PM_RUNTIME |444 DL_FLAG_RPM_ACTIVE);445 if (!link) {446 dev_err(dev, "Failed to add device_link to %d\n", i);447 return -EINVAL;448 }449 450 fsl_chan->pd_dev = pd_chan;451 452 pm_runtime_use_autosuspend(fsl_chan->pd_dev);453 pm_runtime_set_autosuspend_delay(fsl_chan->pd_dev, 200);454 pm_runtime_set_active(fsl_chan->pd_dev);455 }456 457 return 0;458}459 460static int fsl_edma_probe(struct platform_device *pdev)461{462 struct device_node *np = pdev->dev.of_node;463 struct fsl_edma_engine *fsl_edma;464 const struct fsl_edma_drvdata *drvdata = NULL;465 u32 chan_mask[2] = {0, 0};466 char clk_name[36];467 struct edma_regs *regs;468 int chans;469 int ret, i;470 471 drvdata = device_get_match_data(&pdev->dev);472 if (!drvdata) {473 dev_err(&pdev->dev, "unable to find driver data\n");474 return -EINVAL;475 }476 477 ret = of_property_read_u32(np, "dma-channels", &chans);478 if (ret) {479 dev_err(&pdev->dev, "Can't get dma-channels.\n");480 return ret;481 }482 483 fsl_edma = devm_kzalloc(&pdev->dev, struct_size(fsl_edma, chans, chans),484 GFP_KERNEL);485 if (!fsl_edma)486 return -ENOMEM;487 488 fsl_edma->drvdata = drvdata;489 fsl_edma->n_chans = chans;490 mutex_init(&fsl_edma->fsl_edma_mutex);491 492 fsl_edma->membase = devm_platform_ioremap_resource(pdev, 0);493 if (IS_ERR(fsl_edma->membase))494 return PTR_ERR(fsl_edma->membase);495 496 if (!(drvdata->flags & FSL_EDMA_DRV_SPLIT_REG)) {497 fsl_edma_setup_regs(fsl_edma);498 regs = &fsl_edma->regs;499 }500 501 if (drvdata->flags & FSL_EDMA_DRV_HAS_DMACLK) {502 fsl_edma->dmaclk = devm_clk_get_enabled(&pdev->dev, "dma");503 if (IS_ERR(fsl_edma->dmaclk)) {504 dev_err(&pdev->dev, "Missing DMA block clock.\n");505 return PTR_ERR(fsl_edma->dmaclk);506 }507 }508 509 ret = of_property_read_variable_u32_array(np, "dma-channel-mask", chan_mask, 1, 2);510 511 if (ret > 0) {512 fsl_edma->chan_masked = chan_mask[1];513 fsl_edma->chan_masked <<= 32;514 fsl_edma->chan_masked |= chan_mask[0];515 }516 517 for (i = 0; i < fsl_edma->drvdata->dmamuxs; i++) {518 char clkname[32];519 520 /* eDMAv3 mux register move to TCD area if ch_mux exist */521 if (drvdata->flags & FSL_EDMA_DRV_SPLIT_REG)522 break;523 524 fsl_edma->muxbase[i] = devm_platform_ioremap_resource(pdev,525 1 + i);526 if (IS_ERR(fsl_edma->muxbase[i])) {527 /* on error: disable all previously enabled clks */528 fsl_disable_clocks(fsl_edma, i);529 return PTR_ERR(fsl_edma->muxbase[i]);530 }531 532 sprintf(clkname, "dmamux%d", i);533 fsl_edma->muxclk[i] = devm_clk_get_enabled(&pdev->dev, clkname);534 if (IS_ERR(fsl_edma->muxclk[i])) {535 dev_err(&pdev->dev, "Missing DMAMUX block clock.\n");536 /* on error: disable all previously enabled clks */537 return PTR_ERR(fsl_edma->muxclk[i]);538 }539 }540 541 fsl_edma->big_endian = of_property_read_bool(np, "big-endian");542 543 if (drvdata->flags & FSL_EDMA_DRV_HAS_PD) {544 ret = fsl_edma3_attach_pd(pdev, fsl_edma);545 if (ret)546 return ret;547 }548 549 if (drvdata->flags & FSL_EDMA_DRV_TCD64)550 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));551 552 INIT_LIST_HEAD(&fsl_edma->dma_dev.channels);553 for (i = 0; i < fsl_edma->n_chans; i++) {554 struct fsl_edma_chan *fsl_chan = &fsl_edma->chans[i];555 int len;556 557 if (fsl_edma->chan_masked & BIT(i))558 continue;559 560 snprintf(fsl_chan->chan_name, sizeof(fsl_chan->chan_name), "%s-CH%02d",561 dev_name(&pdev->dev), i);562 563 fsl_chan->edma = fsl_edma;564 fsl_chan->pm_state = RUNNING;565 fsl_chan->srcid = 0;566 fsl_chan->dma_dir = DMA_NONE;567 fsl_chan->vchan.desc_free = fsl_edma_free_desc;568 569 len = (drvdata->flags & FSL_EDMA_DRV_SPLIT_REG) ?570 offsetof(struct fsl_edma3_ch_reg, tcd) : 0;571 fsl_chan->tcd = fsl_edma->membase572 + i * drvdata->chreg_space_sz + drvdata->chreg_off + len;573 fsl_chan->mux_addr = fsl_edma->membase + drvdata->mux_off + i * drvdata->mux_skip;574 575 if (drvdata->flags & FSL_EDMA_DRV_HAS_CHCLK) {576 snprintf(clk_name, sizeof(clk_name), "ch%02d", i);577 fsl_chan->clk = devm_clk_get_enabled(&pdev->dev,578 (const char *)clk_name);579 580 if (IS_ERR(fsl_chan->clk))581 return PTR_ERR(fsl_chan->clk);582 }583 fsl_chan->pdev = pdev;584 vchan_init(&fsl_chan->vchan, &fsl_edma->dma_dev);585 586 edma_write_tcdreg(fsl_chan, cpu_to_le32(0), csr);587 fsl_edma_chan_mux(fsl_chan, 0, false);588 if (fsl_chan->edma->drvdata->flags & FSL_EDMA_DRV_HAS_CHCLK)589 clk_disable_unprepare(fsl_chan->clk);590 }591 592 ret = fsl_edma->drvdata->setup_irq(pdev, fsl_edma);593 if (ret)594 return ret;595 596 dma_cap_set(DMA_PRIVATE, fsl_edma->dma_dev.cap_mask);597 dma_cap_set(DMA_SLAVE, fsl_edma->dma_dev.cap_mask);598 dma_cap_set(DMA_CYCLIC, fsl_edma->dma_dev.cap_mask);599 dma_cap_set(DMA_MEMCPY, fsl_edma->dma_dev.cap_mask);600 601 fsl_edma->dma_dev.dev = &pdev->dev;602 fsl_edma->dma_dev.device_alloc_chan_resources603 = fsl_edma_alloc_chan_resources;604 fsl_edma->dma_dev.device_free_chan_resources605 = fsl_edma_free_chan_resources;606 fsl_edma->dma_dev.device_tx_status = fsl_edma_tx_status;607 fsl_edma->dma_dev.device_prep_slave_sg = fsl_edma_prep_slave_sg;608 fsl_edma->dma_dev.device_prep_dma_cyclic = fsl_edma_prep_dma_cyclic;609 fsl_edma->dma_dev.device_prep_dma_memcpy = fsl_edma_prep_memcpy;610 fsl_edma->dma_dev.device_config = fsl_edma_slave_config;611 fsl_edma->dma_dev.device_pause = fsl_edma_pause;612 fsl_edma->dma_dev.device_resume = fsl_edma_resume;613 fsl_edma->dma_dev.device_terminate_all = fsl_edma_terminate_all;614 fsl_edma->dma_dev.device_synchronize = fsl_edma_synchronize;615 fsl_edma->dma_dev.device_issue_pending = fsl_edma_issue_pending;616 617 fsl_edma->dma_dev.src_addr_widths = FSL_EDMA_BUSWIDTHS;618 fsl_edma->dma_dev.dst_addr_widths = FSL_EDMA_BUSWIDTHS;619 620 if (drvdata->flags & FSL_EDMA_DRV_BUS_8BYTE) {621 fsl_edma->dma_dev.src_addr_widths |= BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);622 fsl_edma->dma_dev.dst_addr_widths |= BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);623 }624 625 fsl_edma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);626 if (drvdata->flags & FSL_EDMA_DRV_DEV_TO_DEV)627 fsl_edma->dma_dev.directions |= BIT(DMA_DEV_TO_DEV);628 629 fsl_edma->dma_dev.copy_align = drvdata->flags & FSL_EDMA_DRV_ALIGN_64BYTE ?630 DMAENGINE_ALIGN_64_BYTES :631 DMAENGINE_ALIGN_32_BYTES;632 633 /* Per worst case 'nbytes = 1' take CITER as the max_seg_size */634 dma_set_max_seg_size(fsl_edma->dma_dev.dev,635 FIELD_GET(EDMA_TCD_ITER_MASK, EDMA_TCD_ITER_MASK));636 637 fsl_edma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;638 639 platform_set_drvdata(pdev, fsl_edma);640 641 ret = dma_async_device_register(&fsl_edma->dma_dev);642 if (ret) {643 dev_err(&pdev->dev,644 "Can't register Freescale eDMA engine. (%d)\n", ret);645 return ret;646 }647 648 ret = of_dma_controller_register(np,649 drvdata->flags & FSL_EDMA_DRV_SPLIT_REG ? fsl_edma3_xlate : fsl_edma_xlate,650 fsl_edma);651 if (ret) {652 dev_err(&pdev->dev,653 "Can't register Freescale eDMA of_dma. (%d)\n", ret);654 dma_async_device_unregister(&fsl_edma->dma_dev);655 return ret;656 }657 658 /* enable round robin arbitration */659 if (!(drvdata->flags & FSL_EDMA_DRV_SPLIT_REG))660 edma_writel(fsl_edma, EDMA_CR_ERGA | EDMA_CR_ERCA, regs->cr);661 662 return 0;663}664 665static void fsl_edma_remove(struct platform_device *pdev)666{667 struct device_node *np = pdev->dev.of_node;668 struct fsl_edma_engine *fsl_edma = platform_get_drvdata(pdev);669 670 fsl_edma_irq_exit(pdev, fsl_edma);671 fsl_edma_cleanup_vchan(&fsl_edma->dma_dev);672 of_dma_controller_free(np);673 dma_async_device_unregister(&fsl_edma->dma_dev);674 fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs);675}676 677static int fsl_edma_suspend_late(struct device *dev)678{679 struct fsl_edma_engine *fsl_edma = dev_get_drvdata(dev);680 struct fsl_edma_chan *fsl_chan;681 unsigned long flags;682 int i;683 684 for (i = 0; i < fsl_edma->n_chans; i++) {685 fsl_chan = &fsl_edma->chans[i];686 if (fsl_edma->chan_masked & BIT(i))687 continue;688 spin_lock_irqsave(&fsl_chan->vchan.lock, flags);689 /* Make sure chan is idle or will force disable. */690 if (unlikely(fsl_chan->status == DMA_IN_PROGRESS)) {691 dev_warn(dev, "WARN: There is non-idle channel.");692 fsl_edma_disable_request(fsl_chan);693 fsl_edma_chan_mux(fsl_chan, 0, false);694 }695 696 fsl_chan->pm_state = SUSPENDED;697 spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);698 }699 700 return 0;701}702 703static int fsl_edma_resume_early(struct device *dev)704{705 struct fsl_edma_engine *fsl_edma = dev_get_drvdata(dev);706 struct fsl_edma_chan *fsl_chan;707 struct edma_regs *regs = &fsl_edma->regs;708 int i;709 710 for (i = 0; i < fsl_edma->n_chans; i++) {711 fsl_chan = &fsl_edma->chans[i];712 if (fsl_edma->chan_masked & BIT(i))713 continue;714 fsl_chan->pm_state = RUNNING;715 edma_write_tcdreg(fsl_chan, 0, csr);716 if (fsl_chan->srcid != 0)717 fsl_edma_chan_mux(fsl_chan, fsl_chan->srcid, true);718 }719 720 if (!(fsl_edma->drvdata->flags & FSL_EDMA_DRV_SPLIT_REG))721 edma_writel(fsl_edma, EDMA_CR_ERGA | EDMA_CR_ERCA, regs->cr);722 723 return 0;724}725 726/*727 * eDMA provides the service to others, so it should be suspend late728 * and resume early. When eDMA suspend, all of the clients should stop729 * the DMA data transmission and let the channel idle.730 */731static const struct dev_pm_ops fsl_edma_pm_ops = {732 .suspend_late = fsl_edma_suspend_late,733 .resume_early = fsl_edma_resume_early,734};735 736static struct platform_driver fsl_edma_driver = {737 .driver = {738 .name = "fsl-edma",739 .of_match_table = fsl_edma_dt_ids,740 .pm = &fsl_edma_pm_ops,741 },742 .probe = fsl_edma_probe,743 .remove_new = fsl_edma_remove,744};745 746static int __init fsl_edma_init(void)747{748 return platform_driver_register(&fsl_edma_driver);749}750subsys_initcall(fsl_edma_init);751 752static void __exit fsl_edma_exit(void)753{754 platform_driver_unregister(&fsl_edma_driver);755}756module_exit(fsl_edma_exit);757 758MODULE_ALIAS("platform:fsl-edma");759MODULE_DESCRIPTION("Freescale eDMA engine driver");760MODULE_LICENSE("GPL v2");761