936 lines · c
1// SPDX-License-Identifier: GPL-2.02// Copyright (c) 2018-2019 MediaTek Inc.3 4/*5 * Driver for MediaTek Command-Queue DMA Controller6 *7 * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com>8 *9 */10 11#include <linux/bitops.h>12#include <linux/clk.h>13#include <linux/dmaengine.h>14#include <linux/dma-mapping.h>15#include <linux/err.h>16#include <linux/iopoll.h>17#include <linux/interrupt.h>18#include <linux/list.h>19#include <linux/module.h>20#include <linux/of.h>21#include <linux/of_dma.h>22#include <linux/platform_device.h>23#include <linux/pm_runtime.h>24#include <linux/refcount.h>25#include <linux/slab.h>26 27#include "../virt-dma.h"28 29#define MTK_CQDMA_USEC_POLL 1030#define MTK_CQDMA_TIMEOUT_POLL 100031#define MTK_CQDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)32#define MTK_CQDMA_ALIGN_SIZE 133 34/* The default number of virtual channel */35#define MTK_CQDMA_NR_VCHANS 3236 37/* The default number of physical channel */38#define MTK_CQDMA_NR_PCHANS 339 40/* Registers for underlying dma manipulation */41#define MTK_CQDMA_INT_FLAG 0x042#define MTK_CQDMA_INT_EN 0x443#define MTK_CQDMA_EN 0x844#define MTK_CQDMA_RESET 0xc45#define MTK_CQDMA_FLUSH 0x1446#define MTK_CQDMA_SRC 0x1c47#define MTK_CQDMA_DST 0x2048#define MTK_CQDMA_LEN1 0x2449#define MTK_CQDMA_LEN2 0x2850#define MTK_CQDMA_SRC2 0x6051#define MTK_CQDMA_DST2 0x6452 53/* Registers setting */54#define MTK_CQDMA_EN_BIT BIT(0)55#define MTK_CQDMA_INT_FLAG_BIT BIT(0)56#define MTK_CQDMA_INT_EN_BIT BIT(0)57#define MTK_CQDMA_FLUSH_BIT BIT(0)58 59#define MTK_CQDMA_WARM_RST_BIT BIT(0)60#define MTK_CQDMA_HARD_RST_BIT BIT(1)61 62#define MTK_CQDMA_MAX_LEN GENMASK(27, 0)63#define MTK_CQDMA_ADDR_LIMIT GENMASK(31, 0)64#define MTK_CQDMA_ADDR2_SHFIT (32)65 66/**67 * struct mtk_cqdma_vdesc - The struct holding info describing virtual68 * descriptor (CVD)69 * @vd: An instance for struct virt_dma_desc70 * @len: The total data size device wants to move71 * @residue: The remaining data size device will move72 * @dest: The destination address device wants to move to73 * @src: The source address device wants to move from74 * @ch: The pointer to the corresponding dma channel75 * @node: The lise_head struct to build link-list for VDs76 * @parent: The pointer to the parent CVD77 */78struct mtk_cqdma_vdesc {79 struct virt_dma_desc vd;80 size_t len;81 size_t residue;82 dma_addr_t dest;83 dma_addr_t src;84 struct dma_chan *ch;85 86 struct list_head node;87 struct mtk_cqdma_vdesc *parent;88};89 90/**91 * struct mtk_cqdma_pchan - The struct holding info describing physical92 * channel (PC)93 * @queue: Queue for the PDs issued to this PC94 * @base: The mapped register I/O base of this PC95 * @irq: The IRQ that this PC are using96 * @refcnt: Track how many VCs are using this PC97 * @tasklet: Tasklet for this PC98 * @lock: Lock protect agaisting multiple VCs access PC99 */100struct mtk_cqdma_pchan {101 struct list_head queue;102 void __iomem *base;103 u32 irq;104 105 refcount_t refcnt;106 107 struct tasklet_struct tasklet;108 109 /* lock to protect PC */110 spinlock_t lock;111};112 113/**114 * struct mtk_cqdma_vchan - The struct holding info describing virtual115 * channel (VC)116 * @vc: An instance for struct virt_dma_chan117 * @pc: The pointer to the underlying PC118 * @issue_completion: The wait for all issued descriptors completited119 * @issue_synchronize: Bool indicating channel synchronization starts120 */121struct mtk_cqdma_vchan {122 struct virt_dma_chan vc;123 struct mtk_cqdma_pchan *pc;124 struct completion issue_completion;125 bool issue_synchronize;126};127 128/**129 * struct mtk_cqdma_device - The struct holding info describing CQDMA130 * device131 * @ddev: An instance for struct dma_device132 * @clk: The clock that device internal is using133 * @dma_requests: The number of VCs the device supports to134 * @dma_channels: The number of PCs the device supports to135 * @vc: The pointer to all available VCs136 * @pc: The pointer to all the underlying PCs137 */138struct mtk_cqdma_device {139 struct dma_device ddev;140 struct clk *clk;141 142 u32 dma_requests;143 u32 dma_channels;144 struct mtk_cqdma_vchan *vc;145 struct mtk_cqdma_pchan **pc;146};147 148static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan)149{150 return container_of(chan->device, struct mtk_cqdma_device, ddev);151}152 153static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan)154{155 return container_of(chan, struct mtk_cqdma_vchan, vc.chan);156}157 158static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd)159{160 return container_of(vd, struct mtk_cqdma_vdesc, vd);161}162 163static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma)164{165 return cqdma->ddev.dev;166}167 168static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg)169{170 return readl(pc->base + reg);171}172 173static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)174{175 writel_relaxed(val, pc->base + reg);176}177 178static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg,179 u32 mask, u32 set)180{181 u32 val;182 183 val = mtk_dma_read(pc, reg);184 val &= ~mask;185 val |= set;186 mtk_dma_write(pc, reg, val);187}188 189static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)190{191 mtk_dma_rmw(pc, reg, 0, val);192}193 194static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)195{196 mtk_dma_rmw(pc, reg, val, 0);197}198 199static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd)200{201 kfree(to_cqdma_vdesc(vd));202}203 204static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic)205{206 u32 status = 0;207 208 if (!atomic)209 return readl_poll_timeout(pc->base + MTK_CQDMA_EN,210 status,211 !(status & MTK_CQDMA_EN_BIT),212 MTK_CQDMA_USEC_POLL,213 MTK_CQDMA_TIMEOUT_POLL);214 215 return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN,216 status,217 !(status & MTK_CQDMA_EN_BIT),218 MTK_CQDMA_USEC_POLL,219 MTK_CQDMA_TIMEOUT_POLL);220}221 222static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc)223{224 mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);225 mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);226 227 return mtk_cqdma_poll_engine_done(pc, true);228}229 230static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc,231 struct mtk_cqdma_vdesc *cvd)232{233 /* wait for the previous transaction done */234 if (mtk_cqdma_poll_engine_done(pc, true) < 0)235 dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n");236 237 /* warm reset the dma engine for the new transaction */238 mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT);239 if (mtk_cqdma_poll_engine_done(pc, true) < 0)240 dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n");241 242 /* setup the source */243 mtk_dma_set(pc, MTK_CQDMA_SRC, cvd->src & MTK_CQDMA_ADDR_LIMIT);244#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT245 mtk_dma_set(pc, MTK_CQDMA_SRC2, cvd->src >> MTK_CQDMA_ADDR2_SHFIT);246#else247 mtk_dma_set(pc, MTK_CQDMA_SRC2, 0);248#endif249 250 /* setup the destination */251 mtk_dma_set(pc, MTK_CQDMA_DST, cvd->dest & MTK_CQDMA_ADDR_LIMIT);252#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT253 mtk_dma_set(pc, MTK_CQDMA_DST2, cvd->dest >> MTK_CQDMA_ADDR2_SHFIT);254#else255 mtk_dma_set(pc, MTK_CQDMA_DST2, 0);256#endif257 258 /* setup the length */259 mtk_dma_set(pc, MTK_CQDMA_LEN1, cvd->len);260 261 /* start dma engine */262 mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT);263}264 265static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc)266{267 struct virt_dma_desc *vd, *vd2;268 struct mtk_cqdma_pchan *pc = cvc->pc;269 struct mtk_cqdma_vdesc *cvd;270 bool trigger_engine = false;271 272 lockdep_assert_held(&cvc->vc.lock);273 lockdep_assert_held(&pc->lock);274 275 list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) {276 /* need to trigger dma engine if PC's queue is empty */277 if (list_empty(&pc->queue))278 trigger_engine = true;279 280 cvd = to_cqdma_vdesc(vd);281 282 /* add VD into PC's queue */283 list_add_tail(&cvd->node, &pc->queue);284 285 /* start the dma engine */286 if (trigger_engine)287 mtk_cqdma_start(pc, cvd);288 289 /* remove VD from list desc_issued */290 list_del(&vd->node);291 }292}293 294/*295 * return true if this VC is active,296 * meaning that there are VDs under processing by the PC297 */298static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc)299{300 struct mtk_cqdma_vdesc *cvd;301 302 list_for_each_entry(cvd, &cvc->pc->queue, node)303 if (cvc == to_cqdma_vchan(cvd->ch))304 return true;305 306 return false;307}308 309/*310 * return the pointer of the CVD that is just consumed by the PC311 */312static struct mtk_cqdma_vdesc313*mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc)314{315 struct mtk_cqdma_vchan *cvc;316 struct mtk_cqdma_vdesc *cvd, *ret = NULL;317 318 /* consume a CVD from PC's queue */319 cvd = list_first_entry_or_null(&pc->queue,320 struct mtk_cqdma_vdesc, node);321 if (unlikely(!cvd || !cvd->parent))322 return NULL;323 324 cvc = to_cqdma_vchan(cvd->ch);325 ret = cvd;326 327 /* update residue of the parent CVD */328 cvd->parent->residue -= cvd->len;329 330 /* delete CVD from PC's queue */331 list_del(&cvd->node);332 333 spin_lock(&cvc->vc.lock);334 335 /* check whether all the child CVDs completed */336 if (!cvd->parent->residue) {337 /* add the parent VD into list desc_completed */338 vchan_cookie_complete(&cvd->parent->vd);339 340 /* setup completion if this VC is under synchronization */341 if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) {342 complete(&cvc->issue_completion);343 cvc->issue_synchronize = false;344 }345 }346 347 spin_unlock(&cvc->vc.lock);348 349 /* start transaction for next CVD in the queue */350 cvd = list_first_entry_or_null(&pc->queue,351 struct mtk_cqdma_vdesc, node);352 if (cvd)353 mtk_cqdma_start(pc, cvd);354 355 return ret;356}357 358static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t)359{360 struct mtk_cqdma_pchan *pc = from_tasklet(pc, t, tasklet);361 struct mtk_cqdma_vdesc *cvd = NULL;362 unsigned long flags;363 364 spin_lock_irqsave(&pc->lock, flags);365 /* consume the queue */366 cvd = mtk_cqdma_consume_work_queue(pc);367 spin_unlock_irqrestore(&pc->lock, flags);368 369 /* submit the next CVD */370 if (cvd) {371 dma_run_dependencies(&cvd->vd.tx);372 373 /*374 * free child CVD after completion.375 * the parent CVD would be freed with desc_free by user.376 */377 if (cvd->parent != cvd)378 kfree(cvd);379 }380 381 /* re-enable interrupt before leaving tasklet */382 enable_irq(pc->irq);383}384 385static irqreturn_t mtk_cqdma_irq(int irq, void *devid)386{387 struct mtk_cqdma_device *cqdma = devid;388 irqreturn_t ret = IRQ_NONE;389 bool schedule_tasklet = false;390 u32 i;391 392 /* clear interrupt flags for each PC */393 for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) {394 spin_lock(&cqdma->pc[i]->lock);395 if (mtk_dma_read(cqdma->pc[i],396 MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) {397 /* clear interrupt */398 mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_FLAG,399 MTK_CQDMA_INT_FLAG_BIT);400 401 schedule_tasklet = true;402 ret = IRQ_HANDLED;403 }404 spin_unlock(&cqdma->pc[i]->lock);405 406 if (schedule_tasklet) {407 /* disable interrupt */408 disable_irq_nosync(cqdma->pc[i]->irq);409 410 /* schedule the tasklet to handle the transactions */411 tasklet_schedule(&cqdma->pc[i]->tasklet);412 }413 }414 415 return ret;416}417 418static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c,419 dma_cookie_t cookie)420{421 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);422 struct virt_dma_desc *vd;423 unsigned long flags;424 425 spin_lock_irqsave(&cvc->pc->lock, flags);426 list_for_each_entry(vd, &cvc->pc->queue, node)427 if (vd->tx.cookie == cookie) {428 spin_unlock_irqrestore(&cvc->pc->lock, flags);429 return vd;430 }431 spin_unlock_irqrestore(&cvc->pc->lock, flags);432 433 list_for_each_entry(vd, &cvc->vc.desc_issued, node)434 if (vd->tx.cookie == cookie)435 return vd;436 437 return NULL;438}439 440static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c,441 dma_cookie_t cookie,442 struct dma_tx_state *txstate)443{444 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);445 struct mtk_cqdma_vdesc *cvd;446 struct virt_dma_desc *vd;447 enum dma_status ret;448 unsigned long flags;449 size_t bytes = 0;450 451 ret = dma_cookie_status(c, cookie, txstate);452 if (ret == DMA_COMPLETE || !txstate)453 return ret;454 455 spin_lock_irqsave(&cvc->vc.lock, flags);456 vd = mtk_cqdma_find_active_desc(c, cookie);457 spin_unlock_irqrestore(&cvc->vc.lock, flags);458 459 if (vd) {460 cvd = to_cqdma_vdesc(vd);461 bytes = cvd->residue;462 }463 464 dma_set_residue(txstate, bytes);465 466 return ret;467}468 469static void mtk_cqdma_issue_pending(struct dma_chan *c)470{471 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);472 unsigned long pc_flags;473 unsigned long vc_flags;474 475 /* acquire PC's lock before VS's lock for lock dependency in tasklet */476 spin_lock_irqsave(&cvc->pc->lock, pc_flags);477 spin_lock_irqsave(&cvc->vc.lock, vc_flags);478 479 if (vchan_issue_pending(&cvc->vc))480 mtk_cqdma_issue_vchan_pending(cvc);481 482 spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);483 spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);484}485 486static struct dma_async_tx_descriptor *487mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,488 dma_addr_t src, size_t len, unsigned long flags)489{490 struct mtk_cqdma_vdesc **cvd;491 struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL;492 size_t i, tlen, nr_vd;493 494 /*495 * In the case that trsanction length is larger than the496 * DMA engine supports, a single memcpy transaction needs497 * to be separated into several DMA transactions.498 * Each DMA transaction would be described by a CVD,499 * and the first one is referred as the parent CVD,500 * while the others are child CVDs.501 * The parent CVD's tx descriptor is the only tx descriptor502 * returned to the DMA user, and it should not be completed503 * until all the child CVDs completed.504 */505 nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN);506 cvd = kcalloc(nr_vd, sizeof(*cvd), GFP_NOWAIT);507 if (!cvd)508 return NULL;509 510 for (i = 0; i < nr_vd; ++i) {511 cvd[i] = kzalloc(sizeof(*cvd[i]), GFP_NOWAIT);512 if (!cvd[i]) {513 for (; i > 0; --i)514 kfree(cvd[i - 1]);515 return NULL;516 }517 518 /* setup dma channel */519 cvd[i]->ch = c;520 521 /* setup source, destination, and length */522 tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len;523 cvd[i]->len = tlen;524 cvd[i]->src = src;525 cvd[i]->dest = dest;526 527 /* setup tx descriptor */528 tx = vchan_tx_prep(to_virt_chan(c), &cvd[i]->vd, flags);529 tx->next = NULL;530 531 if (!i) {532 cvd[0]->residue = len;533 } else {534 prev_tx->next = tx;535 cvd[i]->residue = tlen;536 }537 538 cvd[i]->parent = cvd[0];539 540 /* update the src, dest, len, prev_tx for the next CVD */541 src += tlen;542 dest += tlen;543 len -= tlen;544 prev_tx = tx;545 }546 547 return &cvd[0]->vd.tx;548}549 550static void mtk_cqdma_free_inactive_desc(struct dma_chan *c)551{552 struct virt_dma_chan *vc = to_virt_chan(c);553 unsigned long flags;554 LIST_HEAD(head);555 556 /*557 * set desc_allocated, desc_submitted,558 * and desc_issued as the candicates to be freed559 */560 spin_lock_irqsave(&vc->lock, flags);561 list_splice_tail_init(&vc->desc_allocated, &head);562 list_splice_tail_init(&vc->desc_submitted, &head);563 list_splice_tail_init(&vc->desc_issued, &head);564 spin_unlock_irqrestore(&vc->lock, flags);565 566 /* free descriptor lists */567 vchan_dma_desc_free_list(vc, &head);568}569 570static void mtk_cqdma_free_active_desc(struct dma_chan *c)571{572 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);573 bool sync_needed = false;574 unsigned long pc_flags;575 unsigned long vc_flags;576 577 /* acquire PC's lock first due to lock dependency in dma ISR */578 spin_lock_irqsave(&cvc->pc->lock, pc_flags);579 spin_lock_irqsave(&cvc->vc.lock, vc_flags);580 581 /* synchronization is required if this VC is active */582 if (mtk_cqdma_is_vchan_active(cvc)) {583 cvc->issue_synchronize = true;584 sync_needed = true;585 }586 587 spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);588 spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);589 590 /* waiting for the completion of this VC */591 if (sync_needed)592 wait_for_completion(&cvc->issue_completion);593 594 /* free all descriptors in list desc_completed */595 vchan_synchronize(&cvc->vc);596 597 WARN_ONCE(!list_empty(&cvc->vc.desc_completed),598 "Desc pending still in list desc_completed\n");599}600 601static int mtk_cqdma_terminate_all(struct dma_chan *c)602{603 /* free descriptors not processed yet by hardware */604 mtk_cqdma_free_inactive_desc(c);605 606 /* free descriptors being processed by hardware */607 mtk_cqdma_free_active_desc(c);608 609 return 0;610}611 612static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c)613{614 struct mtk_cqdma_device *cqdma = to_cqdma_dev(c);615 struct mtk_cqdma_vchan *vc = to_cqdma_vchan(c);616 struct mtk_cqdma_pchan *pc = NULL;617 u32 i, min_refcnt = U32_MAX, refcnt;618 unsigned long flags;619 620 /* allocate PC with the minimum refcount */621 for (i = 0; i < cqdma->dma_channels; ++i) {622 refcnt = refcount_read(&cqdma->pc[i]->refcnt);623 if (refcnt < min_refcnt) {624 pc = cqdma->pc[i];625 min_refcnt = refcnt;626 }627 }628 629 if (!pc)630 return -ENOSPC;631 632 spin_lock_irqsave(&pc->lock, flags);633 634 if (!refcount_read(&pc->refcnt)) {635 /* allocate PC when the refcount is zero */636 mtk_cqdma_hard_reset(pc);637 638 /* enable interrupt for this PC */639 mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);640 641 /*642 * refcount_inc would complain increment on 0; use-after-free.643 * Thus, we need to explicitly set it as 1 initially.644 */645 refcount_set(&pc->refcnt, 1);646 } else {647 refcount_inc(&pc->refcnt);648 }649 650 spin_unlock_irqrestore(&pc->lock, flags);651 652 vc->pc = pc;653 654 return 0;655}656 657static void mtk_cqdma_free_chan_resources(struct dma_chan *c)658{659 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);660 unsigned long flags;661 662 /* free all descriptors in all lists on the VC */663 mtk_cqdma_terminate_all(c);664 665 spin_lock_irqsave(&cvc->pc->lock, flags);666 667 /* PC is not freed until there is no VC mapped to it */668 if (refcount_dec_and_test(&cvc->pc->refcnt)) {669 /* start the flush operation and stop the engine */670 mtk_dma_set(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);671 672 /* wait for the completion of flush operation */673 if (mtk_cqdma_poll_engine_done(cvc->pc, true) < 0)674 dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n");675 676 /* clear the flush bit and interrupt flag */677 mtk_dma_clr(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);678 mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_FLAG,679 MTK_CQDMA_INT_FLAG_BIT);680 681 /* disable interrupt for this PC */682 mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);683 }684 685 spin_unlock_irqrestore(&cvc->pc->lock, flags);686}687 688static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma)689{690 unsigned long flags;691 int err;692 u32 i;693 694 pm_runtime_enable(cqdma2dev(cqdma));695 pm_runtime_get_sync(cqdma2dev(cqdma));696 697 err = clk_prepare_enable(cqdma->clk);698 699 if (err) {700 pm_runtime_put_sync(cqdma2dev(cqdma));701 pm_runtime_disable(cqdma2dev(cqdma));702 return err;703 }704 705 /* reset all PCs */706 for (i = 0; i < cqdma->dma_channels; ++i) {707 spin_lock_irqsave(&cqdma->pc[i]->lock, flags);708 if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) {709 dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");710 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);711 712 clk_disable_unprepare(cqdma->clk);713 pm_runtime_put_sync(cqdma2dev(cqdma));714 pm_runtime_disable(cqdma2dev(cqdma));715 return -EINVAL;716 }717 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);718 }719 720 return 0;721}722 723static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma)724{725 unsigned long flags;726 u32 i;727 728 /* reset all PCs */729 for (i = 0; i < cqdma->dma_channels; ++i) {730 spin_lock_irqsave(&cqdma->pc[i]->lock, flags);731 if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0)732 dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");733 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);734 }735 736 clk_disable_unprepare(cqdma->clk);737 738 pm_runtime_put_sync(cqdma2dev(cqdma));739 pm_runtime_disable(cqdma2dev(cqdma));740}741 742static const struct of_device_id mtk_cqdma_match[] = {743 { .compatible = "mediatek,mt6765-cqdma" },744 { /* sentinel */ }745};746MODULE_DEVICE_TABLE(of, mtk_cqdma_match);747 748static int mtk_cqdma_probe(struct platform_device *pdev)749{750 struct mtk_cqdma_device *cqdma;751 struct mtk_cqdma_vchan *vc;752 struct dma_device *dd;753 int err;754 u32 i;755 756 cqdma = devm_kzalloc(&pdev->dev, sizeof(*cqdma), GFP_KERNEL);757 if (!cqdma)758 return -ENOMEM;759 760 dd = &cqdma->ddev;761 762 cqdma->clk = devm_clk_get(&pdev->dev, "cqdma");763 if (IS_ERR(cqdma->clk)) {764 dev_err(&pdev->dev, "No clock for %s\n",765 dev_name(&pdev->dev));766 return PTR_ERR(cqdma->clk);767 }768 769 dma_cap_set(DMA_MEMCPY, dd->cap_mask);770 771 dd->copy_align = MTK_CQDMA_ALIGN_SIZE;772 dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources;773 dd->device_free_chan_resources = mtk_cqdma_free_chan_resources;774 dd->device_tx_status = mtk_cqdma_tx_status;775 dd->device_issue_pending = mtk_cqdma_issue_pending;776 dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy;777 dd->device_terminate_all = mtk_cqdma_terminate_all;778 dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;779 dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;780 dd->directions = BIT(DMA_MEM_TO_MEM);781 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;782 dd->dev = &pdev->dev;783 INIT_LIST_HEAD(&dd->channels);784 785 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,786 "dma-requests",787 &cqdma->dma_requests)) {788 dev_info(&pdev->dev,789 "Using %u as missing dma-requests property\n",790 MTK_CQDMA_NR_VCHANS);791 792 cqdma->dma_requests = MTK_CQDMA_NR_VCHANS;793 }794 795 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,796 "dma-channels",797 &cqdma->dma_channels)) {798 dev_info(&pdev->dev,799 "Using %u as missing dma-channels property\n",800 MTK_CQDMA_NR_PCHANS);801 802 cqdma->dma_channels = MTK_CQDMA_NR_PCHANS;803 }804 805 cqdma->pc = devm_kcalloc(&pdev->dev, cqdma->dma_channels,806 sizeof(*cqdma->pc), GFP_KERNEL);807 if (!cqdma->pc)808 return -ENOMEM;809 810 /* initialization for PCs */811 for (i = 0; i < cqdma->dma_channels; ++i) {812 cqdma->pc[i] = devm_kcalloc(&pdev->dev, 1,813 sizeof(**cqdma->pc), GFP_KERNEL);814 if (!cqdma->pc[i])815 return -ENOMEM;816 817 INIT_LIST_HEAD(&cqdma->pc[i]->queue);818 spin_lock_init(&cqdma->pc[i]->lock);819 refcount_set(&cqdma->pc[i]->refcnt, 0);820 cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, i);821 if (IS_ERR(cqdma->pc[i]->base))822 return PTR_ERR(cqdma->pc[i]->base);823 824 /* allocate IRQ resource */825 err = platform_get_irq(pdev, i);826 if (err < 0)827 return err;828 cqdma->pc[i]->irq = err;829 830 err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq,831 mtk_cqdma_irq, 0, dev_name(&pdev->dev),832 cqdma);833 if (err) {834 dev_err(&pdev->dev,835 "request_irq failed with err %d\n", err);836 return -EINVAL;837 }838 }839 840 /* allocate resource for VCs */841 cqdma->vc = devm_kcalloc(&pdev->dev, cqdma->dma_requests,842 sizeof(*cqdma->vc), GFP_KERNEL);843 if (!cqdma->vc)844 return -ENOMEM;845 846 for (i = 0; i < cqdma->dma_requests; i++) {847 vc = &cqdma->vc[i];848 vc->vc.desc_free = mtk_cqdma_vdesc_free;849 vchan_init(&vc->vc, dd);850 init_completion(&vc->issue_completion);851 }852 853 err = dma_async_device_register(dd);854 if (err)855 return err;856 857 err = of_dma_controller_register(pdev->dev.of_node,858 of_dma_xlate_by_chan_id, cqdma);859 if (err) {860 dev_err(&pdev->dev,861 "MediaTek CQDMA OF registration failed %d\n", err);862 goto err_unregister;863 }864 865 err = mtk_cqdma_hw_init(cqdma);866 if (err) {867 dev_err(&pdev->dev,868 "MediaTek CQDMA HW initialization failed %d\n", err);869 goto err_unregister;870 }871 872 platform_set_drvdata(pdev, cqdma);873 874 /* initialize tasklet for each PC */875 for (i = 0; i < cqdma->dma_channels; ++i)876 tasklet_setup(&cqdma->pc[i]->tasklet, mtk_cqdma_tasklet_cb);877 878 dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n");879 880 return 0;881 882err_unregister:883 dma_async_device_unregister(dd);884 885 return err;886}887 888static void mtk_cqdma_remove(struct platform_device *pdev)889{890 struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev);891 struct mtk_cqdma_vchan *vc;892 unsigned long flags;893 int i;894 895 /* kill VC task */896 for (i = 0; i < cqdma->dma_requests; i++) {897 vc = &cqdma->vc[i];898 899 list_del(&vc->vc.chan.device_node);900 tasklet_kill(&vc->vc.task);901 }902 903 /* disable interrupt */904 for (i = 0; i < cqdma->dma_channels; i++) {905 spin_lock_irqsave(&cqdma->pc[i]->lock, flags);906 mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_EN,907 MTK_CQDMA_INT_EN_BIT);908 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);909 910 /* Waits for any pending IRQ handlers to complete */911 synchronize_irq(cqdma->pc[i]->irq);912 913 tasklet_kill(&cqdma->pc[i]->tasklet);914 }915 916 /* disable hardware */917 mtk_cqdma_hw_deinit(cqdma);918 919 dma_async_device_unregister(&cqdma->ddev);920 of_dma_controller_free(pdev->dev.of_node);921}922 923static struct platform_driver mtk_cqdma_driver = {924 .probe = mtk_cqdma_probe,925 .remove_new = mtk_cqdma_remove,926 .driver = {927 .name = KBUILD_MODNAME,928 .of_match_table = mtk_cqdma_match,929 },930};931module_platform_driver(mtk_cqdma_driver);932 933MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver");934MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>");935MODULE_LICENSE("GPL v2");936