1052 lines · c
1// SPDX-License-Identifier: GPL-2.02// Copyright (c) 2017-2018 MediaTek Inc.3 4/*5 * Driver for MediaTek High-Speed DMA Controller6 *7 * Author: Sean Wang <sean.wang@mediatek.com>8 *9 */10 11#include <linux/bitops.h>12#include <linux/clk.h>13#include <linux/dmaengine.h>14#include <linux/dma-mapping.h>15#include <linux/err.h>16#include <linux/iopoll.h>17#include <linux/list.h>18#include <linux/module.h>19#include <linux/of.h>20#include <linux/of_dma.h>21#include <linux/platform_device.h>22#include <linux/pm_runtime.h>23#include <linux/refcount.h>24#include <linux/slab.h>25 26#include "../virt-dma.h"27 28#define MTK_HSDMA_USEC_POLL 2029#define MTK_HSDMA_TIMEOUT_POLL 20000030#define MTK_HSDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)31 32/* The default number of virtual channel */33#define MTK_HSDMA_NR_VCHANS 334 35/* Only one physical channel supported */36#define MTK_HSDMA_NR_MAX_PCHANS 137 38/* Macro for physical descriptor (PD) manipulation */39/* The number of PD which must be 2 of power */40#define MTK_DMA_SIZE 6441#define MTK_HSDMA_NEXT_DESP_IDX(x, y) (((x) + 1) & ((y) - 1))42#define MTK_HSDMA_LAST_DESP_IDX(x, y) (((x) - 1) & ((y) - 1))43#define MTK_HSDMA_MAX_LEN 0x3f8044#define MTK_HSDMA_ALIGN_SIZE 445#define MTK_HSDMA_PLEN_MASK 0x3fff46#define MTK_HSDMA_DESC_PLEN(x) (((x) & MTK_HSDMA_PLEN_MASK) << 16)47#define MTK_HSDMA_DESC_PLEN_GET(x) (((x) >> 16) & MTK_HSDMA_PLEN_MASK)48 49/* Registers for underlying ring manipulation */50#define MTK_HSDMA_TX_BASE 0x051#define MTK_HSDMA_TX_CNT 0x452#define MTK_HSDMA_TX_CPU 0x853#define MTK_HSDMA_TX_DMA 0xc54#define MTK_HSDMA_RX_BASE 0x10055#define MTK_HSDMA_RX_CNT 0x10456#define MTK_HSDMA_RX_CPU 0x10857#define MTK_HSDMA_RX_DMA 0x10c58 59/* Registers for global setup */60#define MTK_HSDMA_GLO 0x20461#define MTK_HSDMA_GLO_MULTI_DMA BIT(10)62#define MTK_HSDMA_TX_WB_DDONE BIT(6)63#define MTK_HSDMA_BURST_64BYTES (0x2 << 4)64#define MTK_HSDMA_GLO_RX_BUSY BIT(3)65#define MTK_HSDMA_GLO_RX_DMA BIT(2)66#define MTK_HSDMA_GLO_TX_BUSY BIT(1)67#define MTK_HSDMA_GLO_TX_DMA BIT(0)68#define MTK_HSDMA_GLO_DMA (MTK_HSDMA_GLO_TX_DMA | \69 MTK_HSDMA_GLO_RX_DMA)70#define MTK_HSDMA_GLO_BUSY (MTK_HSDMA_GLO_RX_BUSY | \71 MTK_HSDMA_GLO_TX_BUSY)72#define MTK_HSDMA_GLO_DEFAULT (MTK_HSDMA_GLO_TX_DMA | \73 MTK_HSDMA_GLO_RX_DMA | \74 MTK_HSDMA_TX_WB_DDONE | \75 MTK_HSDMA_BURST_64BYTES | \76 MTK_HSDMA_GLO_MULTI_DMA)77 78/* Registers for reset */79#define MTK_HSDMA_RESET 0x20880#define MTK_HSDMA_RST_TX BIT(0)81#define MTK_HSDMA_RST_RX BIT(16)82 83/* Registers for interrupt control */84#define MTK_HSDMA_DLYINT 0x20c85#define MTK_HSDMA_RXDLY_INT_EN BIT(15)86 87/* Interrupt fires when the pending number's more than the specified */88#define MTK_HSDMA_RXMAX_PINT(x) (((x) & 0x7f) << 8)89 90/* Interrupt fires when the pending time's more than the specified in 20 us */91#define MTK_HSDMA_RXMAX_PTIME(x) ((x) & 0x7f)92#define MTK_HSDMA_DLYINT_DEFAULT (MTK_HSDMA_RXDLY_INT_EN | \93 MTK_HSDMA_RXMAX_PINT(20) | \94 MTK_HSDMA_RXMAX_PTIME(20))95#define MTK_HSDMA_INT_STATUS 0x22096#define MTK_HSDMA_INT_ENABLE 0x22897#define MTK_HSDMA_INT_RXDONE BIT(16)98 99enum mtk_hsdma_vdesc_flag {100 MTK_HSDMA_VDESC_FINISHED = 0x01,101};102 103#define IS_MTK_HSDMA_VDESC_FINISHED(x) ((x) == MTK_HSDMA_VDESC_FINISHED)104 105/**106 * struct mtk_hsdma_pdesc - This is the struct holding info describing physical107 * descriptor (PD) and its placement must be kept at108 * 4-bytes alignment in little endian order.109 * @desc1: | The control pad used to indicate hardware how to110 * @desc2: | deal with the descriptor such as source and111 * @desc3: | destination address and data length. The maximum112 * @desc4: | data length each pdesc can handle is 0x3f80 bytes113 */114struct mtk_hsdma_pdesc {115 __le32 desc1;116 __le32 desc2;117 __le32 desc3;118 __le32 desc4;119} __packed __aligned(4);120 121/**122 * struct mtk_hsdma_vdesc - This is the struct holding info describing virtual123 * descriptor (VD)124 * @vd: An instance for struct virt_dma_desc125 * @len: The total data size device wants to move126 * @residue: The remaining data size device will move127 * @dest: The destination address device wants to move to128 * @src: The source address device wants to move from129 */130struct mtk_hsdma_vdesc {131 struct virt_dma_desc vd;132 size_t len;133 size_t residue;134 dma_addr_t dest;135 dma_addr_t src;136};137 138/**139 * struct mtk_hsdma_cb - This is the struct holding extra info required for RX140 * ring to know what relevant VD the PD is being141 * mapped to.142 * @vd: Pointer to the relevant VD.143 * @flag: Flag indicating what action should be taken when VD144 * is completed.145 */146struct mtk_hsdma_cb {147 struct virt_dma_desc *vd;148 enum mtk_hsdma_vdesc_flag flag;149};150 151/**152 * struct mtk_hsdma_ring - This struct holds info describing underlying ring153 * space154 * @txd: The descriptor TX ring which describes DMA source155 * information156 * @rxd: The descriptor RX ring which describes DMA157 * destination information158 * @cb: The extra information pointed at by RX ring159 * @tphys: The physical addr of TX ring160 * @rphys: The physical addr of RX ring161 * @cur_tptr: Pointer to the next free descriptor used by the host162 * @cur_rptr: Pointer to the last done descriptor by the device163 */164struct mtk_hsdma_ring {165 struct mtk_hsdma_pdesc *txd;166 struct mtk_hsdma_pdesc *rxd;167 struct mtk_hsdma_cb *cb;168 dma_addr_t tphys;169 dma_addr_t rphys;170 u16 cur_tptr;171 u16 cur_rptr;172};173 174/**175 * struct mtk_hsdma_pchan - This is the struct holding info describing physical176 * channel (PC)177 * @ring: An instance for the underlying ring178 * @sz_ring: Total size allocated for the ring179 * @nr_free: Total number of free rooms in the ring. It would180 * be accessed and updated frequently between IRQ181 * context and user context to reflect whether ring182 * can accept requests from VD.183 */184struct mtk_hsdma_pchan {185 struct mtk_hsdma_ring ring;186 size_t sz_ring;187 atomic_t nr_free;188};189 190/**191 * struct mtk_hsdma_vchan - This is the struct holding info describing virtual192 * channel (VC)193 * @vc: An instance for struct virt_dma_chan194 * @issue_completion: The wait for all issued descriptors completited195 * @issue_synchronize: Bool indicating channel synchronization starts196 * @desc_hw_processing: List those descriptors the hardware is processing,197 * which is protected by vc.lock198 */199struct mtk_hsdma_vchan {200 struct virt_dma_chan vc;201 struct completion issue_completion;202 bool issue_synchronize;203 struct list_head desc_hw_processing;204};205 206/**207 * struct mtk_hsdma_soc - This is the struct holding differences among SoCs208 * @ddone: Bit mask for DDONE209 * @ls0: Bit mask for LS0210 */211struct mtk_hsdma_soc {212 __le32 ddone;213 __le32 ls0;214};215 216/**217 * struct mtk_hsdma_device - This is the struct holding info describing HSDMA218 * device219 * @ddev: An instance for struct dma_device220 * @base: The mapped register I/O base221 * @clk: The clock that device internal is using222 * @irq: The IRQ that device are using223 * @dma_requests: The number of VCs the device supports to224 * @vc: The pointer to all available VCs225 * @pc: The pointer to the underlying PC226 * @pc_refcnt: Track how many VCs are using the PC227 * @lock: Lock protect agaisting multiple VCs access PC228 * @soc: The pointer to area holding differences among229 * various platform230 */231struct mtk_hsdma_device {232 struct dma_device ddev;233 void __iomem *base;234 struct clk *clk;235 u32 irq;236 237 u32 dma_requests;238 struct mtk_hsdma_vchan *vc;239 struct mtk_hsdma_pchan *pc;240 refcount_t pc_refcnt;241 242 /* Lock used to protect against multiple VCs access PC */243 spinlock_t lock;244 245 const struct mtk_hsdma_soc *soc;246};247 248static struct mtk_hsdma_device *to_hsdma_dev(struct dma_chan *chan)249{250 return container_of(chan->device, struct mtk_hsdma_device, ddev);251}252 253static inline struct mtk_hsdma_vchan *to_hsdma_vchan(struct dma_chan *chan)254{255 return container_of(chan, struct mtk_hsdma_vchan, vc.chan);256}257 258static struct mtk_hsdma_vdesc *to_hsdma_vdesc(struct virt_dma_desc *vd)259{260 return container_of(vd, struct mtk_hsdma_vdesc, vd);261}262 263static struct device *hsdma2dev(struct mtk_hsdma_device *hsdma)264{265 return hsdma->ddev.dev;266}267 268static u32 mtk_dma_read(struct mtk_hsdma_device *hsdma, u32 reg)269{270 return readl(hsdma->base + reg);271}272 273static void mtk_dma_write(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)274{275 writel(val, hsdma->base + reg);276}277 278static void mtk_dma_rmw(struct mtk_hsdma_device *hsdma, u32 reg,279 u32 mask, u32 set)280{281 u32 val;282 283 val = mtk_dma_read(hsdma, reg);284 val &= ~mask;285 val |= set;286 mtk_dma_write(hsdma, reg, val);287}288 289static void mtk_dma_set(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)290{291 mtk_dma_rmw(hsdma, reg, 0, val);292}293 294static void mtk_dma_clr(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)295{296 mtk_dma_rmw(hsdma, reg, val, 0);297}298 299static void mtk_hsdma_vdesc_free(struct virt_dma_desc *vd)300{301 kfree(container_of(vd, struct mtk_hsdma_vdesc, vd));302}303 304static int mtk_hsdma_busy_wait(struct mtk_hsdma_device *hsdma)305{306 u32 status = 0;307 308 return readl_poll_timeout(hsdma->base + MTK_HSDMA_GLO, status,309 !(status & MTK_HSDMA_GLO_BUSY),310 MTK_HSDMA_USEC_POLL,311 MTK_HSDMA_TIMEOUT_POLL);312}313 314static int mtk_hsdma_alloc_pchan(struct mtk_hsdma_device *hsdma,315 struct mtk_hsdma_pchan *pc)316{317 struct mtk_hsdma_ring *ring = &pc->ring;318 int err;319 320 memset(pc, 0, sizeof(*pc));321 322 /*323 * Allocate ring space where [0 ... MTK_DMA_SIZE - 1] is for TX ring324 * and [MTK_DMA_SIZE ... 2 * MTK_DMA_SIZE - 1] is for RX ring.325 */326 pc->sz_ring = 2 * MTK_DMA_SIZE * sizeof(*ring->txd);327 ring->txd = dma_alloc_coherent(hsdma2dev(hsdma), pc->sz_ring,328 &ring->tphys, GFP_NOWAIT);329 if (!ring->txd)330 return -ENOMEM;331 332 ring->rxd = &ring->txd[MTK_DMA_SIZE];333 ring->rphys = ring->tphys + MTK_DMA_SIZE * sizeof(*ring->txd);334 ring->cur_tptr = 0;335 ring->cur_rptr = MTK_DMA_SIZE - 1;336 337 ring->cb = kcalloc(MTK_DMA_SIZE, sizeof(*ring->cb), GFP_NOWAIT);338 if (!ring->cb) {339 err = -ENOMEM;340 goto err_free_dma;341 }342 343 atomic_set(&pc->nr_free, MTK_DMA_SIZE - 1);344 345 /* Disable HSDMA and wait for the completion */346 mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);347 err = mtk_hsdma_busy_wait(hsdma);348 if (err)349 goto err_free_cb;350 351 /* Reset */352 mtk_dma_set(hsdma, MTK_HSDMA_RESET,353 MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX);354 mtk_dma_clr(hsdma, MTK_HSDMA_RESET,355 MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX);356 357 /* Setup HSDMA initial pointer in the ring */358 mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, ring->tphys);359 mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, MTK_DMA_SIZE);360 mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr);361 mtk_dma_write(hsdma, MTK_HSDMA_TX_DMA, 0);362 mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, ring->rphys);363 mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, MTK_DMA_SIZE);364 mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, ring->cur_rptr);365 mtk_dma_write(hsdma, MTK_HSDMA_RX_DMA, 0);366 367 /* Enable HSDMA */368 mtk_dma_set(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);369 370 /* Setup delayed interrupt */371 mtk_dma_write(hsdma, MTK_HSDMA_DLYINT, MTK_HSDMA_DLYINT_DEFAULT);372 373 /* Enable interrupt */374 mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);375 376 return 0;377 378err_free_cb:379 kfree(ring->cb);380 381err_free_dma:382 dma_free_coherent(hsdma2dev(hsdma),383 pc->sz_ring, ring->txd, ring->tphys);384 return err;385}386 387static void mtk_hsdma_free_pchan(struct mtk_hsdma_device *hsdma,388 struct mtk_hsdma_pchan *pc)389{390 struct mtk_hsdma_ring *ring = &pc->ring;391 392 /* Disable HSDMA and then wait for the completion */393 mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);394 mtk_hsdma_busy_wait(hsdma);395 396 /* Reset pointer in the ring */397 mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);398 mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, 0);399 mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, 0);400 mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, 0);401 mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, 0);402 mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, 0);403 mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, MTK_DMA_SIZE - 1);404 405 kfree(ring->cb);406 407 dma_free_coherent(hsdma2dev(hsdma),408 pc->sz_ring, ring->txd, ring->tphys);409}410 411static int mtk_hsdma_issue_pending_vdesc(struct mtk_hsdma_device *hsdma,412 struct mtk_hsdma_pchan *pc,413 struct mtk_hsdma_vdesc *hvd)414{415 struct mtk_hsdma_ring *ring = &pc->ring;416 struct mtk_hsdma_pdesc *txd, *rxd;417 u16 reserved, prev, tlen, num_sgs;418 unsigned long flags;419 420 /* Protect against PC is accessed by multiple VCs simultaneously */421 spin_lock_irqsave(&hsdma->lock, flags);422 423 /*424 * Reserve rooms, where pc->nr_free is used to track how many free425 * rooms in the ring being updated in user and IRQ context.426 */427 num_sgs = DIV_ROUND_UP(hvd->len, MTK_HSDMA_MAX_LEN);428 reserved = min_t(u16, num_sgs, atomic_read(&pc->nr_free));429 430 if (!reserved) {431 spin_unlock_irqrestore(&hsdma->lock, flags);432 return -ENOSPC;433 }434 435 atomic_sub(reserved, &pc->nr_free);436 437 while (reserved--) {438 /* Limit size by PD capability for valid data moving */439 tlen = (hvd->len > MTK_HSDMA_MAX_LEN) ?440 MTK_HSDMA_MAX_LEN : hvd->len;441 442 /*443 * Setup PDs using the remaining VD info mapped on those444 * reserved rooms. And since RXD is shared memory between the445 * host and the device allocated by dma_alloc_coherent call,446 * the helper macro WRITE_ONCE can ensure the data written to447 * RAM would really happens.448 */449 txd = &ring->txd[ring->cur_tptr];450 WRITE_ONCE(txd->desc1, hvd->src);451 WRITE_ONCE(txd->desc2,452 hsdma->soc->ls0 | MTK_HSDMA_DESC_PLEN(tlen));453 454 rxd = &ring->rxd[ring->cur_tptr];455 WRITE_ONCE(rxd->desc1, hvd->dest);456 WRITE_ONCE(rxd->desc2, MTK_HSDMA_DESC_PLEN(tlen));457 458 /* Associate VD, the PD belonged to */459 ring->cb[ring->cur_tptr].vd = &hvd->vd;460 461 /* Move forward the pointer of TX ring */462 ring->cur_tptr = MTK_HSDMA_NEXT_DESP_IDX(ring->cur_tptr,463 MTK_DMA_SIZE);464 465 /* Update VD with remaining data */466 hvd->src += tlen;467 hvd->dest += tlen;468 hvd->len -= tlen;469 }470 471 /*472 * Tagging flag for the last PD for VD will be responsible for473 * completing VD.474 */475 if (!hvd->len) {476 prev = MTK_HSDMA_LAST_DESP_IDX(ring->cur_tptr, MTK_DMA_SIZE);477 ring->cb[prev].flag = MTK_HSDMA_VDESC_FINISHED;478 }479 480 /* Ensure all changes indeed done before we're going on */481 wmb();482 483 /*484 * Updating into hardware the pointer of TX ring lets HSDMA to take485 * action for those pending PDs.486 */487 mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr);488 489 spin_unlock_irqrestore(&hsdma->lock, flags);490 491 return 0;492}493 494static void mtk_hsdma_issue_vchan_pending(struct mtk_hsdma_device *hsdma,495 struct mtk_hsdma_vchan *hvc)496{497 struct virt_dma_desc *vd, *vd2;498 int err;499 500 lockdep_assert_held(&hvc->vc.lock);501 502 list_for_each_entry_safe(vd, vd2, &hvc->vc.desc_issued, node) {503 struct mtk_hsdma_vdesc *hvd;504 505 hvd = to_hsdma_vdesc(vd);506 507 /* Map VD into PC and all VCs shares a single PC */508 err = mtk_hsdma_issue_pending_vdesc(hsdma, hsdma->pc, hvd);509 510 /*511 * Move VD from desc_issued to desc_hw_processing when entire512 * VD is fit into available PDs. Otherwise, the uncompleted513 * VDs would stay in list desc_issued and then restart the514 * processing as soon as possible once underlying ring space515 * got freed.516 */517 if (err == -ENOSPC || hvd->len > 0)518 break;519 520 /*521 * The extra list desc_hw_processing is used because522 * hardware can't provide sufficient information allowing us523 * to know what VDs are still working on the underlying ring.524 * Through the additional list, it can help us to implement525 * terminate_all, residue calculation and such thing needed526 * to know detail descriptor status on the hardware.527 */528 list_move_tail(&vd->node, &hvc->desc_hw_processing);529 }530}531 532static void mtk_hsdma_free_rooms_in_ring(struct mtk_hsdma_device *hsdma)533{534 struct mtk_hsdma_vchan *hvc;535 struct mtk_hsdma_pdesc *rxd;536 struct mtk_hsdma_vdesc *hvd;537 struct mtk_hsdma_pchan *pc;538 struct mtk_hsdma_cb *cb;539 int i = MTK_DMA_SIZE;540 __le32 desc2;541 u32 status;542 u16 next;543 544 /* Read IRQ status */545 status = mtk_dma_read(hsdma, MTK_HSDMA_INT_STATUS);546 if (unlikely(!(status & MTK_HSDMA_INT_RXDONE)))547 goto rx_done;548 549 pc = hsdma->pc;550 551 /*552 * Using a fail-safe loop with iterations of up to MTK_DMA_SIZE to553 * reclaim these finished descriptors: The most number of PDs the ISR554 * can handle at one time shouldn't be more than MTK_DMA_SIZE so we555 * take it as limited count instead of just using a dangerous infinite556 * poll.557 */558 while (i--) {559 next = MTK_HSDMA_NEXT_DESP_IDX(pc->ring.cur_rptr,560 MTK_DMA_SIZE);561 rxd = &pc->ring.rxd[next];562 563 /*564 * If MTK_HSDMA_DESC_DDONE is no specified, that means data565 * moving for the PD is still under going.566 */567 desc2 = READ_ONCE(rxd->desc2);568 if (!(desc2 & hsdma->soc->ddone))569 break;570 571 cb = &pc->ring.cb[next];572 if (unlikely(!cb->vd)) {573 dev_err(hsdma2dev(hsdma), "cb->vd cannot be null\n");574 break;575 }576 577 /* Update residue of VD the associated PD belonged to */578 hvd = to_hsdma_vdesc(cb->vd);579 hvd->residue -= MTK_HSDMA_DESC_PLEN_GET(rxd->desc2);580 581 /* Complete VD until the relevant last PD is finished */582 if (IS_MTK_HSDMA_VDESC_FINISHED(cb->flag)) {583 hvc = to_hsdma_vchan(cb->vd->tx.chan);584 585 spin_lock(&hvc->vc.lock);586 587 /* Remove VD from list desc_hw_processing */588 list_del(&cb->vd->node);589 590 /* Add VD into list desc_completed */591 vchan_cookie_complete(cb->vd);592 593 if (hvc->issue_synchronize &&594 list_empty(&hvc->desc_hw_processing)) {595 complete(&hvc->issue_completion);596 hvc->issue_synchronize = false;597 }598 spin_unlock(&hvc->vc.lock);599 600 cb->flag = 0;601 }602 603 cb->vd = NULL;604 605 /*606 * Recycle the RXD with the helper WRITE_ONCE that can ensure607 * data written into RAM would really happens.608 */609 WRITE_ONCE(rxd->desc1, 0);610 WRITE_ONCE(rxd->desc2, 0);611 pc->ring.cur_rptr = next;612 613 /* Release rooms */614 atomic_inc(&pc->nr_free);615 }616 617 /* Ensure all changes indeed done before we're going on */618 wmb();619 620 /* Update CPU pointer for those completed PDs */621 mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, pc->ring.cur_rptr);622 623 /*624 * Acking the pending IRQ allows hardware no longer to keep the used625 * IRQ line in certain trigger state when software has completed all626 * the finished physical descriptors.627 */628 if (atomic_read(&pc->nr_free) >= MTK_DMA_SIZE - 1)629 mtk_dma_write(hsdma, MTK_HSDMA_INT_STATUS, status);630 631 /* ASAP handles pending VDs in all VCs after freeing some rooms */632 for (i = 0; i < hsdma->dma_requests; i++) {633 hvc = &hsdma->vc[i];634 spin_lock(&hvc->vc.lock);635 mtk_hsdma_issue_vchan_pending(hsdma, hvc);636 spin_unlock(&hvc->vc.lock);637 }638 639rx_done:640 /* All completed PDs are cleaned up, so enable interrupt again */641 mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);642}643 644static irqreturn_t mtk_hsdma_irq(int irq, void *devid)645{646 struct mtk_hsdma_device *hsdma = devid;647 648 /*649 * Disable interrupt until all completed PDs are cleaned up in650 * mtk_hsdma_free_rooms call.651 */652 mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);653 654 mtk_hsdma_free_rooms_in_ring(hsdma);655 656 return IRQ_HANDLED;657}658 659static struct virt_dma_desc *mtk_hsdma_find_active_desc(struct dma_chan *c,660 dma_cookie_t cookie)661{662 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);663 struct virt_dma_desc *vd;664 665 list_for_each_entry(vd, &hvc->desc_hw_processing, node)666 if (vd->tx.cookie == cookie)667 return vd;668 669 list_for_each_entry(vd, &hvc->vc.desc_issued, node)670 if (vd->tx.cookie == cookie)671 return vd;672 673 return NULL;674}675 676static enum dma_status mtk_hsdma_tx_status(struct dma_chan *c,677 dma_cookie_t cookie,678 struct dma_tx_state *txstate)679{680 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);681 struct mtk_hsdma_vdesc *hvd;682 struct virt_dma_desc *vd;683 enum dma_status ret;684 unsigned long flags;685 size_t bytes = 0;686 687 ret = dma_cookie_status(c, cookie, txstate);688 if (ret == DMA_COMPLETE || !txstate)689 return ret;690 691 spin_lock_irqsave(&hvc->vc.lock, flags);692 vd = mtk_hsdma_find_active_desc(c, cookie);693 spin_unlock_irqrestore(&hvc->vc.lock, flags);694 695 if (vd) {696 hvd = to_hsdma_vdesc(vd);697 bytes = hvd->residue;698 }699 700 dma_set_residue(txstate, bytes);701 702 return ret;703}704 705static void mtk_hsdma_issue_pending(struct dma_chan *c)706{707 struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);708 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);709 unsigned long flags;710 711 spin_lock_irqsave(&hvc->vc.lock, flags);712 713 if (vchan_issue_pending(&hvc->vc))714 mtk_hsdma_issue_vchan_pending(hsdma, hvc);715 716 spin_unlock_irqrestore(&hvc->vc.lock, flags);717}718 719static struct dma_async_tx_descriptor *720mtk_hsdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,721 dma_addr_t src, size_t len, unsigned long flags)722{723 struct mtk_hsdma_vdesc *hvd;724 725 hvd = kzalloc(sizeof(*hvd), GFP_NOWAIT);726 if (!hvd)727 return NULL;728 729 hvd->len = len;730 hvd->residue = len;731 hvd->src = src;732 hvd->dest = dest;733 734 return vchan_tx_prep(to_virt_chan(c), &hvd->vd, flags);735}736 737static int mtk_hsdma_free_inactive_desc(struct dma_chan *c)738{739 struct virt_dma_chan *vc = to_virt_chan(c);740 unsigned long flags;741 LIST_HEAD(head);742 743 spin_lock_irqsave(&vc->lock, flags);744 list_splice_tail_init(&vc->desc_allocated, &head);745 list_splice_tail_init(&vc->desc_submitted, &head);746 list_splice_tail_init(&vc->desc_issued, &head);747 spin_unlock_irqrestore(&vc->lock, flags);748 749 /* At the point, we don't expect users put descriptor into VC again */750 vchan_dma_desc_free_list(vc, &head);751 752 return 0;753}754 755static void mtk_hsdma_free_active_desc(struct dma_chan *c)756{757 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);758 bool sync_needed = false;759 760 /*761 * Once issue_synchronize is being set, which means once the hardware762 * consumes all descriptors for the channel in the ring, the763 * synchronization must be notified immediately it is completed.764 */765 spin_lock(&hvc->vc.lock);766 if (!list_empty(&hvc->desc_hw_processing)) {767 hvc->issue_synchronize = true;768 sync_needed = true;769 }770 spin_unlock(&hvc->vc.lock);771 772 if (sync_needed)773 wait_for_completion(&hvc->issue_completion);774 /*775 * At the point, we expect that all remaining descriptors in the ring776 * for the channel should be all processing done.777 */778 WARN_ONCE(!list_empty(&hvc->desc_hw_processing),779 "Desc pending still in list desc_hw_processing\n");780 781 /* Free all descriptors in list desc_completed */782 vchan_synchronize(&hvc->vc);783 784 WARN_ONCE(!list_empty(&hvc->vc.desc_completed),785 "Desc pending still in list desc_completed\n");786}787 788static int mtk_hsdma_terminate_all(struct dma_chan *c)789{790 /*791 * Free pending descriptors not processed yet by hardware that have792 * previously been submitted to the channel.793 */794 mtk_hsdma_free_inactive_desc(c);795 796 /*797 * However, the DMA engine doesn't provide any way to stop these798 * descriptors being processed currently by hardware. The only way is799 * to just waiting until these descriptors are all processed completely800 * through mtk_hsdma_free_active_desc call.801 */802 mtk_hsdma_free_active_desc(c);803 804 return 0;805}806 807static int mtk_hsdma_alloc_chan_resources(struct dma_chan *c)808{809 struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);810 int err;811 812 /*813 * Since HSDMA has only one PC, the resource for PC is being allocated814 * when the first VC is being created and the other VCs would run on815 * the same PC.816 */817 if (!refcount_read(&hsdma->pc_refcnt)) {818 err = mtk_hsdma_alloc_pchan(hsdma, hsdma->pc);819 if (err)820 return err;821 /*822 * refcount_inc would complain increment on 0; use-after-free.823 * Thus, we need to explicitly set it as 1 initially.824 */825 refcount_set(&hsdma->pc_refcnt, 1);826 } else {827 refcount_inc(&hsdma->pc_refcnt);828 }829 830 return 0;831}832 833static void mtk_hsdma_free_chan_resources(struct dma_chan *c)834{835 struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);836 837 /* Free all descriptors in all lists on the VC */838 mtk_hsdma_terminate_all(c);839 840 /* The resource for PC is not freed until all the VCs are destroyed */841 if (!refcount_dec_and_test(&hsdma->pc_refcnt))842 return;843 844 mtk_hsdma_free_pchan(hsdma, hsdma->pc);845}846 847static int mtk_hsdma_hw_init(struct mtk_hsdma_device *hsdma)848{849 int err;850 851 pm_runtime_enable(hsdma2dev(hsdma));852 pm_runtime_get_sync(hsdma2dev(hsdma));853 854 err = clk_prepare_enable(hsdma->clk);855 if (err)856 return err;857 858 mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0);859 mtk_dma_write(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DEFAULT);860 861 return 0;862}863 864static int mtk_hsdma_hw_deinit(struct mtk_hsdma_device *hsdma)865{866 mtk_dma_write(hsdma, MTK_HSDMA_GLO, 0);867 868 clk_disable_unprepare(hsdma->clk);869 870 pm_runtime_put_sync(hsdma2dev(hsdma));871 pm_runtime_disable(hsdma2dev(hsdma));872 873 return 0;874}875 876static const struct mtk_hsdma_soc mt7623_soc = {877 .ddone = BIT(31),878 .ls0 = BIT(30),879};880 881static const struct mtk_hsdma_soc mt7622_soc = {882 .ddone = BIT(15),883 .ls0 = BIT(14),884};885 886static const struct of_device_id mtk_hsdma_match[] = {887 { .compatible = "mediatek,mt7623-hsdma", .data = &mt7623_soc},888 { .compatible = "mediatek,mt7622-hsdma", .data = &mt7622_soc},889 { /* sentinel */ }890};891MODULE_DEVICE_TABLE(of, mtk_hsdma_match);892 893static int mtk_hsdma_probe(struct platform_device *pdev)894{895 struct mtk_hsdma_device *hsdma;896 struct mtk_hsdma_vchan *vc;897 struct dma_device *dd;898 int i, err;899 900 hsdma = devm_kzalloc(&pdev->dev, sizeof(*hsdma), GFP_KERNEL);901 if (!hsdma)902 return -ENOMEM;903 904 dd = &hsdma->ddev;905 906 hsdma->base = devm_platform_ioremap_resource(pdev, 0);907 if (IS_ERR(hsdma->base))908 return PTR_ERR(hsdma->base);909 910 hsdma->soc = of_device_get_match_data(&pdev->dev);911 if (!hsdma->soc) {912 dev_err(&pdev->dev, "No device match found\n");913 return -ENODEV;914 }915 916 hsdma->clk = devm_clk_get(&pdev->dev, "hsdma");917 if (IS_ERR(hsdma->clk)) {918 dev_err(&pdev->dev, "No clock for %s\n",919 dev_name(&pdev->dev));920 return PTR_ERR(hsdma->clk);921 }922 923 err = platform_get_irq(pdev, 0);924 if (err < 0)925 return err;926 hsdma->irq = err;927 928 refcount_set(&hsdma->pc_refcnt, 0);929 spin_lock_init(&hsdma->lock);930 931 dma_cap_set(DMA_MEMCPY, dd->cap_mask);932 933 dd->copy_align = MTK_HSDMA_ALIGN_SIZE;934 dd->device_alloc_chan_resources = mtk_hsdma_alloc_chan_resources;935 dd->device_free_chan_resources = mtk_hsdma_free_chan_resources;936 dd->device_tx_status = mtk_hsdma_tx_status;937 dd->device_issue_pending = mtk_hsdma_issue_pending;938 dd->device_prep_dma_memcpy = mtk_hsdma_prep_dma_memcpy;939 dd->device_terminate_all = mtk_hsdma_terminate_all;940 dd->src_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS;941 dd->dst_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS;942 dd->directions = BIT(DMA_MEM_TO_MEM);943 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;944 dd->dev = &pdev->dev;945 INIT_LIST_HEAD(&dd->channels);946 947 hsdma->dma_requests = MTK_HSDMA_NR_VCHANS;948 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,949 "dma-requests",950 &hsdma->dma_requests)) {951 dev_info(&pdev->dev,952 "Using %u as missing dma-requests property\n",953 MTK_HSDMA_NR_VCHANS);954 }955 956 hsdma->pc = devm_kcalloc(&pdev->dev, MTK_HSDMA_NR_MAX_PCHANS,957 sizeof(*hsdma->pc), GFP_KERNEL);958 if (!hsdma->pc)959 return -ENOMEM;960 961 hsdma->vc = devm_kcalloc(&pdev->dev, hsdma->dma_requests,962 sizeof(*hsdma->vc), GFP_KERNEL);963 if (!hsdma->vc)964 return -ENOMEM;965 966 for (i = 0; i < hsdma->dma_requests; i++) {967 vc = &hsdma->vc[i];968 vc->vc.desc_free = mtk_hsdma_vdesc_free;969 vchan_init(&vc->vc, dd);970 init_completion(&vc->issue_completion);971 INIT_LIST_HEAD(&vc->desc_hw_processing);972 }973 974 err = dma_async_device_register(dd);975 if (err)976 return err;977 978 err = of_dma_controller_register(pdev->dev.of_node,979 of_dma_xlate_by_chan_id, hsdma);980 if (err) {981 dev_err(&pdev->dev,982 "MediaTek HSDMA OF registration failed %d\n", err);983 goto err_unregister;984 }985 986 mtk_hsdma_hw_init(hsdma);987 988 err = devm_request_irq(&pdev->dev, hsdma->irq,989 mtk_hsdma_irq, 0,990 dev_name(&pdev->dev), hsdma);991 if (err) {992 dev_err(&pdev->dev,993 "request_irq failed with err %d\n", err);994 goto err_free;995 }996 997 platform_set_drvdata(pdev, hsdma);998 999 dev_info(&pdev->dev, "MediaTek HSDMA driver registered\n");1000 1001 return 0;1002 1003err_free:1004 mtk_hsdma_hw_deinit(hsdma);1005 of_dma_controller_free(pdev->dev.of_node);1006err_unregister:1007 dma_async_device_unregister(dd);1008 1009 return err;1010}1011 1012static void mtk_hsdma_remove(struct platform_device *pdev)1013{1014 struct mtk_hsdma_device *hsdma = platform_get_drvdata(pdev);1015 struct mtk_hsdma_vchan *vc;1016 int i;1017 1018 /* Kill VC task */1019 for (i = 0; i < hsdma->dma_requests; i++) {1020 vc = &hsdma->vc[i];1021 1022 list_del(&vc->vc.chan.device_node);1023 tasklet_kill(&vc->vc.task);1024 }1025 1026 /* Disable DMA interrupt */1027 mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0);1028 1029 /* Waits for any pending IRQ handlers to complete */1030 synchronize_irq(hsdma->irq);1031 1032 /* Disable hardware */1033 mtk_hsdma_hw_deinit(hsdma);1034 1035 dma_async_device_unregister(&hsdma->ddev);1036 of_dma_controller_free(pdev->dev.of_node);1037}1038 1039static struct platform_driver mtk_hsdma_driver = {1040 .probe = mtk_hsdma_probe,1041 .remove_new = mtk_hsdma_remove,1042 .driver = {1043 .name = KBUILD_MODNAME,1044 .of_match_table = mtk_hsdma_match,1045 },1046};1047module_platform_driver(mtk_hsdma_driver);1048 1049MODULE_DESCRIPTION("MediaTek High-Speed DMA Controller Driver");1050MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>");1051MODULE_LICENSE("GPL v2");1052