1086 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * IMG Multi-threaded DMA Controller (MDC)4 *5 * Copyright (C) 2009,2012,2013 Imagination Technologies Ltd.6 * Copyright (C) 2014 Google, Inc.7 */8 9#include <linux/clk.h>10#include <linux/dma-mapping.h>11#include <linux/dmaengine.h>12#include <linux/dmapool.h>13#include <linux/interrupt.h>14#include <linux/io.h>15#include <linux/irq.h>16#include <linux/kernel.h>17#include <linux/mfd/syscon.h>18#include <linux/module.h>19#include <linux/of.h>20#include <linux/of_dma.h>21#include <linux/platform_device.h>22#include <linux/pm_runtime.h>23#include <linux/regmap.h>24#include <linux/slab.h>25#include <linux/spinlock.h>26 27#include "dmaengine.h"28#include "virt-dma.h"29 30#define MDC_MAX_DMA_CHANNELS 3231 32#define MDC_GENERAL_CONFIG 0x00033#define MDC_GENERAL_CONFIG_LIST_IEN BIT(31)34#define MDC_GENERAL_CONFIG_IEN BIT(29)35#define MDC_GENERAL_CONFIG_LEVEL_INT BIT(28)36#define MDC_GENERAL_CONFIG_INC_W BIT(12)37#define MDC_GENERAL_CONFIG_INC_R BIT(8)38#define MDC_GENERAL_CONFIG_PHYSICAL_W BIT(7)39#define MDC_GENERAL_CONFIG_WIDTH_W_SHIFT 440#define MDC_GENERAL_CONFIG_WIDTH_W_MASK 0x741#define MDC_GENERAL_CONFIG_PHYSICAL_R BIT(3)42#define MDC_GENERAL_CONFIG_WIDTH_R_SHIFT 043#define MDC_GENERAL_CONFIG_WIDTH_R_MASK 0x744 45#define MDC_READ_PORT_CONFIG 0x00446#define MDC_READ_PORT_CONFIG_STHREAD_SHIFT 2847#define MDC_READ_PORT_CONFIG_STHREAD_MASK 0xf48#define MDC_READ_PORT_CONFIG_RTHREAD_SHIFT 2449#define MDC_READ_PORT_CONFIG_RTHREAD_MASK 0xf50#define MDC_READ_PORT_CONFIG_WTHREAD_SHIFT 1651#define MDC_READ_PORT_CONFIG_WTHREAD_MASK 0xf52#define MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT 453#define MDC_READ_PORT_CONFIG_BURST_SIZE_MASK 0xff54#define MDC_READ_PORT_CONFIG_DREQ_ENABLE BIT(1)55 56#define MDC_READ_ADDRESS 0x00857 58#define MDC_WRITE_ADDRESS 0x00c59 60#define MDC_TRANSFER_SIZE 0x01061#define MDC_TRANSFER_SIZE_MASK 0xffffff62 63#define MDC_LIST_NODE_ADDRESS 0x01464 65#define MDC_CMDS_PROCESSED 0x01866#define MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT 1667#define MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK 0x3f68#define MDC_CMDS_PROCESSED_INT_ACTIVE BIT(8)69#define MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT 070#define MDC_CMDS_PROCESSED_CMDS_DONE_MASK 0x3f71 72#define MDC_CONTROL_AND_STATUS 0x01c73#define MDC_CONTROL_AND_STATUS_CANCEL BIT(20)74#define MDC_CONTROL_AND_STATUS_LIST_EN BIT(4)75#define MDC_CONTROL_AND_STATUS_EN BIT(0)76 77#define MDC_ACTIVE_TRANSFER_SIZE 0x03078 79#define MDC_GLOBAL_CONFIG_A 0x90080#define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT 1681#define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK 0xff82#define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT 883#define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK 0xff84#define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT 085#define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK 0xff86 87struct mdc_hw_list_desc {88 u32 gen_conf;89 u32 readport_conf;90 u32 read_addr;91 u32 write_addr;92 u32 xfer_size;93 u32 node_addr;94 u32 cmds_done;95 u32 ctrl_status;96 /*97 * Not part of the list descriptor, but instead used by the CPU to98 * traverse the list.99 */100 struct mdc_hw_list_desc *next_desc;101};102 103struct mdc_tx_desc {104 struct mdc_chan *chan;105 struct virt_dma_desc vd;106 dma_addr_t list_phys;107 struct mdc_hw_list_desc *list;108 bool cyclic;109 bool cmd_loaded;110 unsigned int list_len;111 unsigned int list_period_len;112 size_t list_xfer_size;113 unsigned int list_cmds_done;114};115 116struct mdc_chan {117 struct mdc_dma *mdma;118 struct virt_dma_chan vc;119 struct dma_slave_config config;120 struct mdc_tx_desc *desc;121 int irq;122 unsigned int periph;123 unsigned int thread;124 unsigned int chan_nr;125};126 127struct mdc_dma_soc_data {128 void (*enable_chan)(struct mdc_chan *mchan);129 void (*disable_chan)(struct mdc_chan *mchan);130};131 132struct mdc_dma {133 struct dma_device dma_dev;134 void __iomem *regs;135 struct clk *clk;136 struct dma_pool *desc_pool;137 struct regmap *periph_regs;138 spinlock_t lock;139 unsigned int nr_threads;140 unsigned int nr_channels;141 unsigned int bus_width;142 unsigned int max_burst_mult;143 unsigned int max_xfer_size;144 const struct mdc_dma_soc_data *soc;145 struct mdc_chan channels[MDC_MAX_DMA_CHANNELS];146};147 148static inline u32 mdc_readl(struct mdc_dma *mdma, u32 reg)149{150 return readl(mdma->regs + reg);151}152 153static inline void mdc_writel(struct mdc_dma *mdma, u32 val, u32 reg)154{155 writel(val, mdma->regs + reg);156}157 158static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg)159{160 return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg);161}162 163static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg)164{165 mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg);166}167 168static inline struct mdc_chan *to_mdc_chan(struct dma_chan *c)169{170 return container_of(to_virt_chan(c), struct mdc_chan, vc);171}172 173static inline struct mdc_tx_desc *to_mdc_desc(struct dma_async_tx_descriptor *t)174{175 struct virt_dma_desc *vdesc = container_of(t, struct virt_dma_desc, tx);176 177 return container_of(vdesc, struct mdc_tx_desc, vd);178}179 180static inline struct device *mdma2dev(struct mdc_dma *mdma)181{182 return mdma->dma_dev.dev;183}184 185static inline unsigned int to_mdc_width(unsigned int bytes)186{187 return ffs(bytes) - 1;188}189 190static inline void mdc_set_read_width(struct mdc_hw_list_desc *ldesc,191 unsigned int bytes)192{193 ldesc->gen_conf |= to_mdc_width(bytes) <<194 MDC_GENERAL_CONFIG_WIDTH_R_SHIFT;195}196 197static inline void mdc_set_write_width(struct mdc_hw_list_desc *ldesc,198 unsigned int bytes)199{200 ldesc->gen_conf |= to_mdc_width(bytes) <<201 MDC_GENERAL_CONFIG_WIDTH_W_SHIFT;202}203 204static void mdc_list_desc_config(struct mdc_chan *mchan,205 struct mdc_hw_list_desc *ldesc,206 enum dma_transfer_direction dir,207 dma_addr_t src, dma_addr_t dst, size_t len)208{209 struct mdc_dma *mdma = mchan->mdma;210 unsigned int max_burst, burst_size;211 212 ldesc->gen_conf = MDC_GENERAL_CONFIG_IEN | MDC_GENERAL_CONFIG_LIST_IEN |213 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |214 MDC_GENERAL_CONFIG_PHYSICAL_R;215 ldesc->readport_conf =216 (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |217 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |218 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);219 ldesc->read_addr = src;220 ldesc->write_addr = dst;221 ldesc->xfer_size = len - 1;222 ldesc->node_addr = 0;223 ldesc->cmds_done = 0;224 ldesc->ctrl_status = MDC_CONTROL_AND_STATUS_LIST_EN |225 MDC_CONTROL_AND_STATUS_EN;226 ldesc->next_desc = NULL;227 228 if (IS_ALIGNED(dst, mdma->bus_width) &&229 IS_ALIGNED(src, mdma->bus_width))230 max_burst = mdma->bus_width * mdma->max_burst_mult;231 else232 max_burst = mdma->bus_width * (mdma->max_burst_mult - 1);233 234 if (dir == DMA_MEM_TO_DEV) {235 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R;236 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;237 mdc_set_read_width(ldesc, mdma->bus_width);238 mdc_set_write_width(ldesc, mchan->config.dst_addr_width);239 burst_size = min(max_burst, mchan->config.dst_maxburst *240 mchan->config.dst_addr_width);241 } else if (dir == DMA_DEV_TO_MEM) {242 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_W;243 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;244 mdc_set_read_width(ldesc, mchan->config.src_addr_width);245 mdc_set_write_width(ldesc, mdma->bus_width);246 burst_size = min(max_burst, mchan->config.src_maxburst *247 mchan->config.src_addr_width);248 } else {249 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R |250 MDC_GENERAL_CONFIG_INC_W;251 mdc_set_read_width(ldesc, mdma->bus_width);252 mdc_set_write_width(ldesc, mdma->bus_width);253 burst_size = max_burst;254 }255 ldesc->readport_conf |= (burst_size - 1) <<256 MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT;257}258 259static void mdc_list_desc_free(struct mdc_tx_desc *mdesc)260{261 struct mdc_dma *mdma = mdesc->chan->mdma;262 struct mdc_hw_list_desc *curr, *next;263 dma_addr_t curr_phys, next_phys;264 265 curr = mdesc->list;266 curr_phys = mdesc->list_phys;267 while (curr) {268 next = curr->next_desc;269 next_phys = curr->node_addr;270 dma_pool_free(mdma->desc_pool, curr, curr_phys);271 curr = next;272 curr_phys = next_phys;273 }274}275 276static void mdc_desc_free(struct virt_dma_desc *vd)277{278 struct mdc_tx_desc *mdesc = to_mdc_desc(&vd->tx);279 280 mdc_list_desc_free(mdesc);281 kfree(mdesc);282}283 284static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy(285 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len,286 unsigned long flags)287{288 struct mdc_chan *mchan = to_mdc_chan(chan);289 struct mdc_dma *mdma = mchan->mdma;290 struct mdc_tx_desc *mdesc;291 struct mdc_hw_list_desc *curr, *prev = NULL;292 dma_addr_t curr_phys;293 294 if (!len)295 return NULL;296 297 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);298 if (!mdesc)299 return NULL;300 mdesc->chan = mchan;301 mdesc->list_xfer_size = len;302 303 while (len > 0) {304 size_t xfer_size;305 306 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, &curr_phys);307 if (!curr)308 goto free_desc;309 310 if (prev) {311 prev->node_addr = curr_phys;312 prev->next_desc = curr;313 } else {314 mdesc->list_phys = curr_phys;315 mdesc->list = curr;316 }317 318 xfer_size = min_t(size_t, mdma->max_xfer_size, len);319 320 mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest,321 xfer_size);322 323 prev = curr;324 325 mdesc->list_len++;326 src += xfer_size;327 dest += xfer_size;328 len -= xfer_size;329 }330 331 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);332 333free_desc:334 mdc_desc_free(&mdesc->vd);335 336 return NULL;337}338 339static int mdc_check_slave_width(struct mdc_chan *mchan,340 enum dma_transfer_direction dir)341{342 enum dma_slave_buswidth width;343 344 if (dir == DMA_MEM_TO_DEV)345 width = mchan->config.dst_addr_width;346 else347 width = mchan->config.src_addr_width;348 349 switch (width) {350 case DMA_SLAVE_BUSWIDTH_1_BYTE:351 case DMA_SLAVE_BUSWIDTH_2_BYTES:352 case DMA_SLAVE_BUSWIDTH_4_BYTES:353 case DMA_SLAVE_BUSWIDTH_8_BYTES:354 break;355 default:356 return -EINVAL;357 }358 359 if (width > mchan->mdma->bus_width)360 return -EINVAL;361 362 return 0;363}364 365static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic(366 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,367 size_t period_len, enum dma_transfer_direction dir,368 unsigned long flags)369{370 struct mdc_chan *mchan = to_mdc_chan(chan);371 struct mdc_dma *mdma = mchan->mdma;372 struct mdc_tx_desc *mdesc;373 struct mdc_hw_list_desc *curr, *prev = NULL;374 dma_addr_t curr_phys;375 376 if (!buf_len && !period_len)377 return NULL;378 379 if (!is_slave_direction(dir))380 return NULL;381 382 if (mdc_check_slave_width(mchan, dir) < 0)383 return NULL;384 385 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);386 if (!mdesc)387 return NULL;388 mdesc->chan = mchan;389 mdesc->cyclic = true;390 mdesc->list_xfer_size = buf_len;391 mdesc->list_period_len = DIV_ROUND_UP(period_len,392 mdma->max_xfer_size);393 394 while (buf_len > 0) {395 size_t remainder = min(period_len, buf_len);396 397 while (remainder > 0) {398 size_t xfer_size;399 400 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,401 &curr_phys);402 if (!curr)403 goto free_desc;404 405 if (!prev) {406 mdesc->list_phys = curr_phys;407 mdesc->list = curr;408 } else {409 prev->node_addr = curr_phys;410 prev->next_desc = curr;411 }412 413 xfer_size = min_t(size_t, mdma->max_xfer_size,414 remainder);415 416 if (dir == DMA_MEM_TO_DEV) {417 mdc_list_desc_config(mchan, curr, dir,418 buf_addr,419 mchan->config.dst_addr,420 xfer_size);421 } else {422 mdc_list_desc_config(mchan, curr, dir,423 mchan->config.src_addr,424 buf_addr,425 xfer_size);426 }427 428 prev = curr;429 430 mdesc->list_len++;431 buf_addr += xfer_size;432 buf_len -= xfer_size;433 remainder -= xfer_size;434 }435 }436 prev->node_addr = mdesc->list_phys;437 438 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);439 440free_desc:441 mdc_desc_free(&mdesc->vd);442 443 return NULL;444}445 446static struct dma_async_tx_descriptor *mdc_prep_slave_sg(447 struct dma_chan *chan, struct scatterlist *sgl,448 unsigned int sg_len, enum dma_transfer_direction dir,449 unsigned long flags, void *context)450{451 struct mdc_chan *mchan = to_mdc_chan(chan);452 struct mdc_dma *mdma = mchan->mdma;453 struct mdc_tx_desc *mdesc;454 struct scatterlist *sg;455 struct mdc_hw_list_desc *curr, *prev = NULL;456 dma_addr_t curr_phys;457 unsigned int i;458 459 if (!sgl)460 return NULL;461 462 if (!is_slave_direction(dir))463 return NULL;464 465 if (mdc_check_slave_width(mchan, dir) < 0)466 return NULL;467 468 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);469 if (!mdesc)470 return NULL;471 mdesc->chan = mchan;472 473 for_each_sg(sgl, sg, sg_len, i) {474 dma_addr_t buf = sg_dma_address(sg);475 size_t buf_len = sg_dma_len(sg);476 477 while (buf_len > 0) {478 size_t xfer_size;479 480 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,481 &curr_phys);482 if (!curr)483 goto free_desc;484 485 if (!prev) {486 mdesc->list_phys = curr_phys;487 mdesc->list = curr;488 } else {489 prev->node_addr = curr_phys;490 prev->next_desc = curr;491 }492 493 xfer_size = min_t(size_t, mdma->max_xfer_size,494 buf_len);495 496 if (dir == DMA_MEM_TO_DEV) {497 mdc_list_desc_config(mchan, curr, dir, buf,498 mchan->config.dst_addr,499 xfer_size);500 } else {501 mdc_list_desc_config(mchan, curr, dir,502 mchan->config.src_addr,503 buf, xfer_size);504 }505 506 prev = curr;507 508 mdesc->list_len++;509 mdesc->list_xfer_size += xfer_size;510 buf += xfer_size;511 buf_len -= xfer_size;512 }513 }514 515 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);516 517free_desc:518 mdc_desc_free(&mdesc->vd);519 520 return NULL;521}522 523static void mdc_issue_desc(struct mdc_chan *mchan)524{525 struct mdc_dma *mdma = mchan->mdma;526 struct virt_dma_desc *vd;527 struct mdc_tx_desc *mdesc;528 u32 val;529 530 vd = vchan_next_desc(&mchan->vc);531 if (!vd)532 return;533 534 list_del(&vd->node);535 536 mdesc = to_mdc_desc(&vd->tx);537 mchan->desc = mdesc;538 539 dev_dbg(mdma2dev(mdma), "Issuing descriptor on channel %d\n",540 mchan->chan_nr);541 542 mdma->soc->enable_chan(mchan);543 544 val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG);545 val |= MDC_GENERAL_CONFIG_LIST_IEN | MDC_GENERAL_CONFIG_IEN |546 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |547 MDC_GENERAL_CONFIG_PHYSICAL_R;548 mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG);549 val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |550 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |551 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);552 mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG);553 mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS);554 val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS);555 val |= MDC_CONTROL_AND_STATUS_LIST_EN;556 mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS);557}558 559static void mdc_issue_pending(struct dma_chan *chan)560{561 struct mdc_chan *mchan = to_mdc_chan(chan);562 unsigned long flags;563 564 spin_lock_irqsave(&mchan->vc.lock, flags);565 if (vchan_issue_pending(&mchan->vc) && !mchan->desc)566 mdc_issue_desc(mchan);567 spin_unlock_irqrestore(&mchan->vc.lock, flags);568}569 570static enum dma_status mdc_tx_status(struct dma_chan *chan,571 dma_cookie_t cookie, struct dma_tx_state *txstate)572{573 struct mdc_chan *mchan = to_mdc_chan(chan);574 struct mdc_tx_desc *mdesc;575 struct virt_dma_desc *vd;576 unsigned long flags;577 size_t bytes = 0;578 int ret;579 580 ret = dma_cookie_status(chan, cookie, txstate);581 if (ret == DMA_COMPLETE)582 return ret;583 584 if (!txstate)585 return ret;586 587 spin_lock_irqsave(&mchan->vc.lock, flags);588 vd = vchan_find_desc(&mchan->vc, cookie);589 if (vd) {590 mdesc = to_mdc_desc(&vd->tx);591 bytes = mdesc->list_xfer_size;592 } else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) {593 struct mdc_hw_list_desc *ldesc;594 u32 val1, val2, done, processed, residue;595 int i, cmds;596 597 mdesc = mchan->desc;598 599 /*600 * Determine the number of commands that haven't been601 * processed (handled by the IRQ handler) yet.602 */603 do {604 val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &605 ~MDC_CMDS_PROCESSED_INT_ACTIVE;606 residue = mdc_chan_readl(mchan,607 MDC_ACTIVE_TRANSFER_SIZE);608 val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &609 ~MDC_CMDS_PROCESSED_INT_ACTIVE;610 } while (val1 != val2);611 612 done = (val1 >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &613 MDC_CMDS_PROCESSED_CMDS_DONE_MASK;614 processed = (val1 >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &615 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;616 cmds = (done - processed) %617 (MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1);618 619 /*620 * If the command loaded event hasn't been processed yet, then621 * the difference above includes an extra command.622 */623 if (!mdesc->cmd_loaded)624 cmds--;625 else626 cmds += mdesc->list_cmds_done;627 628 bytes = mdesc->list_xfer_size;629 ldesc = mdesc->list;630 for (i = 0; i < cmds; i++) {631 bytes -= ldesc->xfer_size + 1;632 ldesc = ldesc->next_desc;633 }634 if (ldesc) {635 if (residue != MDC_TRANSFER_SIZE_MASK)636 bytes -= ldesc->xfer_size - residue;637 else638 bytes -= ldesc->xfer_size + 1;639 }640 }641 spin_unlock_irqrestore(&mchan->vc.lock, flags);642 643 dma_set_residue(txstate, bytes);644 645 return ret;646}647 648static unsigned int mdc_get_new_events(struct mdc_chan *mchan)649{650 u32 val, processed, done1, done2;651 unsigned int ret;652 653 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);654 processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &655 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;656 /*657 * CMDS_DONE may have incremented between reading CMDS_PROCESSED658 * and clearing INT_ACTIVE. Re-read CMDS_PROCESSED to ensure we659 * didn't miss a command completion.660 */661 do {662 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);663 664 done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &665 MDC_CMDS_PROCESSED_CMDS_DONE_MASK;666 667 val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK <<668 MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) |669 MDC_CMDS_PROCESSED_INT_ACTIVE);670 671 val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT;672 673 mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED);674 675 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);676 677 done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &678 MDC_CMDS_PROCESSED_CMDS_DONE_MASK;679 } while (done1 != done2);680 681 if (done1 >= processed)682 ret = done1 - processed;683 else684 ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) -685 processed) + done1;686 687 return ret;688}689 690static int mdc_terminate_all(struct dma_chan *chan)691{692 struct mdc_chan *mchan = to_mdc_chan(chan);693 unsigned long flags;694 LIST_HEAD(head);695 696 spin_lock_irqsave(&mchan->vc.lock, flags);697 698 mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL,699 MDC_CONTROL_AND_STATUS);700 701 if (mchan->desc) {702 vchan_terminate_vdesc(&mchan->desc->vd);703 mchan->desc = NULL;704 }705 vchan_get_all_descriptors(&mchan->vc, &head);706 707 mdc_get_new_events(mchan);708 709 spin_unlock_irqrestore(&mchan->vc.lock, flags);710 711 vchan_dma_desc_free_list(&mchan->vc, &head);712 713 return 0;714}715 716static void mdc_synchronize(struct dma_chan *chan)717{718 struct mdc_chan *mchan = to_mdc_chan(chan);719 720 vchan_synchronize(&mchan->vc);721}722 723static int mdc_slave_config(struct dma_chan *chan,724 struct dma_slave_config *config)725{726 struct mdc_chan *mchan = to_mdc_chan(chan);727 unsigned long flags;728 729 spin_lock_irqsave(&mchan->vc.lock, flags);730 mchan->config = *config;731 spin_unlock_irqrestore(&mchan->vc.lock, flags);732 733 return 0;734}735 736static int mdc_alloc_chan_resources(struct dma_chan *chan)737{738 struct mdc_chan *mchan = to_mdc_chan(chan);739 struct device *dev = mdma2dev(mchan->mdma);740 741 return pm_runtime_get_sync(dev);742}743 744static void mdc_free_chan_resources(struct dma_chan *chan)745{746 struct mdc_chan *mchan = to_mdc_chan(chan);747 struct mdc_dma *mdma = mchan->mdma;748 struct device *dev = mdma2dev(mdma);749 750 mdc_terminate_all(chan);751 mdma->soc->disable_chan(mchan);752 pm_runtime_put(dev);753}754 755static irqreturn_t mdc_chan_irq(int irq, void *dev_id)756{757 struct mdc_chan *mchan = (struct mdc_chan *)dev_id;758 struct mdc_tx_desc *mdesc;759 unsigned int i, new_events;760 761 spin_lock(&mchan->vc.lock);762 763 dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr);764 765 new_events = mdc_get_new_events(mchan);766 767 if (!new_events)768 goto out;769 770 mdesc = mchan->desc;771 if (!mdesc) {772 dev_warn(mdma2dev(mchan->mdma),773 "IRQ with no active descriptor on channel %d\n",774 mchan->chan_nr);775 goto out;776 }777 778 for (i = 0; i < new_events; i++) {779 /*780 * The first interrupt in a transfer indicates that the781 * command list has been loaded, not that a command has782 * been completed.783 */784 if (!mdesc->cmd_loaded) {785 mdesc->cmd_loaded = true;786 continue;787 }788 789 mdesc->list_cmds_done++;790 if (mdesc->cyclic) {791 mdesc->list_cmds_done %= mdesc->list_len;792 if (mdesc->list_cmds_done % mdesc->list_period_len == 0)793 vchan_cyclic_callback(&mdesc->vd);794 } else if (mdesc->list_cmds_done == mdesc->list_len) {795 mchan->desc = NULL;796 vchan_cookie_complete(&mdesc->vd);797 mdc_issue_desc(mchan);798 break;799 }800 }801out:802 spin_unlock(&mchan->vc.lock);803 804 return IRQ_HANDLED;805}806 807static struct dma_chan *mdc_of_xlate(struct of_phandle_args *dma_spec,808 struct of_dma *ofdma)809{810 struct mdc_dma *mdma = ofdma->of_dma_data;811 struct dma_chan *chan;812 813 if (dma_spec->args_count != 3)814 return NULL;815 816 list_for_each_entry(chan, &mdma->dma_dev.channels, device_node) {817 struct mdc_chan *mchan = to_mdc_chan(chan);818 819 if (!(dma_spec->args[1] & BIT(mchan->chan_nr)))820 continue;821 if (dma_get_slave_channel(chan)) {822 mchan->periph = dma_spec->args[0];823 mchan->thread = dma_spec->args[2];824 return chan;825 }826 }827 828 return NULL;829}830 831#define PISTACHIO_CR_PERIPH_DMA_ROUTE(ch) (0x120 + 0x4 * ((ch) / 4))832#define PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(ch) (8 * ((ch) % 4))833#define PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK 0x3f834 835static void pistachio_mdc_enable_chan(struct mdc_chan *mchan)836{837 struct mdc_dma *mdma = mchan->mdma;838 839 regmap_update_bits(mdma->periph_regs,840 PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),841 PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<842 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),843 mchan->periph <<844 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr));845}846 847static void pistachio_mdc_disable_chan(struct mdc_chan *mchan)848{849 struct mdc_dma *mdma = mchan->mdma;850 851 regmap_update_bits(mdma->periph_regs,852 PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),853 PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<854 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),855 0);856}857 858static const struct mdc_dma_soc_data pistachio_mdc_data = {859 .enable_chan = pistachio_mdc_enable_chan,860 .disable_chan = pistachio_mdc_disable_chan,861};862 863static const struct of_device_id mdc_dma_of_match[] = {864 { .compatible = "img,pistachio-mdc-dma", .data = &pistachio_mdc_data, },865 { },866};867MODULE_DEVICE_TABLE(of, mdc_dma_of_match);868 869static int img_mdc_runtime_suspend(struct device *dev)870{871 struct mdc_dma *mdma = dev_get_drvdata(dev);872 873 clk_disable_unprepare(mdma->clk);874 875 return 0;876}877 878static int img_mdc_runtime_resume(struct device *dev)879{880 struct mdc_dma *mdma = dev_get_drvdata(dev);881 882 return clk_prepare_enable(mdma->clk);883}884 885static int mdc_dma_probe(struct platform_device *pdev)886{887 struct mdc_dma *mdma;888 unsigned int i;889 u32 val;890 int ret;891 892 mdma = devm_kzalloc(&pdev->dev, sizeof(*mdma), GFP_KERNEL);893 if (!mdma)894 return -ENOMEM;895 platform_set_drvdata(pdev, mdma);896 897 mdma->soc = of_device_get_match_data(&pdev->dev);898 899 mdma->regs = devm_platform_ioremap_resource(pdev, 0);900 if (IS_ERR(mdma->regs))901 return PTR_ERR(mdma->regs);902 903 mdma->periph_regs = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,904 "img,cr-periph");905 if (IS_ERR(mdma->periph_regs))906 return PTR_ERR(mdma->periph_regs);907 908 mdma->clk = devm_clk_get(&pdev->dev, "sys");909 if (IS_ERR(mdma->clk))910 return PTR_ERR(mdma->clk);911 912 dma_cap_zero(mdma->dma_dev.cap_mask);913 dma_cap_set(DMA_SLAVE, mdma->dma_dev.cap_mask);914 dma_cap_set(DMA_PRIVATE, mdma->dma_dev.cap_mask);915 dma_cap_set(DMA_CYCLIC, mdma->dma_dev.cap_mask);916 dma_cap_set(DMA_MEMCPY, mdma->dma_dev.cap_mask);917 918 val = mdc_readl(mdma, MDC_GLOBAL_CONFIG_A);919 mdma->nr_channels = (val >> MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT) &920 MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK;921 mdma->nr_threads =922 1 << ((val >> MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT) &923 MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK);924 mdma->bus_width =925 (1 << ((val >> MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT) &926 MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK)) / 8;927 /*928 * Although transfer sizes of up to MDC_TRANSFER_SIZE_MASK + 1 bytes929 * are supported, this makes it possible for the value reported in930 * MDC_ACTIVE_TRANSFER_SIZE to be ambiguous - an active transfer size931 * of MDC_TRANSFER_SIZE_MASK may indicate either that 0 bytes or932 * MDC_TRANSFER_SIZE_MASK + 1 bytes are remaining. To eliminate this933 * ambiguity, restrict transfer sizes to one bus-width less than the934 * actual maximum.935 */936 mdma->max_xfer_size = MDC_TRANSFER_SIZE_MASK + 1 - mdma->bus_width;937 938 of_property_read_u32(pdev->dev.of_node, "dma-channels",939 &mdma->nr_channels);940 ret = of_property_read_u32(pdev->dev.of_node,941 "img,max-burst-multiplier",942 &mdma->max_burst_mult);943 if (ret)944 return ret;945 946 mdma->dma_dev.dev = &pdev->dev;947 mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg;948 mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic;949 mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy;950 mdma->dma_dev.device_alloc_chan_resources = mdc_alloc_chan_resources;951 mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources;952 mdma->dma_dev.device_tx_status = mdc_tx_status;953 mdma->dma_dev.device_issue_pending = mdc_issue_pending;954 mdma->dma_dev.device_terminate_all = mdc_terminate_all;955 mdma->dma_dev.device_synchronize = mdc_synchronize;956 mdma->dma_dev.device_config = mdc_slave_config;957 958 mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);959 mdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;960 for (i = 1; i <= mdma->bus_width; i <<= 1) {961 mdma->dma_dev.src_addr_widths |= BIT(i);962 mdma->dma_dev.dst_addr_widths |= BIT(i);963 }964 965 INIT_LIST_HEAD(&mdma->dma_dev.channels);966 for (i = 0; i < mdma->nr_channels; i++) {967 struct mdc_chan *mchan = &mdma->channels[i];968 969 mchan->mdma = mdma;970 mchan->chan_nr = i;971 mchan->irq = platform_get_irq(pdev, i);972 if (mchan->irq < 0)973 return mchan->irq;974 975 ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq,976 IRQ_TYPE_LEVEL_HIGH,977 dev_name(&pdev->dev), mchan);978 if (ret < 0)979 return ret;980 981 mchan->vc.desc_free = mdc_desc_free;982 vchan_init(&mchan->vc, &mdma->dma_dev);983 }984 985 mdma->desc_pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,986 sizeof(struct mdc_hw_list_desc),987 4, 0);988 if (!mdma->desc_pool)989 return -ENOMEM;990 991 pm_runtime_enable(&pdev->dev);992 if (!pm_runtime_enabled(&pdev->dev)) {993 ret = img_mdc_runtime_resume(&pdev->dev);994 if (ret)995 return ret;996 }997 998 ret = dma_async_device_register(&mdma->dma_dev);999 if (ret)1000 goto suspend;1001 1002 ret = of_dma_controller_register(pdev->dev.of_node, mdc_of_xlate, mdma);1003 if (ret)1004 goto unregister;1005 1006 dev_info(&pdev->dev, "MDC with %u channels and %u threads\n",1007 mdma->nr_channels, mdma->nr_threads);1008 1009 return 0;1010 1011unregister:1012 dma_async_device_unregister(&mdma->dma_dev);1013suspend:1014 if (!pm_runtime_enabled(&pdev->dev))1015 img_mdc_runtime_suspend(&pdev->dev);1016 pm_runtime_disable(&pdev->dev);1017 return ret;1018}1019 1020static void mdc_dma_remove(struct platform_device *pdev)1021{1022 struct mdc_dma *mdma = platform_get_drvdata(pdev);1023 struct mdc_chan *mchan, *next;1024 1025 of_dma_controller_free(pdev->dev.of_node);1026 dma_async_device_unregister(&mdma->dma_dev);1027 1028 list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels,1029 vc.chan.device_node) {1030 list_del(&mchan->vc.chan.device_node);1031 1032 devm_free_irq(&pdev->dev, mchan->irq, mchan);1033 1034 tasklet_kill(&mchan->vc.task);1035 }1036 1037 pm_runtime_disable(&pdev->dev);1038 if (!pm_runtime_status_suspended(&pdev->dev))1039 img_mdc_runtime_suspend(&pdev->dev);1040}1041 1042#ifdef CONFIG_PM_SLEEP1043static int img_mdc_suspend_late(struct device *dev)1044{1045 struct mdc_dma *mdma = dev_get_drvdata(dev);1046 int i;1047 1048 /* Check that all channels are idle */1049 for (i = 0; i < mdma->nr_channels; i++) {1050 struct mdc_chan *mchan = &mdma->channels[i];1051 1052 if (unlikely(mchan->desc))1053 return -EBUSY;1054 }1055 1056 return pm_runtime_force_suspend(dev);1057}1058 1059static int img_mdc_resume_early(struct device *dev)1060{1061 return pm_runtime_force_resume(dev);1062}1063#endif /* CONFIG_PM_SLEEP */1064 1065static const struct dev_pm_ops img_mdc_pm_ops = {1066 SET_RUNTIME_PM_OPS(img_mdc_runtime_suspend,1067 img_mdc_runtime_resume, NULL)1068 SET_LATE_SYSTEM_SLEEP_PM_OPS(img_mdc_suspend_late,1069 img_mdc_resume_early)1070};1071 1072static struct platform_driver mdc_dma_driver = {1073 .driver = {1074 .name = "img-mdc-dma",1075 .pm = &img_mdc_pm_ops,1076 .of_match_table = of_match_ptr(mdc_dma_of_match),1077 },1078 .probe = mdc_dma_probe,1079 .remove_new = mdc_dma_remove,1080};1081module_platform_driver(mdc_dma_driver);1082 1083MODULE_DESCRIPTION("IMG Multi-threaded DMA Controller (MDC) driver");1084MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");1085MODULE_LICENSE("GPL v2");1086