1691 lines · c
1// SPDX-License-Identifier: GPL-2.02// (C) 2017-2018 Synopsys, Inc. (www.synopsys.com)3 4/*5 * Synopsys DesignWare AXI DMA Controller driver.6 *7 * Author: Eugeniy Paltsev <Eugeniy.Paltsev@synopsys.com>8 */9 10#include <linux/bitops.h>11#include <linux/delay.h>12#include <linux/device.h>13#include <linux/dmaengine.h>14#include <linux/dmapool.h>15#include <linux/dma-mapping.h>16#include <linux/err.h>17#include <linux/interrupt.h>18#include <linux/io.h>19#include <linux/iopoll.h>20#include <linux/io-64-nonatomic-lo-hi.h>21#include <linux/kernel.h>22#include <linux/module.h>23#include <linux/of.h>24#include <linux/of_dma.h>25#include <linux/platform_device.h>26#include <linux/pm_runtime.h>27#include <linux/property.h>28#include <linux/reset.h>29#include <linux/slab.h>30#include <linux/types.h>31 32#include "dw-axi-dmac.h"33#include "../dmaengine.h"34#include "../virt-dma.h"35 36/*37 * The set of bus widths supported by the DMA controller. DW AXI DMAC supports38 * master data bus width up to 512 bits (for both AXI master interfaces), but39 * it depends on IP block configuration.40 */41#define AXI_DMA_BUSWIDTHS \42 (DMA_SLAVE_BUSWIDTH_1_BYTE | \43 DMA_SLAVE_BUSWIDTH_2_BYTES | \44 DMA_SLAVE_BUSWIDTH_4_BYTES | \45 DMA_SLAVE_BUSWIDTH_8_BYTES | \46 DMA_SLAVE_BUSWIDTH_16_BYTES | \47 DMA_SLAVE_BUSWIDTH_32_BYTES | \48 DMA_SLAVE_BUSWIDTH_64_BYTES)49 50#define AXI_DMA_FLAG_HAS_APB_REGS BIT(0)51#define AXI_DMA_FLAG_HAS_RESETS BIT(1)52#define AXI_DMA_FLAG_USE_CFG2 BIT(2)53 54static inline void55axi_dma_iowrite32(struct axi_dma_chip *chip, u32 reg, u32 val)56{57 iowrite32(val, chip->regs + reg);58}59 60static inline u32 axi_dma_ioread32(struct axi_dma_chip *chip, u32 reg)61{62 return ioread32(chip->regs + reg);63}64 65static inline void66axi_dma_iowrite64(struct axi_dma_chip *chip, u32 reg, u64 val)67{68 iowrite64(val, chip->regs + reg);69}70 71static inline u64 axi_dma_ioread64(struct axi_dma_chip *chip, u32 reg)72{73 return ioread64(chip->regs + reg);74}75 76static inline void77axi_chan_iowrite32(struct axi_dma_chan *chan, u32 reg, u32 val)78{79 iowrite32(val, chan->chan_regs + reg);80}81 82static inline u32 axi_chan_ioread32(struct axi_dma_chan *chan, u32 reg)83{84 return ioread32(chan->chan_regs + reg);85}86 87static inline void88axi_chan_iowrite64(struct axi_dma_chan *chan, u32 reg, u64 val)89{90 /*91 * We split one 64 bit write for two 32 bit write as some HW doesn't92 * support 64 bit access.93 */94 iowrite32(lower_32_bits(val), chan->chan_regs + reg);95 iowrite32(upper_32_bits(val), chan->chan_regs + reg + 4);96}97 98static inline void axi_chan_config_write(struct axi_dma_chan *chan,99 struct axi_dma_chan_config *config)100{101 u32 cfg_lo, cfg_hi;102 103 cfg_lo = (config->dst_multblk_type << CH_CFG_L_DST_MULTBLK_TYPE_POS |104 config->src_multblk_type << CH_CFG_L_SRC_MULTBLK_TYPE_POS);105 if (chan->chip->dw->hdata->reg_map_8_channels &&106 !chan->chip->dw->hdata->use_cfg2) {107 cfg_hi = config->tt_fc << CH_CFG_H_TT_FC_POS |108 config->hs_sel_src << CH_CFG_H_HS_SEL_SRC_POS |109 config->hs_sel_dst << CH_CFG_H_HS_SEL_DST_POS |110 config->src_per << CH_CFG_H_SRC_PER_POS |111 config->dst_per << CH_CFG_H_DST_PER_POS |112 config->prior << CH_CFG_H_PRIORITY_POS;113 } else {114 cfg_lo |= config->src_per << CH_CFG2_L_SRC_PER_POS |115 config->dst_per << CH_CFG2_L_DST_PER_POS;116 cfg_hi = config->tt_fc << CH_CFG2_H_TT_FC_POS |117 config->hs_sel_src << CH_CFG2_H_HS_SEL_SRC_POS |118 config->hs_sel_dst << CH_CFG2_H_HS_SEL_DST_POS |119 config->prior << CH_CFG2_H_PRIORITY_POS;120 }121 axi_chan_iowrite32(chan, CH_CFG_L, cfg_lo);122 axi_chan_iowrite32(chan, CH_CFG_H, cfg_hi);123}124 125static inline void axi_dma_disable(struct axi_dma_chip *chip)126{127 u32 val;128 129 val = axi_dma_ioread32(chip, DMAC_CFG);130 val &= ~DMAC_EN_MASK;131 axi_dma_iowrite32(chip, DMAC_CFG, val);132}133 134static inline void axi_dma_enable(struct axi_dma_chip *chip)135{136 u32 val;137 138 val = axi_dma_ioread32(chip, DMAC_CFG);139 val |= DMAC_EN_MASK;140 axi_dma_iowrite32(chip, DMAC_CFG, val);141}142 143static inline void axi_dma_irq_disable(struct axi_dma_chip *chip)144{145 u32 val;146 147 val = axi_dma_ioread32(chip, DMAC_CFG);148 val &= ~INT_EN_MASK;149 axi_dma_iowrite32(chip, DMAC_CFG, val);150}151 152static inline void axi_dma_irq_enable(struct axi_dma_chip *chip)153{154 u32 val;155 156 val = axi_dma_ioread32(chip, DMAC_CFG);157 val |= INT_EN_MASK;158 axi_dma_iowrite32(chip, DMAC_CFG, val);159}160 161static inline void axi_chan_irq_disable(struct axi_dma_chan *chan, u32 irq_mask)162{163 u32 val;164 165 if (likely(irq_mask == DWAXIDMAC_IRQ_ALL)) {166 axi_chan_iowrite32(chan, CH_INTSTATUS_ENA, DWAXIDMAC_IRQ_NONE);167 } else {168 val = axi_chan_ioread32(chan, CH_INTSTATUS_ENA);169 val &= ~irq_mask;170 axi_chan_iowrite32(chan, CH_INTSTATUS_ENA, val);171 }172}173 174static inline void axi_chan_irq_set(struct axi_dma_chan *chan, u32 irq_mask)175{176 axi_chan_iowrite32(chan, CH_INTSTATUS_ENA, irq_mask);177}178 179static inline void axi_chan_irq_sig_set(struct axi_dma_chan *chan, u32 irq_mask)180{181 axi_chan_iowrite32(chan, CH_INTSIGNAL_ENA, irq_mask);182}183 184static inline void axi_chan_irq_clear(struct axi_dma_chan *chan, u32 irq_mask)185{186 axi_chan_iowrite32(chan, CH_INTCLEAR, irq_mask);187}188 189static inline u32 axi_chan_irq_read(struct axi_dma_chan *chan)190{191 return axi_chan_ioread32(chan, CH_INTSTATUS);192}193 194static inline void axi_chan_disable(struct axi_dma_chan *chan)195{196 u64 val;197 198 if (chan->chip->dw->hdata->nr_channels >= DMAC_CHAN_16) {199 val = axi_dma_ioread64(chan->chip, DMAC_CHEN);200 if (chan->id >= DMAC_CHAN_16) {201 val &= ~((u64)(BIT(chan->id) >> DMAC_CHAN_16)202 << (DMAC_CHAN_EN_SHIFT + DMAC_CHAN_BLOCK_SHIFT));203 val |= (u64)(BIT(chan->id) >> DMAC_CHAN_16)204 << (DMAC_CHAN_EN2_WE_SHIFT + DMAC_CHAN_BLOCK_SHIFT);205 } else {206 val &= ~(BIT(chan->id) << DMAC_CHAN_EN_SHIFT);207 val |= BIT(chan->id) << DMAC_CHAN_EN2_WE_SHIFT;208 }209 axi_dma_iowrite64(chan->chip, DMAC_CHEN, val);210 } else {211 val = axi_dma_ioread32(chan->chip, DMAC_CHEN);212 val &= ~(BIT(chan->id) << DMAC_CHAN_EN_SHIFT);213 if (chan->chip->dw->hdata->reg_map_8_channels)214 val |= BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT;215 else216 val |= BIT(chan->id) << DMAC_CHAN_EN2_WE_SHIFT;217 axi_dma_iowrite32(chan->chip, DMAC_CHEN, (u32)val);218 }219}220 221static inline void axi_chan_enable(struct axi_dma_chan *chan)222{223 u64 val;224 225 if (chan->chip->dw->hdata->nr_channels >= DMAC_CHAN_16) {226 val = axi_dma_ioread64(chan->chip, DMAC_CHEN);227 if (chan->id >= DMAC_CHAN_16) {228 val |= (u64)(BIT(chan->id) >> DMAC_CHAN_16)229 << (DMAC_CHAN_EN_SHIFT + DMAC_CHAN_BLOCK_SHIFT) |230 (u64)(BIT(chan->id) >> DMAC_CHAN_16)231 << (DMAC_CHAN_EN2_WE_SHIFT + DMAC_CHAN_BLOCK_SHIFT);232 } else {233 val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT |234 BIT(chan->id) << DMAC_CHAN_EN2_WE_SHIFT;235 }236 axi_dma_iowrite64(chan->chip, DMAC_CHEN, val);237 } else {238 val = axi_dma_ioread32(chan->chip, DMAC_CHEN);239 if (chan->chip->dw->hdata->reg_map_8_channels) {240 val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT |241 BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT;242 } else {243 val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT |244 BIT(chan->id) << DMAC_CHAN_EN2_WE_SHIFT;245 }246 axi_dma_iowrite32(chan->chip, DMAC_CHEN, (u32)val);247 }248}249 250static inline bool axi_chan_is_hw_enable(struct axi_dma_chan *chan)251{252 u64 val;253 254 if (chan->chip->dw->hdata->nr_channels >= DMAC_CHAN_16)255 val = axi_dma_ioread64(chan->chip, DMAC_CHEN);256 else257 val = axi_dma_ioread32(chan->chip, DMAC_CHEN);258 259 if (chan->id >= DMAC_CHAN_16)260 return !!(val & ((u64)(BIT(chan->id) >> DMAC_CHAN_16) << DMAC_CHAN_BLOCK_SHIFT));261 else262 return !!(val & (BIT(chan->id) << DMAC_CHAN_EN_SHIFT));263}264 265static void axi_dma_hw_init(struct axi_dma_chip *chip)266{267 int ret;268 u32 i;269 270 for (i = 0; i < chip->dw->hdata->nr_channels; i++) {271 axi_chan_irq_disable(&chip->dw->chan[i], DWAXIDMAC_IRQ_ALL);272 axi_chan_disable(&chip->dw->chan[i]);273 }274 ret = dma_set_mask_and_coherent(chip->dev, DMA_BIT_MASK(64));275 if (ret)276 dev_warn(chip->dev, "Unable to set coherent mask\n");277}278 279static u32 axi_chan_get_xfer_width(struct axi_dma_chan *chan, dma_addr_t src,280 dma_addr_t dst, size_t len)281{282 u32 max_width = chan->chip->dw->hdata->m_data_width;283 284 return __ffs(src | dst | len | BIT(max_width));285}286 287static inline const char *axi_chan_name(struct axi_dma_chan *chan)288{289 return dma_chan_name(&chan->vc.chan);290}291 292static struct axi_dma_desc *axi_desc_alloc(u32 num)293{294 struct axi_dma_desc *desc;295 296 desc = kzalloc(sizeof(*desc), GFP_NOWAIT);297 if (!desc)298 return NULL;299 300 desc->hw_desc = kcalloc(num, sizeof(*desc->hw_desc), GFP_NOWAIT);301 if (!desc->hw_desc) {302 kfree(desc);303 return NULL;304 }305 desc->nr_hw_descs = num;306 307 return desc;308}309 310static struct axi_dma_lli *axi_desc_get(struct axi_dma_chan *chan,311 dma_addr_t *addr)312{313 struct axi_dma_lli *lli;314 dma_addr_t phys;315 316 lli = dma_pool_zalloc(chan->desc_pool, GFP_NOWAIT, &phys);317 if (unlikely(!lli)) {318 dev_err(chan2dev(chan), "%s: not enough descriptors available\n",319 axi_chan_name(chan));320 return NULL;321 }322 323 atomic_inc(&chan->descs_allocated);324 *addr = phys;325 326 return lli;327}328 329static void axi_desc_put(struct axi_dma_desc *desc)330{331 struct axi_dma_chan *chan = desc->chan;332 int count = desc->nr_hw_descs;333 struct axi_dma_hw_desc *hw_desc;334 int descs_put;335 336 for (descs_put = 0; descs_put < count; descs_put++) {337 hw_desc = &desc->hw_desc[descs_put];338 dma_pool_free(chan->desc_pool, hw_desc->lli, hw_desc->llp);339 }340 341 kfree(desc->hw_desc);342 kfree(desc);343 atomic_sub(descs_put, &chan->descs_allocated);344 dev_vdbg(chan2dev(chan), "%s: %d descs put, %d still allocated\n",345 axi_chan_name(chan), descs_put,346 atomic_read(&chan->descs_allocated));347}348 349static void vchan_desc_put(struct virt_dma_desc *vdesc)350{351 axi_desc_put(vd_to_axi_desc(vdesc));352}353 354static enum dma_status355dma_chan_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,356 struct dma_tx_state *txstate)357{358 struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);359 struct virt_dma_desc *vdesc;360 enum dma_status status;361 u32 completed_length;362 unsigned long flags;363 u32 completed_blocks;364 size_t bytes = 0;365 u32 length;366 u32 len;367 368 status = dma_cookie_status(dchan, cookie, txstate);369 if (status == DMA_COMPLETE || !txstate)370 return status;371 372 spin_lock_irqsave(&chan->vc.lock, flags);373 374 vdesc = vchan_find_desc(&chan->vc, cookie);375 if (vdesc) {376 length = vd_to_axi_desc(vdesc)->length;377 completed_blocks = vd_to_axi_desc(vdesc)->completed_blocks;378 len = vd_to_axi_desc(vdesc)->hw_desc[0].len;379 completed_length = completed_blocks * len;380 bytes = length - completed_length;381 }382 383 spin_unlock_irqrestore(&chan->vc.lock, flags);384 dma_set_residue(txstate, bytes);385 386 return status;387}388 389static void write_desc_llp(struct axi_dma_hw_desc *desc, dma_addr_t adr)390{391 desc->lli->llp = cpu_to_le64(adr);392}393 394static void write_chan_llp(struct axi_dma_chan *chan, dma_addr_t adr)395{396 axi_chan_iowrite64(chan, CH_LLP, adr);397}398 399static void dw_axi_dma_set_byte_halfword(struct axi_dma_chan *chan, bool set)400{401 u32 offset = DMAC_APB_BYTE_WR_CH_EN;402 u32 reg_width, val;403 404 if (!chan->chip->apb_regs) {405 dev_dbg(chan->chip->dev, "apb_regs not initialized\n");406 return;407 }408 409 reg_width = __ffs(chan->config.dst_addr_width);410 if (reg_width == DWAXIDMAC_TRANS_WIDTH_16)411 offset = DMAC_APB_HALFWORD_WR_CH_EN;412 413 val = ioread32(chan->chip->apb_regs + offset);414 415 if (set)416 val |= BIT(chan->id);417 else418 val &= ~BIT(chan->id);419 420 iowrite32(val, chan->chip->apb_regs + offset);421}422/* Called in chan locked context */423static void axi_chan_block_xfer_start(struct axi_dma_chan *chan,424 struct axi_dma_desc *first)425{426 u32 priority = chan->chip->dw->hdata->priority[chan->id];427 struct axi_dma_chan_config config = {};428 u32 irq_mask;429 u8 lms = 0; /* Select AXI0 master for LLI fetching */430 431 if (unlikely(axi_chan_is_hw_enable(chan))) {432 dev_err(chan2dev(chan), "%s is non-idle!\n",433 axi_chan_name(chan));434 435 return;436 }437 438 axi_dma_enable(chan->chip);439 440 config.dst_multblk_type = DWAXIDMAC_MBLK_TYPE_LL;441 config.src_multblk_type = DWAXIDMAC_MBLK_TYPE_LL;442 config.tt_fc = DWAXIDMAC_TT_FC_MEM_TO_MEM_DMAC;443 config.prior = priority;444 config.hs_sel_dst = DWAXIDMAC_HS_SEL_HW;445 config.hs_sel_src = DWAXIDMAC_HS_SEL_HW;446 switch (chan->direction) {447 case DMA_MEM_TO_DEV:448 dw_axi_dma_set_byte_halfword(chan, true);449 config.tt_fc = chan->config.device_fc ?450 DWAXIDMAC_TT_FC_MEM_TO_PER_DST :451 DWAXIDMAC_TT_FC_MEM_TO_PER_DMAC;452 if (chan->chip->apb_regs)453 config.dst_per = chan->id;454 else455 config.dst_per = chan->hw_handshake_num;456 break;457 case DMA_DEV_TO_MEM:458 config.tt_fc = chan->config.device_fc ?459 DWAXIDMAC_TT_FC_PER_TO_MEM_SRC :460 DWAXIDMAC_TT_FC_PER_TO_MEM_DMAC;461 if (chan->chip->apb_regs)462 config.src_per = chan->id;463 else464 config.src_per = chan->hw_handshake_num;465 break;466 default:467 break;468 }469 axi_chan_config_write(chan, &config);470 471 write_chan_llp(chan, first->hw_desc[0].llp | lms);472 473 irq_mask = DWAXIDMAC_IRQ_DMA_TRF | DWAXIDMAC_IRQ_ALL_ERR;474 axi_chan_irq_sig_set(chan, irq_mask);475 476 /* Generate 'suspend' status but don't generate interrupt */477 irq_mask |= DWAXIDMAC_IRQ_SUSPENDED;478 axi_chan_irq_set(chan, irq_mask);479 480 axi_chan_enable(chan);481}482 483static void axi_chan_start_first_queued(struct axi_dma_chan *chan)484{485 struct axi_dma_desc *desc;486 struct virt_dma_desc *vd;487 488 vd = vchan_next_desc(&chan->vc);489 if (!vd)490 return;491 492 desc = vd_to_axi_desc(vd);493 dev_vdbg(chan2dev(chan), "%s: started %u\n", axi_chan_name(chan),494 vd->tx.cookie);495 axi_chan_block_xfer_start(chan, desc);496}497 498static void dma_chan_issue_pending(struct dma_chan *dchan)499{500 struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);501 unsigned long flags;502 503 spin_lock_irqsave(&chan->vc.lock, flags);504 if (vchan_issue_pending(&chan->vc))505 axi_chan_start_first_queued(chan);506 spin_unlock_irqrestore(&chan->vc.lock, flags);507}508 509static void dw_axi_dma_synchronize(struct dma_chan *dchan)510{511 struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);512 513 vchan_synchronize(&chan->vc);514}515 516static int dma_chan_alloc_chan_resources(struct dma_chan *dchan)517{518 struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);519 520 /* ASSERT: channel is idle */521 if (axi_chan_is_hw_enable(chan)) {522 dev_err(chan2dev(chan), "%s is non-idle!\n",523 axi_chan_name(chan));524 return -EBUSY;525 }526 527 /* LLI address must be aligned to a 64-byte boundary */528 chan->desc_pool = dma_pool_create(dev_name(chan2dev(chan)),529 chan->chip->dev,530 sizeof(struct axi_dma_lli),531 64, 0);532 if (!chan->desc_pool) {533 dev_err(chan2dev(chan), "No memory for descriptors\n");534 return -ENOMEM;535 }536 dev_vdbg(dchan2dev(dchan), "%s: allocating\n", axi_chan_name(chan));537 538 pm_runtime_get(chan->chip->dev);539 540 return 0;541}542 543static void dma_chan_free_chan_resources(struct dma_chan *dchan)544{545 struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);546 547 /* ASSERT: channel is idle */548 if (axi_chan_is_hw_enable(chan))549 dev_err(dchan2dev(dchan), "%s is non-idle!\n",550 axi_chan_name(chan));551 552 axi_chan_disable(chan);553 axi_chan_irq_disable(chan, DWAXIDMAC_IRQ_ALL);554 555 vchan_free_chan_resources(&chan->vc);556 557 dma_pool_destroy(chan->desc_pool);558 chan->desc_pool = NULL;559 dev_vdbg(dchan2dev(dchan),560 "%s: free resources, descriptor still allocated: %u\n",561 axi_chan_name(chan), atomic_read(&chan->descs_allocated));562 563 pm_runtime_put(chan->chip->dev);564}565 566static void dw_axi_dma_set_hw_channel(struct axi_dma_chan *chan, bool set)567{568 struct axi_dma_chip *chip = chan->chip;569 unsigned long reg_value, val;570 571 if (!chip->apb_regs) {572 dev_err(chip->dev, "apb_regs not initialized\n");573 return;574 }575 576 /*577 * An unused DMA channel has a default value of 0x3F.578 * Lock the DMA channel by assign a handshake number to the channel.579 * Unlock the DMA channel by assign 0x3F to the channel.580 */581 if (set)582 val = chan->hw_handshake_num;583 else584 val = UNUSED_CHANNEL;585 586 reg_value = lo_hi_readq(chip->apb_regs + DMAC_APB_HW_HS_SEL_0);587 588 /* Channel is already allocated, set handshake as per channel ID */589 /* 64 bit write should handle for 8 channels */590 591 reg_value &= ~(DMA_APB_HS_SEL_MASK <<592 (chan->id * DMA_APB_HS_SEL_BIT_SIZE));593 reg_value |= (val << (chan->id * DMA_APB_HS_SEL_BIT_SIZE));594 lo_hi_writeq(reg_value, chip->apb_regs + DMAC_APB_HW_HS_SEL_0);595 596 return;597}598 599/*600 * If DW_axi_dmac sees CHx_CTL.ShadowReg_Or_LLI_Last bit of the fetched LLI601 * as 1, it understands that the current block is the final block in the602 * transfer and completes the DMA transfer operation at the end of current603 * block transfer.604 */605static void set_desc_last(struct axi_dma_hw_desc *desc)606{607 u32 val;608 609 val = le32_to_cpu(desc->lli->ctl_hi);610 val |= CH_CTL_H_LLI_LAST;611 desc->lli->ctl_hi = cpu_to_le32(val);612}613 614static void write_desc_sar(struct axi_dma_hw_desc *desc, dma_addr_t adr)615{616 desc->lli->sar = cpu_to_le64(adr);617}618 619static void write_desc_dar(struct axi_dma_hw_desc *desc, dma_addr_t adr)620{621 desc->lli->dar = cpu_to_le64(adr);622}623 624static void set_desc_src_master(struct axi_dma_hw_desc *desc)625{626 u32 val;627 628 /* Select AXI0 for source master */629 val = le32_to_cpu(desc->lli->ctl_lo);630 val &= ~CH_CTL_L_SRC_MAST;631 desc->lli->ctl_lo = cpu_to_le32(val);632}633 634static void set_desc_dest_master(struct axi_dma_hw_desc *hw_desc,635 struct axi_dma_desc *desc)636{637 u32 val;638 639 /* Select AXI1 for source master if available */640 val = le32_to_cpu(hw_desc->lli->ctl_lo);641 if (desc->chan->chip->dw->hdata->nr_masters > 1)642 val |= CH_CTL_L_DST_MAST;643 else644 val &= ~CH_CTL_L_DST_MAST;645 646 hw_desc->lli->ctl_lo = cpu_to_le32(val);647}648 649static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan,650 struct axi_dma_hw_desc *hw_desc,651 dma_addr_t mem_addr, size_t len)652{653 unsigned int data_width = BIT(chan->chip->dw->hdata->m_data_width);654 unsigned int reg_width;655 unsigned int mem_width;656 dma_addr_t device_addr;657 size_t axi_block_ts;658 size_t block_ts;659 u32 ctllo, ctlhi;660 u32 burst_len;661 662 axi_block_ts = chan->chip->dw->hdata->block_size[chan->id];663 664 mem_width = __ffs(data_width | mem_addr | len);665 if (mem_width > DWAXIDMAC_TRANS_WIDTH_32)666 mem_width = DWAXIDMAC_TRANS_WIDTH_32;667 668 if (!IS_ALIGNED(mem_addr, 4)) {669 dev_err(chan->chip->dev, "invalid buffer alignment\n");670 return -EINVAL;671 }672 673 switch (chan->direction) {674 case DMA_MEM_TO_DEV:675 reg_width = __ffs(chan->config.dst_addr_width);676 device_addr = chan->config.dst_addr;677 ctllo = reg_width << CH_CTL_L_DST_WIDTH_POS |678 mem_width << CH_CTL_L_SRC_WIDTH_POS |679 DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_DST_INC_POS |680 DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS;681 block_ts = len >> mem_width;682 break;683 case DMA_DEV_TO_MEM:684 reg_width = __ffs(chan->config.src_addr_width);685 device_addr = chan->config.src_addr;686 ctllo = reg_width << CH_CTL_L_SRC_WIDTH_POS |687 mem_width << CH_CTL_L_DST_WIDTH_POS |688 DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS |689 DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_SRC_INC_POS;690 block_ts = len >> reg_width;691 break;692 default:693 return -EINVAL;694 }695 696 if (block_ts > axi_block_ts)697 return -EINVAL;698 699 hw_desc->lli = axi_desc_get(chan, &hw_desc->llp);700 if (unlikely(!hw_desc->lli))701 return -ENOMEM;702 703 ctlhi = CH_CTL_H_LLI_VALID;704 705 if (chan->chip->dw->hdata->restrict_axi_burst_len) {706 burst_len = chan->chip->dw->hdata->axi_rw_burst_len;707 ctlhi |= CH_CTL_H_ARLEN_EN | CH_CTL_H_AWLEN_EN |708 burst_len << CH_CTL_H_ARLEN_POS |709 burst_len << CH_CTL_H_AWLEN_POS;710 }711 712 hw_desc->lli->ctl_hi = cpu_to_le32(ctlhi);713 714 if (chan->direction == DMA_MEM_TO_DEV) {715 write_desc_sar(hw_desc, mem_addr);716 write_desc_dar(hw_desc, device_addr);717 } else {718 write_desc_sar(hw_desc, device_addr);719 write_desc_dar(hw_desc, mem_addr);720 }721 722 hw_desc->lli->block_ts_lo = cpu_to_le32(block_ts - 1);723 724 ctllo |= DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS |725 DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS;726 hw_desc->lli->ctl_lo = cpu_to_le32(ctllo);727 728 set_desc_src_master(hw_desc);729 730 hw_desc->len = len;731 return 0;732}733 734static size_t calculate_block_len(struct axi_dma_chan *chan,735 dma_addr_t dma_addr, size_t buf_len,736 enum dma_transfer_direction direction)737{738 u32 data_width, reg_width, mem_width;739 size_t axi_block_ts, block_len;740 741 axi_block_ts = chan->chip->dw->hdata->block_size[chan->id];742 743 switch (direction) {744 case DMA_MEM_TO_DEV:745 data_width = BIT(chan->chip->dw->hdata->m_data_width);746 mem_width = __ffs(data_width | dma_addr | buf_len);747 if (mem_width > DWAXIDMAC_TRANS_WIDTH_32)748 mem_width = DWAXIDMAC_TRANS_WIDTH_32;749 750 block_len = axi_block_ts << mem_width;751 break;752 case DMA_DEV_TO_MEM:753 reg_width = __ffs(chan->config.src_addr_width);754 block_len = axi_block_ts << reg_width;755 break;756 default:757 block_len = 0;758 }759 760 return block_len;761}762 763static struct dma_async_tx_descriptor *764dw_axi_dma_chan_prep_cyclic(struct dma_chan *dchan, dma_addr_t dma_addr,765 size_t buf_len, size_t period_len,766 enum dma_transfer_direction direction,767 unsigned long flags)768{769 struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);770 struct axi_dma_hw_desc *hw_desc = NULL;771 struct axi_dma_desc *desc = NULL;772 dma_addr_t src_addr = dma_addr;773 u32 num_periods, num_segments;774 size_t axi_block_len;775 u32 total_segments;776 u32 segment_len;777 unsigned int i;778 int status;779 u64 llp = 0;780 u8 lms = 0; /* Select AXI0 master for LLI fetching */781 782 num_periods = buf_len / period_len;783 784 axi_block_len = calculate_block_len(chan, dma_addr, buf_len, direction);785 if (axi_block_len == 0)786 return NULL;787 788 num_segments = DIV_ROUND_UP(period_len, axi_block_len);789 segment_len = DIV_ROUND_UP(period_len, num_segments);790 791 total_segments = num_periods * num_segments;792 793 desc = axi_desc_alloc(total_segments);794 if (unlikely(!desc))795 goto err_desc_get;796 797 chan->direction = direction;798 desc->chan = chan;799 chan->cyclic = true;800 desc->length = 0;801 desc->period_len = period_len;802 803 for (i = 0; i < total_segments; i++) {804 hw_desc = &desc->hw_desc[i];805 806 status = dw_axi_dma_set_hw_desc(chan, hw_desc, src_addr,807 segment_len);808 if (status < 0)809 goto err_desc_get;810 811 desc->length += hw_desc->len;812 /* Set end-of-link to the linked descriptor, so that cyclic813 * callback function can be triggered during interrupt.814 */815 set_desc_last(hw_desc);816 817 src_addr += segment_len;818 }819 820 llp = desc->hw_desc[0].llp;821 822 /* Managed transfer list */823 do {824 hw_desc = &desc->hw_desc[--total_segments];825 write_desc_llp(hw_desc, llp | lms);826 llp = hw_desc->llp;827 } while (total_segments);828 829 dw_axi_dma_set_hw_channel(chan, true);830 831 return vchan_tx_prep(&chan->vc, &desc->vd, flags);832 833err_desc_get:834 if (desc)835 axi_desc_put(desc);836 837 return NULL;838}839 840static struct dma_async_tx_descriptor *841dw_axi_dma_chan_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,842 unsigned int sg_len,843 enum dma_transfer_direction direction,844 unsigned long flags, void *context)845{846 struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);847 struct axi_dma_hw_desc *hw_desc = NULL;848 struct axi_dma_desc *desc = NULL;849 u32 num_segments, segment_len;850 unsigned int loop = 0;851 struct scatterlist *sg;852 size_t axi_block_len;853 u32 len, num_sgs = 0;854 unsigned int i;855 dma_addr_t mem;856 int status;857 u64 llp = 0;858 u8 lms = 0; /* Select AXI0 master for LLI fetching */859 860 if (unlikely(!is_slave_direction(direction) || !sg_len))861 return NULL;862 863 mem = sg_dma_address(sgl);864 len = sg_dma_len(sgl);865 866 axi_block_len = calculate_block_len(chan, mem, len, direction);867 if (axi_block_len == 0)868 return NULL;869 870 for_each_sg(sgl, sg, sg_len, i)871 num_sgs += DIV_ROUND_UP(sg_dma_len(sg), axi_block_len);872 873 desc = axi_desc_alloc(num_sgs);874 if (unlikely(!desc))875 goto err_desc_get;876 877 desc->chan = chan;878 desc->length = 0;879 chan->direction = direction;880 881 for_each_sg(sgl, sg, sg_len, i) {882 mem = sg_dma_address(sg);883 len = sg_dma_len(sg);884 num_segments = DIV_ROUND_UP(sg_dma_len(sg), axi_block_len);885 segment_len = DIV_ROUND_UP(sg_dma_len(sg), num_segments);886 887 do {888 hw_desc = &desc->hw_desc[loop++];889 status = dw_axi_dma_set_hw_desc(chan, hw_desc, mem, segment_len);890 if (status < 0)891 goto err_desc_get;892 893 desc->length += hw_desc->len;894 len -= segment_len;895 mem += segment_len;896 } while (len >= segment_len);897 }898 899 /* Set end-of-link to the last link descriptor of list */900 set_desc_last(&desc->hw_desc[num_sgs - 1]);901 902 /* Managed transfer list */903 do {904 hw_desc = &desc->hw_desc[--num_sgs];905 write_desc_llp(hw_desc, llp | lms);906 llp = hw_desc->llp;907 } while (num_sgs);908 909 dw_axi_dma_set_hw_channel(chan, true);910 911 return vchan_tx_prep(&chan->vc, &desc->vd, flags);912 913err_desc_get:914 if (desc)915 axi_desc_put(desc);916 917 return NULL;918}919 920static struct dma_async_tx_descriptor *921dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr,922 dma_addr_t src_adr, size_t len, unsigned long flags)923{924 struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);925 size_t block_ts, max_block_ts, xfer_len;926 struct axi_dma_hw_desc *hw_desc = NULL;927 struct axi_dma_desc *desc = NULL;928 u32 xfer_width, reg, num;929 u64 llp = 0;930 u8 lms = 0; /* Select AXI0 master for LLI fetching */931 932 dev_dbg(chan2dev(chan), "%s: memcpy: src: %pad dst: %pad length: %zd flags: %#lx",933 axi_chan_name(chan), &src_adr, &dst_adr, len, flags);934 935 max_block_ts = chan->chip->dw->hdata->block_size[chan->id];936 xfer_width = axi_chan_get_xfer_width(chan, src_adr, dst_adr, len);937 num = DIV_ROUND_UP(len, max_block_ts << xfer_width);938 desc = axi_desc_alloc(num);939 if (unlikely(!desc))940 goto err_desc_get;941 942 desc->chan = chan;943 num = 0;944 desc->length = 0;945 while (len) {946 xfer_len = len;947 948 hw_desc = &desc->hw_desc[num];949 /*950 * Take care for the alignment.951 * Actually source and destination widths can be different, but952 * make them same to be simpler.953 */954 xfer_width = axi_chan_get_xfer_width(chan, src_adr, dst_adr, xfer_len);955 956 /*957 * block_ts indicates the total number of data of width958 * to be transferred in a DMA block transfer.959 * BLOCK_TS register should be set to block_ts - 1960 */961 block_ts = xfer_len >> xfer_width;962 if (block_ts > max_block_ts) {963 block_ts = max_block_ts;964 xfer_len = max_block_ts << xfer_width;965 }966 967 hw_desc->lli = axi_desc_get(chan, &hw_desc->llp);968 if (unlikely(!hw_desc->lli))969 goto err_desc_get;970 971 write_desc_sar(hw_desc, src_adr);972 write_desc_dar(hw_desc, dst_adr);973 hw_desc->lli->block_ts_lo = cpu_to_le32(block_ts - 1);974 975 reg = CH_CTL_H_LLI_VALID;976 if (chan->chip->dw->hdata->restrict_axi_burst_len) {977 u32 burst_len = chan->chip->dw->hdata->axi_rw_burst_len;978 979 reg |= (CH_CTL_H_ARLEN_EN |980 burst_len << CH_CTL_H_ARLEN_POS |981 CH_CTL_H_AWLEN_EN |982 burst_len << CH_CTL_H_AWLEN_POS);983 }984 hw_desc->lli->ctl_hi = cpu_to_le32(reg);985 986 reg = (DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS |987 DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS |988 xfer_width << CH_CTL_L_DST_WIDTH_POS |989 xfer_width << CH_CTL_L_SRC_WIDTH_POS |990 DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS |991 DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS);992 hw_desc->lli->ctl_lo = cpu_to_le32(reg);993 994 set_desc_src_master(hw_desc);995 set_desc_dest_master(hw_desc, desc);996 997 hw_desc->len = xfer_len;998 desc->length += hw_desc->len;999 /* update the length and addresses for the next loop cycle */1000 len -= xfer_len;1001 dst_adr += xfer_len;1002 src_adr += xfer_len;1003 num++;1004 }1005 1006 /* Set end-of-link to the last link descriptor of list */1007 set_desc_last(&desc->hw_desc[num - 1]);1008 /* Managed transfer list */1009 do {1010 hw_desc = &desc->hw_desc[--num];1011 write_desc_llp(hw_desc, llp | lms);1012 llp = hw_desc->llp;1013 } while (num);1014 1015 return vchan_tx_prep(&chan->vc, &desc->vd, flags);1016 1017err_desc_get:1018 if (desc)1019 axi_desc_put(desc);1020 return NULL;1021}1022 1023static int dw_axi_dma_chan_slave_config(struct dma_chan *dchan,1024 struct dma_slave_config *config)1025{1026 struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);1027 1028 memcpy(&chan->config, config, sizeof(*config));1029 1030 return 0;1031}1032 1033static void axi_chan_dump_lli(struct axi_dma_chan *chan,1034 struct axi_dma_hw_desc *desc)1035{1036 if (!desc->lli) {1037 dev_err(dchan2dev(&chan->vc.chan), "NULL LLI\n");1038 return;1039 }1040 1041 dev_err(dchan2dev(&chan->vc.chan),1042 "SAR: 0x%llx DAR: 0x%llx LLP: 0x%llx BTS 0x%x CTL: 0x%x:%08x",1043 le64_to_cpu(desc->lli->sar),1044 le64_to_cpu(desc->lli->dar),1045 le64_to_cpu(desc->lli->llp),1046 le32_to_cpu(desc->lli->block_ts_lo),1047 le32_to_cpu(desc->lli->ctl_hi),1048 le32_to_cpu(desc->lli->ctl_lo));1049}1050 1051static void axi_chan_list_dump_lli(struct axi_dma_chan *chan,1052 struct axi_dma_desc *desc_head)1053{1054 int count = atomic_read(&chan->descs_allocated);1055 int i;1056 1057 for (i = 0; i < count; i++)1058 axi_chan_dump_lli(chan, &desc_head->hw_desc[i]);1059}1060 1061static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status)1062{1063 struct virt_dma_desc *vd;1064 unsigned long flags;1065 1066 spin_lock_irqsave(&chan->vc.lock, flags);1067 1068 axi_chan_disable(chan);1069 1070 /* The bad descriptor currently is in the head of vc list */1071 vd = vchan_next_desc(&chan->vc);1072 if (!vd) {1073 dev_err(chan2dev(chan), "BUG: %s, IRQ with no descriptors\n",1074 axi_chan_name(chan));1075 goto out;1076 }1077 /* Remove the completed descriptor from issued list */1078 list_del(&vd->node);1079 1080 /* WARN about bad descriptor */1081 dev_err(chan2dev(chan),1082 "Bad descriptor submitted for %s, cookie: %d, irq: 0x%08x\n",1083 axi_chan_name(chan), vd->tx.cookie, status);1084 axi_chan_list_dump_lli(chan, vd_to_axi_desc(vd));1085 1086 vchan_cookie_complete(vd);1087 1088 /* Try to restart the controller */1089 axi_chan_start_first_queued(chan);1090 1091out:1092 spin_unlock_irqrestore(&chan->vc.lock, flags);1093}1094 1095static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan)1096{1097 int count = atomic_read(&chan->descs_allocated);1098 struct axi_dma_hw_desc *hw_desc;1099 struct axi_dma_desc *desc;1100 struct virt_dma_desc *vd;1101 unsigned long flags;1102 u64 llp;1103 int i;1104 1105 spin_lock_irqsave(&chan->vc.lock, flags);1106 if (unlikely(axi_chan_is_hw_enable(chan))) {1107 dev_err(chan2dev(chan), "BUG: %s caught DWAXIDMAC_IRQ_DMA_TRF, but channel not idle!\n",1108 axi_chan_name(chan));1109 axi_chan_disable(chan);1110 }1111 1112 /* The completed descriptor currently is in the head of vc list */1113 vd = vchan_next_desc(&chan->vc);1114 if (!vd) {1115 dev_err(chan2dev(chan), "BUG: %s, IRQ with no descriptors\n",1116 axi_chan_name(chan));1117 goto out;1118 }1119 1120 if (chan->cyclic) {1121 desc = vd_to_axi_desc(vd);1122 if (desc) {1123 llp = lo_hi_readq(chan->chan_regs + CH_LLP);1124 for (i = 0; i < count; i++) {1125 hw_desc = &desc->hw_desc[i];1126 if (hw_desc->llp == llp) {1127 axi_chan_irq_clear(chan, hw_desc->lli->status_lo);1128 hw_desc->lli->ctl_hi |= CH_CTL_H_LLI_VALID;1129 desc->completed_blocks = i;1130 1131 if (((hw_desc->len * (i + 1)) % desc->period_len) == 0)1132 vchan_cyclic_callback(vd);1133 break;1134 }1135 }1136 1137 axi_chan_enable(chan);1138 }1139 } else {1140 /* Remove the completed descriptor from issued list before completing */1141 list_del(&vd->node);1142 vchan_cookie_complete(vd);1143 }1144 1145out:1146 spin_unlock_irqrestore(&chan->vc.lock, flags);1147}1148 1149static irqreturn_t dw_axi_dma_interrupt(int irq, void *dev_id)1150{1151 struct axi_dma_chip *chip = dev_id;1152 struct dw_axi_dma *dw = chip->dw;1153 struct axi_dma_chan *chan;1154 1155 u32 status, i;1156 1157 /* Disable DMAC interrupts. We'll enable them after processing channels */1158 axi_dma_irq_disable(chip);1159 1160 /* Poll, clear and process every channel interrupt status */1161 for (i = 0; i < dw->hdata->nr_channels; i++) {1162 chan = &dw->chan[i];1163 status = axi_chan_irq_read(chan);1164 axi_chan_irq_clear(chan, status);1165 1166 dev_vdbg(chip->dev, "%s %u IRQ status: 0x%08x\n",1167 axi_chan_name(chan), i, status);1168 1169 if (status & DWAXIDMAC_IRQ_ALL_ERR)1170 axi_chan_handle_err(chan, status);1171 else if (status & DWAXIDMAC_IRQ_DMA_TRF)1172 axi_chan_block_xfer_complete(chan);1173 }1174 1175 /* Re-enable interrupts */1176 axi_dma_irq_enable(chip);1177 1178 return IRQ_HANDLED;1179}1180 1181static int dma_chan_terminate_all(struct dma_chan *dchan)1182{1183 struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);1184 u32 chan_active = BIT(chan->id) << DMAC_CHAN_EN_SHIFT;1185 unsigned long flags;1186 u32 val;1187 int ret;1188 LIST_HEAD(head);1189 1190 axi_chan_disable(chan);1191 1192 ret = readl_poll_timeout_atomic(chan->chip->regs + DMAC_CHEN, val,1193 !(val & chan_active), 1000, 50000);1194 if (ret == -ETIMEDOUT)1195 dev_warn(dchan2dev(dchan),1196 "%s failed to stop\n", axi_chan_name(chan));1197 1198 if (chan->direction != DMA_MEM_TO_MEM)1199 dw_axi_dma_set_hw_channel(chan, false);1200 if (chan->direction == DMA_MEM_TO_DEV)1201 dw_axi_dma_set_byte_halfword(chan, false);1202 1203 spin_lock_irqsave(&chan->vc.lock, flags);1204 1205 vchan_get_all_descriptors(&chan->vc, &head);1206 1207 chan->cyclic = false;1208 spin_unlock_irqrestore(&chan->vc.lock, flags);1209 1210 vchan_dma_desc_free_list(&chan->vc, &head);1211 1212 dev_vdbg(dchan2dev(dchan), "terminated: %s\n", axi_chan_name(chan));1213 1214 return 0;1215}1216 1217static int dma_chan_pause(struct dma_chan *dchan)1218{1219 struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);1220 unsigned long flags;1221 unsigned int timeout = 20; /* timeout iterations */1222 u64 val;1223 1224 spin_lock_irqsave(&chan->vc.lock, flags);1225 1226 if (chan->chip->dw->hdata->nr_channels >= DMAC_CHAN_16) {1227 val = axi_dma_ioread64(chan->chip, DMAC_CHSUSPREG);1228 if (chan->id >= DMAC_CHAN_16) {1229 val |= (u64)(BIT(chan->id) >> DMAC_CHAN_16)1230 << (DMAC_CHAN_SUSP2_SHIFT + DMAC_CHAN_BLOCK_SHIFT) |1231 (u64)(BIT(chan->id) >> DMAC_CHAN_16)1232 << (DMAC_CHAN_SUSP2_WE_SHIFT + DMAC_CHAN_BLOCK_SHIFT);1233 } else {1234 val |= BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT |1235 BIT(chan->id) << DMAC_CHAN_SUSP2_WE_SHIFT;1236 }1237 axi_dma_iowrite64(chan->chip, DMAC_CHSUSPREG, val);1238 } else {1239 if (chan->chip->dw->hdata->reg_map_8_channels) {1240 val = axi_dma_ioread32(chan->chip, DMAC_CHEN);1241 val |= BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT |1242 BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT;1243 axi_dma_iowrite32(chan->chip, DMAC_CHEN, (u32)val);1244 } else {1245 val = axi_dma_ioread32(chan->chip, DMAC_CHSUSPREG);1246 val |= BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT |1247 BIT(chan->id) << DMAC_CHAN_SUSP2_WE_SHIFT;1248 axi_dma_iowrite32(chan->chip, DMAC_CHSUSPREG, (u32)val);1249 }1250 }1251 1252 do {1253 if (axi_chan_irq_read(chan) & DWAXIDMAC_IRQ_SUSPENDED)1254 break;1255 1256 udelay(2);1257 } while (--timeout);1258 1259 axi_chan_irq_clear(chan, DWAXIDMAC_IRQ_SUSPENDED);1260 1261 chan->is_paused = true;1262 1263 spin_unlock_irqrestore(&chan->vc.lock, flags);1264 1265 return timeout ? 0 : -EAGAIN;1266}1267 1268/* Called in chan locked context */1269static inline void axi_chan_resume(struct axi_dma_chan *chan)1270{1271 u64 val;1272 1273 if (chan->chip->dw->hdata->nr_channels >= DMAC_CHAN_16) {1274 val = axi_dma_ioread64(chan->chip, DMAC_CHSUSPREG);1275 if (chan->id >= DMAC_CHAN_16) {1276 val &= ~((u64)(BIT(chan->id) >> DMAC_CHAN_16)1277 << (DMAC_CHAN_SUSP2_SHIFT + DMAC_CHAN_BLOCK_SHIFT));1278 val |= ((u64)(BIT(chan->id) >> DMAC_CHAN_16)1279 << (DMAC_CHAN_SUSP2_WE_SHIFT + DMAC_CHAN_BLOCK_SHIFT));1280 } else {1281 val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT);1282 val |= (BIT(chan->id) << DMAC_CHAN_SUSP2_WE_SHIFT);1283 }1284 axi_dma_iowrite64(chan->chip, DMAC_CHSUSPREG, val);1285 } else {1286 if (chan->chip->dw->hdata->reg_map_8_channels) {1287 val = axi_dma_ioread32(chan->chip, DMAC_CHEN);1288 val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT);1289 val |= (BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT);1290 axi_dma_iowrite32(chan->chip, DMAC_CHEN, (u32)val);1291 } else {1292 val = axi_dma_ioread32(chan->chip, DMAC_CHSUSPREG);1293 val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT);1294 val |= (BIT(chan->id) << DMAC_CHAN_SUSP2_WE_SHIFT);1295 axi_dma_iowrite32(chan->chip, DMAC_CHSUSPREG, (u32)val);1296 }1297 }1298 1299 chan->is_paused = false;1300}1301 1302static int dma_chan_resume(struct dma_chan *dchan)1303{1304 struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);1305 unsigned long flags;1306 1307 spin_lock_irqsave(&chan->vc.lock, flags);1308 1309 if (chan->is_paused)1310 axi_chan_resume(chan);1311 1312 spin_unlock_irqrestore(&chan->vc.lock, flags);1313 1314 return 0;1315}1316 1317static int axi_dma_suspend(struct axi_dma_chip *chip)1318{1319 axi_dma_irq_disable(chip);1320 axi_dma_disable(chip);1321 1322 clk_disable_unprepare(chip->core_clk);1323 clk_disable_unprepare(chip->cfgr_clk);1324 1325 return 0;1326}1327 1328static int axi_dma_resume(struct axi_dma_chip *chip)1329{1330 int ret;1331 1332 ret = clk_prepare_enable(chip->cfgr_clk);1333 if (ret < 0)1334 return ret;1335 1336 ret = clk_prepare_enable(chip->core_clk);1337 if (ret < 0)1338 return ret;1339 1340 axi_dma_enable(chip);1341 axi_dma_irq_enable(chip);1342 1343 return 0;1344}1345 1346static int __maybe_unused axi_dma_runtime_suspend(struct device *dev)1347{1348 struct axi_dma_chip *chip = dev_get_drvdata(dev);1349 1350 return axi_dma_suspend(chip);1351}1352 1353static int __maybe_unused axi_dma_runtime_resume(struct device *dev)1354{1355 struct axi_dma_chip *chip = dev_get_drvdata(dev);1356 1357 return axi_dma_resume(chip);1358}1359 1360static struct dma_chan *dw_axi_dma_of_xlate(struct of_phandle_args *dma_spec,1361 struct of_dma *ofdma)1362{1363 struct dw_axi_dma *dw = ofdma->of_dma_data;1364 struct axi_dma_chan *chan;1365 struct dma_chan *dchan;1366 1367 dchan = dma_get_any_slave_channel(&dw->dma);1368 if (!dchan)1369 return NULL;1370 1371 chan = dchan_to_axi_dma_chan(dchan);1372 chan->hw_handshake_num = dma_spec->args[0];1373 return dchan;1374}1375 1376static int parse_device_properties(struct axi_dma_chip *chip)1377{1378 struct device *dev = chip->dev;1379 u32 tmp, carr[DMAC_MAX_CHANNELS];1380 int ret;1381 1382 ret = device_property_read_u32(dev, "dma-channels", &tmp);1383 if (ret)1384 return ret;1385 if (tmp == 0 || tmp > DMAC_MAX_CHANNELS)1386 return -EINVAL;1387 1388 chip->dw->hdata->nr_channels = tmp;1389 if (tmp <= DMA_REG_MAP_CH_REF)1390 chip->dw->hdata->reg_map_8_channels = true;1391 1392 ret = device_property_read_u32(dev, "snps,dma-masters", &tmp);1393 if (ret)1394 return ret;1395 if (tmp == 0 || tmp > DMAC_MAX_MASTERS)1396 return -EINVAL;1397 1398 chip->dw->hdata->nr_masters = tmp;1399 1400 ret = device_property_read_u32(dev, "snps,data-width", &tmp);1401 if (ret)1402 return ret;1403 if (tmp > DWAXIDMAC_TRANS_WIDTH_MAX)1404 return -EINVAL;1405 1406 chip->dw->hdata->m_data_width = tmp;1407 1408 ret = device_property_read_u32_array(dev, "snps,block-size", carr,1409 chip->dw->hdata->nr_channels);1410 if (ret)1411 return ret;1412 for (tmp = 0; tmp < chip->dw->hdata->nr_channels; tmp++) {1413 if (carr[tmp] == 0 || carr[tmp] > DMAC_MAX_BLK_SIZE)1414 return -EINVAL;1415 1416 chip->dw->hdata->block_size[tmp] = carr[tmp];1417 }1418 1419 ret = device_property_read_u32_array(dev, "snps,priority", carr,1420 chip->dw->hdata->nr_channels);1421 if (ret)1422 return ret;1423 /* Priority value must be programmed within [0:nr_channels-1] range */1424 for (tmp = 0; tmp < chip->dw->hdata->nr_channels; tmp++) {1425 if (carr[tmp] >= chip->dw->hdata->nr_channels)1426 return -EINVAL;1427 1428 chip->dw->hdata->priority[tmp] = carr[tmp];1429 }1430 1431 /* axi-max-burst-len is optional property */1432 ret = device_property_read_u32(dev, "snps,axi-max-burst-len", &tmp);1433 if (!ret) {1434 if (tmp > DWAXIDMAC_ARWLEN_MAX + 1)1435 return -EINVAL;1436 if (tmp < DWAXIDMAC_ARWLEN_MIN + 1)1437 return -EINVAL;1438 1439 chip->dw->hdata->restrict_axi_burst_len = true;1440 chip->dw->hdata->axi_rw_burst_len = tmp;1441 }1442 1443 return 0;1444}1445 1446static int axi_req_irqs(struct platform_device *pdev, struct axi_dma_chip *chip)1447{1448 int irq_count = platform_irq_count(pdev);1449 int ret;1450 1451 for (int i = 0; i < irq_count; i++) {1452 chip->irq[i] = platform_get_irq(pdev, i);1453 if (chip->irq[i] < 0)1454 return chip->irq[i];1455 ret = devm_request_irq(chip->dev, chip->irq[i], dw_axi_dma_interrupt,1456 IRQF_SHARED, KBUILD_MODNAME, chip);1457 if (ret < 0)1458 return ret;1459 }1460 1461 return 0;1462}1463 1464static int dw_probe(struct platform_device *pdev)1465{1466 struct axi_dma_chip *chip;1467 struct dw_axi_dma *dw;1468 struct dw_axi_dma_hcfg *hdata;1469 struct reset_control *resets;1470 unsigned int flags;1471 u32 i;1472 int ret;1473 1474 chip = devm_kzalloc(&pdev->dev, sizeof(*chip), GFP_KERNEL);1475 if (!chip)1476 return -ENOMEM;1477 1478 dw = devm_kzalloc(&pdev->dev, sizeof(*dw), GFP_KERNEL);1479 if (!dw)1480 return -ENOMEM;1481 1482 hdata = devm_kzalloc(&pdev->dev, sizeof(*hdata), GFP_KERNEL);1483 if (!hdata)1484 return -ENOMEM;1485 1486 chip->dw = dw;1487 chip->dev = &pdev->dev;1488 chip->dw->hdata = hdata;1489 1490 chip->regs = devm_platform_ioremap_resource(pdev, 0);1491 if (IS_ERR(chip->regs))1492 return PTR_ERR(chip->regs);1493 1494 flags = (uintptr_t)of_device_get_match_data(&pdev->dev);1495 if (flags & AXI_DMA_FLAG_HAS_APB_REGS) {1496 chip->apb_regs = devm_platform_ioremap_resource(pdev, 1);1497 if (IS_ERR(chip->apb_regs))1498 return PTR_ERR(chip->apb_regs);1499 }1500 1501 if (flags & AXI_DMA_FLAG_HAS_RESETS) {1502 resets = devm_reset_control_array_get_exclusive(&pdev->dev);1503 if (IS_ERR(resets))1504 return PTR_ERR(resets);1505 1506 ret = reset_control_deassert(resets);1507 if (ret)1508 return ret;1509 }1510 1511 chip->dw->hdata->use_cfg2 = !!(flags & AXI_DMA_FLAG_USE_CFG2);1512 1513 chip->core_clk = devm_clk_get(chip->dev, "core-clk");1514 if (IS_ERR(chip->core_clk))1515 return PTR_ERR(chip->core_clk);1516 1517 chip->cfgr_clk = devm_clk_get(chip->dev, "cfgr-clk");1518 if (IS_ERR(chip->cfgr_clk))1519 return PTR_ERR(chip->cfgr_clk);1520 1521 ret = parse_device_properties(chip);1522 if (ret)1523 return ret;1524 1525 dw->chan = devm_kcalloc(chip->dev, hdata->nr_channels,1526 sizeof(*dw->chan), GFP_KERNEL);1527 if (!dw->chan)1528 return -ENOMEM;1529 1530 ret = axi_req_irqs(pdev, chip);1531 if (ret)1532 return ret;1533 1534 INIT_LIST_HEAD(&dw->dma.channels);1535 for (i = 0; i < hdata->nr_channels; i++) {1536 struct axi_dma_chan *chan = &dw->chan[i];1537 1538 chan->chip = chip;1539 chan->id = i;1540 chan->chan_regs = chip->regs + COMMON_REG_LEN + i * CHAN_REG_LEN;1541 atomic_set(&chan->descs_allocated, 0);1542 1543 chan->vc.desc_free = vchan_desc_put;1544 vchan_init(&chan->vc, &dw->dma);1545 }1546 1547 /* Set capabilities */1548 dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask);1549 dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);1550 dma_cap_set(DMA_CYCLIC, dw->dma.cap_mask);1551 1552 /* DMA capabilities */1553 dw->dma.max_burst = hdata->axi_rw_burst_len;1554 dw->dma.src_addr_widths = AXI_DMA_BUSWIDTHS;1555 dw->dma.dst_addr_widths = AXI_DMA_BUSWIDTHS;1556 dw->dma.directions = BIT(DMA_MEM_TO_MEM);1557 dw->dma.directions |= BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);1558 dw->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;1559 1560 dw->dma.dev = chip->dev;1561 dw->dma.device_tx_status = dma_chan_tx_status;1562 dw->dma.device_issue_pending = dma_chan_issue_pending;1563 dw->dma.device_terminate_all = dma_chan_terminate_all;1564 dw->dma.device_pause = dma_chan_pause;1565 dw->dma.device_resume = dma_chan_resume;1566 1567 dw->dma.device_alloc_chan_resources = dma_chan_alloc_chan_resources;1568 dw->dma.device_free_chan_resources = dma_chan_free_chan_resources;1569 1570 dw->dma.device_prep_dma_memcpy = dma_chan_prep_dma_memcpy;1571 dw->dma.device_synchronize = dw_axi_dma_synchronize;1572 dw->dma.device_config = dw_axi_dma_chan_slave_config;1573 dw->dma.device_prep_slave_sg = dw_axi_dma_chan_prep_slave_sg;1574 dw->dma.device_prep_dma_cyclic = dw_axi_dma_chan_prep_cyclic;1575 1576 /*1577 * Synopsis DesignWare AxiDMA datasheet mentioned Maximum1578 * supported blocks is 1024. Device register width is 4 bytes.1579 * Therefore, set constraint to 1024 * 4.1580 */1581 dw->dma.dev->dma_parms = &dw->dma_parms;1582 dma_set_max_seg_size(&pdev->dev, MAX_BLOCK_SIZE);1583 platform_set_drvdata(pdev, chip);1584 1585 pm_runtime_enable(chip->dev);1586 1587 /*1588 * We can't just call pm_runtime_get here instead of1589 * pm_runtime_get_noresume + axi_dma_resume because we need1590 * driver to work also without Runtime PM.1591 */1592 pm_runtime_get_noresume(chip->dev);1593 ret = axi_dma_resume(chip);1594 if (ret < 0)1595 goto err_pm_disable;1596 1597 axi_dma_hw_init(chip);1598 1599 pm_runtime_put(chip->dev);1600 1601 ret = dmaenginem_async_device_register(&dw->dma);1602 if (ret)1603 goto err_pm_disable;1604 1605 /* Register with OF helpers for DMA lookups */1606 ret = of_dma_controller_register(pdev->dev.of_node,1607 dw_axi_dma_of_xlate, dw);1608 if (ret < 0)1609 dev_warn(&pdev->dev,1610 "Failed to register OF DMA controller, fallback to MEM_TO_MEM mode\n");1611 1612 dev_info(chip->dev, "DesignWare AXI DMA Controller, %d channels\n",1613 dw->hdata->nr_channels);1614 1615 return 0;1616 1617err_pm_disable:1618 pm_runtime_disable(chip->dev);1619 1620 return ret;1621}1622 1623static void dw_remove(struct platform_device *pdev)1624{1625 struct axi_dma_chip *chip = platform_get_drvdata(pdev);1626 struct dw_axi_dma *dw = chip->dw;1627 struct axi_dma_chan *chan, *_chan;1628 u32 i;1629 1630 /* Enable clk before accessing to registers */1631 clk_prepare_enable(chip->cfgr_clk);1632 clk_prepare_enable(chip->core_clk);1633 axi_dma_irq_disable(chip);1634 for (i = 0; i < dw->hdata->nr_channels; i++) {1635 axi_chan_disable(&chip->dw->chan[i]);1636 axi_chan_irq_disable(&chip->dw->chan[i], DWAXIDMAC_IRQ_ALL);1637 }1638 axi_dma_disable(chip);1639 1640 pm_runtime_disable(chip->dev);1641 axi_dma_suspend(chip);1642 1643 for (i = 0; i < DMAC_MAX_CHANNELS; i++)1644 if (chip->irq[i] > 0)1645 devm_free_irq(chip->dev, chip->irq[i], chip);1646 1647 of_dma_controller_free(chip->dev->of_node);1648 1649 list_for_each_entry_safe(chan, _chan, &dw->dma.channels,1650 vc.chan.device_node) {1651 list_del(&chan->vc.chan.device_node);1652 tasklet_kill(&chan->vc.task);1653 }1654}1655 1656static const struct dev_pm_ops dw_axi_dma_pm_ops = {1657 SET_RUNTIME_PM_OPS(axi_dma_runtime_suspend, axi_dma_runtime_resume, NULL)1658};1659 1660static const struct of_device_id dw_dma_of_id_table[] = {1661 {1662 .compatible = "snps,axi-dma-1.01a"1663 }, {1664 .compatible = "intel,kmb-axi-dma",1665 .data = (void *)AXI_DMA_FLAG_HAS_APB_REGS,1666 }, {1667 .compatible = "starfive,jh7110-axi-dma",1668 .data = (void *)(AXI_DMA_FLAG_HAS_RESETS | AXI_DMA_FLAG_USE_CFG2),1669 }, {1670 .compatible = "starfive,jh8100-axi-dma",1671 .data = (void *)AXI_DMA_FLAG_HAS_RESETS,1672 },1673 {}1674};1675MODULE_DEVICE_TABLE(of, dw_dma_of_id_table);1676 1677static struct platform_driver dw_driver = {1678 .probe = dw_probe,1679 .remove_new = dw_remove,1680 .driver = {1681 .name = KBUILD_MODNAME,1682 .of_match_table = dw_dma_of_id_table,1683 .pm = &dw_axi_dma_pm_ops,1684 },1685};1686module_platform_driver(dw_driver);1687 1688MODULE_LICENSE("GPL v2");1689MODULE_DESCRIPTION("Synopsys DesignWare AXI DMA Controller platform driver");1690MODULE_AUTHOR("Eugeniy Paltsev <Eugeniy.Paltsev@synopsys.com>");1691