1021 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * ADMA driver for Nvidia's Tegra210 ADMA controller.4 *5 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.6 */7 8#include <linux/clk.h>9#include <linux/iopoll.h>10#include <linux/module.h>11#include <linux/of.h>12#include <linux/of_dma.h>13#include <linux/of_irq.h>14#include <linux/platform_device.h>15#include <linux/pm_runtime.h>16#include <linux/slab.h>17 18#include "virt-dma.h"19 20#define ADMA_CH_CMD 0x0021#define ADMA_CH_STATUS 0x0c22#define ADMA_CH_STATUS_XFER_EN BIT(0)23#define ADMA_CH_STATUS_XFER_PAUSED BIT(1)24 25#define ADMA_CH_INT_STATUS 0x1026#define ADMA_CH_INT_STATUS_XFER_DONE BIT(0)27 28#define ADMA_CH_INT_CLEAR 0x1c29#define ADMA_CH_CTRL 0x2430#define ADMA_CH_CTRL_DIR(val) (((val) & 0xf) << 12)31#define ADMA_CH_CTRL_DIR_AHUB2MEM 232#define ADMA_CH_CTRL_DIR_MEM2AHUB 433#define ADMA_CH_CTRL_MODE_CONTINUOUS (2 << 8)34#define ADMA_CH_CTRL_FLOWCTRL_EN BIT(1)35#define ADMA_CH_CTRL_XFER_PAUSE_SHIFT 036 37#define ADMA_CH_CONFIG 0x2838#define ADMA_CH_CONFIG_SRC_BUF(val) (((val) & 0x7) << 28)39#define ADMA_CH_CONFIG_TRG_BUF(val) (((val) & 0x7) << 24)40#define ADMA_CH_CONFIG_BURST_SIZE_SHIFT 2041#define ADMA_CH_CONFIG_MAX_BURST_SIZE 1642#define ADMA_CH_CONFIG_WEIGHT_FOR_WRR(val) ((val) & 0xf)43#define ADMA_CH_CONFIG_MAX_BUFS 844#define TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(reqs) (reqs << 4)45 46#define ADMA_CH_FIFO_CTRL 0x2c47#define ADMA_CH_TX_FIFO_SIZE_SHIFT 848#define ADMA_CH_RX_FIFO_SIZE_SHIFT 049 50#define ADMA_CH_LOWER_SRC_ADDR 0x3451#define ADMA_CH_LOWER_TRG_ADDR 0x3c52#define ADMA_CH_TC 0x4453#define ADMA_CH_TC_COUNT_MASK 0x3ffffffc54 55#define ADMA_CH_XFER_STATUS 0x5456#define ADMA_CH_XFER_STATUS_COUNT_MASK 0xffff57 58#define ADMA_GLOBAL_CMD 0x0059#define ADMA_GLOBAL_SOFT_RESET 0x0460 61#define TEGRA_ADMA_BURST_COMPLETE_TIME 2062 63#define ADMA_CH_REG_FIELD_VAL(val, mask, shift) (((val) & mask) << shift)64 65struct tegra_adma;66 67/*68 * struct tegra_adma_chip_data - Tegra chip specific data69 * @adma_get_burst_config: Function callback used to set DMA burst size.70 * @global_reg_offset: Register offset of DMA global register.71 * @global_int_clear: Register offset of DMA global interrupt clear.72 * @ch_req_tx_shift: Register offset for AHUB transmit channel select.73 * @ch_req_rx_shift: Register offset for AHUB receive channel select.74 * @ch_base_offset: Register offset of DMA channel registers.75 * @ch_fifo_ctrl: Default value for channel FIFO CTRL register.76 * @ch_req_mask: Mask for Tx or Rx channel select.77 * @ch_req_max: Maximum number of Tx or Rx channels available.78 * @ch_reg_size: Size of DMA channel register space.79 * @nr_channels: Number of DMA channels available.80 * @ch_fifo_size_mask: Mask for FIFO size field.81 * @sreq_index_offset: Slave channel index offset.82 * @has_outstanding_reqs: If DMA channel can have outstanding requests.83 */84struct tegra_adma_chip_data {85 unsigned int (*adma_get_burst_config)(unsigned int burst_size);86 unsigned int global_reg_offset;87 unsigned int global_int_clear;88 unsigned int ch_req_tx_shift;89 unsigned int ch_req_rx_shift;90 unsigned int ch_base_offset;91 unsigned int ch_fifo_ctrl;92 unsigned int ch_req_mask;93 unsigned int ch_req_max;94 unsigned int ch_reg_size;95 unsigned int nr_channels;96 unsigned int ch_fifo_size_mask;97 unsigned int sreq_index_offset;98 bool has_outstanding_reqs;99};100 101/*102 * struct tegra_adma_chan_regs - Tegra ADMA channel registers103 */104struct tegra_adma_chan_regs {105 unsigned int ctrl;106 unsigned int config;107 unsigned int src_addr;108 unsigned int trg_addr;109 unsigned int fifo_ctrl;110 unsigned int cmd;111 unsigned int tc;112};113 114/*115 * struct tegra_adma_desc - Tegra ADMA descriptor to manage transfer requests.116 */117struct tegra_adma_desc {118 struct virt_dma_desc vd;119 struct tegra_adma_chan_regs ch_regs;120 size_t buf_len;121 size_t period_len;122 size_t num_periods;123};124 125/*126 * struct tegra_adma_chan - Tegra ADMA channel information127 */128struct tegra_adma_chan {129 struct virt_dma_chan vc;130 struct tegra_adma_desc *desc;131 struct tegra_adma *tdma;132 int irq;133 void __iomem *chan_addr;134 135 /* Slave channel configuration info */136 struct dma_slave_config sconfig;137 enum dma_transfer_direction sreq_dir;138 unsigned int sreq_index;139 bool sreq_reserved;140 struct tegra_adma_chan_regs ch_regs;141 142 /* Transfer count and position info */143 unsigned int tx_buf_count;144 unsigned int tx_buf_pos;145};146 147/*148 * struct tegra_adma - Tegra ADMA controller information149 */150struct tegra_adma {151 struct dma_device dma_dev;152 struct device *dev;153 void __iomem *base_addr;154 struct clk *ahub_clk;155 unsigned int nr_channels;156 unsigned long *dma_chan_mask;157 unsigned long rx_requests_reserved;158 unsigned long tx_requests_reserved;159 160 /* Used to store global command register state when suspending */161 unsigned int global_cmd;162 163 const struct tegra_adma_chip_data *cdata;164 165 /* Last member of the structure */166 struct tegra_adma_chan channels[] __counted_by(nr_channels);167};168 169static inline void tdma_write(struct tegra_adma *tdma, u32 reg, u32 val)170{171 writel(val, tdma->base_addr + tdma->cdata->global_reg_offset + reg);172}173 174static inline u32 tdma_read(struct tegra_adma *tdma, u32 reg)175{176 return readl(tdma->base_addr + tdma->cdata->global_reg_offset + reg);177}178 179static inline void tdma_ch_write(struct tegra_adma_chan *tdc, u32 reg, u32 val)180{181 writel(val, tdc->chan_addr + reg);182}183 184static inline u32 tdma_ch_read(struct tegra_adma_chan *tdc, u32 reg)185{186 return readl(tdc->chan_addr + reg);187}188 189static inline struct tegra_adma_chan *to_tegra_adma_chan(struct dma_chan *dc)190{191 return container_of(dc, struct tegra_adma_chan, vc.chan);192}193 194static inline struct tegra_adma_desc *to_tegra_adma_desc(195 struct dma_async_tx_descriptor *td)196{197 return container_of(td, struct tegra_adma_desc, vd.tx);198}199 200static inline struct device *tdc2dev(struct tegra_adma_chan *tdc)201{202 return tdc->tdma->dev;203}204 205static void tegra_adma_desc_free(struct virt_dma_desc *vd)206{207 kfree(container_of(vd, struct tegra_adma_desc, vd));208}209 210static int tegra_adma_slave_config(struct dma_chan *dc,211 struct dma_slave_config *sconfig)212{213 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);214 215 memcpy(&tdc->sconfig, sconfig, sizeof(*sconfig));216 217 return 0;218}219 220static int tegra_adma_init(struct tegra_adma *tdma)221{222 u32 status;223 int ret;224 225 /* Clear any interrupts */226 tdma_write(tdma, tdma->cdata->ch_base_offset + tdma->cdata->global_int_clear, 0x1);227 228 /* Assert soft reset */229 tdma_write(tdma, ADMA_GLOBAL_SOFT_RESET, 0x1);230 231 /* Wait for reset to clear */232 ret = readx_poll_timeout(readl,233 tdma->base_addr +234 tdma->cdata->global_reg_offset +235 ADMA_GLOBAL_SOFT_RESET,236 status, status == 0, 20, 10000);237 if (ret)238 return ret;239 240 /* Enable global ADMA registers */241 tdma_write(tdma, ADMA_GLOBAL_CMD, 1);242 243 return 0;244}245 246static int tegra_adma_request_alloc(struct tegra_adma_chan *tdc,247 enum dma_transfer_direction direction)248{249 struct tegra_adma *tdma = tdc->tdma;250 unsigned int sreq_index = tdc->sreq_index;251 252 if (tdc->sreq_reserved)253 return tdc->sreq_dir == direction ? 0 : -EINVAL;254 255 if (sreq_index > tdma->cdata->ch_req_max) {256 dev_err(tdma->dev, "invalid DMA request\n");257 return -EINVAL;258 }259 260 switch (direction) {261 case DMA_MEM_TO_DEV:262 if (test_and_set_bit(sreq_index, &tdma->tx_requests_reserved)) {263 dev_err(tdma->dev, "DMA request reserved\n");264 return -EINVAL;265 }266 break;267 268 case DMA_DEV_TO_MEM:269 if (test_and_set_bit(sreq_index, &tdma->rx_requests_reserved)) {270 dev_err(tdma->dev, "DMA request reserved\n");271 return -EINVAL;272 }273 break;274 275 default:276 dev_WARN(tdma->dev, "channel %s has invalid transfer type\n",277 dma_chan_name(&tdc->vc.chan));278 return -EINVAL;279 }280 281 tdc->sreq_dir = direction;282 tdc->sreq_reserved = true;283 284 return 0;285}286 287static void tegra_adma_request_free(struct tegra_adma_chan *tdc)288{289 struct tegra_adma *tdma = tdc->tdma;290 291 if (!tdc->sreq_reserved)292 return;293 294 switch (tdc->sreq_dir) {295 case DMA_MEM_TO_DEV:296 clear_bit(tdc->sreq_index, &tdma->tx_requests_reserved);297 break;298 299 case DMA_DEV_TO_MEM:300 clear_bit(tdc->sreq_index, &tdma->rx_requests_reserved);301 break;302 303 default:304 dev_WARN(tdma->dev, "channel %s has invalid transfer type\n",305 dma_chan_name(&tdc->vc.chan));306 return;307 }308 309 tdc->sreq_reserved = false;310}311 312static u32 tegra_adma_irq_status(struct tegra_adma_chan *tdc)313{314 u32 status = tdma_ch_read(tdc, ADMA_CH_INT_STATUS);315 316 return status & ADMA_CH_INT_STATUS_XFER_DONE;317}318 319static u32 tegra_adma_irq_clear(struct tegra_adma_chan *tdc)320{321 u32 status = tegra_adma_irq_status(tdc);322 323 if (status)324 tdma_ch_write(tdc, ADMA_CH_INT_CLEAR, status);325 326 return status;327}328 329static void tegra_adma_stop(struct tegra_adma_chan *tdc)330{331 unsigned int status;332 333 /* Disable ADMA */334 tdma_ch_write(tdc, ADMA_CH_CMD, 0);335 336 /* Clear interrupt status */337 tegra_adma_irq_clear(tdc);338 339 if (readx_poll_timeout_atomic(readl, tdc->chan_addr + ADMA_CH_STATUS,340 status, !(status & ADMA_CH_STATUS_XFER_EN),341 20, 10000)) {342 dev_err(tdc2dev(tdc), "unable to stop DMA channel\n");343 return;344 }345 346 kfree(tdc->desc);347 tdc->desc = NULL;348}349 350static void tegra_adma_start(struct tegra_adma_chan *tdc)351{352 struct virt_dma_desc *vd = vchan_next_desc(&tdc->vc);353 struct tegra_adma_chan_regs *ch_regs;354 struct tegra_adma_desc *desc;355 356 if (!vd)357 return;358 359 list_del(&vd->node);360 361 desc = to_tegra_adma_desc(&vd->tx);362 363 if (!desc) {364 dev_warn(tdc2dev(tdc), "unable to start DMA, no descriptor\n");365 return;366 }367 368 ch_regs = &desc->ch_regs;369 370 tdc->tx_buf_pos = 0;371 tdc->tx_buf_count = 0;372 tdma_ch_write(tdc, ADMA_CH_TC, ch_regs->tc);373 tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);374 tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, ch_regs->src_addr);375 tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, ch_regs->trg_addr);376 tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_regs->fifo_ctrl);377 tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_regs->config);378 379 /* Start ADMA */380 tdma_ch_write(tdc, ADMA_CH_CMD, 1);381 382 tdc->desc = desc;383}384 385static unsigned int tegra_adma_get_residue(struct tegra_adma_chan *tdc)386{387 struct tegra_adma_desc *desc = tdc->desc;388 unsigned int max = ADMA_CH_XFER_STATUS_COUNT_MASK + 1;389 unsigned int pos = tdma_ch_read(tdc, ADMA_CH_XFER_STATUS);390 unsigned int periods_remaining;391 392 /*393 * Handle wrap around of buffer count register394 */395 if (pos < tdc->tx_buf_pos)396 tdc->tx_buf_count += pos + (max - tdc->tx_buf_pos);397 else398 tdc->tx_buf_count += pos - tdc->tx_buf_pos;399 400 periods_remaining = tdc->tx_buf_count % desc->num_periods;401 tdc->tx_buf_pos = pos;402 403 return desc->buf_len - (periods_remaining * desc->period_len);404}405 406static irqreturn_t tegra_adma_isr(int irq, void *dev_id)407{408 struct tegra_adma_chan *tdc = dev_id;409 unsigned long status;410 411 spin_lock(&tdc->vc.lock);412 413 status = tegra_adma_irq_clear(tdc);414 if (status == 0 || !tdc->desc) {415 spin_unlock(&tdc->vc.lock);416 return IRQ_NONE;417 }418 419 vchan_cyclic_callback(&tdc->desc->vd);420 421 spin_unlock(&tdc->vc.lock);422 423 return IRQ_HANDLED;424}425 426static void tegra_adma_issue_pending(struct dma_chan *dc)427{428 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);429 unsigned long flags;430 431 spin_lock_irqsave(&tdc->vc.lock, flags);432 433 if (vchan_issue_pending(&tdc->vc)) {434 if (!tdc->desc)435 tegra_adma_start(tdc);436 }437 438 spin_unlock_irqrestore(&tdc->vc.lock, flags);439}440 441static bool tegra_adma_is_paused(struct tegra_adma_chan *tdc)442{443 u32 csts;444 445 csts = tdma_ch_read(tdc, ADMA_CH_STATUS);446 csts &= ADMA_CH_STATUS_XFER_PAUSED;447 448 return csts ? true : false;449}450 451static int tegra_adma_pause(struct dma_chan *dc)452{453 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);454 struct tegra_adma_desc *desc = tdc->desc;455 struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;456 int dcnt = 10;457 458 ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);459 ch_regs->ctrl |= (1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT);460 tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);461 462 while (dcnt-- && !tegra_adma_is_paused(tdc))463 udelay(TEGRA_ADMA_BURST_COMPLETE_TIME);464 465 if (dcnt < 0) {466 dev_err(tdc2dev(tdc), "unable to pause DMA channel\n");467 return -EBUSY;468 }469 470 return 0;471}472 473static int tegra_adma_resume(struct dma_chan *dc)474{475 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);476 struct tegra_adma_desc *desc = tdc->desc;477 struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;478 479 ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);480 ch_regs->ctrl &= ~(1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT);481 tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);482 483 return 0;484}485 486static int tegra_adma_terminate_all(struct dma_chan *dc)487{488 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);489 unsigned long flags;490 LIST_HEAD(head);491 492 spin_lock_irqsave(&tdc->vc.lock, flags);493 494 if (tdc->desc)495 tegra_adma_stop(tdc);496 497 tegra_adma_request_free(tdc);498 vchan_get_all_descriptors(&tdc->vc, &head);499 spin_unlock_irqrestore(&tdc->vc.lock, flags);500 vchan_dma_desc_free_list(&tdc->vc, &head);501 502 return 0;503}504 505static enum dma_status tegra_adma_tx_status(struct dma_chan *dc,506 dma_cookie_t cookie,507 struct dma_tx_state *txstate)508{509 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);510 struct tegra_adma_desc *desc;511 struct virt_dma_desc *vd;512 enum dma_status ret;513 unsigned long flags;514 unsigned int residual;515 516 ret = dma_cookie_status(dc, cookie, txstate);517 if (ret == DMA_COMPLETE || !txstate)518 return ret;519 520 spin_lock_irqsave(&tdc->vc.lock, flags);521 522 vd = vchan_find_desc(&tdc->vc, cookie);523 if (vd) {524 desc = to_tegra_adma_desc(&vd->tx);525 residual = desc->ch_regs.tc;526 } else if (tdc->desc && tdc->desc->vd.tx.cookie == cookie) {527 residual = tegra_adma_get_residue(tdc);528 } else {529 residual = 0;530 }531 532 spin_unlock_irqrestore(&tdc->vc.lock, flags);533 534 dma_set_residue(txstate, residual);535 536 return ret;537}538 539static unsigned int tegra210_adma_get_burst_config(unsigned int burst_size)540{541 if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE)542 burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE;543 544 return fls(burst_size) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT;545}546 547static unsigned int tegra186_adma_get_burst_config(unsigned int burst_size)548{549 if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE)550 burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE;551 552 return (burst_size - 1) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT;553}554 555static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc,556 struct tegra_adma_desc *desc,557 dma_addr_t buf_addr,558 enum dma_transfer_direction direction)559{560 struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;561 const struct tegra_adma_chip_data *cdata = tdc->tdma->cdata;562 unsigned int burst_size, adma_dir, fifo_size_shift;563 564 if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS)565 return -EINVAL;566 567 switch (direction) {568 case DMA_MEM_TO_DEV:569 fifo_size_shift = ADMA_CH_TX_FIFO_SIZE_SHIFT;570 adma_dir = ADMA_CH_CTRL_DIR_MEM2AHUB;571 burst_size = tdc->sconfig.dst_maxburst;572 ch_regs->config = ADMA_CH_CONFIG_SRC_BUF(desc->num_periods - 1);573 ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index,574 cdata->ch_req_mask,575 cdata->ch_req_tx_shift);576 ch_regs->src_addr = buf_addr;577 break;578 579 case DMA_DEV_TO_MEM:580 fifo_size_shift = ADMA_CH_RX_FIFO_SIZE_SHIFT;581 adma_dir = ADMA_CH_CTRL_DIR_AHUB2MEM;582 burst_size = tdc->sconfig.src_maxburst;583 ch_regs->config = ADMA_CH_CONFIG_TRG_BUF(desc->num_periods - 1);584 ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index,585 cdata->ch_req_mask,586 cdata->ch_req_rx_shift);587 ch_regs->trg_addr = buf_addr;588 break;589 590 default:591 dev_err(tdc2dev(tdc), "DMA direction is not supported\n");592 return -EINVAL;593 }594 595 ch_regs->ctrl |= ADMA_CH_CTRL_DIR(adma_dir) |596 ADMA_CH_CTRL_MODE_CONTINUOUS |597 ADMA_CH_CTRL_FLOWCTRL_EN;598 ch_regs->config |= cdata->adma_get_burst_config(burst_size);599 ch_regs->config |= ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1);600 if (cdata->has_outstanding_reqs)601 ch_regs->config |= TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(8);602 603 /*604 * 'sreq_index' represents the current ADMAIF channel number and as per605 * HW recommendation its FIFO size should match with the corresponding606 * ADMA channel.607 *608 * ADMA FIFO size is set as per below (based on default ADMAIF channel609 * FIFO sizes):610 * fifo_size = 0x2 (sreq_index > sreq_index_offset)611 * fifo_size = 0x3 (sreq_index <= sreq_index_offset)612 *613 */614 if (tdc->sreq_index > cdata->sreq_index_offset)615 ch_regs->fifo_ctrl =616 ADMA_CH_REG_FIELD_VAL(2, cdata->ch_fifo_size_mask,617 fifo_size_shift);618 else619 ch_regs->fifo_ctrl =620 ADMA_CH_REG_FIELD_VAL(3, cdata->ch_fifo_size_mask,621 fifo_size_shift);622 623 ch_regs->tc = desc->period_len & ADMA_CH_TC_COUNT_MASK;624 625 return tegra_adma_request_alloc(tdc, direction);626}627 628static struct dma_async_tx_descriptor *tegra_adma_prep_dma_cyclic(629 struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len,630 size_t period_len, enum dma_transfer_direction direction,631 unsigned long flags)632{633 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);634 struct tegra_adma_desc *desc = NULL;635 636 if (!buf_len || !period_len || period_len > ADMA_CH_TC_COUNT_MASK) {637 dev_err(tdc2dev(tdc), "invalid buffer/period len\n");638 return NULL;639 }640 641 if (buf_len % period_len) {642 dev_err(tdc2dev(tdc), "buf_len not a multiple of period_len\n");643 return NULL;644 }645 646 if (!IS_ALIGNED(buf_addr, 4)) {647 dev_err(tdc2dev(tdc), "invalid buffer alignment\n");648 return NULL;649 }650 651 desc = kzalloc(sizeof(*desc), GFP_NOWAIT);652 if (!desc)653 return NULL;654 655 desc->buf_len = buf_len;656 desc->period_len = period_len;657 desc->num_periods = buf_len / period_len;658 659 if (tegra_adma_set_xfer_params(tdc, desc, buf_addr, direction)) {660 kfree(desc);661 return NULL;662 }663 664 return vchan_tx_prep(&tdc->vc, &desc->vd, flags);665}666 667static int tegra_adma_alloc_chan_resources(struct dma_chan *dc)668{669 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);670 int ret;671 672 ret = request_irq(tdc->irq, tegra_adma_isr, 0, dma_chan_name(dc), tdc);673 if (ret) {674 dev_err(tdc2dev(tdc), "failed to get interrupt for %s\n",675 dma_chan_name(dc));676 return ret;677 }678 679 ret = pm_runtime_resume_and_get(tdc2dev(tdc));680 if (ret < 0) {681 free_irq(tdc->irq, tdc);682 return ret;683 }684 685 dma_cookie_init(&tdc->vc.chan);686 687 return 0;688}689 690static void tegra_adma_free_chan_resources(struct dma_chan *dc)691{692 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);693 694 tegra_adma_terminate_all(dc);695 vchan_free_chan_resources(&tdc->vc);696 tasklet_kill(&tdc->vc.task);697 free_irq(tdc->irq, tdc);698 pm_runtime_put(tdc2dev(tdc));699 700 tdc->sreq_index = 0;701 tdc->sreq_dir = DMA_TRANS_NONE;702}703 704static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec,705 struct of_dma *ofdma)706{707 struct tegra_adma *tdma = ofdma->of_dma_data;708 struct tegra_adma_chan *tdc;709 struct dma_chan *chan;710 unsigned int sreq_index;711 712 if (dma_spec->args_count != 1)713 return NULL;714 715 sreq_index = dma_spec->args[0];716 717 if (sreq_index == 0) {718 dev_err(tdma->dev, "DMA request must not be 0\n");719 return NULL;720 }721 722 chan = dma_get_any_slave_channel(&tdma->dma_dev);723 if (!chan)724 return NULL;725 726 tdc = to_tegra_adma_chan(chan);727 tdc->sreq_index = sreq_index;728 729 return chan;730}731 732static int __maybe_unused tegra_adma_runtime_suspend(struct device *dev)733{734 struct tegra_adma *tdma = dev_get_drvdata(dev);735 struct tegra_adma_chan_regs *ch_reg;736 struct tegra_adma_chan *tdc;737 int i;738 739 tdma->global_cmd = tdma_read(tdma, ADMA_GLOBAL_CMD);740 if (!tdma->global_cmd)741 goto clk_disable;742 743 for (i = 0; i < tdma->nr_channels; i++) {744 tdc = &tdma->channels[i];745 /* skip for reserved channels */746 if (!tdc->tdma)747 continue;748 749 ch_reg = &tdc->ch_regs;750 ch_reg->cmd = tdma_ch_read(tdc, ADMA_CH_CMD);751 /* skip if channel is not active */752 if (!ch_reg->cmd)753 continue;754 ch_reg->tc = tdma_ch_read(tdc, ADMA_CH_TC);755 ch_reg->src_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_SRC_ADDR);756 ch_reg->trg_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_TRG_ADDR);757 ch_reg->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);758 ch_reg->fifo_ctrl = tdma_ch_read(tdc, ADMA_CH_FIFO_CTRL);759 ch_reg->config = tdma_ch_read(tdc, ADMA_CH_CONFIG);760 }761 762clk_disable:763 clk_disable_unprepare(tdma->ahub_clk);764 765 return 0;766}767 768static int __maybe_unused tegra_adma_runtime_resume(struct device *dev)769{770 struct tegra_adma *tdma = dev_get_drvdata(dev);771 struct tegra_adma_chan_regs *ch_reg;772 struct tegra_adma_chan *tdc;773 int ret, i;774 775 ret = clk_prepare_enable(tdma->ahub_clk);776 if (ret) {777 dev_err(dev, "ahub clk_enable failed: %d\n", ret);778 return ret;779 }780 tdma_write(tdma, ADMA_GLOBAL_CMD, tdma->global_cmd);781 782 if (!tdma->global_cmd)783 return 0;784 785 for (i = 0; i < tdma->nr_channels; i++) {786 tdc = &tdma->channels[i];787 /* skip for reserved channels */788 if (!tdc->tdma)789 continue;790 ch_reg = &tdc->ch_regs;791 /* skip if channel was not active earlier */792 if (!ch_reg->cmd)793 continue;794 tdma_ch_write(tdc, ADMA_CH_TC, ch_reg->tc);795 tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, ch_reg->src_addr);796 tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, ch_reg->trg_addr);797 tdma_ch_write(tdc, ADMA_CH_CTRL, ch_reg->ctrl);798 tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_reg->fifo_ctrl);799 tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_reg->config);800 tdma_ch_write(tdc, ADMA_CH_CMD, ch_reg->cmd);801 }802 803 return 0;804}805 806static const struct tegra_adma_chip_data tegra210_chip_data = {807 .adma_get_burst_config = tegra210_adma_get_burst_config,808 .global_reg_offset = 0xc00,809 .global_int_clear = 0x20,810 .ch_req_tx_shift = 28,811 .ch_req_rx_shift = 24,812 .ch_base_offset = 0,813 .ch_req_mask = 0xf,814 .ch_req_max = 10,815 .ch_reg_size = 0x80,816 .nr_channels = 22,817 .ch_fifo_size_mask = 0xf,818 .sreq_index_offset = 2,819 .has_outstanding_reqs = false,820};821 822static const struct tegra_adma_chip_data tegra186_chip_data = {823 .adma_get_burst_config = tegra186_adma_get_burst_config,824 .global_reg_offset = 0,825 .global_int_clear = 0x402c,826 .ch_req_tx_shift = 27,827 .ch_req_rx_shift = 22,828 .ch_base_offset = 0x10000,829 .ch_req_mask = 0x1f,830 .ch_req_max = 20,831 .ch_reg_size = 0x100,832 .nr_channels = 32,833 .ch_fifo_size_mask = 0x1f,834 .sreq_index_offset = 4,835 .has_outstanding_reqs = true,836};837 838static const struct of_device_id tegra_adma_of_match[] = {839 { .compatible = "nvidia,tegra210-adma", .data = &tegra210_chip_data },840 { .compatible = "nvidia,tegra186-adma", .data = &tegra186_chip_data },841 { },842};843MODULE_DEVICE_TABLE(of, tegra_adma_of_match);844 845static int tegra_adma_probe(struct platform_device *pdev)846{847 const struct tegra_adma_chip_data *cdata;848 struct tegra_adma *tdma;849 int ret, i;850 851 cdata = of_device_get_match_data(&pdev->dev);852 if (!cdata) {853 dev_err(&pdev->dev, "device match data not found\n");854 return -ENODEV;855 }856 857 tdma = devm_kzalloc(&pdev->dev,858 struct_size(tdma, channels, cdata->nr_channels),859 GFP_KERNEL);860 if (!tdma)861 return -ENOMEM;862 863 tdma->dev = &pdev->dev;864 tdma->cdata = cdata;865 tdma->nr_channels = cdata->nr_channels;866 platform_set_drvdata(pdev, tdma);867 868 tdma->base_addr = devm_platform_ioremap_resource(pdev, 0);869 if (IS_ERR(tdma->base_addr))870 return PTR_ERR(tdma->base_addr);871 872 tdma->ahub_clk = devm_clk_get(&pdev->dev, "d_audio");873 if (IS_ERR(tdma->ahub_clk)) {874 dev_err(&pdev->dev, "Error: Missing ahub controller clock\n");875 return PTR_ERR(tdma->ahub_clk);876 }877 878 tdma->dma_chan_mask = devm_kzalloc(&pdev->dev,879 BITS_TO_LONGS(tdma->nr_channels) * sizeof(unsigned long),880 GFP_KERNEL);881 if (!tdma->dma_chan_mask)882 return -ENOMEM;883 884 /* Enable all channels by default */885 bitmap_fill(tdma->dma_chan_mask, tdma->nr_channels);886 887 ret = of_property_read_u32_array(pdev->dev.of_node, "dma-channel-mask",888 (u32 *)tdma->dma_chan_mask,889 BITS_TO_U32(tdma->nr_channels));890 if (ret < 0 && (ret != -EINVAL)) {891 dev_err(&pdev->dev, "dma-channel-mask is not complete.\n");892 return ret;893 }894 895 INIT_LIST_HEAD(&tdma->dma_dev.channels);896 for (i = 0; i < tdma->nr_channels; i++) {897 struct tegra_adma_chan *tdc = &tdma->channels[i];898 899 /* skip for reserved channels */900 if (!test_bit(i, tdma->dma_chan_mask))901 continue;902 903 tdc->chan_addr = tdma->base_addr + cdata->ch_base_offset904 + (cdata->ch_reg_size * i);905 906 tdc->irq = of_irq_get(pdev->dev.of_node, i);907 if (tdc->irq <= 0) {908 ret = tdc->irq ?: -ENXIO;909 goto irq_dispose;910 }911 912 vchan_init(&tdc->vc, &tdma->dma_dev);913 tdc->vc.desc_free = tegra_adma_desc_free;914 tdc->tdma = tdma;915 }916 917 pm_runtime_enable(&pdev->dev);918 919 ret = pm_runtime_resume_and_get(&pdev->dev);920 if (ret < 0)921 goto rpm_disable;922 923 ret = tegra_adma_init(tdma);924 if (ret)925 goto rpm_put;926 927 dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask);928 dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);929 dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask);930 931 tdma->dma_dev.dev = &pdev->dev;932 tdma->dma_dev.device_alloc_chan_resources =933 tegra_adma_alloc_chan_resources;934 tdma->dma_dev.device_free_chan_resources =935 tegra_adma_free_chan_resources;936 tdma->dma_dev.device_issue_pending = tegra_adma_issue_pending;937 tdma->dma_dev.device_prep_dma_cyclic = tegra_adma_prep_dma_cyclic;938 tdma->dma_dev.device_config = tegra_adma_slave_config;939 tdma->dma_dev.device_tx_status = tegra_adma_tx_status;940 tdma->dma_dev.device_terminate_all = tegra_adma_terminate_all;941 tdma->dma_dev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);942 tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);943 tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);944 tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;945 tdma->dma_dev.device_pause = tegra_adma_pause;946 tdma->dma_dev.device_resume = tegra_adma_resume;947 948 ret = dma_async_device_register(&tdma->dma_dev);949 if (ret < 0) {950 dev_err(&pdev->dev, "ADMA registration failed: %d\n", ret);951 goto rpm_put;952 }953 954 ret = of_dma_controller_register(pdev->dev.of_node,955 tegra_dma_of_xlate, tdma);956 if (ret < 0) {957 dev_err(&pdev->dev, "ADMA OF registration failed %d\n", ret);958 goto dma_remove;959 }960 961 pm_runtime_put(&pdev->dev);962 963 dev_info(&pdev->dev, "Tegra210 ADMA driver registered %d channels\n",964 tdma->nr_channels);965 966 return 0;967 968dma_remove:969 dma_async_device_unregister(&tdma->dma_dev);970rpm_put:971 pm_runtime_put_sync(&pdev->dev);972rpm_disable:973 pm_runtime_disable(&pdev->dev);974irq_dispose:975 while (--i >= 0)976 irq_dispose_mapping(tdma->channels[i].irq);977 978 return ret;979}980 981static void tegra_adma_remove(struct platform_device *pdev)982{983 struct tegra_adma *tdma = platform_get_drvdata(pdev);984 int i;985 986 of_dma_controller_free(pdev->dev.of_node);987 dma_async_device_unregister(&tdma->dma_dev);988 989 for (i = 0; i < tdma->nr_channels; ++i) {990 if (tdma->channels[i].irq)991 irq_dispose_mapping(tdma->channels[i].irq);992 }993 994 pm_runtime_disable(&pdev->dev);995}996 997static const struct dev_pm_ops tegra_adma_dev_pm_ops = {998 SET_RUNTIME_PM_OPS(tegra_adma_runtime_suspend,999 tegra_adma_runtime_resume, NULL)1000 SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,1001 pm_runtime_force_resume)1002};1003 1004static struct platform_driver tegra_admac_driver = {1005 .driver = {1006 .name = "tegra-adma",1007 .pm = &tegra_adma_dev_pm_ops,1008 .of_match_table = tegra_adma_of_match,1009 },1010 .probe = tegra_adma_probe,1011 .remove_new = tegra_adma_remove,1012};1013 1014module_platform_driver(tegra_admac_driver);1015 1016MODULE_ALIAS("platform:tegra210-adma");1017MODULE_DESCRIPTION("NVIDIA Tegra ADMA driver");1018MODULE_AUTHOR("Dara Ramesh <dramesh@nvidia.com>");1019MODULE_AUTHOR("Jon Hunter <jonathanh@nvidia.com>");1020MODULE_LICENSE("GPL v2");1021