1544 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * DMA driver for NVIDIA Tegra GPC DMA controller.4 *5 * Copyright (c) 2014-2022, NVIDIA CORPORATION. All rights reserved.6 */7 8#include <linux/bitfield.h>9#include <linux/dmaengine.h>10#include <linux/dma-mapping.h>11#include <linux/interrupt.h>12#include <linux/iommu.h>13#include <linux/iopoll.h>14#include <linux/minmax.h>15#include <linux/module.h>16#include <linux/of.h>17#include <linux/of_dma.h>18#include <linux/platform_device.h>19#include <linux/reset.h>20#include <linux/slab.h>21#include <dt-bindings/memory/tegra186-mc.h>22#include "virt-dma.h"23 24/* CSR register */25#define TEGRA_GPCDMA_CHAN_CSR 0x0026#define TEGRA_GPCDMA_CSR_ENB BIT(31)27#define TEGRA_GPCDMA_CSR_IE_EOC BIT(30)28#define TEGRA_GPCDMA_CSR_ONCE BIT(27)29 30#define TEGRA_GPCDMA_CSR_FC_MODE GENMASK(25, 24)31#define TEGRA_GPCDMA_CSR_FC_MODE_NO_MMIO \32 FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 0)33#define TEGRA_GPCDMA_CSR_FC_MODE_ONE_MMIO \34 FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 1)35#define TEGRA_GPCDMA_CSR_FC_MODE_TWO_MMIO \36 FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 2)37#define TEGRA_GPCDMA_CSR_FC_MODE_FOUR_MMIO \38 FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 3)39 40#define TEGRA_GPCDMA_CSR_DMA GENMASK(23, 21)41#define TEGRA_GPCDMA_CSR_DMA_IO2MEM_NO_FC \42 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 0)43#define TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC \44 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 1)45#define TEGRA_GPCDMA_CSR_DMA_MEM2IO_NO_FC \46 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 2)47#define TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC \48 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 3)49#define TEGRA_GPCDMA_CSR_DMA_MEM2MEM \50 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 4)51#define TEGRA_GPCDMA_CSR_DMA_FIXED_PAT \52 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 6)53 54#define TEGRA_GPCDMA_CSR_REQ_SEL_MASK GENMASK(20, 16)55#define TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED \56 FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, 4)57#define TEGRA_GPCDMA_CSR_IRQ_MASK BIT(15)58#define TEGRA_GPCDMA_CSR_WEIGHT GENMASK(13, 10)59 60/* STATUS register */61#define TEGRA_GPCDMA_CHAN_STATUS 0x00462#define TEGRA_GPCDMA_STATUS_BUSY BIT(31)63#define TEGRA_GPCDMA_STATUS_ISE_EOC BIT(30)64#define TEGRA_GPCDMA_STATUS_PING_PONG BIT(28)65#define TEGRA_GPCDMA_STATUS_DMA_ACTIVITY BIT(27)66#define TEGRA_GPCDMA_STATUS_CHANNEL_PAUSE BIT(26)67#define TEGRA_GPCDMA_STATUS_CHANNEL_RX BIT(25)68#define TEGRA_GPCDMA_STATUS_CHANNEL_TX BIT(24)69#define TEGRA_GPCDMA_STATUS_IRQ_INTR_STA BIT(23)70#define TEGRA_GPCDMA_STATUS_IRQ_STA BIT(21)71#define TEGRA_GPCDMA_STATUS_IRQ_TRIG_STA BIT(20)72 73#define TEGRA_GPCDMA_CHAN_CSRE 0x00874#define TEGRA_GPCDMA_CHAN_CSRE_PAUSE BIT(31)75 76/* Source address */77#define TEGRA_GPCDMA_CHAN_SRC_PTR 0x00C78 79/* Destination address */80#define TEGRA_GPCDMA_CHAN_DST_PTR 0x01081 82/* High address pointer */83#define TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR 0x01484#define TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR GENMASK(7, 0)85#define TEGRA_GPCDMA_HIGH_ADDR_DST_PTR GENMASK(23, 16)86 87/* MC sequence register */88#define TEGRA_GPCDMA_CHAN_MCSEQ 0x1889#define TEGRA_GPCDMA_MCSEQ_DATA_SWAP BIT(31)90#define TEGRA_GPCDMA_MCSEQ_REQ_COUNT GENMASK(30, 25)91#define TEGRA_GPCDMA_MCSEQ_BURST GENMASK(24, 23)92#define TEGRA_GPCDMA_MCSEQ_BURST_2 \93 FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 0)94#define TEGRA_GPCDMA_MCSEQ_BURST_16 \95 FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 3)96#define TEGRA_GPCDMA_MCSEQ_WRAP1 GENMASK(22, 20)97#define TEGRA_GPCDMA_MCSEQ_WRAP0 GENMASK(19, 17)98#define TEGRA_GPCDMA_MCSEQ_WRAP_NONE 099 100#define TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK GENMASK(13, 7)101#define TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK GENMASK(6, 0)102 103/* MMIO sequence register */104#define TEGRA_GPCDMA_CHAN_MMIOSEQ 0x01c105#define TEGRA_GPCDMA_MMIOSEQ_DBL_BUF BIT(31)106#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH GENMASK(30, 28)107#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8 \108 FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 0)109#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16 \110 FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 1)111#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32 \112 FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 2)113#define TEGRA_GPCDMA_MMIOSEQ_DATA_SWAP BIT(27)114#define TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT 23115#define TEGRA_GPCDMA_MMIOSEQ_BURST_MIN 2U116#define TEGRA_GPCDMA_MMIOSEQ_BURST_MAX 32U117#define TEGRA_GPCDMA_MMIOSEQ_BURST(bs) \118 (GENMASK((fls(bs) - 2), 0) << TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT)119#define TEGRA_GPCDMA_MMIOSEQ_MASTER_ID GENMASK(22, 19)120#define TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD GENMASK(18, 16)121#define TEGRA_GPCDMA_MMIOSEQ_MMIO_PROT GENMASK(8, 7)122 123/* Channel WCOUNT */124#define TEGRA_GPCDMA_CHAN_WCOUNT 0x20125 126/* Transfer count */127#define TEGRA_GPCDMA_CHAN_XFER_COUNT 0x24128 129/* DMA byte count status */130#define TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS 0x28131 132/* Error Status Register */133#define TEGRA_GPCDMA_CHAN_ERR_STATUS 0x30134#define TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT 8135#define TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK 0xF136#define TEGRA_GPCDMA_CHAN_ERR_TYPE(err) ( \137 ((err) >> TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT) & \138 TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK)139#define TEGRA_DMA_BM_FIFO_FULL_ERR 0xF140#define TEGRA_DMA_PERIPH_FIFO_FULL_ERR 0xE141#define TEGRA_DMA_PERIPH_ID_ERR 0xD142#define TEGRA_DMA_STREAM_ID_ERR 0xC143#define TEGRA_DMA_MC_SLAVE_ERR 0xB144#define TEGRA_DMA_MMIO_SLAVE_ERR 0xA145 146/* Fixed Pattern */147#define TEGRA_GPCDMA_CHAN_FIXED_PATTERN 0x34148 149#define TEGRA_GPCDMA_CHAN_TZ 0x38150#define TEGRA_GPCDMA_CHAN_TZ_MMIO_PROT_1 BIT(0)151#define TEGRA_GPCDMA_CHAN_TZ_MC_PROT_1 BIT(1)152 153#define TEGRA_GPCDMA_CHAN_SPARE 0x3c154#define TEGRA_GPCDMA_CHAN_SPARE_EN_LEGACY_FC BIT(16)155 156/*157 * If any burst is in flight and DMA paused then this is the time to complete158 * on-flight burst and update DMA status register.159 */160#define TEGRA_GPCDMA_BURST_COMPLETE_TIME 10161#define TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT 5000 /* 5 msec */162 163/* Channel base address offset from GPCDMA base address */164#define TEGRA_GPCDMA_CHANNEL_BASE_ADDR_OFFSET 0x10000165 166/* Default channel mask reserving channel0 */167#define TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK 0xfffffffe168 169struct tegra_dma;170struct tegra_dma_channel;171 172/*173 * tegra_dma_chip_data Tegra chip specific DMA data174 * @nr_channels: Number of channels available in the controller.175 * @channel_reg_size: Channel register size.176 * @max_dma_count: Maximum DMA transfer count supported by DMA controller.177 * @hw_support_pause: DMA HW engine support pause of the channel.178 */179struct tegra_dma_chip_data {180 bool hw_support_pause;181 unsigned int nr_channels;182 unsigned int channel_reg_size;183 unsigned int max_dma_count;184 int (*terminate)(struct tegra_dma_channel *tdc);185};186 187/* DMA channel registers */188struct tegra_dma_channel_regs {189 u32 csr;190 u32 src_ptr;191 u32 dst_ptr;192 u32 high_addr_ptr;193 u32 mc_seq;194 u32 mmio_seq;195 u32 wcount;196 u32 fixed_pattern;197};198 199/*200 * tegra_dma_sg_req: DMA request details to configure hardware. This201 * contains the details for one transfer to configure DMA hw.202 * The client's request for data transfer can be broken into multiple203 * sub-transfer as per requester details and hw support. This sub transfer204 * get added as an array in Tegra DMA desc which manages the transfer details.205 */206struct tegra_dma_sg_req {207 unsigned int len;208 struct tegra_dma_channel_regs ch_regs;209};210 211/*212 * tegra_dma_desc: Tegra DMA descriptors which uses virt_dma_desc to213 * manage client request and keep track of transfer status, callbacks214 * and request counts etc.215 */216struct tegra_dma_desc {217 bool cyclic;218 unsigned int bytes_req;219 unsigned int bytes_xfer;220 unsigned int sg_idx;221 unsigned int sg_count;222 struct virt_dma_desc vd;223 struct tegra_dma_channel *tdc;224 struct tegra_dma_sg_req sg_req[] __counted_by(sg_count);225};226 227/*228 * tegra_dma_channel: Channel specific information229 */230struct tegra_dma_channel {231 bool config_init;232 char name[30];233 enum dma_transfer_direction sid_dir;234 int id;235 int irq;236 int slave_id;237 struct tegra_dma *tdma;238 struct virt_dma_chan vc;239 struct tegra_dma_desc *dma_desc;240 struct dma_slave_config dma_sconfig;241 unsigned int stream_id;242 unsigned long chan_base_offset;243};244 245/*246 * tegra_dma: Tegra DMA specific information247 */248struct tegra_dma {249 const struct tegra_dma_chip_data *chip_data;250 unsigned long sid_m2d_reserved;251 unsigned long sid_d2m_reserved;252 u32 chan_mask;253 void __iomem *base_addr;254 struct device *dev;255 struct dma_device dma_dev;256 struct reset_control *rst;257 struct tegra_dma_channel channels[];258};259 260static inline void tdc_write(struct tegra_dma_channel *tdc,261 u32 reg, u32 val)262{263 writel_relaxed(val, tdc->tdma->base_addr + tdc->chan_base_offset + reg);264}265 266static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg)267{268 return readl_relaxed(tdc->tdma->base_addr + tdc->chan_base_offset + reg);269}270 271static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc)272{273 return container_of(dc, struct tegra_dma_channel, vc.chan);274}275 276static inline struct tegra_dma_desc *vd_to_tegra_dma_desc(struct virt_dma_desc *vd)277{278 return container_of(vd, struct tegra_dma_desc, vd);279}280 281static inline struct device *tdc2dev(struct tegra_dma_channel *tdc)282{283 return tdc->vc.chan.device->dev;284}285 286static void tegra_dma_dump_chan_regs(struct tegra_dma_channel *tdc)287{288 dev_dbg(tdc2dev(tdc), "DMA Channel %d name %s register dump:\n",289 tdc->id, tdc->name);290 dev_dbg(tdc2dev(tdc), "CSR %x STA %x CSRE %x SRC %x DST %x\n",291 tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR),292 tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS),293 tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE),294 tdc_read(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR),295 tdc_read(tdc, TEGRA_GPCDMA_CHAN_DST_PTR)296 );297 dev_dbg(tdc2dev(tdc), "MCSEQ %x IOSEQ %x WCNT %x XFER %x BSTA %x\n",298 tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ),299 tdc_read(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ),300 tdc_read(tdc, TEGRA_GPCDMA_CHAN_WCOUNT),301 tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT),302 tdc_read(tdc, TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS)303 );304 dev_dbg(tdc2dev(tdc), "DMA ERR_STA %x\n",305 tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS));306}307 308static int tegra_dma_sid_reserve(struct tegra_dma_channel *tdc,309 enum dma_transfer_direction direction)310{311 struct tegra_dma *tdma = tdc->tdma;312 int sid = tdc->slave_id;313 314 if (!is_slave_direction(direction))315 return 0;316 317 switch (direction) {318 case DMA_MEM_TO_DEV:319 if (test_and_set_bit(sid, &tdma->sid_m2d_reserved)) {320 dev_err(tdma->dev, "slave id already in use\n");321 return -EINVAL;322 }323 break;324 case DMA_DEV_TO_MEM:325 if (test_and_set_bit(sid, &tdma->sid_d2m_reserved)) {326 dev_err(tdma->dev, "slave id already in use\n");327 return -EINVAL;328 }329 break;330 default:331 break;332 }333 334 tdc->sid_dir = direction;335 336 return 0;337}338 339static void tegra_dma_sid_free(struct tegra_dma_channel *tdc)340{341 struct tegra_dma *tdma = tdc->tdma;342 int sid = tdc->slave_id;343 344 switch (tdc->sid_dir) {345 case DMA_MEM_TO_DEV:346 clear_bit(sid, &tdma->sid_m2d_reserved);347 break;348 case DMA_DEV_TO_MEM:349 clear_bit(sid, &tdma->sid_d2m_reserved);350 break;351 default:352 break;353 }354 355 tdc->sid_dir = DMA_TRANS_NONE;356}357 358static void tegra_dma_desc_free(struct virt_dma_desc *vd)359{360 kfree(container_of(vd, struct tegra_dma_desc, vd));361}362 363static int tegra_dma_slave_config(struct dma_chan *dc,364 struct dma_slave_config *sconfig)365{366 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);367 368 memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig));369 tdc->config_init = true;370 371 return 0;372}373 374static int tegra_dma_pause(struct tegra_dma_channel *tdc)375{376 int ret;377 u32 val;378 379 val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE);380 val |= TEGRA_GPCDMA_CHAN_CSRE_PAUSE;381 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val);382 383 /* Wait until busy bit is de-asserted */384 ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +385 tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS,386 val,387 !(val & TEGRA_GPCDMA_STATUS_BUSY),388 TEGRA_GPCDMA_BURST_COMPLETE_TIME,389 TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);390 391 if (ret) {392 dev_err(tdc2dev(tdc), "DMA pause timed out\n");393 tegra_dma_dump_chan_regs(tdc);394 }395 396 return ret;397}398 399static int tegra_dma_device_pause(struct dma_chan *dc)400{401 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);402 unsigned long flags;403 int ret;404 405 if (!tdc->tdma->chip_data->hw_support_pause)406 return -ENOSYS;407 408 spin_lock_irqsave(&tdc->vc.lock, flags);409 ret = tegra_dma_pause(tdc);410 spin_unlock_irqrestore(&tdc->vc.lock, flags);411 412 return ret;413}414 415static void tegra_dma_resume(struct tegra_dma_channel *tdc)416{417 u32 val;418 419 val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE);420 val &= ~TEGRA_GPCDMA_CHAN_CSRE_PAUSE;421 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val);422}423 424static int tegra_dma_device_resume(struct dma_chan *dc)425{426 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);427 unsigned long flags;428 429 if (!tdc->tdma->chip_data->hw_support_pause)430 return -ENOSYS;431 432 spin_lock_irqsave(&tdc->vc.lock, flags);433 tegra_dma_resume(tdc);434 spin_unlock_irqrestore(&tdc->vc.lock, flags);435 436 return 0;437}438 439static inline int tegra_dma_pause_noerr(struct tegra_dma_channel *tdc)440{441 /* Return 0 irrespective of PAUSE status.442 * This is useful to recover channels that can exit out of flush443 * state when the channel is disabled.444 */445 446 tegra_dma_pause(tdc);447 return 0;448}449 450static void tegra_dma_disable(struct tegra_dma_channel *tdc)451{452 u32 csr, status;453 454 csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR);455 456 /* Disable interrupts */457 csr &= ~TEGRA_GPCDMA_CSR_IE_EOC;458 459 /* Disable DMA */460 csr &= ~TEGRA_GPCDMA_CSR_ENB;461 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr);462 463 /* Clear interrupt status if it is there */464 status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);465 if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) {466 dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__);467 tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS, status);468 }469}470 471static void tegra_dma_configure_next_sg(struct tegra_dma_channel *tdc)472{473 struct tegra_dma_desc *dma_desc = tdc->dma_desc;474 struct tegra_dma_channel_regs *ch_regs;475 int ret;476 u32 val;477 478 dma_desc->sg_idx++;479 480 /* Reset the sg index for cyclic transfers */481 if (dma_desc->sg_idx == dma_desc->sg_count)482 dma_desc->sg_idx = 0;483 484 /* Configure next transfer immediately after DMA is busy */485 ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +486 tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS,487 val,488 (val & TEGRA_GPCDMA_STATUS_BUSY), 0,489 TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);490 if (ret)491 return;492 493 ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs;494 495 tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount);496 tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr);497 tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr);498 tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr);499 500 /* Start DMA */501 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR,502 ch_regs->csr | TEGRA_GPCDMA_CSR_ENB);503}504 505static void tegra_dma_start(struct tegra_dma_channel *tdc)506{507 struct tegra_dma_desc *dma_desc = tdc->dma_desc;508 struct tegra_dma_channel_regs *ch_regs;509 struct virt_dma_desc *vdesc;510 511 if (!dma_desc) {512 vdesc = vchan_next_desc(&tdc->vc);513 if (!vdesc)514 return;515 516 dma_desc = vd_to_tegra_dma_desc(vdesc);517 list_del(&vdesc->node);518 dma_desc->tdc = tdc;519 tdc->dma_desc = dma_desc;520 521 tegra_dma_resume(tdc);522 }523 524 ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs;525 526 tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount);527 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, 0);528 tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr);529 tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr);530 tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr);531 tdc_write(tdc, TEGRA_GPCDMA_CHAN_FIXED_PATTERN, ch_regs->fixed_pattern);532 tdc_write(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ, ch_regs->mmio_seq);533 tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, ch_regs->mc_seq);534 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, ch_regs->csr);535 536 /* Start DMA */537 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR,538 ch_regs->csr | TEGRA_GPCDMA_CSR_ENB);539}540 541static void tegra_dma_xfer_complete(struct tegra_dma_channel *tdc)542{543 vchan_cookie_complete(&tdc->dma_desc->vd);544 545 tegra_dma_sid_free(tdc);546 tdc->dma_desc = NULL;547}548 549static void tegra_dma_chan_decode_error(struct tegra_dma_channel *tdc,550 unsigned int err_status)551{552 switch (TEGRA_GPCDMA_CHAN_ERR_TYPE(err_status)) {553 case TEGRA_DMA_BM_FIFO_FULL_ERR:554 dev_err(tdc->tdma->dev,555 "GPCDMA CH%d bm fifo full\n", tdc->id);556 break;557 558 case TEGRA_DMA_PERIPH_FIFO_FULL_ERR:559 dev_err(tdc->tdma->dev,560 "GPCDMA CH%d peripheral fifo full\n", tdc->id);561 break;562 563 case TEGRA_DMA_PERIPH_ID_ERR:564 dev_err(tdc->tdma->dev,565 "GPCDMA CH%d illegal peripheral id\n", tdc->id);566 break;567 568 case TEGRA_DMA_STREAM_ID_ERR:569 dev_err(tdc->tdma->dev,570 "GPCDMA CH%d illegal stream id\n", tdc->id);571 break;572 573 case TEGRA_DMA_MC_SLAVE_ERR:574 dev_err(tdc->tdma->dev,575 "GPCDMA CH%d mc slave error\n", tdc->id);576 break;577 578 case TEGRA_DMA_MMIO_SLAVE_ERR:579 dev_err(tdc->tdma->dev,580 "GPCDMA CH%d mmio slave error\n", tdc->id);581 break;582 583 default:584 dev_err(tdc->tdma->dev,585 "GPCDMA CH%d security violation %x\n", tdc->id,586 err_status);587 }588}589 590static irqreturn_t tegra_dma_isr(int irq, void *dev_id)591{592 struct tegra_dma_channel *tdc = dev_id;593 struct tegra_dma_desc *dma_desc = tdc->dma_desc;594 struct tegra_dma_sg_req *sg_req;595 u32 status;596 597 /* Check channel error status register */598 status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS);599 if (status) {600 tegra_dma_chan_decode_error(tdc, status);601 tegra_dma_dump_chan_regs(tdc);602 tdc_write(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS, 0xFFFFFFFF);603 }604 605 spin_lock(&tdc->vc.lock);606 status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);607 if (!(status & TEGRA_GPCDMA_STATUS_ISE_EOC))608 goto irq_done;609 610 tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS,611 TEGRA_GPCDMA_STATUS_ISE_EOC);612 613 if (!dma_desc)614 goto irq_done;615 616 sg_req = dma_desc->sg_req;617 dma_desc->bytes_xfer += sg_req[dma_desc->sg_idx].len;618 619 if (dma_desc->cyclic) {620 vchan_cyclic_callback(&dma_desc->vd);621 tegra_dma_configure_next_sg(tdc);622 } else {623 dma_desc->sg_idx++;624 if (dma_desc->sg_idx == dma_desc->sg_count)625 tegra_dma_xfer_complete(tdc);626 else627 tegra_dma_start(tdc);628 }629 630irq_done:631 spin_unlock(&tdc->vc.lock);632 return IRQ_HANDLED;633}634 635static void tegra_dma_issue_pending(struct dma_chan *dc)636{637 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);638 unsigned long flags;639 640 if (tdc->dma_desc)641 return;642 643 spin_lock_irqsave(&tdc->vc.lock, flags);644 if (vchan_issue_pending(&tdc->vc))645 tegra_dma_start(tdc);646 647 /*648 * For cyclic DMA transfers, program the second649 * transfer parameters as soon as the first DMA650 * transfer is started inorder for the DMA651 * controller to trigger the second transfer652 * with the correct parameters.653 */654 if (tdc->dma_desc && tdc->dma_desc->cyclic)655 tegra_dma_configure_next_sg(tdc);656 657 spin_unlock_irqrestore(&tdc->vc.lock, flags);658}659 660static int tegra_dma_stop_client(struct tegra_dma_channel *tdc)661{662 int ret;663 u32 status, csr;664 665 /*666 * Change the client associated with the DMA channel667 * to stop DMA engine from starting any more bursts for668 * the given client and wait for in flight bursts to complete669 */670 csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR);671 csr &= ~(TEGRA_GPCDMA_CSR_REQ_SEL_MASK);672 csr |= TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED;673 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr);674 675 /* Wait for in flight data transfer to finish */676 udelay(TEGRA_GPCDMA_BURST_COMPLETE_TIME);677 678 /* If TX/RX path is still active wait till it becomes679 * inactive680 */681 682 ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +683 tdc->chan_base_offset +684 TEGRA_GPCDMA_CHAN_STATUS,685 status,686 !(status & (TEGRA_GPCDMA_STATUS_CHANNEL_TX |687 TEGRA_GPCDMA_STATUS_CHANNEL_RX)),688 5,689 TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);690 if (ret) {691 dev_err(tdc2dev(tdc), "Timeout waiting for DMA burst completion!\n");692 tegra_dma_dump_chan_regs(tdc);693 }694 695 return ret;696}697 698static int tegra_dma_terminate_all(struct dma_chan *dc)699{700 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);701 unsigned long flags;702 LIST_HEAD(head);703 int err;704 705 spin_lock_irqsave(&tdc->vc.lock, flags);706 707 if (tdc->dma_desc) {708 err = tdc->tdma->chip_data->terminate(tdc);709 if (err) {710 spin_unlock_irqrestore(&tdc->vc.lock, flags);711 return err;712 }713 714 vchan_terminate_vdesc(&tdc->dma_desc->vd);715 tegra_dma_disable(tdc);716 tdc->dma_desc = NULL;717 }718 719 tegra_dma_sid_free(tdc);720 vchan_get_all_descriptors(&tdc->vc, &head);721 spin_unlock_irqrestore(&tdc->vc.lock, flags);722 723 vchan_dma_desc_free_list(&tdc->vc, &head);724 725 return 0;726}727 728static int tegra_dma_get_residual(struct tegra_dma_channel *tdc)729{730 struct tegra_dma_desc *dma_desc = tdc->dma_desc;731 struct tegra_dma_sg_req *sg_req = dma_desc->sg_req;732 unsigned int bytes_xfer, residual;733 u32 wcount = 0, status;734 735 wcount = tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT);736 737 /*738 * Set wcount = 0 if EOC bit is set. The transfer would have739 * already completed and the CHAN_XFER_COUNT could have updated740 * for the next transfer, specifically in case of cyclic transfers.741 */742 status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);743 if (status & TEGRA_GPCDMA_STATUS_ISE_EOC)744 wcount = 0;745 746 bytes_xfer = dma_desc->bytes_xfer +747 sg_req[dma_desc->sg_idx].len - (wcount * 4);748 749 if (dma_desc->bytes_req == bytes_xfer)750 return 0;751 752 residual = dma_desc->bytes_req - (bytes_xfer % dma_desc->bytes_req);753 754 return residual;755}756 757static enum dma_status tegra_dma_tx_status(struct dma_chan *dc,758 dma_cookie_t cookie,759 struct dma_tx_state *txstate)760{761 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);762 struct tegra_dma_desc *dma_desc;763 struct virt_dma_desc *vd;764 unsigned int residual;765 unsigned long flags;766 enum dma_status ret;767 768 ret = dma_cookie_status(dc, cookie, txstate);769 if (ret == DMA_COMPLETE)770 return ret;771 772 spin_lock_irqsave(&tdc->vc.lock, flags);773 vd = vchan_find_desc(&tdc->vc, cookie);774 if (vd) {775 dma_desc = vd_to_tegra_dma_desc(vd);776 residual = dma_desc->bytes_req;777 dma_set_residue(txstate, residual);778 } else if (tdc->dma_desc && tdc->dma_desc->vd.tx.cookie == cookie) {779 residual = tegra_dma_get_residual(tdc);780 dma_set_residue(txstate, residual);781 } else {782 dev_err(tdc2dev(tdc), "cookie %d is not found\n", cookie);783 }784 spin_unlock_irqrestore(&tdc->vc.lock, flags);785 786 return ret;787}788 789static inline int get_bus_width(struct tegra_dma_channel *tdc,790 enum dma_slave_buswidth slave_bw)791{792 switch (slave_bw) {793 case DMA_SLAVE_BUSWIDTH_1_BYTE:794 return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8;795 case DMA_SLAVE_BUSWIDTH_2_BYTES:796 return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16;797 case DMA_SLAVE_BUSWIDTH_4_BYTES:798 return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32;799 default:800 dev_err(tdc2dev(tdc), "given slave bus width is not supported\n");801 return -EINVAL;802 }803}804 805static unsigned int get_burst_size(struct tegra_dma_channel *tdc,806 u32 burst_size, enum dma_slave_buswidth slave_bw,807 int len)808{809 unsigned int burst_mmio_width, burst_byte;810 811 /*812 * burst_size from client is in terms of the bus_width.813 * convert that into words.814 * If burst_size is not specified from client, then use815 * len to calculate the optimum burst size816 */817 burst_byte = burst_size ? burst_size * slave_bw : len;818 burst_mmio_width = burst_byte / 4;819 820 if (burst_mmio_width < TEGRA_GPCDMA_MMIOSEQ_BURST_MIN)821 return 0;822 823 burst_mmio_width = min(burst_mmio_width, TEGRA_GPCDMA_MMIOSEQ_BURST_MAX);824 825 return TEGRA_GPCDMA_MMIOSEQ_BURST(burst_mmio_width);826}827 828static int get_transfer_param(struct tegra_dma_channel *tdc,829 enum dma_transfer_direction direction,830 u32 *apb_addr,831 u32 *mmio_seq,832 u32 *csr,833 unsigned int *burst_size,834 enum dma_slave_buswidth *slave_bw)835{836 switch (direction) {837 case DMA_MEM_TO_DEV:838 *apb_addr = tdc->dma_sconfig.dst_addr;839 *mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.dst_addr_width);840 *burst_size = tdc->dma_sconfig.dst_maxburst;841 *slave_bw = tdc->dma_sconfig.dst_addr_width;842 *csr = TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC;843 return 0;844 case DMA_DEV_TO_MEM:845 *apb_addr = tdc->dma_sconfig.src_addr;846 *mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.src_addr_width);847 *burst_size = tdc->dma_sconfig.src_maxburst;848 *slave_bw = tdc->dma_sconfig.src_addr_width;849 *csr = TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC;850 return 0;851 default:852 dev_err(tdc2dev(tdc), "DMA direction is not supported\n");853 }854 855 return -EINVAL;856}857 858static struct dma_async_tx_descriptor *859tegra_dma_prep_dma_memset(struct dma_chan *dc, dma_addr_t dest, int value,860 size_t len, unsigned long flags)861{862 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);863 unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count;864 struct tegra_dma_sg_req *sg_req;865 struct tegra_dma_desc *dma_desc;866 u32 csr, mc_seq;867 868 if ((len & 3) || (dest & 3) || len > max_dma_count) {869 dev_err(tdc2dev(tdc),870 "DMA length/memory address is not supported\n");871 return NULL;872 }873 874 /* Set DMA mode to fixed pattern */875 csr = TEGRA_GPCDMA_CSR_DMA_FIXED_PAT;876 /* Enable once or continuous mode */877 csr |= TEGRA_GPCDMA_CSR_ONCE;878 /* Enable IRQ mask */879 csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;880 /* Enable the DMA interrupt */881 if (flags & DMA_PREP_INTERRUPT)882 csr |= TEGRA_GPCDMA_CSR_IE_EOC;883 /* Configure default priority weight for the channel */884 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);885 886 mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);887 /* retain stream-id and clean rest */888 mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;889 890 /* Set the address wrapping */891 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,892 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);893 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,894 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);895 896 /* Program outstanding MC requests */897 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);898 /* Set burst size */899 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;900 901 dma_desc = kzalloc(struct_size(dma_desc, sg_req, 1), GFP_NOWAIT);902 if (!dma_desc)903 return NULL;904 905 dma_desc->bytes_req = len;906 dma_desc->sg_count = 1;907 sg_req = dma_desc->sg_req;908 909 sg_req[0].ch_regs.src_ptr = 0;910 sg_req[0].ch_regs.dst_ptr = dest;911 sg_req[0].ch_regs.high_addr_ptr =912 FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32));913 sg_req[0].ch_regs.fixed_pattern = value;914 /* Word count reg takes value as (N +1) words */915 sg_req[0].ch_regs.wcount = ((len - 4) >> 2);916 sg_req[0].ch_regs.csr = csr;917 sg_req[0].ch_regs.mmio_seq = 0;918 sg_req[0].ch_regs.mc_seq = mc_seq;919 sg_req[0].len = len;920 921 dma_desc->cyclic = false;922 return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);923}924 925static struct dma_async_tx_descriptor *926tegra_dma_prep_dma_memcpy(struct dma_chan *dc, dma_addr_t dest,927 dma_addr_t src, size_t len, unsigned long flags)928{929 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);930 struct tegra_dma_sg_req *sg_req;931 struct tegra_dma_desc *dma_desc;932 unsigned int max_dma_count;933 u32 csr, mc_seq;934 935 max_dma_count = tdc->tdma->chip_data->max_dma_count;936 if ((len & 3) || (src & 3) || (dest & 3) || len > max_dma_count) {937 dev_err(tdc2dev(tdc),938 "DMA length/memory address is not supported\n");939 return NULL;940 }941 942 /* Set DMA mode to memory to memory transfer */943 csr = TEGRA_GPCDMA_CSR_DMA_MEM2MEM;944 /* Enable once or continuous mode */945 csr |= TEGRA_GPCDMA_CSR_ONCE;946 /* Enable IRQ mask */947 csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;948 /* Enable the DMA interrupt */949 if (flags & DMA_PREP_INTERRUPT)950 csr |= TEGRA_GPCDMA_CSR_IE_EOC;951 /* Configure default priority weight for the channel */952 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);953 954 mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);955 /* retain stream-id and clean rest */956 mc_seq &= (TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK) |957 (TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK);958 959 /* Set the address wrapping */960 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,961 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);962 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,963 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);964 965 /* Program outstanding MC requests */966 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);967 /* Set burst size */968 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;969 970 dma_desc = kzalloc(struct_size(dma_desc, sg_req, 1), GFP_NOWAIT);971 if (!dma_desc)972 return NULL;973 974 dma_desc->bytes_req = len;975 dma_desc->sg_count = 1;976 sg_req = dma_desc->sg_req;977 978 sg_req[0].ch_regs.src_ptr = src;979 sg_req[0].ch_regs.dst_ptr = dest;980 sg_req[0].ch_regs.high_addr_ptr =981 FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (src >> 32));982 sg_req[0].ch_regs.high_addr_ptr |=983 FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32));984 /* Word count reg takes value as (N +1) words */985 sg_req[0].ch_regs.wcount = ((len - 4) >> 2);986 sg_req[0].ch_regs.csr = csr;987 sg_req[0].ch_regs.mmio_seq = 0;988 sg_req[0].ch_regs.mc_seq = mc_seq;989 sg_req[0].len = len;990 991 dma_desc->cyclic = false;992 return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);993}994 995static struct dma_async_tx_descriptor *996tegra_dma_prep_slave_sg(struct dma_chan *dc, struct scatterlist *sgl,997 unsigned int sg_len, enum dma_transfer_direction direction,998 unsigned long flags, void *context)999{1000 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);1001 unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count;1002 enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED;1003 u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0;1004 struct tegra_dma_sg_req *sg_req;1005 struct tegra_dma_desc *dma_desc;1006 struct scatterlist *sg;1007 u32 burst_size;1008 unsigned int i;1009 int ret;1010 1011 if (!tdc->config_init) {1012 dev_err(tdc2dev(tdc), "DMA channel is not configured\n");1013 return NULL;1014 }1015 if (sg_len < 1) {1016 dev_err(tdc2dev(tdc), "Invalid segment length %d\n", sg_len);1017 return NULL;1018 }1019 1020 ret = tegra_dma_sid_reserve(tdc, direction);1021 if (ret)1022 return NULL;1023 1024 ret = get_transfer_param(tdc, direction, &apb_ptr, &mmio_seq, &csr,1025 &burst_size, &slave_bw);1026 if (ret < 0)1027 return NULL;1028 1029 /* Enable once or continuous mode */1030 csr |= TEGRA_GPCDMA_CSR_ONCE;1031 /* Program the slave id in requestor select */1032 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, tdc->slave_id);1033 /* Enable IRQ mask */1034 csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;1035 /* Configure default priority weight for the channel*/1036 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);1037 1038 /* Enable the DMA interrupt */1039 if (flags & DMA_PREP_INTERRUPT)1040 csr |= TEGRA_GPCDMA_CSR_IE_EOC;1041 1042 mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);1043 /* retain stream-id and clean rest */1044 mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;1045 1046 /* Set the address wrapping on both MC and MMIO side */1047 1048 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,1049 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);1050 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,1051 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);1052 mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1);1053 1054 /* Program 2 MC outstanding requests by default. */1055 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);1056 1057 /* Setting MC burst size depending on MMIO burst size */1058 if (burst_size == 64)1059 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;1060 else1061 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_2;1062 1063 dma_desc = kzalloc(struct_size(dma_desc, sg_req, sg_len), GFP_NOWAIT);1064 if (!dma_desc)1065 return NULL;1066 1067 dma_desc->sg_count = sg_len;1068 sg_req = dma_desc->sg_req;1069 1070 /* Make transfer requests */1071 for_each_sg(sgl, sg, sg_len, i) {1072 u32 len;1073 dma_addr_t mem;1074 1075 mem = sg_dma_address(sg);1076 len = sg_dma_len(sg);1077 1078 if ((len & 3) || (mem & 3) || len > max_dma_count) {1079 dev_err(tdc2dev(tdc),1080 "DMA length/memory address is not supported\n");1081 kfree(dma_desc);1082 return NULL;1083 }1084 1085 mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len);1086 dma_desc->bytes_req += len;1087 1088 if (direction == DMA_MEM_TO_DEV) {1089 sg_req[i].ch_regs.src_ptr = mem;1090 sg_req[i].ch_regs.dst_ptr = apb_ptr;1091 sg_req[i].ch_regs.high_addr_ptr =1092 FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32));1093 } else if (direction == DMA_DEV_TO_MEM) {1094 sg_req[i].ch_regs.src_ptr = apb_ptr;1095 sg_req[i].ch_regs.dst_ptr = mem;1096 sg_req[i].ch_regs.high_addr_ptr =1097 FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32));1098 }1099 1100 /*1101 * Word count register takes input in words. Writing a value1102 * of N into word count register means a req of (N+1) words.1103 */1104 sg_req[i].ch_regs.wcount = ((len - 4) >> 2);1105 sg_req[i].ch_regs.csr = csr;1106 sg_req[i].ch_regs.mmio_seq = mmio_seq;1107 sg_req[i].ch_regs.mc_seq = mc_seq;1108 sg_req[i].len = len;1109 }1110 1111 dma_desc->cyclic = false;1112 return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);1113}1114 1115static struct dma_async_tx_descriptor *1116tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len,1117 size_t period_len, enum dma_transfer_direction direction,1118 unsigned long flags)1119{1120 enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED;1121 u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0, burst_size;1122 unsigned int max_dma_count, len, period_count, i;1123 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);1124 struct tegra_dma_desc *dma_desc;1125 struct tegra_dma_sg_req *sg_req;1126 dma_addr_t mem = buf_addr;1127 int ret;1128 1129 if (!buf_len || !period_len) {1130 dev_err(tdc2dev(tdc), "Invalid buffer/period len\n");1131 return NULL;1132 }1133 1134 if (!tdc->config_init) {1135 dev_err(tdc2dev(tdc), "DMA slave is not configured\n");1136 return NULL;1137 }1138 1139 ret = tegra_dma_sid_reserve(tdc, direction);1140 if (ret)1141 return NULL;1142 1143 /*1144 * We only support cycle transfer when buf_len is multiple of1145 * period_len.1146 */1147 if (buf_len % period_len) {1148 dev_err(tdc2dev(tdc), "buf_len is not multiple of period_len\n");1149 return NULL;1150 }1151 1152 len = period_len;1153 max_dma_count = tdc->tdma->chip_data->max_dma_count;1154 if ((len & 3) || (buf_addr & 3) || len > max_dma_count) {1155 dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n");1156 return NULL;1157 }1158 1159 ret = get_transfer_param(tdc, direction, &apb_ptr, &mmio_seq, &csr,1160 &burst_size, &slave_bw);1161 if (ret < 0)1162 return NULL;1163 1164 /* Enable once or continuous mode */1165 csr &= ~TEGRA_GPCDMA_CSR_ONCE;1166 /* Program the slave id in requestor select */1167 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, tdc->slave_id);1168 /* Enable IRQ mask */1169 csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;1170 /* Configure default priority weight for the channel*/1171 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);1172 1173 /* Enable the DMA interrupt */1174 if (flags & DMA_PREP_INTERRUPT)1175 csr |= TEGRA_GPCDMA_CSR_IE_EOC;1176 1177 mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1);1178 1179 mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);1180 /* retain stream-id and clean rest */1181 mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;1182 1183 /* Set the address wrapping on both MC and MMIO side */1184 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,1185 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);1186 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,1187 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);1188 1189 /* Program 2 MC outstanding requests by default. */1190 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);1191 /* Setting MC burst size depending on MMIO burst size */1192 if (burst_size == 64)1193 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;1194 else1195 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_2;1196 1197 period_count = buf_len / period_len;1198 dma_desc = kzalloc(struct_size(dma_desc, sg_req, period_count),1199 GFP_NOWAIT);1200 if (!dma_desc)1201 return NULL;1202 1203 dma_desc->bytes_req = buf_len;1204 dma_desc->sg_count = period_count;1205 sg_req = dma_desc->sg_req;1206 1207 /* Split transfer equal to period size */1208 for (i = 0; i < period_count; i++) {1209 mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len);1210 if (direction == DMA_MEM_TO_DEV) {1211 sg_req[i].ch_regs.src_ptr = mem;1212 sg_req[i].ch_regs.dst_ptr = apb_ptr;1213 sg_req[i].ch_regs.high_addr_ptr =1214 FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32));1215 } else if (direction == DMA_DEV_TO_MEM) {1216 sg_req[i].ch_regs.src_ptr = apb_ptr;1217 sg_req[i].ch_regs.dst_ptr = mem;1218 sg_req[i].ch_regs.high_addr_ptr =1219 FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32));1220 }1221 /*1222 * Word count register takes input in words. Writing a value1223 * of N into word count register means a req of (N+1) words.1224 */1225 sg_req[i].ch_regs.wcount = ((len - 4) >> 2);1226 sg_req[i].ch_regs.csr = csr;1227 sg_req[i].ch_regs.mmio_seq = mmio_seq;1228 sg_req[i].ch_regs.mc_seq = mc_seq;1229 sg_req[i].len = len;1230 1231 mem += len;1232 }1233 1234 dma_desc->cyclic = true;1235 1236 return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);1237}1238 1239static int tegra_dma_alloc_chan_resources(struct dma_chan *dc)1240{1241 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);1242 int ret;1243 1244 ret = request_irq(tdc->irq, tegra_dma_isr, 0, tdc->name, tdc);1245 if (ret) {1246 dev_err(tdc2dev(tdc), "request_irq failed for %s\n", tdc->name);1247 return ret;1248 }1249 1250 dma_cookie_init(&tdc->vc.chan);1251 tdc->config_init = false;1252 return 0;1253}1254 1255static void tegra_dma_chan_synchronize(struct dma_chan *dc)1256{1257 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);1258 1259 synchronize_irq(tdc->irq);1260 vchan_synchronize(&tdc->vc);1261}1262 1263static void tegra_dma_free_chan_resources(struct dma_chan *dc)1264{1265 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);1266 1267 dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id);1268 1269 tegra_dma_terminate_all(dc);1270 synchronize_irq(tdc->irq);1271 1272 tasklet_kill(&tdc->vc.task);1273 tdc->config_init = false;1274 tdc->slave_id = -1;1275 tdc->sid_dir = DMA_TRANS_NONE;1276 free_irq(tdc->irq, tdc);1277 1278 vchan_free_chan_resources(&tdc->vc);1279}1280 1281static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec,1282 struct of_dma *ofdma)1283{1284 struct tegra_dma *tdma = ofdma->of_dma_data;1285 struct tegra_dma_channel *tdc;1286 struct dma_chan *chan;1287 1288 chan = dma_get_any_slave_channel(&tdma->dma_dev);1289 if (!chan)1290 return NULL;1291 1292 tdc = to_tegra_dma_chan(chan);1293 tdc->slave_id = dma_spec->args[0];1294 1295 return chan;1296}1297 1298static const struct tegra_dma_chip_data tegra186_dma_chip_data = {1299 .nr_channels = 32,1300 .channel_reg_size = SZ_64K,1301 .max_dma_count = SZ_1G,1302 .hw_support_pause = false,1303 .terminate = tegra_dma_stop_client,1304};1305 1306static const struct tegra_dma_chip_data tegra194_dma_chip_data = {1307 .nr_channels = 32,1308 .channel_reg_size = SZ_64K,1309 .max_dma_count = SZ_1G,1310 .hw_support_pause = true,1311 .terminate = tegra_dma_pause,1312};1313 1314static const struct tegra_dma_chip_data tegra234_dma_chip_data = {1315 .nr_channels = 32,1316 .channel_reg_size = SZ_64K,1317 .max_dma_count = SZ_1G,1318 .hw_support_pause = true,1319 .terminate = tegra_dma_pause_noerr,1320};1321 1322static const struct of_device_id tegra_dma_of_match[] = {1323 {1324 .compatible = "nvidia,tegra186-gpcdma",1325 .data = &tegra186_dma_chip_data,1326 }, {1327 .compatible = "nvidia,tegra194-gpcdma",1328 .data = &tegra194_dma_chip_data,1329 }, {1330 .compatible = "nvidia,tegra234-gpcdma",1331 .data = &tegra234_dma_chip_data,1332 }, {1333 },1334};1335MODULE_DEVICE_TABLE(of, tegra_dma_of_match);1336 1337static int tegra_dma_program_sid(struct tegra_dma_channel *tdc, int stream_id)1338{1339 unsigned int reg_val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);1340 1341 reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK);1342 reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK);1343 1344 reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK, stream_id);1345 reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK, stream_id);1346 1347 tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, reg_val);1348 return 0;1349}1350 1351static int tegra_dma_probe(struct platform_device *pdev)1352{1353 const struct tegra_dma_chip_data *cdata = NULL;1354 unsigned int i;1355 u32 stream_id;1356 struct tegra_dma *tdma;1357 int ret;1358 1359 cdata = of_device_get_match_data(&pdev->dev);1360 1361 tdma = devm_kzalloc(&pdev->dev,1362 struct_size(tdma, channels, cdata->nr_channels),1363 GFP_KERNEL);1364 if (!tdma)1365 return -ENOMEM;1366 1367 tdma->dev = &pdev->dev;1368 tdma->chip_data = cdata;1369 platform_set_drvdata(pdev, tdma);1370 1371 tdma->base_addr = devm_platform_ioremap_resource(pdev, 0);1372 if (IS_ERR(tdma->base_addr))1373 return PTR_ERR(tdma->base_addr);1374 1375 tdma->rst = devm_reset_control_get_exclusive(&pdev->dev, "gpcdma");1376 if (IS_ERR(tdma->rst)) {1377 return dev_err_probe(&pdev->dev, PTR_ERR(tdma->rst),1378 "Missing controller reset\n");1379 }1380 reset_control_reset(tdma->rst);1381 1382 tdma->dma_dev.dev = &pdev->dev;1383 1384 if (!tegra_dev_iommu_get_stream_id(&pdev->dev, &stream_id)) {1385 dev_err(&pdev->dev, "Missing iommu stream-id\n");1386 return -EINVAL;1387 }1388 1389 ret = device_property_read_u32(&pdev->dev, "dma-channel-mask",1390 &tdma->chan_mask);1391 if (ret) {1392 dev_warn(&pdev->dev,1393 "Missing dma-channel-mask property, using default channel mask %#x\n",1394 TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK);1395 tdma->chan_mask = TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK;1396 }1397 1398 INIT_LIST_HEAD(&tdma->dma_dev.channels);1399 for (i = 0; i < cdata->nr_channels; i++) {1400 struct tegra_dma_channel *tdc = &tdma->channels[i];1401 1402 /* Check for channel mask */1403 if (!(tdma->chan_mask & BIT(i)))1404 continue;1405 1406 tdc->irq = platform_get_irq(pdev, i);1407 if (tdc->irq < 0)1408 return tdc->irq;1409 1410 tdc->chan_base_offset = TEGRA_GPCDMA_CHANNEL_BASE_ADDR_OFFSET +1411 i * cdata->channel_reg_size;1412 snprintf(tdc->name, sizeof(tdc->name), "gpcdma.%d", i);1413 tdc->tdma = tdma;1414 tdc->id = i;1415 tdc->slave_id = -1;1416 1417 vchan_init(&tdc->vc, &tdma->dma_dev);1418 tdc->vc.desc_free = tegra_dma_desc_free;1419 1420 /* program stream-id for this channel */1421 tegra_dma_program_sid(tdc, stream_id);1422 tdc->stream_id = stream_id;1423 }1424 1425 dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask);1426 dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);1427 dma_cap_set(DMA_MEMCPY, tdma->dma_dev.cap_mask);1428 dma_cap_set(DMA_MEMSET, tdma->dma_dev.cap_mask);1429 dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask);1430 1431 /*1432 * Only word aligned transfers are supported. Set the copy1433 * alignment shift.1434 */1435 tdma->dma_dev.copy_align = 2;1436 tdma->dma_dev.fill_align = 2;1437 tdma->dma_dev.device_alloc_chan_resources =1438 tegra_dma_alloc_chan_resources;1439 tdma->dma_dev.device_free_chan_resources =1440 tegra_dma_free_chan_resources;1441 tdma->dma_dev.device_prep_slave_sg = tegra_dma_prep_slave_sg;1442 tdma->dma_dev.device_prep_dma_memcpy = tegra_dma_prep_dma_memcpy;1443 tdma->dma_dev.device_prep_dma_memset = tegra_dma_prep_dma_memset;1444 tdma->dma_dev.device_prep_dma_cyclic = tegra_dma_prep_dma_cyclic;1445 tdma->dma_dev.device_config = tegra_dma_slave_config;1446 tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all;1447 tdma->dma_dev.device_tx_status = tegra_dma_tx_status;1448 tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending;1449 tdma->dma_dev.device_pause = tegra_dma_device_pause;1450 tdma->dma_dev.device_resume = tegra_dma_device_resume;1451 tdma->dma_dev.device_synchronize = tegra_dma_chan_synchronize;1452 tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;1453 1454 ret = dma_async_device_register(&tdma->dma_dev);1455 if (ret < 0) {1456 dev_err_probe(&pdev->dev, ret,1457 "GPC DMA driver registration failed\n");1458 return ret;1459 }1460 1461 ret = of_dma_controller_register(pdev->dev.of_node,1462 tegra_dma_of_xlate, tdma);1463 if (ret < 0) {1464 dev_err_probe(&pdev->dev, ret,1465 "GPC DMA OF registration failed\n");1466 1467 dma_async_device_unregister(&tdma->dma_dev);1468 return ret;1469 }1470 1471 dev_info(&pdev->dev, "GPC DMA driver register %lu channels\n",1472 hweight_long(tdma->chan_mask));1473 1474 return 0;1475}1476 1477static void tegra_dma_remove(struct platform_device *pdev)1478{1479 struct tegra_dma *tdma = platform_get_drvdata(pdev);1480 1481 of_dma_controller_free(pdev->dev.of_node);1482 dma_async_device_unregister(&tdma->dma_dev);1483}1484 1485static int __maybe_unused tegra_dma_pm_suspend(struct device *dev)1486{1487 struct tegra_dma *tdma = dev_get_drvdata(dev);1488 unsigned int i;1489 1490 for (i = 0; i < tdma->chip_data->nr_channels; i++) {1491 struct tegra_dma_channel *tdc = &tdma->channels[i];1492 1493 if (!(tdma->chan_mask & BIT(i)))1494 continue;1495 1496 if (tdc->dma_desc) {1497 dev_err(tdma->dev, "channel %u busy\n", i);1498 return -EBUSY;1499 }1500 }1501 1502 return 0;1503}1504 1505static int __maybe_unused tegra_dma_pm_resume(struct device *dev)1506{1507 struct tegra_dma *tdma = dev_get_drvdata(dev);1508 unsigned int i;1509 1510 reset_control_reset(tdma->rst);1511 1512 for (i = 0; i < tdma->chip_data->nr_channels; i++) {1513 struct tegra_dma_channel *tdc = &tdma->channels[i];1514 1515 if (!(tdma->chan_mask & BIT(i)))1516 continue;1517 1518 tegra_dma_program_sid(tdc, tdc->stream_id);1519 }1520 1521 return 0;1522}1523 1524static const struct dev_pm_ops tegra_dma_dev_pm_ops = {1525 SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_pm_suspend, tegra_dma_pm_resume)1526};1527 1528static struct platform_driver tegra_dma_driver = {1529 .driver = {1530 .name = "tegra-gpcdma",1531 .pm = &tegra_dma_dev_pm_ops,1532 .of_match_table = tegra_dma_of_match,1533 },1534 .probe = tegra_dma_probe,1535 .remove_new = tegra_dma_remove,1536};1537 1538module_platform_driver(tegra_dma_driver);1539 1540MODULE_DESCRIPTION("NVIDIA Tegra GPC DMA Controller driver");1541MODULE_AUTHOR("Pavan Kunapuli <pkunapuli@nvidia.com>");1542MODULE_AUTHOR("Rajesh Gumasta <rgumasta@nvidia.com>");1543MODULE_LICENSE("GPL");1544