brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * DMA driver for Xilinx Video DMA Engine4 *5 * Copyright (C) 2010-2014 Xilinx, Inc. All rights reserved.6 *7 * Based on the Freescale DMA driver.8 *9 * Description:10 * The AXI Video Direct Memory Access (AXI VDMA) core is a soft Xilinx IP11 * core that provides high-bandwidth direct memory access between memory12 * and AXI4-Stream type video target peripherals. The core provides efficient13 * two dimensional DMA operations with independent asynchronous read (S2MM)14 * and write (MM2S) channel operation. It can be configured to have either15 * one channel or two channels. If configured as two channels, one is to16 * transmit to the video device (MM2S) and another is to receive from the17 * video device (S2MM). Initialization, status, interrupt and management18 * registers are accessed through an AXI4-Lite slave interface.19 *20 * The AXI Direct Memory Access (AXI DMA) core is a soft Xilinx IP core that21 * provides high-bandwidth one dimensional direct memory access between memory22 * and AXI4-Stream target peripherals. It supports one receive and one23 * transmit channel, both of them optional at synthesis time.24 *25 * The AXI CDMA, is a soft IP, which provides high-bandwidth Direct Memory26 * Access (DMA) between a memory-mapped source address and a memory-mapped27 * destination address.28 *29 * The AXI Multichannel Direct Memory Access (AXI MCDMA) core is a soft30 * Xilinx IP that provides high-bandwidth direct memory access between31 * memory and AXI4-Stream target peripherals. It provides scatter gather32 * (SG) interface with multiple channels independent configuration support.33 *34 */35 36#include <linux/bitops.h>37#include <linux/dmapool.h>38#include <linux/dma/xilinx_dma.h>39#include <linux/init.h>40#include <linux/interrupt.h>41#include <linux/io.h>42#include <linux/iopoll.h>43#include <linux/module.h>44#include <linux/of.h>45#include <linux/of_dma.h>46#include <linux/of_irq.h>47#include <linux/platform_device.h>48#include <linux/slab.h>49#include <linux/clk.h>50#include <linux/io-64-nonatomic-lo-hi.h>51 52#include "../dmaengine.h"53 54/* Register/Descriptor Offsets */55#define XILINX_DMA_MM2S_CTRL_OFFSET		0x000056#define XILINX_DMA_S2MM_CTRL_OFFSET		0x003057#define XILINX_VDMA_MM2S_DESC_OFFSET		0x005058#define XILINX_VDMA_S2MM_DESC_OFFSET		0x00a059 60/* Control Registers */61#define XILINX_DMA_REG_DMACR			0x000062#define XILINX_DMA_DMACR_DELAY_MAX		0xff63#define XILINX_DMA_DMACR_DELAY_SHIFT		2464#define XILINX_DMA_DMACR_FRAME_COUNT_MAX	0xff65#define XILINX_DMA_DMACR_FRAME_COUNT_SHIFT	1666#define XILINX_DMA_DMACR_ERR_IRQ		BIT(14)67#define XILINX_DMA_DMACR_DLY_CNT_IRQ		BIT(13)68#define XILINX_DMA_DMACR_FRM_CNT_IRQ		BIT(12)69#define XILINX_DMA_DMACR_MASTER_SHIFT		870#define XILINX_DMA_DMACR_FSYNCSRC_SHIFT	571#define XILINX_DMA_DMACR_FRAMECNT_EN		BIT(4)72#define XILINX_DMA_DMACR_GENLOCK_EN		BIT(3)73#define XILINX_DMA_DMACR_RESET			BIT(2)74#define XILINX_DMA_DMACR_CIRC_EN		BIT(1)75#define XILINX_DMA_DMACR_RUNSTOP		BIT(0)76#define XILINX_DMA_DMACR_FSYNCSRC_MASK		GENMASK(6, 5)77#define XILINX_DMA_DMACR_DELAY_MASK		GENMASK(31, 24)78#define XILINX_DMA_DMACR_FRAME_COUNT_MASK	GENMASK(23, 16)79#define XILINX_DMA_DMACR_MASTER_MASK		GENMASK(11, 8)80 81#define XILINX_DMA_REG_DMASR			0x000482#define XILINX_DMA_DMASR_EOL_LATE_ERR		BIT(15)83#define XILINX_DMA_DMASR_ERR_IRQ		BIT(14)84#define XILINX_DMA_DMASR_DLY_CNT_IRQ		BIT(13)85#define XILINX_DMA_DMASR_FRM_CNT_IRQ		BIT(12)86#define XILINX_DMA_DMASR_SOF_LATE_ERR		BIT(11)87#define XILINX_DMA_DMASR_SG_DEC_ERR		BIT(10)88#define XILINX_DMA_DMASR_SG_SLV_ERR		BIT(9)89#define XILINX_DMA_DMASR_EOF_EARLY_ERR		BIT(8)90#define XILINX_DMA_DMASR_SOF_EARLY_ERR		BIT(7)91#define XILINX_DMA_DMASR_DMA_DEC_ERR		BIT(6)92#define XILINX_DMA_DMASR_DMA_SLAVE_ERR		BIT(5)93#define XILINX_DMA_DMASR_DMA_INT_ERR		BIT(4)94#define XILINX_DMA_DMASR_SG_MASK		BIT(3)95#define XILINX_DMA_DMASR_IDLE			BIT(1)96#define XILINX_DMA_DMASR_HALTED		BIT(0)97#define XILINX_DMA_DMASR_DELAY_MASK		GENMASK(31, 24)98#define XILINX_DMA_DMASR_FRAME_COUNT_MASK	GENMASK(23, 16)99 100#define XILINX_DMA_REG_CURDESC			0x0008101#define XILINX_DMA_REG_TAILDESC		0x0010102#define XILINX_DMA_REG_REG_INDEX		0x0014103#define XILINX_DMA_REG_FRMSTORE		0x0018104#define XILINX_DMA_REG_THRESHOLD		0x001c105#define XILINX_DMA_REG_FRMPTR_STS		0x0024106#define XILINX_DMA_REG_PARK_PTR		0x0028107#define XILINX_DMA_PARK_PTR_WR_REF_SHIFT	8108#define XILINX_DMA_PARK_PTR_WR_REF_MASK		GENMASK(12, 8)109#define XILINX_DMA_PARK_PTR_RD_REF_SHIFT	0110#define XILINX_DMA_PARK_PTR_RD_REF_MASK		GENMASK(4, 0)111#define XILINX_DMA_REG_VDMA_VERSION		0x002c112 113/* Register Direct Mode Registers */114#define XILINX_DMA_REG_VSIZE			0x0000115#define XILINX_DMA_VSIZE_MASK			GENMASK(12, 0)116#define XILINX_DMA_REG_HSIZE			0x0004117#define XILINX_DMA_HSIZE_MASK			GENMASK(15, 0)118 119#define XILINX_DMA_REG_FRMDLY_STRIDE		0x0008120#define XILINX_DMA_FRMDLY_STRIDE_FRMDLY_SHIFT	24121#define XILINX_DMA_FRMDLY_STRIDE_STRIDE_SHIFT	0122 123#define XILINX_VDMA_REG_START_ADDRESS(n)	(0x000c + 4 * (n))124#define XILINX_VDMA_REG_START_ADDRESS_64(n)	(0x000c + 8 * (n))125 126#define XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP	0x00ec127#define XILINX_VDMA_ENABLE_VERTICAL_FLIP	BIT(0)128 129/* HW specific definitions */130#define XILINX_MCDMA_MAX_CHANS_PER_DEVICE	0x20131#define XILINX_DMA_MAX_CHANS_PER_DEVICE		0x2132#define XILINX_CDMA_MAX_CHANS_PER_DEVICE	0x1133 134#define XILINX_DMA_DMAXR_ALL_IRQ_MASK	\135		(XILINX_DMA_DMASR_FRM_CNT_IRQ | \136		 XILINX_DMA_DMASR_DLY_CNT_IRQ | \137		 XILINX_DMA_DMASR_ERR_IRQ)138 139#define XILINX_DMA_DMASR_ALL_ERR_MASK	\140		(XILINX_DMA_DMASR_EOL_LATE_ERR | \141		 XILINX_DMA_DMASR_SOF_LATE_ERR | \142		 XILINX_DMA_DMASR_SG_DEC_ERR | \143		 XILINX_DMA_DMASR_SG_SLV_ERR | \144		 XILINX_DMA_DMASR_EOF_EARLY_ERR | \145		 XILINX_DMA_DMASR_SOF_EARLY_ERR | \146		 XILINX_DMA_DMASR_DMA_DEC_ERR | \147		 XILINX_DMA_DMASR_DMA_SLAVE_ERR | \148		 XILINX_DMA_DMASR_DMA_INT_ERR)149 150/*151 * Recoverable errors are DMA Internal error, SOF Early, EOF Early152 * and SOF Late. They are only recoverable when C_FLUSH_ON_FSYNC153 * is enabled in the h/w system.154 */155#define XILINX_DMA_DMASR_ERR_RECOVER_MASK	\156		(XILINX_DMA_DMASR_SOF_LATE_ERR | \157		 XILINX_DMA_DMASR_EOF_EARLY_ERR | \158		 XILINX_DMA_DMASR_SOF_EARLY_ERR | \159		 XILINX_DMA_DMASR_DMA_INT_ERR)160 161/* Axi VDMA Flush on Fsync bits */162#define XILINX_DMA_FLUSH_S2MM		3163#define XILINX_DMA_FLUSH_MM2S		2164#define XILINX_DMA_FLUSH_BOTH		1165 166/* Delay loop counter to prevent hardware failure */167#define XILINX_DMA_LOOP_COUNT		1000000168 169/* AXI DMA Specific Registers/Offsets */170#define XILINX_DMA_REG_SRCDSTADDR	0x18171#define XILINX_DMA_REG_BTT		0x28172 173/* AXI DMA Specific Masks/Bit fields */174#define XILINX_DMA_MAX_TRANS_LEN_MIN	8175#define XILINX_DMA_MAX_TRANS_LEN_MAX	23176#define XILINX_DMA_V2_MAX_TRANS_LEN_MAX	26177#define XILINX_DMA_CR_COALESCE_MAX	GENMASK(23, 16)178#define XILINX_DMA_CR_DELAY_MAX		GENMASK(31, 24)179#define XILINX_DMA_CR_CYCLIC_BD_EN_MASK	BIT(4)180#define XILINX_DMA_CR_COALESCE_SHIFT	16181#define XILINX_DMA_CR_DELAY_SHIFT	24182#define XILINX_DMA_BD_SOP		BIT(27)183#define XILINX_DMA_BD_EOP		BIT(26)184#define XILINX_DMA_BD_COMP_MASK		BIT(31)185#define XILINX_DMA_COALESCE_MAX		255186#define XILINX_DMA_NUM_DESCS		512187#define XILINX_DMA_NUM_APP_WORDS	5188 189/* AXI CDMA Specific Registers/Offsets */190#define XILINX_CDMA_REG_SRCADDR		0x18191#define XILINX_CDMA_REG_DSTADDR		0x20192 193/* AXI CDMA Specific Masks */194#define XILINX_CDMA_CR_SGMODE          BIT(3)195 196#define xilinx_prep_dma_addr_t(addr)	\197	((dma_addr_t)((u64)addr##_##msb << 32 | (addr)))198 199/* AXI MCDMA Specific Registers/Offsets */200#define XILINX_MCDMA_MM2S_CTRL_OFFSET		0x0000201#define XILINX_MCDMA_S2MM_CTRL_OFFSET		0x0500202#define XILINX_MCDMA_CHEN_OFFSET		0x0008203#define XILINX_MCDMA_CH_ERR_OFFSET		0x0010204#define XILINX_MCDMA_RXINT_SER_OFFSET		0x0020205#define XILINX_MCDMA_TXINT_SER_OFFSET		0x0028206#define XILINX_MCDMA_CHAN_CR_OFFSET(x)		(0x40 + (x) * 0x40)207#define XILINX_MCDMA_CHAN_SR_OFFSET(x)		(0x44 + (x) * 0x40)208#define XILINX_MCDMA_CHAN_CDESC_OFFSET(x)	(0x48 + (x) * 0x40)209#define XILINX_MCDMA_CHAN_TDESC_OFFSET(x)	(0x50 + (x) * 0x40)210 211/* AXI MCDMA Specific Masks/Shifts */212#define XILINX_MCDMA_COALESCE_SHIFT		16213#define XILINX_MCDMA_COALESCE_MAX		24214#define XILINX_MCDMA_IRQ_ALL_MASK		GENMASK(7, 5)215#define XILINX_MCDMA_COALESCE_MASK		GENMASK(23, 16)216#define XILINX_MCDMA_CR_RUNSTOP_MASK		BIT(0)217#define XILINX_MCDMA_IRQ_IOC_MASK		BIT(5)218#define XILINX_MCDMA_IRQ_DELAY_MASK		BIT(6)219#define XILINX_MCDMA_IRQ_ERR_MASK		BIT(7)220#define XILINX_MCDMA_BD_EOP			BIT(30)221#define XILINX_MCDMA_BD_SOP			BIT(31)222 223/**224 * struct xilinx_vdma_desc_hw - Hardware Descriptor225 * @next_desc: Next Descriptor Pointer @0x00226 * @pad1: Reserved @0x04227 * @buf_addr: Buffer address @0x08228 * @buf_addr_msb: MSB of Buffer address @0x0C229 * @vsize: Vertical Size @0x10230 * @hsize: Horizontal Size @0x14231 * @stride: Number of bytes between the first232 *	    pixels of each horizontal line @0x18233 */234struct xilinx_vdma_desc_hw {235	u32 next_desc;236	u32 pad1;237	u32 buf_addr;238	u32 buf_addr_msb;239	u32 vsize;240	u32 hsize;241	u32 stride;242} __aligned(64);243 244/**245 * struct xilinx_axidma_desc_hw - Hardware Descriptor for AXI DMA246 * @next_desc: Next Descriptor Pointer @0x00247 * @next_desc_msb: MSB of Next Descriptor Pointer @0x04248 * @buf_addr: Buffer address @0x08249 * @buf_addr_msb: MSB of Buffer address @0x0C250 * @reserved1: Reserved @0x10251 * @reserved2: Reserved @0x14252 * @control: Control field @0x18253 * @status: Status field @0x1C254 * @app: APP Fields @0x20 - 0x30255 */256struct xilinx_axidma_desc_hw {257	u32 next_desc;258	u32 next_desc_msb;259	u32 buf_addr;260	u32 buf_addr_msb;261	u32 reserved1;262	u32 reserved2;263	u32 control;264	u32 status;265	u32 app[XILINX_DMA_NUM_APP_WORDS];266} __aligned(64);267 268/**269 * struct xilinx_aximcdma_desc_hw - Hardware Descriptor for AXI MCDMA270 * @next_desc: Next Descriptor Pointer @0x00271 * @next_desc_msb: MSB of Next Descriptor Pointer @0x04272 * @buf_addr: Buffer address @0x08273 * @buf_addr_msb: MSB of Buffer address @0x0C274 * @rsvd: Reserved field @0x10275 * @control: Control Information field @0x14276 * @status: Status field @0x18277 * @sideband_status: Status of sideband signals @0x1C278 * @app: APP Fields @0x20 - 0x30279 */280struct xilinx_aximcdma_desc_hw {281	u32 next_desc;282	u32 next_desc_msb;283	u32 buf_addr;284	u32 buf_addr_msb;285	u32 rsvd;286	u32 control;287	u32 status;288	u32 sideband_status;289	u32 app[XILINX_DMA_NUM_APP_WORDS];290} __aligned(64);291 292/**293 * struct xilinx_cdma_desc_hw - Hardware Descriptor294 * @next_desc: Next Descriptor Pointer @0x00295 * @next_desc_msb: Next Descriptor Pointer MSB @0x04296 * @src_addr: Source address @0x08297 * @src_addr_msb: Source address MSB @0x0C298 * @dest_addr: Destination address @0x10299 * @dest_addr_msb: Destination address MSB @0x14300 * @control: Control field @0x18301 * @status: Status field @0x1C302 */303struct xilinx_cdma_desc_hw {304	u32 next_desc;305	u32 next_desc_msb;306	u32 src_addr;307	u32 src_addr_msb;308	u32 dest_addr;309	u32 dest_addr_msb;310	u32 control;311	u32 status;312} __aligned(64);313 314/**315 * struct xilinx_vdma_tx_segment - Descriptor segment316 * @hw: Hardware descriptor317 * @node: Node in the descriptor segments list318 * @phys: Physical address of segment319 */320struct xilinx_vdma_tx_segment {321	struct xilinx_vdma_desc_hw hw;322	struct list_head node;323	dma_addr_t phys;324} __aligned(64);325 326/**327 * struct xilinx_axidma_tx_segment - Descriptor segment328 * @hw: Hardware descriptor329 * @node: Node in the descriptor segments list330 * @phys: Physical address of segment331 */332struct xilinx_axidma_tx_segment {333	struct xilinx_axidma_desc_hw hw;334	struct list_head node;335	dma_addr_t phys;336} __aligned(64);337 338/**339 * struct xilinx_aximcdma_tx_segment - Descriptor segment340 * @hw: Hardware descriptor341 * @node: Node in the descriptor segments list342 * @phys: Physical address of segment343 */344struct xilinx_aximcdma_tx_segment {345	struct xilinx_aximcdma_desc_hw hw;346	struct list_head node;347	dma_addr_t phys;348} __aligned(64);349 350/**351 * struct xilinx_cdma_tx_segment - Descriptor segment352 * @hw: Hardware descriptor353 * @node: Node in the descriptor segments list354 * @phys: Physical address of segment355 */356struct xilinx_cdma_tx_segment {357	struct xilinx_cdma_desc_hw hw;358	struct list_head node;359	dma_addr_t phys;360} __aligned(64);361 362/**363 * struct xilinx_dma_tx_descriptor - Per Transaction structure364 * @async_tx: Async transaction descriptor365 * @segments: TX segments list366 * @node: Node in the channel descriptors list367 * @cyclic: Check for cyclic transfers.368 * @err: Whether the descriptor has an error.369 * @residue: Residue of the completed descriptor370 */371struct xilinx_dma_tx_descriptor {372	struct dma_async_tx_descriptor async_tx;373	struct list_head segments;374	struct list_head node;375	bool cyclic;376	bool err;377	u32 residue;378};379 380/**381 * struct xilinx_dma_chan - Driver specific DMA channel structure382 * @xdev: Driver specific device structure383 * @ctrl_offset: Control registers offset384 * @desc_offset: TX descriptor registers offset385 * @lock: Descriptor operation lock386 * @pending_list: Descriptors waiting387 * @active_list: Descriptors ready to submit388 * @done_list: Complete descriptors389 * @free_seg_list: Free descriptors390 * @common: DMA common channel391 * @desc_pool: Descriptors pool392 * @dev: The dma device393 * @irq: Channel IRQ394 * @id: Channel ID395 * @direction: Transfer direction396 * @num_frms: Number of frames397 * @has_sg: Support scatter transfers398 * @cyclic: Check for cyclic transfers.399 * @genlock: Support genlock mode400 * @err: Channel has errors401 * @idle: Check for channel idle402 * @terminating: Check for channel being synchronized by user403 * @tasklet: Cleanup work after irq404 * @config: Device configuration info405 * @flush_on_fsync: Flush on Frame sync406 * @desc_pendingcount: Descriptor pending count407 * @ext_addr: Indicates 64 bit addressing is supported by dma channel408 * @desc_submitcount: Descriptor h/w submitted count409 * @seg_v: Statically allocated segments base410 * @seg_mv: Statically allocated segments base for MCDMA411 * @seg_p: Physical allocated segments base412 * @cyclic_seg_v: Statically allocated segment base for cyclic transfers413 * @cyclic_seg_p: Physical allocated segments base for cyclic dma414 * @start_transfer: Differentiate b/w DMA IP's transfer415 * @stop_transfer: Differentiate b/w DMA IP's quiesce416 * @tdest: TDEST value for mcdma417 * @has_vflip: S2MM vertical flip418 * @irq_delay: Interrupt delay timeout419 */420struct xilinx_dma_chan {421	struct xilinx_dma_device *xdev;422	u32 ctrl_offset;423	u32 desc_offset;424	spinlock_t lock;425	struct list_head pending_list;426	struct list_head active_list;427	struct list_head done_list;428	struct list_head free_seg_list;429	struct dma_chan common;430	struct dma_pool *desc_pool;431	struct device *dev;432	int irq;433	int id;434	enum dma_transfer_direction direction;435	int num_frms;436	bool has_sg;437	bool cyclic;438	bool genlock;439	bool err;440	bool idle;441	bool terminating;442	struct tasklet_struct tasklet;443	struct xilinx_vdma_config config;444	bool flush_on_fsync;445	u32 desc_pendingcount;446	bool ext_addr;447	u32 desc_submitcount;448	struct xilinx_axidma_tx_segment *seg_v;449	struct xilinx_aximcdma_tx_segment *seg_mv;450	dma_addr_t seg_p;451	struct xilinx_axidma_tx_segment *cyclic_seg_v;452	dma_addr_t cyclic_seg_p;453	void (*start_transfer)(struct xilinx_dma_chan *chan);454	int (*stop_transfer)(struct xilinx_dma_chan *chan);455	u16 tdest;456	bool has_vflip;457	u8 irq_delay;458};459 460/**461 * enum xdma_ip_type - DMA IP type.462 *463 * @XDMA_TYPE_AXIDMA: Axi dma ip.464 * @XDMA_TYPE_CDMA: Axi cdma ip.465 * @XDMA_TYPE_VDMA: Axi vdma ip.466 * @XDMA_TYPE_AXIMCDMA: Axi MCDMA ip.467 *468 */469enum xdma_ip_type {470	XDMA_TYPE_AXIDMA = 0,471	XDMA_TYPE_CDMA,472	XDMA_TYPE_VDMA,473	XDMA_TYPE_AXIMCDMA474};475 476struct xilinx_dma_config {477	enum xdma_ip_type dmatype;478	int (*clk_init)(struct platform_device *pdev, struct clk **axi_clk,479			struct clk **tx_clk, struct clk **txs_clk,480			struct clk **rx_clk, struct clk **rxs_clk);481	irqreturn_t (*irq_handler)(int irq, void *data);482	const int max_channels;483};484 485/**486 * struct xilinx_dma_device - DMA device structure487 * @regs: I/O mapped base address488 * @dev: Device Structure489 * @common: DMA device structure490 * @chan: Driver specific DMA channel491 * @flush_on_fsync: Flush on frame sync492 * @ext_addr: Indicates 64 bit addressing is supported by dma device493 * @pdev: Platform device structure pointer494 * @dma_config: DMA config structure495 * @axi_clk: DMA Axi4-lite interace clock496 * @tx_clk: DMA mm2s clock497 * @txs_clk: DMA mm2s stream clock498 * @rx_clk: DMA s2mm clock499 * @rxs_clk: DMA s2mm stream clock500 * @s2mm_chan_id: DMA s2mm channel identifier501 * @mm2s_chan_id: DMA mm2s channel identifier502 * @max_buffer_len: Max buffer length503 * @has_axistream_connected: AXI DMA connected to AXI Stream IP504 */505struct xilinx_dma_device {506	void __iomem *regs;507	struct device *dev;508	struct dma_device common;509	struct xilinx_dma_chan *chan[XILINX_MCDMA_MAX_CHANS_PER_DEVICE];510	u32 flush_on_fsync;511	bool ext_addr;512	struct platform_device  *pdev;513	const struct xilinx_dma_config *dma_config;514	struct clk *axi_clk;515	struct clk *tx_clk;516	struct clk *txs_clk;517	struct clk *rx_clk;518	struct clk *rxs_clk;519	u32 s2mm_chan_id;520	u32 mm2s_chan_id;521	u32 max_buffer_len;522	bool has_axistream_connected;523};524 525/* Macros */526#define to_xilinx_chan(chan) \527	container_of(chan, struct xilinx_dma_chan, common)528#define to_dma_tx_descriptor(tx) \529	container_of(tx, struct xilinx_dma_tx_descriptor, async_tx)530#define xilinx_dma_poll_timeout(chan, reg, val, cond, delay_us, timeout_us) \531	readl_poll_timeout_atomic(chan->xdev->regs + chan->ctrl_offset + reg, \532				  val, cond, delay_us, timeout_us)533 534/* IO accessors */535static inline u32 dma_read(struct xilinx_dma_chan *chan, u32 reg)536{537	return ioread32(chan->xdev->regs + reg);538}539 540static inline void dma_write(struct xilinx_dma_chan *chan, u32 reg, u32 value)541{542	iowrite32(value, chan->xdev->regs + reg);543}544 545static inline void vdma_desc_write(struct xilinx_dma_chan *chan, u32 reg,546				   u32 value)547{548	dma_write(chan, chan->desc_offset + reg, value);549}550 551static inline u32 dma_ctrl_read(struct xilinx_dma_chan *chan, u32 reg)552{553	return dma_read(chan, chan->ctrl_offset + reg);554}555 556static inline void dma_ctrl_write(struct xilinx_dma_chan *chan, u32 reg,557				   u32 value)558{559	dma_write(chan, chan->ctrl_offset + reg, value);560}561 562static inline void dma_ctrl_clr(struct xilinx_dma_chan *chan, u32 reg,563				 u32 clr)564{565	dma_ctrl_write(chan, reg, dma_ctrl_read(chan, reg) & ~clr);566}567 568static inline void dma_ctrl_set(struct xilinx_dma_chan *chan, u32 reg,569				 u32 set)570{571	dma_ctrl_write(chan, reg, dma_ctrl_read(chan, reg) | set);572}573 574/**575 * vdma_desc_write_64 - 64-bit descriptor write576 * @chan: Driver specific VDMA channel577 * @reg: Register to write578 * @value_lsb: lower address of the descriptor.579 * @value_msb: upper address of the descriptor.580 *581 * Since vdma driver is trying to write to a register offset which is not a582 * multiple of 64 bits(ex : 0x5c), we are writing as two separate 32 bits583 * instead of a single 64 bit register write.584 */585static inline void vdma_desc_write_64(struct xilinx_dma_chan *chan, u32 reg,586				      u32 value_lsb, u32 value_msb)587{588	/* Write the lsb 32 bits*/589	writel(value_lsb, chan->xdev->regs + chan->desc_offset + reg);590 591	/* Write the msb 32 bits */592	writel(value_msb, chan->xdev->regs + chan->desc_offset + reg + 4);593}594 595static inline void dma_writeq(struct xilinx_dma_chan *chan, u32 reg, u64 value)596{597	lo_hi_writeq(value, chan->xdev->regs + chan->ctrl_offset + reg);598}599 600static inline void xilinx_write(struct xilinx_dma_chan *chan, u32 reg,601				dma_addr_t addr)602{603	if (chan->ext_addr)604		dma_writeq(chan, reg, addr);605	else606		dma_ctrl_write(chan, reg, addr);607}608 609static inline void xilinx_axidma_buf(struct xilinx_dma_chan *chan,610				     struct xilinx_axidma_desc_hw *hw,611				     dma_addr_t buf_addr, size_t sg_used,612				     size_t period_len)613{614	if (chan->ext_addr) {615		hw->buf_addr = lower_32_bits(buf_addr + sg_used + period_len);616		hw->buf_addr_msb = upper_32_bits(buf_addr + sg_used +617						 period_len);618	} else {619		hw->buf_addr = buf_addr + sg_used + period_len;620	}621}622 623static inline void xilinx_aximcdma_buf(struct xilinx_dma_chan *chan,624				       struct xilinx_aximcdma_desc_hw *hw,625				       dma_addr_t buf_addr, size_t sg_used)626{627	if (chan->ext_addr) {628		hw->buf_addr = lower_32_bits(buf_addr + sg_used);629		hw->buf_addr_msb = upper_32_bits(buf_addr + sg_used);630	} else {631		hw->buf_addr = buf_addr + sg_used;632	}633}634 635/**636 * xilinx_dma_get_metadata_ptr- Populate metadata pointer and payload length637 * @tx: async transaction descriptor638 * @payload_len: metadata payload length639 * @max_len: metadata max length640 * Return: The app field pointer.641 */642static void *xilinx_dma_get_metadata_ptr(struct dma_async_tx_descriptor *tx,643					 size_t *payload_len, size_t *max_len)644{645	struct xilinx_dma_tx_descriptor *desc = to_dma_tx_descriptor(tx);646	struct xilinx_axidma_tx_segment *seg;647 648	*max_len = *payload_len = sizeof(u32) * XILINX_DMA_NUM_APP_WORDS;649	seg = list_first_entry(&desc->segments,650			       struct xilinx_axidma_tx_segment, node);651	return seg->hw.app;652}653 654static struct dma_descriptor_metadata_ops xilinx_dma_metadata_ops = {655	.get_ptr = xilinx_dma_get_metadata_ptr,656};657 658/* -----------------------------------------------------------------------------659 * Descriptors and segments alloc and free660 */661 662/**663 * xilinx_vdma_alloc_tx_segment - Allocate transaction segment664 * @chan: Driver specific DMA channel665 *666 * Return: The allocated segment on success and NULL on failure.667 */668static struct xilinx_vdma_tx_segment *669xilinx_vdma_alloc_tx_segment(struct xilinx_dma_chan *chan)670{671	struct xilinx_vdma_tx_segment *segment;672	dma_addr_t phys;673 674	segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys);675	if (!segment)676		return NULL;677 678	segment->phys = phys;679 680	return segment;681}682 683/**684 * xilinx_cdma_alloc_tx_segment - Allocate transaction segment685 * @chan: Driver specific DMA channel686 *687 * Return: The allocated segment on success and NULL on failure.688 */689static struct xilinx_cdma_tx_segment *690xilinx_cdma_alloc_tx_segment(struct xilinx_dma_chan *chan)691{692	struct xilinx_cdma_tx_segment *segment;693	dma_addr_t phys;694 695	segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys);696	if (!segment)697		return NULL;698 699	segment->phys = phys;700 701	return segment;702}703 704/**705 * xilinx_axidma_alloc_tx_segment - Allocate transaction segment706 * @chan: Driver specific DMA channel707 *708 * Return: The allocated segment on success and NULL on failure.709 */710static struct xilinx_axidma_tx_segment *711xilinx_axidma_alloc_tx_segment(struct xilinx_dma_chan *chan)712{713	struct xilinx_axidma_tx_segment *segment = NULL;714	unsigned long flags;715 716	spin_lock_irqsave(&chan->lock, flags);717	if (!list_empty(&chan->free_seg_list)) {718		segment = list_first_entry(&chan->free_seg_list,719					   struct xilinx_axidma_tx_segment,720					   node);721		list_del(&segment->node);722	}723	spin_unlock_irqrestore(&chan->lock, flags);724 725	if (!segment)726		dev_dbg(chan->dev, "Could not find free tx segment\n");727 728	return segment;729}730 731/**732 * xilinx_aximcdma_alloc_tx_segment - Allocate transaction segment733 * @chan: Driver specific DMA channel734 *735 * Return: The allocated segment on success and NULL on failure.736 */737static struct xilinx_aximcdma_tx_segment *738xilinx_aximcdma_alloc_tx_segment(struct xilinx_dma_chan *chan)739{740	struct xilinx_aximcdma_tx_segment *segment = NULL;741	unsigned long flags;742 743	spin_lock_irqsave(&chan->lock, flags);744	if (!list_empty(&chan->free_seg_list)) {745		segment = list_first_entry(&chan->free_seg_list,746					   struct xilinx_aximcdma_tx_segment,747					   node);748		list_del(&segment->node);749	}750	spin_unlock_irqrestore(&chan->lock, flags);751 752	return segment;753}754 755static void xilinx_dma_clean_hw_desc(struct xilinx_axidma_desc_hw *hw)756{757	u32 next_desc = hw->next_desc;758	u32 next_desc_msb = hw->next_desc_msb;759 760	memset(hw, 0, sizeof(struct xilinx_axidma_desc_hw));761 762	hw->next_desc = next_desc;763	hw->next_desc_msb = next_desc_msb;764}765 766static void xilinx_mcdma_clean_hw_desc(struct xilinx_aximcdma_desc_hw *hw)767{768	u32 next_desc = hw->next_desc;769	u32 next_desc_msb = hw->next_desc_msb;770 771	memset(hw, 0, sizeof(struct xilinx_aximcdma_desc_hw));772 773	hw->next_desc = next_desc;774	hw->next_desc_msb = next_desc_msb;775}776 777/**778 * xilinx_dma_free_tx_segment - Free transaction segment779 * @chan: Driver specific DMA channel780 * @segment: DMA transaction segment781 */782static void xilinx_dma_free_tx_segment(struct xilinx_dma_chan *chan,783				struct xilinx_axidma_tx_segment *segment)784{785	xilinx_dma_clean_hw_desc(&segment->hw);786 787	list_add_tail(&segment->node, &chan->free_seg_list);788}789 790/**791 * xilinx_mcdma_free_tx_segment - Free transaction segment792 * @chan: Driver specific DMA channel793 * @segment: DMA transaction segment794 */795static void xilinx_mcdma_free_tx_segment(struct xilinx_dma_chan *chan,796					 struct xilinx_aximcdma_tx_segment *797					 segment)798{799	xilinx_mcdma_clean_hw_desc(&segment->hw);800 801	list_add_tail(&segment->node, &chan->free_seg_list);802}803 804/**805 * xilinx_cdma_free_tx_segment - Free transaction segment806 * @chan: Driver specific DMA channel807 * @segment: DMA transaction segment808 */809static void xilinx_cdma_free_tx_segment(struct xilinx_dma_chan *chan,810				struct xilinx_cdma_tx_segment *segment)811{812	dma_pool_free(chan->desc_pool, segment, segment->phys);813}814 815/**816 * xilinx_vdma_free_tx_segment - Free transaction segment817 * @chan: Driver specific DMA channel818 * @segment: DMA transaction segment819 */820static void xilinx_vdma_free_tx_segment(struct xilinx_dma_chan *chan,821					struct xilinx_vdma_tx_segment *segment)822{823	dma_pool_free(chan->desc_pool, segment, segment->phys);824}825 826/**827 * xilinx_dma_alloc_tx_descriptor - Allocate transaction descriptor828 * @chan: Driver specific DMA channel829 *830 * Return: The allocated descriptor on success and NULL on failure.831 */832static struct xilinx_dma_tx_descriptor *833xilinx_dma_alloc_tx_descriptor(struct xilinx_dma_chan *chan)834{835	struct xilinx_dma_tx_descriptor *desc;836 837	desc = kzalloc(sizeof(*desc), GFP_NOWAIT);838	if (!desc)839		return NULL;840 841	INIT_LIST_HEAD(&desc->segments);842 843	return desc;844}845 846/**847 * xilinx_dma_free_tx_descriptor - Free transaction descriptor848 * @chan: Driver specific DMA channel849 * @desc: DMA transaction descriptor850 */851static void852xilinx_dma_free_tx_descriptor(struct xilinx_dma_chan *chan,853			       struct xilinx_dma_tx_descriptor *desc)854{855	struct xilinx_vdma_tx_segment *segment, *next;856	struct xilinx_cdma_tx_segment *cdma_segment, *cdma_next;857	struct xilinx_axidma_tx_segment *axidma_segment, *axidma_next;858	struct xilinx_aximcdma_tx_segment *aximcdma_segment, *aximcdma_next;859 860	if (!desc)861		return;862 863	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {864		list_for_each_entry_safe(segment, next, &desc->segments, node) {865			list_del(&segment->node);866			xilinx_vdma_free_tx_segment(chan, segment);867		}868	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {869		list_for_each_entry_safe(cdma_segment, cdma_next,870					 &desc->segments, node) {871			list_del(&cdma_segment->node);872			xilinx_cdma_free_tx_segment(chan, cdma_segment);873		}874	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {875		list_for_each_entry_safe(axidma_segment, axidma_next,876					 &desc->segments, node) {877			list_del(&axidma_segment->node);878			xilinx_dma_free_tx_segment(chan, axidma_segment);879		}880	} else {881		list_for_each_entry_safe(aximcdma_segment, aximcdma_next,882					 &desc->segments, node) {883			list_del(&aximcdma_segment->node);884			xilinx_mcdma_free_tx_segment(chan, aximcdma_segment);885		}886	}887 888	kfree(desc);889}890 891/* Required functions */892 893/**894 * xilinx_dma_free_desc_list - Free descriptors list895 * @chan: Driver specific DMA channel896 * @list: List to parse and delete the descriptor897 */898static void xilinx_dma_free_desc_list(struct xilinx_dma_chan *chan,899					struct list_head *list)900{901	struct xilinx_dma_tx_descriptor *desc, *next;902 903	list_for_each_entry_safe(desc, next, list, node) {904		list_del(&desc->node);905		xilinx_dma_free_tx_descriptor(chan, desc);906	}907}908 909/**910 * xilinx_dma_free_descriptors - Free channel descriptors911 * @chan: Driver specific DMA channel912 */913static void xilinx_dma_free_descriptors(struct xilinx_dma_chan *chan)914{915	unsigned long flags;916 917	spin_lock_irqsave(&chan->lock, flags);918 919	xilinx_dma_free_desc_list(chan, &chan->pending_list);920	xilinx_dma_free_desc_list(chan, &chan->done_list);921	xilinx_dma_free_desc_list(chan, &chan->active_list);922 923	spin_unlock_irqrestore(&chan->lock, flags);924}925 926/**927 * xilinx_dma_free_chan_resources - Free channel resources928 * @dchan: DMA channel929 */930static void xilinx_dma_free_chan_resources(struct dma_chan *dchan)931{932	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);933	unsigned long flags;934 935	dev_dbg(chan->dev, "Free all channel resources.\n");936 937	xilinx_dma_free_descriptors(chan);938 939	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {940		spin_lock_irqsave(&chan->lock, flags);941		INIT_LIST_HEAD(&chan->free_seg_list);942		spin_unlock_irqrestore(&chan->lock, flags);943 944		/* Free memory that is allocated for BD */945		dma_free_coherent(chan->dev, sizeof(*chan->seg_v) *946				  XILINX_DMA_NUM_DESCS, chan->seg_v,947				  chan->seg_p);948 949		/* Free Memory that is allocated for cyclic DMA Mode */950		dma_free_coherent(chan->dev, sizeof(*chan->cyclic_seg_v),951				  chan->cyclic_seg_v, chan->cyclic_seg_p);952	}953 954	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {955		spin_lock_irqsave(&chan->lock, flags);956		INIT_LIST_HEAD(&chan->free_seg_list);957		spin_unlock_irqrestore(&chan->lock, flags);958 959		/* Free memory that is allocated for BD */960		dma_free_coherent(chan->dev, sizeof(*chan->seg_mv) *961				  XILINX_DMA_NUM_DESCS, chan->seg_mv,962				  chan->seg_p);963	}964 965	if (chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA &&966	    chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIMCDMA) {967		dma_pool_destroy(chan->desc_pool);968		chan->desc_pool = NULL;969	}970 971}972 973/**974 * xilinx_dma_get_residue - Compute residue for a given descriptor975 * @chan: Driver specific dma channel976 * @desc: dma transaction descriptor977 *978 * Return: The number of residue bytes for the descriptor.979 */980static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan,981				  struct xilinx_dma_tx_descriptor *desc)982{983	struct xilinx_cdma_tx_segment *cdma_seg;984	struct xilinx_axidma_tx_segment *axidma_seg;985	struct xilinx_aximcdma_tx_segment *aximcdma_seg;986	struct xilinx_cdma_desc_hw *cdma_hw;987	struct xilinx_axidma_desc_hw *axidma_hw;988	struct xilinx_aximcdma_desc_hw *aximcdma_hw;989	struct list_head *entry;990	u32 residue = 0;991 992	list_for_each(entry, &desc->segments) {993		if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {994			cdma_seg = list_entry(entry,995					      struct xilinx_cdma_tx_segment,996					      node);997			cdma_hw = &cdma_seg->hw;998			residue += (cdma_hw->control - cdma_hw->status) &999				   chan->xdev->max_buffer_len;1000		} else if (chan->xdev->dma_config->dmatype ==1001			   XDMA_TYPE_AXIDMA) {1002			axidma_seg = list_entry(entry,1003						struct xilinx_axidma_tx_segment,1004						node);1005			axidma_hw = &axidma_seg->hw;1006			residue += (axidma_hw->control - axidma_hw->status) &1007				   chan->xdev->max_buffer_len;1008		} else {1009			aximcdma_seg =1010				list_entry(entry,1011					   struct xilinx_aximcdma_tx_segment,1012					   node);1013			aximcdma_hw = &aximcdma_seg->hw;1014			residue +=1015				(aximcdma_hw->control - aximcdma_hw->status) &1016				chan->xdev->max_buffer_len;1017		}1018	}1019 1020	return residue;1021}1022 1023/**1024 * xilinx_dma_chan_handle_cyclic - Cyclic dma callback1025 * @chan: Driver specific dma channel1026 * @desc: dma transaction descriptor1027 * @flags: flags for spin lock1028 */1029static void xilinx_dma_chan_handle_cyclic(struct xilinx_dma_chan *chan,1030					  struct xilinx_dma_tx_descriptor *desc,1031					  unsigned long *flags)1032{1033	struct dmaengine_desc_callback cb;1034 1035	dmaengine_desc_get_callback(&desc->async_tx, &cb);1036	if (dmaengine_desc_callback_valid(&cb)) {1037		spin_unlock_irqrestore(&chan->lock, *flags);1038		dmaengine_desc_callback_invoke(&cb, NULL);1039		spin_lock_irqsave(&chan->lock, *flags);1040	}1041}1042 1043/**1044 * xilinx_dma_chan_desc_cleanup - Clean channel descriptors1045 * @chan: Driver specific DMA channel1046 */1047static void xilinx_dma_chan_desc_cleanup(struct xilinx_dma_chan *chan)1048{1049	struct xilinx_dma_tx_descriptor *desc, *next;1050	unsigned long flags;1051 1052	spin_lock_irqsave(&chan->lock, flags);1053 1054	list_for_each_entry_safe(desc, next, &chan->done_list, node) {1055		struct dmaengine_result result;1056 1057		if (desc->cyclic) {1058			xilinx_dma_chan_handle_cyclic(chan, desc, &flags);1059			break;1060		}1061 1062		/* Remove from the list of running transactions */1063		list_del(&desc->node);1064 1065		if (unlikely(desc->err)) {1066			if (chan->direction == DMA_DEV_TO_MEM)1067				result.result = DMA_TRANS_READ_FAILED;1068			else1069				result.result = DMA_TRANS_WRITE_FAILED;1070		} else {1071			result.result = DMA_TRANS_NOERROR;1072		}1073 1074		result.residue = desc->residue;1075 1076		/* Run the link descriptor callback function */1077		spin_unlock_irqrestore(&chan->lock, flags);1078		dmaengine_desc_get_callback_invoke(&desc->async_tx, &result);1079		spin_lock_irqsave(&chan->lock, flags);1080 1081		/* Run any dependencies, then free the descriptor */1082		dma_run_dependencies(&desc->async_tx);1083		xilinx_dma_free_tx_descriptor(chan, desc);1084 1085		/*1086		 * While we ran a callback the user called a terminate function,1087		 * which takes care of cleaning up any remaining descriptors1088		 */1089		if (chan->terminating)1090			break;1091	}1092 1093	spin_unlock_irqrestore(&chan->lock, flags);1094}1095 1096/**1097 * xilinx_dma_do_tasklet - Schedule completion tasklet1098 * @t: Pointer to the Xilinx DMA channel structure1099 */1100static void xilinx_dma_do_tasklet(struct tasklet_struct *t)1101{1102	struct xilinx_dma_chan *chan = from_tasklet(chan, t, tasklet);1103 1104	xilinx_dma_chan_desc_cleanup(chan);1105}1106 1107/**1108 * xilinx_dma_alloc_chan_resources - Allocate channel resources1109 * @dchan: DMA channel1110 *1111 * Return: '0' on success and failure value on error1112 */1113static int xilinx_dma_alloc_chan_resources(struct dma_chan *dchan)1114{1115	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);1116	int i;1117 1118	/* Has this channel already been allocated? */1119	if (chan->desc_pool)1120		return 0;1121 1122	/*1123	 * We need the descriptor to be aligned to 64bytes1124	 * for meeting Xilinx VDMA specification requirement.1125	 */1126	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {1127		/* Allocate the buffer descriptors. */1128		chan->seg_v = dma_alloc_coherent(chan->dev,1129						 sizeof(*chan->seg_v) * XILINX_DMA_NUM_DESCS,1130						 &chan->seg_p, GFP_KERNEL);1131		if (!chan->seg_v) {1132			dev_err(chan->dev,1133				"unable to allocate channel %d descriptors\n",1134				chan->id);1135			return -ENOMEM;1136		}1137		/*1138		 * For cyclic DMA mode we need to program the tail Descriptor1139		 * register with a value which is not a part of the BD chain1140		 * so allocating a desc segment during channel allocation for1141		 * programming tail descriptor.1142		 */1143		chan->cyclic_seg_v = dma_alloc_coherent(chan->dev,1144							sizeof(*chan->cyclic_seg_v),1145							&chan->cyclic_seg_p,1146							GFP_KERNEL);1147		if (!chan->cyclic_seg_v) {1148			dev_err(chan->dev,1149				"unable to allocate desc segment for cyclic DMA\n");1150			dma_free_coherent(chan->dev, sizeof(*chan->seg_v) *1151				XILINX_DMA_NUM_DESCS, chan->seg_v,1152				chan->seg_p);1153			return -ENOMEM;1154		}1155		chan->cyclic_seg_v->phys = chan->cyclic_seg_p;1156 1157		for (i = 0; i < XILINX_DMA_NUM_DESCS; i++) {1158			chan->seg_v[i].hw.next_desc =1159			lower_32_bits(chan->seg_p + sizeof(*chan->seg_v) *1160				((i + 1) % XILINX_DMA_NUM_DESCS));1161			chan->seg_v[i].hw.next_desc_msb =1162			upper_32_bits(chan->seg_p + sizeof(*chan->seg_v) *1163				((i + 1) % XILINX_DMA_NUM_DESCS));1164			chan->seg_v[i].phys = chan->seg_p +1165				sizeof(*chan->seg_v) * i;1166			list_add_tail(&chan->seg_v[i].node,1167				      &chan->free_seg_list);1168		}1169	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {1170		/* Allocate the buffer descriptors. */1171		chan->seg_mv = dma_alloc_coherent(chan->dev,1172						  sizeof(*chan->seg_mv) *1173						  XILINX_DMA_NUM_DESCS,1174						  &chan->seg_p, GFP_KERNEL);1175		if (!chan->seg_mv) {1176			dev_err(chan->dev,1177				"unable to allocate channel %d descriptors\n",1178				chan->id);1179			return -ENOMEM;1180		}1181		for (i = 0; i < XILINX_DMA_NUM_DESCS; i++) {1182			chan->seg_mv[i].hw.next_desc =1183			lower_32_bits(chan->seg_p + sizeof(*chan->seg_mv) *1184				((i + 1) % XILINX_DMA_NUM_DESCS));1185			chan->seg_mv[i].hw.next_desc_msb =1186			upper_32_bits(chan->seg_p + sizeof(*chan->seg_mv) *1187				((i + 1) % XILINX_DMA_NUM_DESCS));1188			chan->seg_mv[i].phys = chan->seg_p +1189				sizeof(*chan->seg_mv) * i;1190			list_add_tail(&chan->seg_mv[i].node,1191				      &chan->free_seg_list);1192		}1193	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {1194		chan->desc_pool = dma_pool_create("xilinx_cdma_desc_pool",1195				   chan->dev,1196				   sizeof(struct xilinx_cdma_tx_segment),1197				   __alignof__(struct xilinx_cdma_tx_segment),1198				   0);1199	} else {1200		chan->desc_pool = dma_pool_create("xilinx_vdma_desc_pool",1201				     chan->dev,1202				     sizeof(struct xilinx_vdma_tx_segment),1203				     __alignof__(struct xilinx_vdma_tx_segment),1204				     0);1205	}1206 1207	if (!chan->desc_pool &&1208	    ((chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA) &&1209		chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIMCDMA)) {1210		dev_err(chan->dev,1211			"unable to allocate channel %d descriptor pool\n",1212			chan->id);1213		return -ENOMEM;1214	}1215 1216	dma_cookie_init(dchan);1217 1218	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {1219		/* For AXI DMA resetting once channel will reset the1220		 * other channel as well so enable the interrupts here.1221		 */1222		dma_ctrl_set(chan, XILINX_DMA_REG_DMACR,1223			      XILINX_DMA_DMAXR_ALL_IRQ_MASK);1224	}1225 1226	if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg)1227		dma_ctrl_set(chan, XILINX_DMA_REG_DMACR,1228			     XILINX_CDMA_CR_SGMODE);1229 1230	return 0;1231}1232 1233/**1234 * xilinx_dma_calc_copysize - Calculate the amount of data to copy1235 * @chan: Driver specific DMA channel1236 * @size: Total data that needs to be copied1237 * @done: Amount of data that has been already copied1238 *1239 * Return: Amount of data that has to be copied1240 */1241static int xilinx_dma_calc_copysize(struct xilinx_dma_chan *chan,1242				    int size, int done)1243{1244	size_t copy;1245 1246	copy = min_t(size_t, size - done,1247		     chan->xdev->max_buffer_len);1248 1249	if ((copy + done < size) &&1250	    chan->xdev->common.copy_align) {1251		/*1252		 * If this is not the last descriptor, make sure1253		 * the next one will be properly aligned1254		 */1255		copy = rounddown(copy,1256				 (1 << chan->xdev->common.copy_align));1257	}1258	return copy;1259}1260 1261/**1262 * xilinx_dma_tx_status - Get DMA transaction status1263 * @dchan: DMA channel1264 * @cookie: Transaction identifier1265 * @txstate: Transaction state1266 *1267 * Return: DMA transaction status1268 */1269static enum dma_status xilinx_dma_tx_status(struct dma_chan *dchan,1270					dma_cookie_t cookie,1271					struct dma_tx_state *txstate)1272{1273	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);1274	struct xilinx_dma_tx_descriptor *desc;1275	enum dma_status ret;1276	unsigned long flags;1277	u32 residue = 0;1278 1279	ret = dma_cookie_status(dchan, cookie, txstate);1280	if (ret == DMA_COMPLETE || !txstate)1281		return ret;1282 1283	spin_lock_irqsave(&chan->lock, flags);1284	if (!list_empty(&chan->active_list)) {1285		desc = list_last_entry(&chan->active_list,1286				       struct xilinx_dma_tx_descriptor, node);1287		/*1288		 * VDMA and simple mode do not support residue reporting, so the1289		 * residue field will always be 0.1290		 */1291		if (chan->has_sg && chan->xdev->dma_config->dmatype != XDMA_TYPE_VDMA)1292			residue = xilinx_dma_get_residue(chan, desc);1293	}1294	spin_unlock_irqrestore(&chan->lock, flags);1295 1296	dma_set_residue(txstate, residue);1297 1298	return ret;1299}1300 1301/**1302 * xilinx_dma_stop_transfer - Halt DMA channel1303 * @chan: Driver specific DMA channel1304 *1305 * Return: '0' on success and failure value on error1306 */1307static int xilinx_dma_stop_transfer(struct xilinx_dma_chan *chan)1308{1309	u32 val;1310 1311	dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RUNSTOP);1312 1313	/* Wait for the hardware to halt */1314	return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val,1315				       val & XILINX_DMA_DMASR_HALTED, 0,1316				       XILINX_DMA_LOOP_COUNT);1317}1318 1319/**1320 * xilinx_cdma_stop_transfer - Wait for the current transfer to complete1321 * @chan: Driver specific DMA channel1322 *1323 * Return: '0' on success and failure value on error1324 */1325static int xilinx_cdma_stop_transfer(struct xilinx_dma_chan *chan)1326{1327	u32 val;1328 1329	return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val,1330				       val & XILINX_DMA_DMASR_IDLE, 0,1331				       XILINX_DMA_LOOP_COUNT);1332}1333 1334/**1335 * xilinx_dma_start - Start DMA channel1336 * @chan: Driver specific DMA channel1337 */1338static void xilinx_dma_start(struct xilinx_dma_chan *chan)1339{1340	int err;1341	u32 val;1342 1343	dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RUNSTOP);1344 1345	/* Wait for the hardware to start */1346	err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val,1347				      !(val & XILINX_DMA_DMASR_HALTED), 0,1348				      XILINX_DMA_LOOP_COUNT);1349 1350	if (err) {1351		dev_err(chan->dev, "Cannot start channel %p: %x\n",1352			chan, dma_ctrl_read(chan, XILINX_DMA_REG_DMASR));1353 1354		chan->err = true;1355	}1356}1357 1358/**1359 * xilinx_vdma_start_transfer - Starts VDMA transfer1360 * @chan: Driver specific channel struct pointer1361 */1362static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan)1363{1364	struct xilinx_vdma_config *config = &chan->config;1365	struct xilinx_dma_tx_descriptor *desc;1366	u32 reg, j;1367	struct xilinx_vdma_tx_segment *segment, *last = NULL;1368	int i = 0;1369 1370	/* This function was invoked with lock held */1371	if (chan->err)1372		return;1373 1374	if (!chan->idle)1375		return;1376 1377	if (list_empty(&chan->pending_list))1378		return;1379 1380	desc = list_first_entry(&chan->pending_list,1381				struct xilinx_dma_tx_descriptor, node);1382 1383	/* Configure the hardware using info in the config structure */1384	if (chan->has_vflip) {1385		reg = dma_read(chan, XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP);1386		reg &= ~XILINX_VDMA_ENABLE_VERTICAL_FLIP;1387		reg |= config->vflip_en;1388		dma_write(chan, XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP,1389			  reg);1390	}1391 1392	reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);1393 1394	if (config->frm_cnt_en)1395		reg |= XILINX_DMA_DMACR_FRAMECNT_EN;1396	else1397		reg &= ~XILINX_DMA_DMACR_FRAMECNT_EN;1398 1399	/* If not parking, enable circular mode */1400	if (config->park)1401		reg &= ~XILINX_DMA_DMACR_CIRC_EN;1402	else1403		reg |= XILINX_DMA_DMACR_CIRC_EN;1404 1405	dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);1406 1407	j = chan->desc_submitcount;1408	reg = dma_read(chan, XILINX_DMA_REG_PARK_PTR);1409	if (chan->direction == DMA_MEM_TO_DEV) {1410		reg &= ~XILINX_DMA_PARK_PTR_RD_REF_MASK;1411		reg |= j << XILINX_DMA_PARK_PTR_RD_REF_SHIFT;1412	} else {1413		reg &= ~XILINX_DMA_PARK_PTR_WR_REF_MASK;1414		reg |= j << XILINX_DMA_PARK_PTR_WR_REF_SHIFT;1415	}1416	dma_write(chan, XILINX_DMA_REG_PARK_PTR, reg);1417 1418	/* Start the hardware */1419	xilinx_dma_start(chan);1420 1421	if (chan->err)1422		return;1423 1424	/* Start the transfer */1425	if (chan->desc_submitcount < chan->num_frms)1426		i = chan->desc_submitcount;1427 1428	list_for_each_entry(segment, &desc->segments, node) {1429		if (chan->ext_addr)1430			vdma_desc_write_64(chan,1431				   XILINX_VDMA_REG_START_ADDRESS_64(i++),1432				   segment->hw.buf_addr,1433				   segment->hw.buf_addr_msb);1434		else1435			vdma_desc_write(chan,1436					XILINX_VDMA_REG_START_ADDRESS(i++),1437					segment->hw.buf_addr);1438 1439		last = segment;1440	}1441 1442	if (!last)1443		return;1444 1445	/* HW expects these parameters to be same for one transaction */1446	vdma_desc_write(chan, XILINX_DMA_REG_HSIZE, last->hw.hsize);1447	vdma_desc_write(chan, XILINX_DMA_REG_FRMDLY_STRIDE,1448			last->hw.stride);1449	vdma_desc_write(chan, XILINX_DMA_REG_VSIZE, last->hw.vsize);1450 1451	chan->desc_submitcount++;1452	chan->desc_pendingcount--;1453	list_move_tail(&desc->node, &chan->active_list);1454	if (chan->desc_submitcount == chan->num_frms)1455		chan->desc_submitcount = 0;1456 1457	chan->idle = false;1458}1459 1460/**1461 * xilinx_cdma_start_transfer - Starts cdma transfer1462 * @chan: Driver specific channel struct pointer1463 */1464static void xilinx_cdma_start_transfer(struct xilinx_dma_chan *chan)1465{1466	struct xilinx_dma_tx_descriptor *head_desc, *tail_desc;1467	struct xilinx_cdma_tx_segment *tail_segment;1468	u32 ctrl_reg = dma_read(chan, XILINX_DMA_REG_DMACR);1469 1470	if (chan->err)1471		return;1472 1473	if (!chan->idle)1474		return;1475 1476	if (list_empty(&chan->pending_list))1477		return;1478 1479	head_desc = list_first_entry(&chan->pending_list,1480				     struct xilinx_dma_tx_descriptor, node);1481	tail_desc = list_last_entry(&chan->pending_list,1482				    struct xilinx_dma_tx_descriptor, node);1483	tail_segment = list_last_entry(&tail_desc->segments,1484				       struct xilinx_cdma_tx_segment, node);1485 1486	if (chan->desc_pendingcount <= XILINX_DMA_COALESCE_MAX) {1487		ctrl_reg &= ~XILINX_DMA_CR_COALESCE_MAX;1488		ctrl_reg |= chan->desc_pendingcount <<1489				XILINX_DMA_CR_COALESCE_SHIFT;1490		dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, ctrl_reg);1491	}1492 1493	if (chan->has_sg) {1494		dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR,1495			     XILINX_CDMA_CR_SGMODE);1496 1497		dma_ctrl_set(chan, XILINX_DMA_REG_DMACR,1498			     XILINX_CDMA_CR_SGMODE);1499 1500		xilinx_write(chan, XILINX_DMA_REG_CURDESC,1501			     head_desc->async_tx.phys);1502 1503		/* Update tail ptr register which will start the transfer */1504		xilinx_write(chan, XILINX_DMA_REG_TAILDESC,1505			     tail_segment->phys);1506	} else {1507		/* In simple mode */1508		struct xilinx_cdma_tx_segment *segment;1509		struct xilinx_cdma_desc_hw *hw;1510 1511		segment = list_first_entry(&head_desc->segments,1512					   struct xilinx_cdma_tx_segment,1513					   node);1514 1515		hw = &segment->hw;1516 1517		xilinx_write(chan, XILINX_CDMA_REG_SRCADDR,1518			     xilinx_prep_dma_addr_t(hw->src_addr));1519		xilinx_write(chan, XILINX_CDMA_REG_DSTADDR,1520			     xilinx_prep_dma_addr_t(hw->dest_addr));1521 1522		/* Start the transfer */1523		dma_ctrl_write(chan, XILINX_DMA_REG_BTT,1524				hw->control & chan->xdev->max_buffer_len);1525	}1526 1527	list_splice_tail_init(&chan->pending_list, &chan->active_list);1528	chan->desc_pendingcount = 0;1529	chan->idle = false;1530}1531 1532/**1533 * xilinx_dma_start_transfer - Starts DMA transfer1534 * @chan: Driver specific channel struct pointer1535 */1536static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan)1537{1538	struct xilinx_dma_tx_descriptor *head_desc, *tail_desc;1539	struct xilinx_axidma_tx_segment *tail_segment;1540	u32 reg;1541 1542	if (chan->err)1543		return;1544 1545	if (list_empty(&chan->pending_list))1546		return;1547 1548	if (!chan->idle)1549		return;1550 1551	head_desc = list_first_entry(&chan->pending_list,1552				     struct xilinx_dma_tx_descriptor, node);1553	tail_desc = list_last_entry(&chan->pending_list,1554				    struct xilinx_dma_tx_descriptor, node);1555	tail_segment = list_last_entry(&tail_desc->segments,1556				       struct xilinx_axidma_tx_segment, node);1557 1558	reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);1559 1560	if (chan->desc_pendingcount <= XILINX_DMA_COALESCE_MAX) {1561		reg &= ~XILINX_DMA_CR_COALESCE_MAX;1562		reg |= chan->desc_pendingcount <<1563				  XILINX_DMA_CR_COALESCE_SHIFT;1564		dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);1565	}1566 1567	if (chan->has_sg)1568		xilinx_write(chan, XILINX_DMA_REG_CURDESC,1569			     head_desc->async_tx.phys);1570	reg  &= ~XILINX_DMA_CR_DELAY_MAX;1571	reg  |= chan->irq_delay << XILINX_DMA_CR_DELAY_SHIFT;1572	dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);1573 1574	xilinx_dma_start(chan);1575 1576	if (chan->err)1577		return;1578 1579	/* Start the transfer */1580	if (chan->has_sg) {1581		if (chan->cyclic)1582			xilinx_write(chan, XILINX_DMA_REG_TAILDESC,1583				     chan->cyclic_seg_v->phys);1584		else1585			xilinx_write(chan, XILINX_DMA_REG_TAILDESC,1586				     tail_segment->phys);1587	} else {1588		struct xilinx_axidma_tx_segment *segment;1589		struct xilinx_axidma_desc_hw *hw;1590 1591		segment = list_first_entry(&head_desc->segments,1592					   struct xilinx_axidma_tx_segment,1593					   node);1594		hw = &segment->hw;1595 1596		xilinx_write(chan, XILINX_DMA_REG_SRCDSTADDR,1597			     xilinx_prep_dma_addr_t(hw->buf_addr));1598 1599		/* Start the transfer */1600		dma_ctrl_write(chan, XILINX_DMA_REG_BTT,1601			       hw->control & chan->xdev->max_buffer_len);1602	}1603 1604	list_splice_tail_init(&chan->pending_list, &chan->active_list);1605	chan->desc_pendingcount = 0;1606	chan->idle = false;1607}1608 1609/**1610 * xilinx_mcdma_start_transfer - Starts MCDMA transfer1611 * @chan: Driver specific channel struct pointer1612 */1613static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan)1614{1615	struct xilinx_dma_tx_descriptor *head_desc, *tail_desc;1616	struct xilinx_aximcdma_tx_segment *tail_segment;1617	u32 reg;1618 1619	/*1620	 * lock has been held by calling functions, so we don't need it1621	 * to take it here again.1622	 */1623 1624	if (chan->err)1625		return;1626 1627	if (!chan->idle)1628		return;1629 1630	if (list_empty(&chan->pending_list))1631		return;1632 1633	head_desc = list_first_entry(&chan->pending_list,1634				     struct xilinx_dma_tx_descriptor, node);1635	tail_desc = list_last_entry(&chan->pending_list,1636				    struct xilinx_dma_tx_descriptor, node);1637	tail_segment = list_last_entry(&tail_desc->segments,1638				       struct xilinx_aximcdma_tx_segment, node);1639 1640	reg = dma_ctrl_read(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest));1641 1642	if (chan->desc_pendingcount <= XILINX_MCDMA_COALESCE_MAX) {1643		reg &= ~XILINX_MCDMA_COALESCE_MASK;1644		reg |= chan->desc_pendingcount <<1645			XILINX_MCDMA_COALESCE_SHIFT;1646	}1647 1648	reg |= XILINX_MCDMA_IRQ_ALL_MASK;1649	dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg);1650 1651	/* Program current descriptor */1652	xilinx_write(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET(chan->tdest),1653		     head_desc->async_tx.phys);1654 1655	/* Program channel enable register */1656	reg = dma_ctrl_read(chan, XILINX_MCDMA_CHEN_OFFSET);1657	reg |= BIT(chan->tdest);1658	dma_ctrl_write(chan, XILINX_MCDMA_CHEN_OFFSET, reg);1659 1660	/* Start the fetch of BDs for the channel */1661	reg = dma_ctrl_read(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest));1662	reg |= XILINX_MCDMA_CR_RUNSTOP_MASK;1663	dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg);1664 1665	xilinx_dma_start(chan);1666 1667	if (chan->err)1668		return;1669 1670	/* Start the transfer */1671	xilinx_write(chan, XILINX_MCDMA_CHAN_TDESC_OFFSET(chan->tdest),1672		     tail_segment->phys);1673 1674	list_splice_tail_init(&chan->pending_list, &chan->active_list);1675	chan->desc_pendingcount = 0;1676	chan->idle = false;1677}1678 1679/**1680 * xilinx_dma_issue_pending - Issue pending transactions1681 * @dchan: DMA channel1682 */1683static void xilinx_dma_issue_pending(struct dma_chan *dchan)1684{1685	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);1686	unsigned long flags;1687 1688	spin_lock_irqsave(&chan->lock, flags);1689	chan->start_transfer(chan);1690	spin_unlock_irqrestore(&chan->lock, flags);1691}1692 1693/**1694 * xilinx_dma_device_config - Configure the DMA channel1695 * @dchan: DMA channel1696 * @config: channel configuration1697 *1698 * Return: 0 always.1699 */1700static int xilinx_dma_device_config(struct dma_chan *dchan,1701				    struct dma_slave_config *config)1702{1703	return 0;1704}1705 1706/**1707 * xilinx_dma_complete_descriptor - Mark the active descriptor as complete1708 * @chan : xilinx DMA channel1709 *1710 * CONTEXT: hardirq1711 */1712static void xilinx_dma_complete_descriptor(struct xilinx_dma_chan *chan)1713{1714	struct xilinx_dma_tx_descriptor *desc, *next;1715 1716	/* This function was invoked with lock held */1717	if (list_empty(&chan->active_list))1718		return;1719 1720	list_for_each_entry_safe(desc, next, &chan->active_list, node) {1721		if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {1722			struct xilinx_axidma_tx_segment *seg;1723 1724			seg = list_last_entry(&desc->segments,1725					      struct xilinx_axidma_tx_segment, node);1726			if (!(seg->hw.status & XILINX_DMA_BD_COMP_MASK) && chan->has_sg)1727				break;1728		}1729		if (chan->has_sg && chan->xdev->dma_config->dmatype !=1730		    XDMA_TYPE_VDMA)1731			desc->residue = xilinx_dma_get_residue(chan, desc);1732		else1733			desc->residue = 0;1734		desc->err = chan->err;1735 1736		list_del(&desc->node);1737		if (!desc->cyclic)1738			dma_cookie_complete(&desc->async_tx);1739		list_add_tail(&desc->node, &chan->done_list);1740	}1741}1742 1743/**1744 * xilinx_dma_reset - Reset DMA channel1745 * @chan: Driver specific DMA channel1746 *1747 * Return: '0' on success and failure value on error1748 */1749static int xilinx_dma_reset(struct xilinx_dma_chan *chan)1750{1751	int err;1752	u32 tmp;1753 1754	dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RESET);1755 1756	/* Wait for the hardware to finish reset */1757	err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMACR, tmp,1758				      !(tmp & XILINX_DMA_DMACR_RESET), 0,1759				      XILINX_DMA_LOOP_COUNT);1760 1761	if (err) {1762		dev_err(chan->dev, "reset timeout, cr %x, sr %x\n",1763			dma_ctrl_read(chan, XILINX_DMA_REG_DMACR),1764			dma_ctrl_read(chan, XILINX_DMA_REG_DMASR));1765		return -ETIMEDOUT;1766	}1767 1768	chan->err = false;1769	chan->idle = true;1770	chan->desc_pendingcount = 0;1771	chan->desc_submitcount = 0;1772 1773	return err;1774}1775 1776/**1777 * xilinx_dma_chan_reset - Reset DMA channel and enable interrupts1778 * @chan: Driver specific DMA channel1779 *1780 * Return: '0' on success and failure value on error1781 */1782static int xilinx_dma_chan_reset(struct xilinx_dma_chan *chan)1783{1784	int err;1785 1786	/* Reset VDMA */1787	err = xilinx_dma_reset(chan);1788	if (err)1789		return err;1790 1791	/* Enable interrupts */1792	dma_ctrl_set(chan, XILINX_DMA_REG_DMACR,1793		      XILINX_DMA_DMAXR_ALL_IRQ_MASK);1794 1795	return 0;1796}1797 1798/**1799 * xilinx_mcdma_irq_handler - MCDMA Interrupt handler1800 * @irq: IRQ number1801 * @data: Pointer to the Xilinx MCDMA channel structure1802 *1803 * Return: IRQ_HANDLED/IRQ_NONE1804 */1805static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data)1806{1807	struct xilinx_dma_chan *chan = data;1808	u32 status, ser_offset, chan_sermask, chan_offset = 0, chan_id;1809 1810	if (chan->direction == DMA_DEV_TO_MEM)1811		ser_offset = XILINX_MCDMA_RXINT_SER_OFFSET;1812	else1813		ser_offset = XILINX_MCDMA_TXINT_SER_OFFSET;1814 1815	/* Read the channel id raising the interrupt*/1816	chan_sermask = dma_ctrl_read(chan, ser_offset);1817	chan_id = ffs(chan_sermask);1818 1819	if (!chan_id)1820		return IRQ_NONE;1821 1822	if (chan->direction == DMA_DEV_TO_MEM)1823		chan_offset = chan->xdev->dma_config->max_channels / 2;1824 1825	chan_offset = chan_offset + (chan_id - 1);1826	chan = chan->xdev->chan[chan_offset];1827	/* Read the status and ack the interrupts. */1828	status = dma_ctrl_read(chan, XILINX_MCDMA_CHAN_SR_OFFSET(chan->tdest));1829	if (!(status & XILINX_MCDMA_IRQ_ALL_MASK))1830		return IRQ_NONE;1831 1832	dma_ctrl_write(chan, XILINX_MCDMA_CHAN_SR_OFFSET(chan->tdest),1833		       status & XILINX_MCDMA_IRQ_ALL_MASK);1834 1835	if (status & XILINX_MCDMA_IRQ_ERR_MASK) {1836		dev_err(chan->dev, "Channel %p has errors %x cdr %x tdr %x\n",1837			chan,1838			dma_ctrl_read(chan, XILINX_MCDMA_CH_ERR_OFFSET),1839			dma_ctrl_read(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET1840				      (chan->tdest)),1841			dma_ctrl_read(chan, XILINX_MCDMA_CHAN_TDESC_OFFSET1842				      (chan->tdest)));1843		chan->err = true;1844	}1845 1846	if (status & XILINX_MCDMA_IRQ_DELAY_MASK) {1847		/*1848		 * Device takes too long to do the transfer when user requires1849		 * responsiveness.1850		 */1851		dev_dbg(chan->dev, "Inter-packet latency too long\n");1852	}1853 1854	if (status & XILINX_MCDMA_IRQ_IOC_MASK) {1855		spin_lock(&chan->lock);1856		xilinx_dma_complete_descriptor(chan);1857		chan->idle = true;1858		chan->start_transfer(chan);1859		spin_unlock(&chan->lock);1860	}1861 1862	tasklet_hi_schedule(&chan->tasklet);1863	return IRQ_HANDLED;1864}1865 1866/**1867 * xilinx_dma_irq_handler - DMA Interrupt handler1868 * @irq: IRQ number1869 * @data: Pointer to the Xilinx DMA channel structure1870 *1871 * Return: IRQ_HANDLED/IRQ_NONE1872 */1873static irqreturn_t xilinx_dma_irq_handler(int irq, void *data)1874{1875	struct xilinx_dma_chan *chan = data;1876	u32 status;1877 1878	/* Read the status and ack the interrupts. */1879	status = dma_ctrl_read(chan, XILINX_DMA_REG_DMASR);1880	if (!(status & XILINX_DMA_DMAXR_ALL_IRQ_MASK))1881		return IRQ_NONE;1882 1883	dma_ctrl_write(chan, XILINX_DMA_REG_DMASR,1884			status & XILINX_DMA_DMAXR_ALL_IRQ_MASK);1885 1886	if (status & XILINX_DMA_DMASR_ERR_IRQ) {1887		/*1888		 * An error occurred. If C_FLUSH_ON_FSYNC is enabled and the1889		 * error is recoverable, ignore it. Otherwise flag the error.1890		 *1891		 * Only recoverable errors can be cleared in the DMASR register,1892		 * make sure not to write to other error bits to 1.1893		 */1894		u32 errors = status & XILINX_DMA_DMASR_ALL_ERR_MASK;1895 1896		dma_ctrl_write(chan, XILINX_DMA_REG_DMASR,1897				errors & XILINX_DMA_DMASR_ERR_RECOVER_MASK);1898 1899		if (!chan->flush_on_fsync ||1900		    (errors & ~XILINX_DMA_DMASR_ERR_RECOVER_MASK)) {1901			dev_err(chan->dev,1902				"Channel %p has errors %x, cdr %x tdr %x\n",1903				chan, errors,1904				dma_ctrl_read(chan, XILINX_DMA_REG_CURDESC),1905				dma_ctrl_read(chan, XILINX_DMA_REG_TAILDESC));1906			chan->err = true;1907		}1908	}1909 1910	if (status & (XILINX_DMA_DMASR_FRM_CNT_IRQ |1911		      XILINX_DMA_DMASR_DLY_CNT_IRQ)) {1912		spin_lock(&chan->lock);1913		xilinx_dma_complete_descriptor(chan);1914		chan->idle = true;1915		chan->start_transfer(chan);1916		spin_unlock(&chan->lock);1917	}1918 1919	tasklet_schedule(&chan->tasklet);1920	return IRQ_HANDLED;1921}1922 1923/**1924 * append_desc_queue - Queuing descriptor1925 * @chan: Driver specific dma channel1926 * @desc: dma transaction descriptor1927 */1928static void append_desc_queue(struct xilinx_dma_chan *chan,1929			      struct xilinx_dma_tx_descriptor *desc)1930{1931	struct xilinx_vdma_tx_segment *tail_segment;1932	struct xilinx_dma_tx_descriptor *tail_desc;1933	struct xilinx_axidma_tx_segment *axidma_tail_segment;1934	struct xilinx_aximcdma_tx_segment *aximcdma_tail_segment;1935	struct xilinx_cdma_tx_segment *cdma_tail_segment;1936 1937	if (list_empty(&chan->pending_list))1938		goto append;1939 1940	/*1941	 * Add the hardware descriptor to the chain of hardware descriptors1942	 * that already exists in memory.1943	 */1944	tail_desc = list_last_entry(&chan->pending_list,1945				    struct xilinx_dma_tx_descriptor, node);1946	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {1947		tail_segment = list_last_entry(&tail_desc->segments,1948					       struct xilinx_vdma_tx_segment,1949					       node);1950		tail_segment->hw.next_desc = (u32)desc->async_tx.phys;1951	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {1952		cdma_tail_segment = list_last_entry(&tail_desc->segments,1953						struct xilinx_cdma_tx_segment,1954						node);1955		cdma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys;1956	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {1957		axidma_tail_segment = list_last_entry(&tail_desc->segments,1958					       struct xilinx_axidma_tx_segment,1959					       node);1960		axidma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys;1961	} else {1962		aximcdma_tail_segment =1963			list_last_entry(&tail_desc->segments,1964					struct xilinx_aximcdma_tx_segment,1965					node);1966		aximcdma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys;1967	}1968 1969	/*1970	 * Add the software descriptor and all children to the list1971	 * of pending transactions1972	 */1973append:1974	list_add_tail(&desc->node, &chan->pending_list);1975	chan->desc_pendingcount++;1976 1977	if (chan->has_sg && (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA)1978	    && unlikely(chan->desc_pendingcount > chan->num_frms)) {1979		dev_dbg(chan->dev, "desc pendingcount is too high\n");1980		chan->desc_pendingcount = chan->num_frms;1981	}1982}1983 1984/**1985 * xilinx_dma_tx_submit - Submit DMA transaction1986 * @tx: Async transaction descriptor1987 *1988 * Return: cookie value on success and failure value on error1989 */1990static dma_cookie_t xilinx_dma_tx_submit(struct dma_async_tx_descriptor *tx)1991{1992	struct xilinx_dma_tx_descriptor *desc = to_dma_tx_descriptor(tx);1993	struct xilinx_dma_chan *chan = to_xilinx_chan(tx->chan);1994	dma_cookie_t cookie;1995	unsigned long flags;1996	int err;1997 1998	if (chan->cyclic) {1999		xilinx_dma_free_tx_descriptor(chan, desc);2000		return -EBUSY;2001	}2002 2003	if (chan->err) {2004		/*2005		 * If reset fails, need to hard reset the system.2006		 * Channel is no longer functional2007		 */2008		err = xilinx_dma_chan_reset(chan);2009		if (err < 0)2010			return err;2011	}2012 2013	spin_lock_irqsave(&chan->lock, flags);2014 2015	cookie = dma_cookie_assign(tx);2016 2017	/* Put this transaction onto the tail of the pending queue */2018	append_desc_queue(chan, desc);2019 2020	if (desc->cyclic)2021		chan->cyclic = true;2022 2023	chan->terminating = false;2024 2025	spin_unlock_irqrestore(&chan->lock, flags);2026 2027	return cookie;2028}2029 2030/**2031 * xilinx_vdma_dma_prep_interleaved - prepare a descriptor for a2032 *	DMA_SLAVE transaction2033 * @dchan: DMA channel2034 * @xt: Interleaved template pointer2035 * @flags: transfer ack flags2036 *2037 * Return: Async transaction descriptor on success and NULL on failure2038 */2039static struct dma_async_tx_descriptor *2040xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan,2041				 struct dma_interleaved_template *xt,2042				 unsigned long flags)2043{2044	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);2045	struct xilinx_dma_tx_descriptor *desc;2046	struct xilinx_vdma_tx_segment *segment;2047	struct xilinx_vdma_desc_hw *hw;2048 2049	if (!is_slave_direction(xt->dir))2050		return NULL;2051 2052	if (!xt->numf || !xt->sgl[0].size)2053		return NULL;2054 2055	if (xt->numf & ~XILINX_DMA_VSIZE_MASK ||2056	    xt->sgl[0].size & ~XILINX_DMA_HSIZE_MASK)2057		return NULL;2058 2059	if (xt->frame_size != 1)2060		return NULL;2061 2062	/* Allocate a transaction descriptor. */2063	desc = xilinx_dma_alloc_tx_descriptor(chan);2064	if (!desc)2065		return NULL;2066 2067	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);2068	desc->async_tx.tx_submit = xilinx_dma_tx_submit;2069	async_tx_ack(&desc->async_tx);2070 2071	/* Allocate the link descriptor from DMA pool */2072	segment = xilinx_vdma_alloc_tx_segment(chan);2073	if (!segment)2074		goto error;2075 2076	/* Fill in the hardware descriptor */2077	hw = &segment->hw;2078	hw->vsize = xt->numf;2079	hw->hsize = xt->sgl[0].size;2080	hw->stride = (xt->sgl[0].icg + xt->sgl[0].size) <<2081			XILINX_DMA_FRMDLY_STRIDE_STRIDE_SHIFT;2082	hw->stride |= chan->config.frm_dly <<2083			XILINX_DMA_FRMDLY_STRIDE_FRMDLY_SHIFT;2084 2085	if (xt->dir != DMA_MEM_TO_DEV) {2086		if (chan->ext_addr) {2087			hw->buf_addr = lower_32_bits(xt->dst_start);2088			hw->buf_addr_msb = upper_32_bits(xt->dst_start);2089		} else {2090			hw->buf_addr = xt->dst_start;2091		}2092	} else {2093		if (chan->ext_addr) {2094			hw->buf_addr = lower_32_bits(xt->src_start);2095			hw->buf_addr_msb = upper_32_bits(xt->src_start);2096		} else {2097			hw->buf_addr = xt->src_start;2098		}2099	}2100 2101	/* Insert the segment into the descriptor segments list. */2102	list_add_tail(&segment->node, &desc->segments);2103 2104	/* Link the last hardware descriptor with the first. */2105	segment = list_first_entry(&desc->segments,2106				   struct xilinx_vdma_tx_segment, node);2107	desc->async_tx.phys = segment->phys;2108 2109	return &desc->async_tx;2110 2111error:2112	xilinx_dma_free_tx_descriptor(chan, desc);2113	return NULL;2114}2115 2116/**2117 * xilinx_cdma_prep_memcpy - prepare descriptors for a memcpy transaction2118 * @dchan: DMA channel2119 * @dma_dst: destination address2120 * @dma_src: source address2121 * @len: transfer length2122 * @flags: transfer ack flags2123 *2124 * Return: Async transaction descriptor on success and NULL on failure2125 */2126static struct dma_async_tx_descriptor *2127xilinx_cdma_prep_memcpy(struct dma_chan *dchan, dma_addr_t dma_dst,2128			dma_addr_t dma_src, size_t len, unsigned long flags)2129{2130	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);2131	struct xilinx_dma_tx_descriptor *desc;2132	struct xilinx_cdma_tx_segment *segment;2133	struct xilinx_cdma_desc_hw *hw;2134 2135	if (!len || len > chan->xdev->max_buffer_len)2136		return NULL;2137 2138	desc = xilinx_dma_alloc_tx_descriptor(chan);2139	if (!desc)2140		return NULL;2141 2142	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);2143	desc->async_tx.tx_submit = xilinx_dma_tx_submit;2144 2145	/* Allocate the link descriptor from DMA pool */2146	segment = xilinx_cdma_alloc_tx_segment(chan);2147	if (!segment)2148		goto error;2149 2150	hw = &segment->hw;2151	hw->control = len;2152	hw->src_addr = dma_src;2153	hw->dest_addr = dma_dst;2154	if (chan->ext_addr) {2155		hw->src_addr_msb = upper_32_bits(dma_src);2156		hw->dest_addr_msb = upper_32_bits(dma_dst);2157	}2158 2159	/* Insert the segment into the descriptor segments list. */2160	list_add_tail(&segment->node, &desc->segments);2161 2162	desc->async_tx.phys = segment->phys;2163	hw->next_desc = segment->phys;2164 2165	return &desc->async_tx;2166 2167error:2168	xilinx_dma_free_tx_descriptor(chan, desc);2169	return NULL;2170}2171 2172/**2173 * xilinx_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction2174 * @dchan: DMA channel2175 * @sgl: scatterlist to transfer to/from2176 * @sg_len: number of entries in @scatterlist2177 * @direction: DMA direction2178 * @flags: transfer ack flags2179 * @context: APP words of the descriptor2180 *2181 * Return: Async transaction descriptor on success and NULL on failure2182 */2183static struct dma_async_tx_descriptor *xilinx_dma_prep_slave_sg(2184	struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len,2185	enum dma_transfer_direction direction, unsigned long flags,2186	void *context)2187{2188	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);2189	struct xilinx_dma_tx_descriptor *desc;2190	struct xilinx_axidma_tx_segment *segment = NULL;2191	u32 *app_w = (u32 *)context;2192	struct scatterlist *sg;2193	size_t copy;2194	size_t sg_used;2195	unsigned int i;2196 2197	if (!is_slave_direction(direction))2198		return NULL;2199 2200	/* Allocate a transaction descriptor. */2201	desc = xilinx_dma_alloc_tx_descriptor(chan);2202	if (!desc)2203		return NULL;2204 2205	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);2206	desc->async_tx.tx_submit = xilinx_dma_tx_submit;2207 2208	/* Build transactions using information in the scatter gather list */2209	for_each_sg(sgl, sg, sg_len, i) {2210		sg_used = 0;2211 2212		/* Loop until the entire scatterlist entry is used */2213		while (sg_used < sg_dma_len(sg)) {2214			struct xilinx_axidma_desc_hw *hw;2215 2216			/* Get a free segment */2217			segment = xilinx_axidma_alloc_tx_segment(chan);2218			if (!segment)2219				goto error;2220 2221			/*2222			 * Calculate the maximum number of bytes to transfer,2223			 * making sure it is less than the hw limit2224			 */2225			copy = xilinx_dma_calc_copysize(chan, sg_dma_len(sg),2226							sg_used);2227			hw = &segment->hw;2228 2229			/* Fill in the descriptor */2230			xilinx_axidma_buf(chan, hw, sg_dma_address(sg),2231					  sg_used, 0);2232 2233			hw->control = copy;2234 2235			if (chan->direction == DMA_MEM_TO_DEV) {2236				if (app_w)2237					memcpy(hw->app, app_w, sizeof(u32) *2238					       XILINX_DMA_NUM_APP_WORDS);2239			}2240 2241			sg_used += copy;2242 2243			/*2244			 * Insert the segment into the descriptor segments2245			 * list.2246			 */2247			list_add_tail(&segment->node, &desc->segments);2248		}2249	}2250 2251	segment = list_first_entry(&desc->segments,2252				   struct xilinx_axidma_tx_segment, node);2253	desc->async_tx.phys = segment->phys;2254 2255	/* For the last DMA_MEM_TO_DEV transfer, set EOP */2256	if (chan->direction == DMA_MEM_TO_DEV) {2257		segment->hw.control |= XILINX_DMA_BD_SOP;2258		segment = list_last_entry(&desc->segments,2259					  struct xilinx_axidma_tx_segment,2260					  node);2261		segment->hw.control |= XILINX_DMA_BD_EOP;2262	}2263 2264	if (chan->xdev->has_axistream_connected)2265		desc->async_tx.metadata_ops = &xilinx_dma_metadata_ops;2266 2267	return &desc->async_tx;2268 2269error:2270	xilinx_dma_free_tx_descriptor(chan, desc);2271	return NULL;2272}2273 2274/**2275 * xilinx_dma_prep_dma_cyclic - prepare descriptors for a DMA_SLAVE transaction2276 * @dchan: DMA channel2277 * @buf_addr: Physical address of the buffer2278 * @buf_len: Total length of the cyclic buffers2279 * @period_len: length of individual cyclic buffer2280 * @direction: DMA direction2281 * @flags: transfer ack flags2282 *2283 * Return: Async transaction descriptor on success and NULL on failure2284 */2285static struct dma_async_tx_descriptor *xilinx_dma_prep_dma_cyclic(2286	struct dma_chan *dchan, dma_addr_t buf_addr, size_t buf_len,2287	size_t period_len, enum dma_transfer_direction direction,2288	unsigned long flags)2289{2290	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);2291	struct xilinx_dma_tx_descriptor *desc;2292	struct xilinx_axidma_tx_segment *segment, *head_segment, *prev = NULL;2293	size_t copy, sg_used;2294	unsigned int num_periods;2295	int i;2296	u32 reg;2297 2298	if (!period_len)2299		return NULL;2300 2301	num_periods = buf_len / period_len;2302 2303	if (!num_periods)2304		return NULL;2305 2306	if (!is_slave_direction(direction))2307		return NULL;2308 2309	/* Allocate a transaction descriptor. */2310	desc = xilinx_dma_alloc_tx_descriptor(chan);2311	if (!desc)2312		return NULL;2313 2314	chan->direction = direction;2315	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);2316	desc->async_tx.tx_submit = xilinx_dma_tx_submit;2317 2318	for (i = 0; i < num_periods; ++i) {2319		sg_used = 0;2320 2321		while (sg_used < period_len) {2322			struct xilinx_axidma_desc_hw *hw;2323 2324			/* Get a free segment */2325			segment = xilinx_axidma_alloc_tx_segment(chan);2326			if (!segment)2327				goto error;2328 2329			/*2330			 * Calculate the maximum number of bytes to transfer,2331			 * making sure it is less than the hw limit2332			 */2333			copy = xilinx_dma_calc_copysize(chan, period_len,2334							sg_used);2335			hw = &segment->hw;2336			xilinx_axidma_buf(chan, hw, buf_addr, sg_used,2337					  period_len * i);2338			hw->control = copy;2339 2340			if (prev)2341				prev->hw.next_desc = segment->phys;2342 2343			prev = segment;2344			sg_used += copy;2345 2346			/*2347			 * Insert the segment into the descriptor segments2348			 * list.2349			 */2350			list_add_tail(&segment->node, &desc->segments);2351		}2352	}2353 2354	head_segment = list_first_entry(&desc->segments,2355				   struct xilinx_axidma_tx_segment, node);2356	desc->async_tx.phys = head_segment->phys;2357 2358	desc->cyclic = true;2359	reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);2360	reg |= XILINX_DMA_CR_CYCLIC_BD_EN_MASK;2361	dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);2362 2363	segment = list_last_entry(&desc->segments,2364				  struct xilinx_axidma_tx_segment,2365				  node);2366	segment->hw.next_desc = (u32) head_segment->phys;2367 2368	/* For the last DMA_MEM_TO_DEV transfer, set EOP */2369	if (direction == DMA_MEM_TO_DEV) {2370		head_segment->hw.control |= XILINX_DMA_BD_SOP;2371		segment->hw.control |= XILINX_DMA_BD_EOP;2372	}2373 2374	return &desc->async_tx;2375 2376error:2377	xilinx_dma_free_tx_descriptor(chan, desc);2378	return NULL;2379}2380 2381/**2382 * xilinx_mcdma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction2383 * @dchan: DMA channel2384 * @sgl: scatterlist to transfer to/from2385 * @sg_len: number of entries in @scatterlist2386 * @direction: DMA direction2387 * @flags: transfer ack flags2388 * @context: APP words of the descriptor2389 *2390 * Return: Async transaction descriptor on success and NULL on failure2391 */2392static struct dma_async_tx_descriptor *2393xilinx_mcdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,2394			   unsigned int sg_len,2395			   enum dma_transfer_direction direction,2396			   unsigned long flags, void *context)2397{2398	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);2399	struct xilinx_dma_tx_descriptor *desc;2400	struct xilinx_aximcdma_tx_segment *segment = NULL;2401	u32 *app_w = (u32 *)context;2402	struct scatterlist *sg;2403	size_t copy;2404	size_t sg_used;2405	unsigned int i;2406 2407	if (!is_slave_direction(direction))2408		return NULL;2409 2410	/* Allocate a transaction descriptor. */2411	desc = xilinx_dma_alloc_tx_descriptor(chan);2412	if (!desc)2413		return NULL;2414 2415	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);2416	desc->async_tx.tx_submit = xilinx_dma_tx_submit;2417 2418	/* Build transactions using information in the scatter gather list */2419	for_each_sg(sgl, sg, sg_len, i) {2420		sg_used = 0;2421 2422		/* Loop until the entire scatterlist entry is used */2423		while (sg_used < sg_dma_len(sg)) {2424			struct xilinx_aximcdma_desc_hw *hw;2425 2426			/* Get a free segment */2427			segment = xilinx_aximcdma_alloc_tx_segment(chan);2428			if (!segment)2429				goto error;2430 2431			/*2432			 * Calculate the maximum number of bytes to transfer,2433			 * making sure it is less than the hw limit2434			 */2435			copy = min_t(size_t, sg_dma_len(sg) - sg_used,2436				     chan->xdev->max_buffer_len);2437			hw = &segment->hw;2438 2439			/* Fill in the descriptor */2440			xilinx_aximcdma_buf(chan, hw, sg_dma_address(sg),2441					    sg_used);2442			hw->control = copy;2443 2444			if (chan->direction == DMA_MEM_TO_DEV && app_w) {2445				memcpy(hw->app, app_w, sizeof(u32) *2446				       XILINX_DMA_NUM_APP_WORDS);2447			}2448 2449			sg_used += copy;2450			/*2451			 * Insert the segment into the descriptor segments2452			 * list.2453			 */2454			list_add_tail(&segment->node, &desc->segments);2455		}2456	}2457 2458	segment = list_first_entry(&desc->segments,2459				   struct xilinx_aximcdma_tx_segment, node);2460	desc->async_tx.phys = segment->phys;2461 2462	/* For the last DMA_MEM_TO_DEV transfer, set EOP */2463	if (chan->direction == DMA_MEM_TO_DEV) {2464		segment->hw.control |= XILINX_MCDMA_BD_SOP;2465		segment = list_last_entry(&desc->segments,2466					  struct xilinx_aximcdma_tx_segment,2467					  node);2468		segment->hw.control |= XILINX_MCDMA_BD_EOP;2469	}2470 2471	return &desc->async_tx;2472 2473error:2474	xilinx_dma_free_tx_descriptor(chan, desc);2475 2476	return NULL;2477}2478 2479/**2480 * xilinx_dma_terminate_all - Halt the channel and free descriptors2481 * @dchan: Driver specific DMA Channel pointer2482 *2483 * Return: '0' always.2484 */2485static int xilinx_dma_terminate_all(struct dma_chan *dchan)2486{2487	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);2488	u32 reg;2489	int err;2490 2491	if (!chan->cyclic) {2492		err = chan->stop_transfer(chan);2493		if (err) {2494			dev_err(chan->dev, "Cannot stop channel %p: %x\n",2495				chan, dma_ctrl_read(chan,2496				XILINX_DMA_REG_DMASR));2497			chan->err = true;2498		}2499	}2500 2501	xilinx_dma_chan_reset(chan);2502	/* Remove and free all of the descriptors in the lists */2503	chan->terminating = true;2504	xilinx_dma_free_descriptors(chan);2505	chan->idle = true;2506 2507	if (chan->cyclic) {2508		reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);2509		reg &= ~XILINX_DMA_CR_CYCLIC_BD_EN_MASK;2510		dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);2511		chan->cyclic = false;2512	}2513 2514	if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg)2515		dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR,2516			     XILINX_CDMA_CR_SGMODE);2517 2518	return 0;2519}2520 2521static void xilinx_dma_synchronize(struct dma_chan *dchan)2522{2523	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);2524 2525	tasklet_kill(&chan->tasklet);2526}2527 2528/**2529 * xilinx_vdma_channel_set_config - Configure VDMA channel2530 * Run-time configuration for Axi VDMA, supports:2531 * . halt the channel2532 * . configure interrupt coalescing and inter-packet delay threshold2533 * . start/stop parking2534 * . enable genlock2535 *2536 * @dchan: DMA channel2537 * @cfg: VDMA device configuration pointer2538 *2539 * Return: '0' on success and failure value on error2540 */2541int xilinx_vdma_channel_set_config(struct dma_chan *dchan,2542					struct xilinx_vdma_config *cfg)2543{2544	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);2545	u32 dmacr;2546 2547	if (cfg->reset)2548		return xilinx_dma_chan_reset(chan);2549 2550	dmacr = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);2551 2552	chan->config.frm_dly = cfg->frm_dly;2553	chan->config.park = cfg->park;2554 2555	/* genlock settings */2556	chan->config.gen_lock = cfg->gen_lock;2557	chan->config.master = cfg->master;2558 2559	dmacr &= ~XILINX_DMA_DMACR_GENLOCK_EN;2560	if (cfg->gen_lock && chan->genlock) {2561		dmacr |= XILINX_DMA_DMACR_GENLOCK_EN;2562		dmacr &= ~XILINX_DMA_DMACR_MASTER_MASK;2563		dmacr |= cfg->master << XILINX_DMA_DMACR_MASTER_SHIFT;2564	}2565 2566	chan->config.frm_cnt_en = cfg->frm_cnt_en;2567	chan->config.vflip_en = cfg->vflip_en;2568 2569	if (cfg->park)2570		chan->config.park_frm = cfg->park_frm;2571	else2572		chan->config.park_frm = -1;2573 2574	chan->config.coalesc = cfg->coalesc;2575	chan->config.delay = cfg->delay;2576 2577	if (cfg->coalesc <= XILINX_DMA_DMACR_FRAME_COUNT_MAX) {2578		dmacr &= ~XILINX_DMA_DMACR_FRAME_COUNT_MASK;2579		dmacr |= cfg->coalesc << XILINX_DMA_DMACR_FRAME_COUNT_SHIFT;2580		chan->config.coalesc = cfg->coalesc;2581	}2582 2583	if (cfg->delay <= XILINX_DMA_DMACR_DELAY_MAX) {2584		dmacr &= ~XILINX_DMA_DMACR_DELAY_MASK;2585		dmacr |= cfg->delay << XILINX_DMA_DMACR_DELAY_SHIFT;2586		chan->config.delay = cfg->delay;2587	}2588 2589	/* FSync Source selection */2590	dmacr &= ~XILINX_DMA_DMACR_FSYNCSRC_MASK;2591	dmacr |= cfg->ext_fsync << XILINX_DMA_DMACR_FSYNCSRC_SHIFT;2592 2593	dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, dmacr);2594 2595	return 0;2596}2597EXPORT_SYMBOL(xilinx_vdma_channel_set_config);2598 2599/* -----------------------------------------------------------------------------2600 * Probe and remove2601 */2602 2603/**2604 * xilinx_dma_chan_remove - Per Channel remove function2605 * @chan: Driver specific DMA channel2606 */2607static void xilinx_dma_chan_remove(struct xilinx_dma_chan *chan)2608{2609	/* Disable all interrupts */2610	dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR,2611		      XILINX_DMA_DMAXR_ALL_IRQ_MASK);2612 2613	if (chan->irq > 0)2614		free_irq(chan->irq, chan);2615 2616	tasklet_kill(&chan->tasklet);2617 2618	list_del(&chan->common.device_node);2619}2620 2621static int axidma_clk_init(struct platform_device *pdev, struct clk **axi_clk,2622			    struct clk **tx_clk, struct clk **rx_clk,2623			    struct clk **sg_clk, struct clk **tmp_clk)2624{2625	int err;2626 2627	*tmp_clk = NULL;2628 2629	*axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk");2630	if (IS_ERR(*axi_clk))2631		return dev_err_probe(&pdev->dev, PTR_ERR(*axi_clk), "failed to get axi_aclk\n");2632 2633	*tx_clk = devm_clk_get(&pdev->dev, "m_axi_mm2s_aclk");2634	if (IS_ERR(*tx_clk))2635		*tx_clk = NULL;2636 2637	*rx_clk = devm_clk_get(&pdev->dev, "m_axi_s2mm_aclk");2638	if (IS_ERR(*rx_clk))2639		*rx_clk = NULL;2640 2641	*sg_clk = devm_clk_get(&pdev->dev, "m_axi_sg_aclk");2642	if (IS_ERR(*sg_clk))2643		*sg_clk = NULL;2644 2645	err = clk_prepare_enable(*axi_clk);2646	if (err) {2647		dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n", err);2648		return err;2649	}2650 2651	err = clk_prepare_enable(*tx_clk);2652	if (err) {2653		dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err);2654		goto err_disable_axiclk;2655	}2656 2657	err = clk_prepare_enable(*rx_clk);2658	if (err) {2659		dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err);2660		goto err_disable_txclk;2661	}2662 2663	err = clk_prepare_enable(*sg_clk);2664	if (err) {2665		dev_err(&pdev->dev, "failed to enable sg_clk (%d)\n", err);2666		goto err_disable_rxclk;2667	}2668 2669	return 0;2670 2671err_disable_rxclk:2672	clk_disable_unprepare(*rx_clk);2673err_disable_txclk:2674	clk_disable_unprepare(*tx_clk);2675err_disable_axiclk:2676	clk_disable_unprepare(*axi_clk);2677 2678	return err;2679}2680 2681static int axicdma_clk_init(struct platform_device *pdev, struct clk **axi_clk,2682			    struct clk **dev_clk, struct clk **tmp_clk,2683			    struct clk **tmp1_clk, struct clk **tmp2_clk)2684{2685	int err;2686 2687	*tmp_clk = NULL;2688	*tmp1_clk = NULL;2689	*tmp2_clk = NULL;2690 2691	*axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk");2692	if (IS_ERR(*axi_clk))2693		return dev_err_probe(&pdev->dev, PTR_ERR(*axi_clk), "failed to get axi_aclk\n");2694 2695	*dev_clk = devm_clk_get(&pdev->dev, "m_axi_aclk");2696	if (IS_ERR(*dev_clk))2697		return dev_err_probe(&pdev->dev, PTR_ERR(*dev_clk), "failed to get dev_clk\n");2698 2699	err = clk_prepare_enable(*axi_clk);2700	if (err) {2701		dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n", err);2702		return err;2703	}2704 2705	err = clk_prepare_enable(*dev_clk);2706	if (err) {2707		dev_err(&pdev->dev, "failed to enable dev_clk (%d)\n", err);2708		goto err_disable_axiclk;2709	}2710 2711	return 0;2712 2713err_disable_axiclk:2714	clk_disable_unprepare(*axi_clk);2715 2716	return err;2717}2718 2719static int axivdma_clk_init(struct platform_device *pdev, struct clk **axi_clk,2720			    struct clk **tx_clk, struct clk **txs_clk,2721			    struct clk **rx_clk, struct clk **rxs_clk)2722{2723	int err;2724 2725	*axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk");2726	if (IS_ERR(*axi_clk))2727		return dev_err_probe(&pdev->dev, PTR_ERR(*axi_clk), "failed to get axi_aclk\n");2728 2729	*tx_clk = devm_clk_get(&pdev->dev, "m_axi_mm2s_aclk");2730	if (IS_ERR(*tx_clk))2731		*tx_clk = NULL;2732 2733	*txs_clk = devm_clk_get(&pdev->dev, "m_axis_mm2s_aclk");2734	if (IS_ERR(*txs_clk))2735		*txs_clk = NULL;2736 2737	*rx_clk = devm_clk_get(&pdev->dev, "m_axi_s2mm_aclk");2738	if (IS_ERR(*rx_clk))2739		*rx_clk = NULL;2740 2741	*rxs_clk = devm_clk_get(&pdev->dev, "s_axis_s2mm_aclk");2742	if (IS_ERR(*rxs_clk))2743		*rxs_clk = NULL;2744 2745	err = clk_prepare_enable(*axi_clk);2746	if (err) {2747		dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n",2748			err);2749		return err;2750	}2751 2752	err = clk_prepare_enable(*tx_clk);2753	if (err) {2754		dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err);2755		goto err_disable_axiclk;2756	}2757 2758	err = clk_prepare_enable(*txs_clk);2759	if (err) {2760		dev_err(&pdev->dev, "failed to enable txs_clk (%d)\n", err);2761		goto err_disable_txclk;2762	}2763 2764	err = clk_prepare_enable(*rx_clk);2765	if (err) {2766		dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err);2767		goto err_disable_txsclk;2768	}2769 2770	err = clk_prepare_enable(*rxs_clk);2771	if (err) {2772		dev_err(&pdev->dev, "failed to enable rxs_clk (%d)\n", err);2773		goto err_disable_rxclk;2774	}2775 2776	return 0;2777 2778err_disable_rxclk:2779	clk_disable_unprepare(*rx_clk);2780err_disable_txsclk:2781	clk_disable_unprepare(*txs_clk);2782err_disable_txclk:2783	clk_disable_unprepare(*tx_clk);2784err_disable_axiclk:2785	clk_disable_unprepare(*axi_clk);2786 2787	return err;2788}2789 2790static void xdma_disable_allclks(struct xilinx_dma_device *xdev)2791{2792	clk_disable_unprepare(xdev->rxs_clk);2793	clk_disable_unprepare(xdev->rx_clk);2794	clk_disable_unprepare(xdev->txs_clk);2795	clk_disable_unprepare(xdev->tx_clk);2796	clk_disable_unprepare(xdev->axi_clk);2797}2798 2799/**2800 * xilinx_dma_chan_probe - Per Channel Probing2801 * It get channel features from the device tree entry and2802 * initialize special channel handling routines2803 *2804 * @xdev: Driver specific device structure2805 * @node: Device node2806 *2807 * Return: '0' on success and failure value on error2808 */2809static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev,2810				  struct device_node *node)2811{2812	struct xilinx_dma_chan *chan;2813	bool has_dre = false;2814	u32 value, width;2815	int err;2816 2817	/* Allocate and initialize the channel structure */2818	chan = devm_kzalloc(xdev->dev, sizeof(*chan), GFP_KERNEL);2819	if (!chan)2820		return -ENOMEM;2821 2822	chan->dev = xdev->dev;2823	chan->xdev = xdev;2824	chan->desc_pendingcount = 0x0;2825	chan->ext_addr = xdev->ext_addr;2826	/* This variable ensures that descriptors are not2827	 * Submitted when dma engine is in progress. This variable is2828	 * Added to avoid polling for a bit in the status register to2829	 * Know dma state in the driver hot path.2830	 */2831	chan->idle = true;2832 2833	spin_lock_init(&chan->lock);2834	INIT_LIST_HEAD(&chan->pending_list);2835	INIT_LIST_HEAD(&chan->done_list);2836	INIT_LIST_HEAD(&chan->active_list);2837	INIT_LIST_HEAD(&chan->free_seg_list);2838 2839	/* Retrieve the channel properties from the device tree */2840	has_dre = of_property_read_bool(node, "xlnx,include-dre");2841 2842	of_property_read_u8(node, "xlnx,irq-delay", &chan->irq_delay);2843 2844	chan->genlock = of_property_read_bool(node, "xlnx,genlock-mode");2845 2846	err = of_property_read_u32(node, "xlnx,datawidth", &value);2847	if (err) {2848		dev_err(xdev->dev, "missing xlnx,datawidth property\n");2849		return err;2850	}2851	width = value >> 3; /* Convert bits to bytes */2852 2853	/* If data width is greater than 8 bytes, DRE is not in hw */2854	if (width > 8)2855		has_dre = false;2856 2857	if (!has_dre)2858		xdev->common.copy_align = (enum dmaengine_alignment)fls(width - 1);2859 2860	if (of_device_is_compatible(node, "xlnx,axi-vdma-mm2s-channel") ||2861	    of_device_is_compatible(node, "xlnx,axi-dma-mm2s-channel") ||2862	    of_device_is_compatible(node, "xlnx,axi-cdma-channel")) {2863		chan->direction = DMA_MEM_TO_DEV;2864		chan->id = xdev->mm2s_chan_id++;2865		chan->tdest = chan->id;2866 2867		chan->ctrl_offset = XILINX_DMA_MM2S_CTRL_OFFSET;2868		if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {2869			chan->desc_offset = XILINX_VDMA_MM2S_DESC_OFFSET;2870			chan->config.park = 1;2871 2872			if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH ||2873			    xdev->flush_on_fsync == XILINX_DMA_FLUSH_MM2S)2874				chan->flush_on_fsync = true;2875		}2876	} else if (of_device_is_compatible(node,2877					   "xlnx,axi-vdma-s2mm-channel") ||2878		   of_device_is_compatible(node,2879					   "xlnx,axi-dma-s2mm-channel")) {2880		chan->direction = DMA_DEV_TO_MEM;2881		chan->id = xdev->s2mm_chan_id++;2882		chan->tdest = chan->id - xdev->dma_config->max_channels / 2;2883		chan->has_vflip = of_property_read_bool(node,2884					"xlnx,enable-vert-flip");2885		if (chan->has_vflip) {2886			chan->config.vflip_en = dma_read(chan,2887				XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP) &2888				XILINX_VDMA_ENABLE_VERTICAL_FLIP;2889		}2890 2891		if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA)2892			chan->ctrl_offset = XILINX_MCDMA_S2MM_CTRL_OFFSET;2893		else2894			chan->ctrl_offset = XILINX_DMA_S2MM_CTRL_OFFSET;2895 2896		if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {2897			chan->desc_offset = XILINX_VDMA_S2MM_DESC_OFFSET;2898			chan->config.park = 1;2899 2900			if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH ||2901			    xdev->flush_on_fsync == XILINX_DMA_FLUSH_S2MM)2902				chan->flush_on_fsync = true;2903		}2904	} else {2905		dev_err(xdev->dev, "Invalid channel compatible node\n");2906		return -EINVAL;2907	}2908 2909	/* Request the interrupt */2910	chan->irq = of_irq_get(node, chan->tdest);2911	if (chan->irq < 0)2912		return dev_err_probe(xdev->dev, chan->irq, "failed to get irq\n");2913	err = request_irq(chan->irq, xdev->dma_config->irq_handler,2914			  IRQF_SHARED, "xilinx-dma-controller", chan);2915	if (err) {2916		dev_err(xdev->dev, "unable to request IRQ %d\n", chan->irq);2917		return err;2918	}2919 2920	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {2921		chan->start_transfer = xilinx_dma_start_transfer;2922		chan->stop_transfer = xilinx_dma_stop_transfer;2923	} else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {2924		chan->start_transfer = xilinx_mcdma_start_transfer;2925		chan->stop_transfer = xilinx_dma_stop_transfer;2926	} else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {2927		chan->start_transfer = xilinx_cdma_start_transfer;2928		chan->stop_transfer = xilinx_cdma_stop_transfer;2929	} else {2930		chan->start_transfer = xilinx_vdma_start_transfer;2931		chan->stop_transfer = xilinx_dma_stop_transfer;2932	}2933 2934	/* check if SG is enabled (only for AXIDMA, AXIMCDMA, and CDMA) */2935	if (xdev->dma_config->dmatype != XDMA_TYPE_VDMA) {2936		if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA ||2937		    dma_ctrl_read(chan, XILINX_DMA_REG_DMASR) &2938			    XILINX_DMA_DMASR_SG_MASK)2939			chan->has_sg = true;2940		dev_dbg(chan->dev, "ch %d: SG %s\n", chan->id,2941			chan->has_sg ? "enabled" : "disabled");2942	}2943 2944	/* Initialize the tasklet */2945	tasklet_setup(&chan->tasklet, xilinx_dma_do_tasklet);2946 2947	/*2948	 * Initialize the DMA channel and add it to the DMA engine channels2949	 * list.2950	 */2951	chan->common.device = &xdev->common;2952 2953	list_add_tail(&chan->common.device_node, &xdev->common.channels);2954	xdev->chan[chan->id] = chan;2955 2956	/* Reset the channel */2957	err = xilinx_dma_chan_reset(chan);2958	if (err < 0) {2959		dev_err(xdev->dev, "Reset channel failed\n");2960		return err;2961	}2962 2963	return 0;2964}2965 2966/**2967 * xilinx_dma_child_probe - Per child node probe2968 * It get number of dma-channels per child node from2969 * device-tree and initializes all the channels.2970 *2971 * @xdev: Driver specific device structure2972 * @node: Device node2973 *2974 * Return: '0' on success and failure value on error.2975 */2976static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev,2977				    struct device_node *node)2978{2979	int ret, i;2980	u32 nr_channels = 1;2981 2982	ret = of_property_read_u32(node, "dma-channels", &nr_channels);2983	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA && ret < 0)2984		dev_warn(xdev->dev, "missing dma-channels property\n");2985 2986	for (i = 0; i < nr_channels; i++) {2987		ret = xilinx_dma_chan_probe(xdev, node);2988		if (ret)2989			return ret;2990	}2991 2992	return 0;2993}2994 2995/**2996 * of_dma_xilinx_xlate - Translation function2997 * @dma_spec: Pointer to DMA specifier as found in the device tree2998 * @ofdma: Pointer to DMA controller data2999 *3000 * Return: DMA channel pointer on success and NULL on error3001 */3002static struct dma_chan *of_dma_xilinx_xlate(struct of_phandle_args *dma_spec,3003						struct of_dma *ofdma)3004{3005	struct xilinx_dma_device *xdev = ofdma->of_dma_data;3006	int chan_id = dma_spec->args[0];3007 3008	if (chan_id >= xdev->dma_config->max_channels || !xdev->chan[chan_id])3009		return NULL;3010 3011	return dma_get_slave_channel(&xdev->chan[chan_id]->common);3012}3013 3014static const struct xilinx_dma_config axidma_config = {3015	.dmatype = XDMA_TYPE_AXIDMA,3016	.clk_init = axidma_clk_init,3017	.irq_handler = xilinx_dma_irq_handler,3018	.max_channels = XILINX_DMA_MAX_CHANS_PER_DEVICE,3019};3020 3021static const struct xilinx_dma_config aximcdma_config = {3022	.dmatype = XDMA_TYPE_AXIMCDMA,3023	.clk_init = axidma_clk_init,3024	.irq_handler = xilinx_mcdma_irq_handler,3025	.max_channels = XILINX_MCDMA_MAX_CHANS_PER_DEVICE,3026};3027static const struct xilinx_dma_config axicdma_config = {3028	.dmatype = XDMA_TYPE_CDMA,3029	.clk_init = axicdma_clk_init,3030	.irq_handler = xilinx_dma_irq_handler,3031	.max_channels = XILINX_CDMA_MAX_CHANS_PER_DEVICE,3032};3033 3034static const struct xilinx_dma_config axivdma_config = {3035	.dmatype = XDMA_TYPE_VDMA,3036	.clk_init = axivdma_clk_init,3037	.irq_handler = xilinx_dma_irq_handler,3038	.max_channels = XILINX_DMA_MAX_CHANS_PER_DEVICE,3039};3040 3041static const struct of_device_id xilinx_dma_of_ids[] = {3042	{ .compatible = "xlnx,axi-dma-1.00.a", .data = &axidma_config },3043	{ .compatible = "xlnx,axi-cdma-1.00.a", .data = &axicdma_config },3044	{ .compatible = "xlnx,axi-vdma-1.00.a", .data = &axivdma_config },3045	{ .compatible = "xlnx,axi-mcdma-1.00.a", .data = &aximcdma_config },3046	{}3047};3048MODULE_DEVICE_TABLE(of, xilinx_dma_of_ids);3049 3050/**3051 * xilinx_dma_probe - Driver probe function3052 * @pdev: Pointer to the platform_device structure3053 *3054 * Return: '0' on success and failure value on error3055 */3056static int xilinx_dma_probe(struct platform_device *pdev)3057{3058	int (*clk_init)(struct platform_device *, struct clk **, struct clk **,3059			struct clk **, struct clk **, struct clk **)3060					= axivdma_clk_init;3061	struct device_node *node = pdev->dev.of_node;3062	struct xilinx_dma_device *xdev;3063	struct device_node *child, *np = pdev->dev.of_node;3064	u32 num_frames, addr_width, len_width;3065	int i, err;3066 3067	/* Allocate and initialize the DMA engine structure */3068	xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL);3069	if (!xdev)3070		return -ENOMEM;3071 3072	xdev->dev = &pdev->dev;3073	if (np) {3074		const struct of_device_id *match;3075 3076		match = of_match_node(xilinx_dma_of_ids, np);3077		if (match && match->data) {3078			xdev->dma_config = match->data;3079			clk_init = xdev->dma_config->clk_init;3080		}3081	}3082 3083	err = clk_init(pdev, &xdev->axi_clk, &xdev->tx_clk, &xdev->txs_clk,3084		       &xdev->rx_clk, &xdev->rxs_clk);3085	if (err)3086		return err;3087 3088	/* Request and map I/O memory */3089	xdev->regs = devm_platform_ioremap_resource(pdev, 0);3090	if (IS_ERR(xdev->regs)) {3091		err = PTR_ERR(xdev->regs);3092		goto disable_clks;3093	}3094	/* Retrieve the DMA engine properties from the device tree */3095	xdev->max_buffer_len = GENMASK(XILINX_DMA_MAX_TRANS_LEN_MAX - 1, 0);3096	xdev->s2mm_chan_id = xdev->dma_config->max_channels / 2;3097 3098	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA ||3099	    xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {3100		if (!of_property_read_u32(node, "xlnx,sg-length-width",3101					  &len_width)) {3102			if (len_width < XILINX_DMA_MAX_TRANS_LEN_MIN ||3103			    len_width > XILINX_DMA_V2_MAX_TRANS_LEN_MAX) {3104				dev_warn(xdev->dev,3105					 "invalid xlnx,sg-length-width property value. Using default width\n");3106			} else {3107				if (len_width > XILINX_DMA_MAX_TRANS_LEN_MAX)3108					dev_warn(xdev->dev, "Please ensure that IP supports buffer length > 23 bits\n");3109				xdev->max_buffer_len =3110					GENMASK(len_width - 1, 0);3111			}3112		}3113	}3114 3115	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {3116		xdev->has_axistream_connected =3117			of_property_read_bool(node, "xlnx,axistream-connected");3118	}3119 3120	if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {3121		err = of_property_read_u32(node, "xlnx,num-fstores",3122					   &num_frames);3123		if (err < 0) {3124			dev_err(xdev->dev,3125				"missing xlnx,num-fstores property\n");3126			goto disable_clks;3127		}3128 3129		err = of_property_read_u32(node, "xlnx,flush-fsync",3130					   &xdev->flush_on_fsync);3131		if (err < 0)3132			dev_warn(xdev->dev,3133				 "missing xlnx,flush-fsync property\n");3134	}3135 3136	err = of_property_read_u32(node, "xlnx,addrwidth", &addr_width);3137	if (err < 0)3138		dev_warn(xdev->dev, "missing xlnx,addrwidth property\n");3139 3140	if (addr_width > 32)3141		xdev->ext_addr = true;3142	else3143		xdev->ext_addr = false;3144 3145	/* Set metadata mode */3146	if (xdev->has_axistream_connected)3147		xdev->common.desc_metadata_modes = DESC_METADATA_ENGINE;3148 3149	/* Set the dma mask bits */3150	err = dma_set_mask_and_coherent(xdev->dev, DMA_BIT_MASK(addr_width));3151	if (err < 0) {3152		dev_err(xdev->dev, "DMA mask error %d\n", err);3153		goto disable_clks;3154	}3155 3156	/* Initialize the DMA engine */3157	xdev->common.dev = &pdev->dev;3158 3159	INIT_LIST_HEAD(&xdev->common.channels);3160	if (!(xdev->dma_config->dmatype == XDMA_TYPE_CDMA)) {3161		dma_cap_set(DMA_SLAVE, xdev->common.cap_mask);3162		dma_cap_set(DMA_PRIVATE, xdev->common.cap_mask);3163	}3164 3165	xdev->common.device_alloc_chan_resources =3166				xilinx_dma_alloc_chan_resources;3167	xdev->common.device_free_chan_resources =3168				xilinx_dma_free_chan_resources;3169	xdev->common.device_terminate_all = xilinx_dma_terminate_all;3170	xdev->common.device_synchronize = xilinx_dma_synchronize;3171	xdev->common.device_tx_status = xilinx_dma_tx_status;3172	xdev->common.device_issue_pending = xilinx_dma_issue_pending;3173	xdev->common.device_config = xilinx_dma_device_config;3174	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {3175		dma_cap_set(DMA_CYCLIC, xdev->common.cap_mask);3176		xdev->common.device_prep_slave_sg = xilinx_dma_prep_slave_sg;3177		xdev->common.device_prep_dma_cyclic =3178					  xilinx_dma_prep_dma_cyclic;3179		/* Residue calculation is supported by only AXI DMA and CDMA */3180		xdev->common.residue_granularity =3181					  DMA_RESIDUE_GRANULARITY_SEGMENT;3182	} else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {3183		dma_cap_set(DMA_MEMCPY, xdev->common.cap_mask);3184		xdev->common.device_prep_dma_memcpy = xilinx_cdma_prep_memcpy;3185		/* Residue calculation is supported by only AXI DMA and CDMA */3186		xdev->common.residue_granularity =3187					  DMA_RESIDUE_GRANULARITY_SEGMENT;3188	} else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {3189		xdev->common.device_prep_slave_sg = xilinx_mcdma_prep_slave_sg;3190	} else {3191		xdev->common.device_prep_interleaved_dma =3192				xilinx_vdma_dma_prep_interleaved;3193	}3194 3195	platform_set_drvdata(pdev, xdev);3196 3197	/* Initialize the channels */3198	for_each_child_of_node(node, child) {3199		err = xilinx_dma_child_probe(xdev, child);3200		if (err < 0) {3201			of_node_put(child);3202			goto error;3203		}3204	}3205 3206	if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {3207		for (i = 0; i < xdev->dma_config->max_channels; i++)3208			if (xdev->chan[i])3209				xdev->chan[i]->num_frms = num_frames;3210	}3211 3212	/* Register the DMA engine with the core */3213	err = dma_async_device_register(&xdev->common);3214	if (err) {3215		dev_err(xdev->dev, "failed to register the dma device\n");3216		goto error;3217	}3218 3219	err = of_dma_controller_register(node, of_dma_xilinx_xlate,3220					 xdev);3221	if (err < 0) {3222		dev_err(&pdev->dev, "Unable to register DMA to DT\n");3223		dma_async_device_unregister(&xdev->common);3224		goto error;3225	}3226 3227	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA)3228		dev_info(&pdev->dev, "Xilinx AXI DMA Engine Driver Probed!!\n");3229	else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA)3230		dev_info(&pdev->dev, "Xilinx AXI CDMA Engine Driver Probed!!\n");3231	else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA)3232		dev_info(&pdev->dev, "Xilinx AXI MCDMA Engine Driver Probed!!\n");3233	else3234		dev_info(&pdev->dev, "Xilinx AXI VDMA Engine Driver Probed!!\n");3235 3236	return 0;3237 3238error:3239	for (i = 0; i < xdev->dma_config->max_channels; i++)3240		if (xdev->chan[i])3241			xilinx_dma_chan_remove(xdev->chan[i]);3242disable_clks:3243	xdma_disable_allclks(xdev);3244 3245	return err;3246}3247 3248/**3249 * xilinx_dma_remove - Driver remove function3250 * @pdev: Pointer to the platform_device structure3251 */3252static void xilinx_dma_remove(struct platform_device *pdev)3253{3254	struct xilinx_dma_device *xdev = platform_get_drvdata(pdev);3255	int i;3256 3257	of_dma_controller_free(pdev->dev.of_node);3258 3259	dma_async_device_unregister(&xdev->common);3260 3261	for (i = 0; i < xdev->dma_config->max_channels; i++)3262		if (xdev->chan[i])3263			xilinx_dma_chan_remove(xdev->chan[i]);3264 3265	xdma_disable_allclks(xdev);3266}3267 3268static struct platform_driver xilinx_vdma_driver = {3269	.driver = {3270		.name = "xilinx-vdma",3271		.of_match_table = xilinx_dma_of_ids,3272	},3273	.probe = xilinx_dma_probe,3274	.remove_new = xilinx_dma_remove,3275};3276 3277module_platform_driver(xilinx_vdma_driver);3278 3279MODULE_AUTHOR("Xilinx, Inc.");3280MODULE_DESCRIPTION("Xilinx VDMA driver");3281MODULE_LICENSE("GPL v2");3282