brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.02/*3 *  Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com4 *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>5 */6 7#include <linux/kernel.h>8#include <linux/module.h>9#include <linux/delay.h>10#include <linux/dmaengine.h>11#include <linux/dma-mapping.h>12#include <linux/dmapool.h>13#include <linux/err.h>14#include <linux/init.h>15#include <linux/interrupt.h>16#include <linux/list.h>17#include <linux/platform_device.h>18#include <linux/slab.h>19#include <linux/spinlock.h>20#include <linux/sys_soc.h>21#include <linux/of.h>22#include <linux/of_dma.h>23#include <linux/of_irq.h>24#include <linux/workqueue.h>25#include <linux/completion.h>26#include <linux/soc/ti/k3-ringacc.h>27#include <linux/soc/ti/ti_sci_protocol.h>28#include <linux/soc/ti/ti_sci_inta_msi.h>29#include <linux/dma/k3-event-router.h>30#include <linux/dma/ti-cppi5.h>31 32#include "../virt-dma.h"33#include "k3-udma.h"34#include "k3-psil-priv.h"35 36struct udma_static_tr {37	u8 elsize; /* RPSTR0 */38	u16 elcnt; /* RPSTR0 */39	u16 bstcnt; /* RPSTR1 */40};41 42#define K3_UDMA_MAX_RFLOWS		102443#define K3_UDMA_DEFAULT_RING_SIZE	1644 45/* How SRC/DST tag should be updated by UDMA in the descriptor's Word 3 */46#define UDMA_RFLOW_SRCTAG_NONE		047#define UDMA_RFLOW_SRCTAG_CFG_TAG	148#define UDMA_RFLOW_SRCTAG_FLOW_ID	249#define UDMA_RFLOW_SRCTAG_SRC_TAG	450 51#define UDMA_RFLOW_DSTTAG_NONE		052#define UDMA_RFLOW_DSTTAG_CFG_TAG	153#define UDMA_RFLOW_DSTTAG_FLOW_ID	254#define UDMA_RFLOW_DSTTAG_DST_TAG_LO	455#define UDMA_RFLOW_DSTTAG_DST_TAG_HI	556 57struct udma_chan;58 59enum k3_dma_type {60	DMA_TYPE_UDMA = 0,61	DMA_TYPE_BCDMA,62	DMA_TYPE_PKTDMA,63};64 65enum udma_mmr {66	MMR_GCFG = 0,67	MMR_BCHANRT,68	MMR_RCHANRT,69	MMR_TCHANRT,70	MMR_LAST,71};72 73static const char * const mmr_names[] = {74	[MMR_GCFG] = "gcfg",75	[MMR_BCHANRT] = "bchanrt",76	[MMR_RCHANRT] = "rchanrt",77	[MMR_TCHANRT] = "tchanrt",78};79 80struct udma_tchan {81	void __iomem *reg_rt;82 83	int id;84	struct k3_ring *t_ring; /* Transmit ring */85	struct k3_ring *tc_ring; /* Transmit Completion ring */86	int tflow_id; /* applicable only for PKTDMA */87 88};89 90#define udma_bchan udma_tchan91 92struct udma_rflow {93	int id;94	struct k3_ring *fd_ring; /* Free Descriptor ring */95	struct k3_ring *r_ring; /* Receive ring */96};97 98struct udma_rchan {99	void __iomem *reg_rt;100 101	int id;102};103 104struct udma_oes_offsets {105	/* K3 UDMA Output Event Offset */106	u32 udma_rchan;107 108	/* BCDMA Output Event Offsets */109	u32 bcdma_bchan_data;110	u32 bcdma_bchan_ring;111	u32 bcdma_tchan_data;112	u32 bcdma_tchan_ring;113	u32 bcdma_rchan_data;114	u32 bcdma_rchan_ring;115 116	/* PKTDMA Output Event Offsets */117	u32 pktdma_tchan_flow;118	u32 pktdma_rchan_flow;119};120 121#define UDMA_FLAG_PDMA_ACC32		BIT(0)122#define UDMA_FLAG_PDMA_BURST		BIT(1)123#define UDMA_FLAG_TDTYPE		BIT(2)124#define UDMA_FLAG_BURST_SIZE		BIT(3)125#define UDMA_FLAGS_J7_CLASS		(UDMA_FLAG_PDMA_ACC32 | \126					 UDMA_FLAG_PDMA_BURST | \127					 UDMA_FLAG_TDTYPE | \128					 UDMA_FLAG_BURST_SIZE)129 130struct udma_match_data {131	enum k3_dma_type type;132	u32 psil_base;133	bool enable_memcpy_support;134	u32 flags;135	u32 statictr_z_mask;136	u8 burst_size[3];137	struct udma_soc_data *soc_data;138};139 140struct udma_soc_data {141	struct udma_oes_offsets oes;142	u32 bcdma_trigger_event_offset;143};144 145struct udma_hwdesc {146	size_t cppi5_desc_size;147	void *cppi5_desc_vaddr;148	dma_addr_t cppi5_desc_paddr;149 150	/* TR descriptor internal pointers */151	void *tr_req_base;152	struct cppi5_tr_resp_t *tr_resp_base;153};154 155struct udma_rx_flush {156	struct udma_hwdesc hwdescs[2];157 158	size_t buffer_size;159	void *buffer_vaddr;160	dma_addr_t buffer_paddr;161};162 163struct udma_tpl {164	u8 levels;165	u32 start_idx[3];166};167 168struct udma_dev {169	struct dma_device ddev;170	struct device *dev;171	void __iomem *mmrs[MMR_LAST];172	const struct udma_match_data *match_data;173	const struct udma_soc_data *soc_data;174 175	struct udma_tpl bchan_tpl;176	struct udma_tpl tchan_tpl;177	struct udma_tpl rchan_tpl;178 179	size_t desc_align; /* alignment to use for descriptors */180 181	struct udma_tisci_rm tisci_rm;182 183	struct k3_ringacc *ringacc;184 185	struct work_struct purge_work;186	struct list_head desc_to_purge;187	spinlock_t lock;188 189	struct udma_rx_flush rx_flush;190 191	int bchan_cnt;192	int tchan_cnt;193	int echan_cnt;194	int rchan_cnt;195	int rflow_cnt;196	int tflow_cnt;197	unsigned long *bchan_map;198	unsigned long *tchan_map;199	unsigned long *rchan_map;200	unsigned long *rflow_gp_map;201	unsigned long *rflow_gp_map_allocated;202	unsigned long *rflow_in_use;203	unsigned long *tflow_map;204 205	struct udma_bchan *bchans;206	struct udma_tchan *tchans;207	struct udma_rchan *rchans;208	struct udma_rflow *rflows;209 210	struct udma_chan *channels;211	u32 psil_base;212	u32 atype;213	u32 asel;214};215 216struct udma_desc {217	struct virt_dma_desc vd;218 219	bool terminated;220 221	enum dma_transfer_direction dir;222 223	struct udma_static_tr static_tr;224	u32 residue;225 226	unsigned int sglen;227	unsigned int desc_idx; /* Only used for cyclic in packet mode */228	unsigned int tr_idx;229 230	u32 metadata_size;231	void *metadata; /* pointer to provided metadata buffer (EPIP, PSdata) */232 233	unsigned int hwdesc_count;234	struct udma_hwdesc hwdesc[];235};236 237enum udma_chan_state {238	UDMA_CHAN_IS_IDLE = 0, /* not active, no teardown is in progress */239	UDMA_CHAN_IS_ACTIVE, /* Normal operation */240	UDMA_CHAN_IS_TERMINATING, /* channel is being terminated */241};242 243struct udma_tx_drain {244	struct delayed_work work;245	ktime_t tstamp;246	u32 residue;247};248 249struct udma_chan_config {250	bool pkt_mode; /* TR or packet */251	bool needs_epib; /* EPIB is needed for the communication or not */252	u32 psd_size; /* size of Protocol Specific Data */253	u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */254	u32 hdesc_size; /* Size of a packet descriptor in packet mode */255	bool notdpkt; /* Suppress sending TDC packet */256	int remote_thread_id;257	u32 atype;258	u32 asel;259	u32 src_thread;260	u32 dst_thread;261	enum psil_endpoint_type ep_type;262	bool enable_acc32;263	bool enable_burst;264	enum udma_tp_level channel_tpl; /* Channel Throughput Level */265 266	u32 tr_trigger_type;267	unsigned long tx_flags;268 269	/* PKDMA mapped channel */270	int mapped_channel_id;271	/* PKTDMA default tflow or rflow for mapped channel */272	int default_flow_id;273 274	enum dma_transfer_direction dir;275};276 277struct udma_chan {278	struct virt_dma_chan vc;279	struct dma_slave_config	cfg;280	struct udma_dev *ud;281	struct device *dma_dev;282	struct udma_desc *desc;283	struct udma_desc *terminated_desc;284	struct udma_static_tr static_tr;285	char *name;286 287	struct udma_bchan *bchan;288	struct udma_tchan *tchan;289	struct udma_rchan *rchan;290	struct udma_rflow *rflow;291 292	bool psil_paired;293 294	int irq_num_ring;295	int irq_num_udma;296 297	bool cyclic;298	bool paused;299 300	enum udma_chan_state state;301	struct completion teardown_completed;302 303	struct udma_tx_drain tx_drain;304 305	/* Channel configuration parameters */306	struct udma_chan_config config;307	/* Channel configuration parameters (backup) */308	struct udma_chan_config backup_config;309 310	/* dmapool for packet mode descriptors */311	bool use_dma_pool;312	struct dma_pool *hdesc_pool;313 314	u32 id;315};316 317static inline struct udma_dev *to_udma_dev(struct dma_device *d)318{319	return container_of(d, struct udma_dev, ddev);320}321 322static inline struct udma_chan *to_udma_chan(struct dma_chan *c)323{324	return container_of(c, struct udma_chan, vc.chan);325}326 327static inline struct udma_desc *to_udma_desc(struct dma_async_tx_descriptor *t)328{329	return container_of(t, struct udma_desc, vd.tx);330}331 332/* Generic register access functions */333static inline u32 udma_read(void __iomem *base, int reg)334{335	return readl(base + reg);336}337 338static inline void udma_write(void __iomem *base, int reg, u32 val)339{340	writel(val, base + reg);341}342 343static inline void udma_update_bits(void __iomem *base, int reg,344				    u32 mask, u32 val)345{346	u32 tmp, orig;347 348	orig = readl(base + reg);349	tmp = orig & ~mask;350	tmp |= (val & mask);351 352	if (tmp != orig)353		writel(tmp, base + reg);354}355 356/* TCHANRT */357static inline u32 udma_tchanrt_read(struct udma_chan *uc, int reg)358{359	if (!uc->tchan)360		return 0;361	return udma_read(uc->tchan->reg_rt, reg);362}363 364static inline void udma_tchanrt_write(struct udma_chan *uc, int reg, u32 val)365{366	if (!uc->tchan)367		return;368	udma_write(uc->tchan->reg_rt, reg, val);369}370 371static inline void udma_tchanrt_update_bits(struct udma_chan *uc, int reg,372					    u32 mask, u32 val)373{374	if (!uc->tchan)375		return;376	udma_update_bits(uc->tchan->reg_rt, reg, mask, val);377}378 379/* RCHANRT */380static inline u32 udma_rchanrt_read(struct udma_chan *uc, int reg)381{382	if (!uc->rchan)383		return 0;384	return udma_read(uc->rchan->reg_rt, reg);385}386 387static inline void udma_rchanrt_write(struct udma_chan *uc, int reg, u32 val)388{389	if (!uc->rchan)390		return;391	udma_write(uc->rchan->reg_rt, reg, val);392}393 394static inline void udma_rchanrt_update_bits(struct udma_chan *uc, int reg,395					    u32 mask, u32 val)396{397	if (!uc->rchan)398		return;399	udma_update_bits(uc->rchan->reg_rt, reg, mask, val);400}401 402static int navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread)403{404	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;405 406	dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;407	return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci,408					      tisci_rm->tisci_navss_dev_id,409					      src_thread, dst_thread);410}411 412static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread,413			     u32 dst_thread)414{415	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;416 417	dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;418	return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci,419						tisci_rm->tisci_navss_dev_id,420						src_thread, dst_thread);421}422 423static void k3_configure_chan_coherency(struct dma_chan *chan, u32 asel)424{425	struct device *chan_dev = &chan->dev->device;426 427	if (asel == 0) {428		/* No special handling for the channel */429		chan->dev->chan_dma_dev = false;430 431		chan_dev->dma_coherent = false;432		chan_dev->dma_parms = NULL;433	} else if (asel == 14 || asel == 15) {434		chan->dev->chan_dma_dev = true;435 436		chan_dev->dma_coherent = true;437		dma_coerce_mask_and_coherent(chan_dev, DMA_BIT_MASK(48));438		chan_dev->dma_parms = chan_dev->parent->dma_parms;439	} else {440		dev_warn(chan->device->dev, "Invalid ASEL value: %u\n", asel);441 442		chan_dev->dma_coherent = false;443		chan_dev->dma_parms = NULL;444	}445}446 447static u8 udma_get_chan_tpl_index(struct udma_tpl *tpl_map, int chan_id)448{449	int i;450 451	for (i = 0; i < tpl_map->levels; i++) {452		if (chan_id >= tpl_map->start_idx[i])453			return i;454	}455 456	return 0;457}458 459static void udma_reset_uchan(struct udma_chan *uc)460{461	memset(&uc->config, 0, sizeof(uc->config));462	uc->config.remote_thread_id = -1;463	uc->config.mapped_channel_id = -1;464	uc->config.default_flow_id = -1;465	uc->state = UDMA_CHAN_IS_IDLE;466}467 468static void udma_dump_chan_stdata(struct udma_chan *uc)469{470	struct device *dev = uc->ud->dev;471	u32 offset;472	int i;473 474	if (uc->config.dir == DMA_MEM_TO_DEV || uc->config.dir == DMA_MEM_TO_MEM) {475		dev_dbg(dev, "TCHAN State data:\n");476		for (i = 0; i < 32; i++) {477			offset = UDMA_CHAN_RT_STDATA_REG + i * 4;478			dev_dbg(dev, "TRT_STDATA[%02d]: 0x%08x\n", i,479				udma_tchanrt_read(uc, offset));480		}481	}482 483	if (uc->config.dir == DMA_DEV_TO_MEM || uc->config.dir == DMA_MEM_TO_MEM) {484		dev_dbg(dev, "RCHAN State data:\n");485		for (i = 0; i < 32; i++) {486			offset = UDMA_CHAN_RT_STDATA_REG + i * 4;487			dev_dbg(dev, "RRT_STDATA[%02d]: 0x%08x\n", i,488				udma_rchanrt_read(uc, offset));489		}490	}491}492 493static inline dma_addr_t udma_curr_cppi5_desc_paddr(struct udma_desc *d,494						    int idx)495{496	return d->hwdesc[idx].cppi5_desc_paddr;497}498 499static inline void *udma_curr_cppi5_desc_vaddr(struct udma_desc *d, int idx)500{501	return d->hwdesc[idx].cppi5_desc_vaddr;502}503 504static struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc,505						   dma_addr_t paddr)506{507	struct udma_desc *d = uc->terminated_desc;508 509	if (d) {510		dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,511								   d->desc_idx);512 513		if (desc_paddr != paddr)514			d = NULL;515	}516 517	if (!d) {518		d = uc->desc;519		if (d) {520			dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,521								d->desc_idx);522 523			if (desc_paddr != paddr)524				d = NULL;525		}526	}527 528	return d;529}530 531static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d)532{533	if (uc->use_dma_pool) {534		int i;535 536		for (i = 0; i < d->hwdesc_count; i++) {537			if (!d->hwdesc[i].cppi5_desc_vaddr)538				continue;539 540			dma_pool_free(uc->hdesc_pool,541				      d->hwdesc[i].cppi5_desc_vaddr,542				      d->hwdesc[i].cppi5_desc_paddr);543 544			d->hwdesc[i].cppi5_desc_vaddr = NULL;545		}546	} else if (d->hwdesc[0].cppi5_desc_vaddr) {547		dma_free_coherent(uc->dma_dev, d->hwdesc[0].cppi5_desc_size,548				  d->hwdesc[0].cppi5_desc_vaddr,549				  d->hwdesc[0].cppi5_desc_paddr);550 551		d->hwdesc[0].cppi5_desc_vaddr = NULL;552	}553}554 555static void udma_purge_desc_work(struct work_struct *work)556{557	struct udma_dev *ud = container_of(work, typeof(*ud), purge_work);558	struct virt_dma_desc *vd, *_vd;559	unsigned long flags;560	LIST_HEAD(head);561 562	spin_lock_irqsave(&ud->lock, flags);563	list_splice_tail_init(&ud->desc_to_purge, &head);564	spin_unlock_irqrestore(&ud->lock, flags);565 566	list_for_each_entry_safe(vd, _vd, &head, node) {567		struct udma_chan *uc = to_udma_chan(vd->tx.chan);568		struct udma_desc *d = to_udma_desc(&vd->tx);569 570		udma_free_hwdesc(uc, d);571		list_del(&vd->node);572		kfree(d);573	}574 575	/* If more to purge, schedule the work again */576	if (!list_empty(&ud->desc_to_purge))577		schedule_work(&ud->purge_work);578}579 580static void udma_desc_free(struct virt_dma_desc *vd)581{582	struct udma_dev *ud = to_udma_dev(vd->tx.chan->device);583	struct udma_chan *uc = to_udma_chan(vd->tx.chan);584	struct udma_desc *d = to_udma_desc(&vd->tx);585	unsigned long flags;586 587	if (uc->terminated_desc == d)588		uc->terminated_desc = NULL;589 590	if (uc->use_dma_pool) {591		udma_free_hwdesc(uc, d);592		kfree(d);593		return;594	}595 596	spin_lock_irqsave(&ud->lock, flags);597	list_add_tail(&vd->node, &ud->desc_to_purge);598	spin_unlock_irqrestore(&ud->lock, flags);599 600	schedule_work(&ud->purge_work);601}602 603static bool udma_is_chan_running(struct udma_chan *uc)604{605	u32 trt_ctl = 0;606	u32 rrt_ctl = 0;607 608	if (uc->tchan)609		trt_ctl = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);610	if (uc->rchan)611		rrt_ctl = udma_rchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);612 613	if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN)614		return true;615 616	return false;617}618 619static bool udma_is_chan_paused(struct udma_chan *uc)620{621	u32 val, pause_mask;622 623	switch (uc->config.dir) {624	case DMA_DEV_TO_MEM:625		val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG);626		pause_mask = UDMA_PEER_RT_EN_PAUSE;627		break;628	case DMA_MEM_TO_DEV:629		val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG);630		pause_mask = UDMA_PEER_RT_EN_PAUSE;631		break;632	case DMA_MEM_TO_MEM:633		val = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);634		pause_mask = UDMA_CHAN_RT_CTL_PAUSE;635		break;636	default:637		return false;638	}639 640	if (val & pause_mask)641		return true;642 643	return false;644}645 646static inline dma_addr_t udma_get_rx_flush_hwdesc_paddr(struct udma_chan *uc)647{648	return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr;649}650 651static int udma_push_to_ring(struct udma_chan *uc, int idx)652{653	struct udma_desc *d = uc->desc;654	struct k3_ring *ring = NULL;655	dma_addr_t paddr;656 657	switch (uc->config.dir) {658	case DMA_DEV_TO_MEM:659		ring = uc->rflow->fd_ring;660		break;661	case DMA_MEM_TO_DEV:662	case DMA_MEM_TO_MEM:663		ring = uc->tchan->t_ring;664		break;665	default:666		return -EINVAL;667	}668 669	/* RX flush packet: idx == -1 is only passed in case of DEV_TO_MEM */670	if (idx == -1) {671		paddr = udma_get_rx_flush_hwdesc_paddr(uc);672	} else {673		paddr = udma_curr_cppi5_desc_paddr(d, idx);674 675		wmb(); /* Ensure that writes are not moved over this point */676	}677 678	return k3_ringacc_ring_push(ring, &paddr);679}680 681static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr)682{683	if (uc->config.dir != DMA_DEV_TO_MEM)684		return false;685 686	if (addr == udma_get_rx_flush_hwdesc_paddr(uc))687		return true;688 689	return false;690}691 692static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr)693{694	struct k3_ring *ring = NULL;695	int ret;696 697	switch (uc->config.dir) {698	case DMA_DEV_TO_MEM:699		ring = uc->rflow->r_ring;700		break;701	case DMA_MEM_TO_DEV:702	case DMA_MEM_TO_MEM:703		ring = uc->tchan->tc_ring;704		break;705	default:706		return -ENOENT;707	}708 709	ret = k3_ringacc_ring_pop(ring, addr);710	if (ret)711		return ret;712 713	rmb(); /* Ensure that reads are not moved before this point */714 715	/* Teardown completion */716	if (cppi5_desc_is_tdcm(*addr))717		return 0;718 719	/* Check for flush descriptor */720	if (udma_desc_is_rx_flush(uc, *addr))721		return -ENOENT;722 723	return 0;724}725 726static void udma_reset_rings(struct udma_chan *uc)727{728	struct k3_ring *ring1 = NULL;729	struct k3_ring *ring2 = NULL;730 731	switch (uc->config.dir) {732	case DMA_DEV_TO_MEM:733		if (uc->rchan) {734			ring1 = uc->rflow->fd_ring;735			ring2 = uc->rflow->r_ring;736		}737		break;738	case DMA_MEM_TO_DEV:739	case DMA_MEM_TO_MEM:740		if (uc->tchan) {741			ring1 = uc->tchan->t_ring;742			ring2 = uc->tchan->tc_ring;743		}744		break;745	default:746		break;747	}748 749	if (ring1)750		k3_ringacc_ring_reset_dma(ring1,751					  k3_ringacc_ring_get_occ(ring1));752	if (ring2)753		k3_ringacc_ring_reset(ring2);754 755	/* make sure we are not leaking memory by stalled descriptor */756	if (uc->terminated_desc) {757		udma_desc_free(&uc->terminated_desc->vd);758		uc->terminated_desc = NULL;759	}760}761 762static void udma_decrement_byte_counters(struct udma_chan *uc, u32 val)763{764	if (uc->desc->dir == DMA_DEV_TO_MEM) {765		udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);766		udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);767		if (uc->config.ep_type != PSIL_EP_NATIVE)768			udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);769	} else {770		udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);771		udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);772		if (!uc->bchan && uc->config.ep_type != PSIL_EP_NATIVE)773			udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);774	}775}776 777static void udma_reset_counters(struct udma_chan *uc)778{779	u32 val;780 781	if (uc->tchan) {782		val = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);783		udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);784 785		val = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);786		udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);787 788		val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG);789		udma_tchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val);790 791		if (!uc->bchan) {792			val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);793			udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);794		}795	}796 797	if (uc->rchan) {798		val = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);799		udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);800 801		val = udma_rchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);802		udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);803 804		val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG);805		udma_rchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val);806 807		val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);808		udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);809	}810}811 812static int udma_reset_chan(struct udma_chan *uc, bool hard)813{814	switch (uc->config.dir) {815	case DMA_DEV_TO_MEM:816		udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0);817		udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);818		break;819	case DMA_MEM_TO_DEV:820		udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);821		udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0);822		break;823	case DMA_MEM_TO_MEM:824		udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);825		udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);826		break;827	default:828		return -EINVAL;829	}830 831	/* Reset all counters */832	udma_reset_counters(uc);833 834	/* Hard reset: re-initialize the channel to reset */835	if (hard) {836		struct udma_chan_config ucc_backup;837		int ret;838 839		memcpy(&ucc_backup, &uc->config, sizeof(uc->config));840		uc->ud->ddev.device_free_chan_resources(&uc->vc.chan);841 842		/* restore the channel configuration */843		memcpy(&uc->config, &ucc_backup, sizeof(uc->config));844		ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan);845		if (ret)846			return ret;847 848		/*849		 * Setting forced teardown after forced reset helps recovering850		 * the rchan.851		 */852		if (uc->config.dir == DMA_DEV_TO_MEM)853			udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,854					   UDMA_CHAN_RT_CTL_EN |855					   UDMA_CHAN_RT_CTL_TDOWN |856					   UDMA_CHAN_RT_CTL_FTDOWN);857	}858	uc->state = UDMA_CHAN_IS_IDLE;859 860	return 0;861}862 863static void udma_start_desc(struct udma_chan *uc)864{865	struct udma_chan_config *ucc = &uc->config;866 867	if (uc->ud->match_data->type == DMA_TYPE_UDMA && ucc->pkt_mode &&868	    (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) {869		int i;870 871		/*872		 * UDMA only: Push all descriptors to ring for packet mode873		 * cyclic or RX874		 * PKTDMA supports pre-linked descriptor and cyclic is not875		 * supported876		 */877		for (i = 0; i < uc->desc->sglen; i++)878			udma_push_to_ring(uc, i);879	} else {880		udma_push_to_ring(uc, 0);881	}882}883 884static bool udma_chan_needs_reconfiguration(struct udma_chan *uc)885{886	/* Only PDMAs have staticTR */887	if (uc->config.ep_type == PSIL_EP_NATIVE)888		return false;889 890	/* Check if the staticTR configuration has changed for TX */891	if (memcmp(&uc->static_tr, &uc->desc->static_tr, sizeof(uc->static_tr)))892		return true;893 894	return false;895}896 897static int udma_start(struct udma_chan *uc)898{899	struct virt_dma_desc *vd = vchan_next_desc(&uc->vc);900 901	if (!vd) {902		uc->desc = NULL;903		return -ENOENT;904	}905 906	list_del(&vd->node);907 908	uc->desc = to_udma_desc(&vd->tx);909 910	/* Channel is already running and does not need reconfiguration */911	if (udma_is_chan_running(uc) && !udma_chan_needs_reconfiguration(uc)) {912		udma_start_desc(uc);913		goto out;914	}915 916	/* Make sure that we clear the teardown bit, if it is set */917	udma_reset_chan(uc, false);918 919	/* Push descriptors before we start the channel */920	udma_start_desc(uc);921 922	switch (uc->desc->dir) {923	case DMA_DEV_TO_MEM:924		/* Config remote TR */925		if (uc->config.ep_type == PSIL_EP_PDMA_XY) {926			u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |927				  PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);928			const struct udma_match_data *match_data =929							uc->ud->match_data;930 931			if (uc->config.enable_acc32)932				val |= PDMA_STATIC_TR_XY_ACC32;933			if (uc->config.enable_burst)934				val |= PDMA_STATIC_TR_XY_BURST;935 936			udma_rchanrt_write(uc,937					   UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG,938					   val);939 940			udma_rchanrt_write(uc,941				UDMA_CHAN_RT_PEER_STATIC_TR_Z_REG,942				PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt,943						 match_data->statictr_z_mask));944 945			/* save the current staticTR configuration */946			memcpy(&uc->static_tr, &uc->desc->static_tr,947			       sizeof(uc->static_tr));948		}949 950		udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,951				   UDMA_CHAN_RT_CTL_EN);952 953		/* Enable remote */954		udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,955				   UDMA_PEER_RT_EN_ENABLE);956 957		break;958	case DMA_MEM_TO_DEV:959		/* Config remote TR */960		if (uc->config.ep_type == PSIL_EP_PDMA_XY) {961			u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |962				  PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);963 964			if (uc->config.enable_acc32)965				val |= PDMA_STATIC_TR_XY_ACC32;966			if (uc->config.enable_burst)967				val |= PDMA_STATIC_TR_XY_BURST;968 969			udma_tchanrt_write(uc,970					   UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG,971					   val);972 973			/* save the current staticTR configuration */974			memcpy(&uc->static_tr, &uc->desc->static_tr,975			       sizeof(uc->static_tr));976		}977 978		/* Enable remote */979		udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,980				   UDMA_PEER_RT_EN_ENABLE);981 982		udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,983				   UDMA_CHAN_RT_CTL_EN);984 985		break;986	case DMA_MEM_TO_MEM:987		udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,988				   UDMA_CHAN_RT_CTL_EN);989		udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,990				   UDMA_CHAN_RT_CTL_EN);991 992		break;993	default:994		return -EINVAL;995	}996 997	uc->state = UDMA_CHAN_IS_ACTIVE;998out:999 1000	return 0;1001}1002 1003static int udma_stop(struct udma_chan *uc)1004{1005	enum udma_chan_state old_state = uc->state;1006 1007	uc->state = UDMA_CHAN_IS_TERMINATING;1008	reinit_completion(&uc->teardown_completed);1009 1010	switch (uc->config.dir) {1011	case DMA_DEV_TO_MEM:1012		if (!uc->cyclic && !uc->desc)1013			udma_push_to_ring(uc, -1);1014 1015		udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,1016				   UDMA_PEER_RT_EN_ENABLE |1017				   UDMA_PEER_RT_EN_TEARDOWN);1018		break;1019	case DMA_MEM_TO_DEV:1020		udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,1021				   UDMA_PEER_RT_EN_ENABLE |1022				   UDMA_PEER_RT_EN_FLUSH);1023		udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,1024				   UDMA_CHAN_RT_CTL_EN |1025				   UDMA_CHAN_RT_CTL_TDOWN);1026		break;1027	case DMA_MEM_TO_MEM:1028		udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,1029				   UDMA_CHAN_RT_CTL_EN |1030				   UDMA_CHAN_RT_CTL_TDOWN);1031		break;1032	default:1033		uc->state = old_state;1034		complete_all(&uc->teardown_completed);1035		return -EINVAL;1036	}1037 1038	return 0;1039}1040 1041static void udma_cyclic_packet_elapsed(struct udma_chan *uc)1042{1043	struct udma_desc *d = uc->desc;1044	struct cppi5_host_desc_t *h_desc;1045 1046	h_desc = d->hwdesc[d->desc_idx].cppi5_desc_vaddr;1047	cppi5_hdesc_reset_to_original(h_desc);1048	udma_push_to_ring(uc, d->desc_idx);1049	d->desc_idx = (d->desc_idx + 1) % d->sglen;1050}1051 1052static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d)1053{1054	struct cppi5_host_desc_t *h_desc = d->hwdesc[0].cppi5_desc_vaddr;1055 1056	memcpy(d->metadata, h_desc->epib, d->metadata_size);1057}1058 1059static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d)1060{1061	u32 peer_bcnt, bcnt;1062 1063	/*1064	 * Only TX towards PDMA is affected.1065	 * If DMA_PREP_INTERRUPT is not set by consumer then skip the transfer1066	 * completion calculation, consumer must ensure that there is no stale1067	 * data in DMA fabric in this case.1068	 */1069	if (uc->config.ep_type == PSIL_EP_NATIVE ||1070	    uc->config.dir != DMA_MEM_TO_DEV || !(uc->config.tx_flags & DMA_PREP_INTERRUPT))1071		return true;1072 1073	peer_bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);1074	bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);1075 1076	/* Transfer is incomplete, store current residue and time stamp */1077	if (peer_bcnt < bcnt) {1078		uc->tx_drain.residue = bcnt - peer_bcnt;1079		uc->tx_drain.tstamp = ktime_get();1080		return false;1081	}1082 1083	return true;1084}1085 1086static void udma_check_tx_completion(struct work_struct *work)1087{1088	struct udma_chan *uc = container_of(work, typeof(*uc),1089					    tx_drain.work.work);1090	bool desc_done = true;1091	u32 residue_diff;1092	ktime_t time_diff;1093	unsigned long delay;1094 1095	while (1) {1096		if (uc->desc) {1097			/* Get previous residue and time stamp */1098			residue_diff = uc->tx_drain.residue;1099			time_diff = uc->tx_drain.tstamp;1100			/*1101			 * Get current residue and time stamp or see if1102			 * transfer is complete1103			 */1104			desc_done = udma_is_desc_really_done(uc, uc->desc);1105		}1106 1107		if (!desc_done) {1108			/*1109			 * Find the time delta and residue delta w.r.t1110			 * previous poll1111			 */1112			time_diff = ktime_sub(uc->tx_drain.tstamp,1113					      time_diff) + 1;1114			residue_diff -= uc->tx_drain.residue;1115			if (residue_diff) {1116				/*1117				 * Try to guess when we should check1118				 * next time by calculating rate at1119				 * which data is being drained at the1120				 * peer device1121				 */1122				delay = (time_diff / residue_diff) *1123					uc->tx_drain.residue;1124			} else {1125				/* No progress, check again in 1 second  */1126				schedule_delayed_work(&uc->tx_drain.work, HZ);1127				break;1128			}1129 1130			usleep_range(ktime_to_us(delay),1131				     ktime_to_us(delay) + 10);1132			continue;1133		}1134 1135		if (uc->desc) {1136			struct udma_desc *d = uc->desc;1137 1138			udma_decrement_byte_counters(uc, d->residue);1139			udma_start(uc);1140			vchan_cookie_complete(&d->vd);1141			break;1142		}1143 1144		break;1145	}1146}1147 1148static irqreturn_t udma_ring_irq_handler(int irq, void *data)1149{1150	struct udma_chan *uc = data;1151	struct udma_desc *d;1152	dma_addr_t paddr = 0;1153 1154	if (udma_pop_from_ring(uc, &paddr) || !paddr)1155		return IRQ_HANDLED;1156 1157	spin_lock(&uc->vc.lock);1158 1159	/* Teardown completion message */1160	if (cppi5_desc_is_tdcm(paddr)) {1161		complete_all(&uc->teardown_completed);1162 1163		if (uc->terminated_desc) {1164			udma_desc_free(&uc->terminated_desc->vd);1165			uc->terminated_desc = NULL;1166		}1167 1168		if (!uc->desc)1169			udma_start(uc);1170 1171		goto out;1172	}1173 1174	d = udma_udma_desc_from_paddr(uc, paddr);1175 1176	if (d) {1177		dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,1178								   d->desc_idx);1179		if (desc_paddr != paddr) {1180			dev_err(uc->ud->dev, "not matching descriptors!\n");1181			goto out;1182		}1183 1184		if (d == uc->desc) {1185			/* active descriptor */1186			if (uc->cyclic) {1187				udma_cyclic_packet_elapsed(uc);1188				vchan_cyclic_callback(&d->vd);1189			} else {1190				if (udma_is_desc_really_done(uc, d)) {1191					udma_decrement_byte_counters(uc, d->residue);1192					udma_start(uc);1193					vchan_cookie_complete(&d->vd);1194				} else {1195					schedule_delayed_work(&uc->tx_drain.work,1196							      0);1197				}1198			}1199		} else {1200			/*1201			 * terminated descriptor, mark the descriptor as1202			 * completed to update the channel's cookie marker1203			 */1204			dma_cookie_complete(&d->vd.tx);1205		}1206	}1207out:1208	spin_unlock(&uc->vc.lock);1209 1210	return IRQ_HANDLED;1211}1212 1213static irqreturn_t udma_udma_irq_handler(int irq, void *data)1214{1215	struct udma_chan *uc = data;1216	struct udma_desc *d;1217 1218	spin_lock(&uc->vc.lock);1219	d = uc->desc;1220	if (d) {1221		d->tr_idx = (d->tr_idx + 1) % d->sglen;1222 1223		if (uc->cyclic) {1224			vchan_cyclic_callback(&d->vd);1225		} else {1226			/* TODO: figure out the real amount of data */1227			udma_decrement_byte_counters(uc, d->residue);1228			udma_start(uc);1229			vchan_cookie_complete(&d->vd);1230		}1231	}1232 1233	spin_unlock(&uc->vc.lock);1234 1235	return IRQ_HANDLED;1236}1237 1238/**1239 * __udma_alloc_gp_rflow_range - alloc range of GP RX flows1240 * @ud: UDMA device1241 * @from: Start the search from this flow id number1242 * @cnt: Number of consecutive flow ids to allocate1243 *1244 * Allocate range of RX flow ids for future use, those flows can be requested1245 * only using explicit flow id number. if @from is set to -1 it will try to find1246 * first free range. if @from is positive value it will force allocation only1247 * of the specified range of flows.1248 *1249 * Returns -ENOMEM if can't find free range.1250 * -EEXIST if requested range is busy.1251 * -EINVAL if wrong input values passed.1252 * Returns flow id on success.1253 */1254static int __udma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt)1255{1256	int start, tmp_from;1257	DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS);1258 1259	tmp_from = from;1260	if (tmp_from < 0)1261		tmp_from = ud->rchan_cnt;1262	/* default flows can't be allocated and accessible only by id */1263	if (tmp_from < ud->rchan_cnt)1264		return -EINVAL;1265 1266	if (tmp_from + cnt > ud->rflow_cnt)1267		return -EINVAL;1268 1269	bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated,1270		  ud->rflow_cnt);1271 1272	start = bitmap_find_next_zero_area(tmp,1273					   ud->rflow_cnt,1274					   tmp_from, cnt, 0);1275	if (start >= ud->rflow_cnt)1276		return -ENOMEM;1277 1278	if (from >= 0 && start != from)1279		return -EEXIST;1280 1281	bitmap_set(ud->rflow_gp_map_allocated, start, cnt);1282	return start;1283}1284 1285static int __udma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt)1286{1287	if (from < ud->rchan_cnt)1288		return -EINVAL;1289	if (from + cnt > ud->rflow_cnt)1290		return -EINVAL;1291 1292	bitmap_clear(ud->rflow_gp_map_allocated, from, cnt);1293	return 0;1294}1295 1296static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id)1297{1298	/*1299	 * Attempt to request rflow by ID can be made for any rflow1300	 * if not in use with assumption that caller knows what's doing.1301	 * TI-SCI FW will perform additional permission check ant way, it's1302	 * safe1303	 */1304 1305	if (id < 0 || id >= ud->rflow_cnt)1306		return ERR_PTR(-ENOENT);1307 1308	if (test_bit(id, ud->rflow_in_use))1309		return ERR_PTR(-ENOENT);1310 1311	if (ud->rflow_gp_map) {1312		/* GP rflow has to be allocated first */1313		if (!test_bit(id, ud->rflow_gp_map) &&1314		    !test_bit(id, ud->rflow_gp_map_allocated))1315			return ERR_PTR(-EINVAL);1316	}1317 1318	dev_dbg(ud->dev, "get rflow%d\n", id);1319	set_bit(id, ud->rflow_in_use);1320	return &ud->rflows[id];1321}1322 1323static void __udma_put_rflow(struct udma_dev *ud, struct udma_rflow *rflow)1324{1325	if (!test_bit(rflow->id, ud->rflow_in_use)) {1326		dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id);1327		return;1328	}1329 1330	dev_dbg(ud->dev, "put rflow%d\n", rflow->id);1331	clear_bit(rflow->id, ud->rflow_in_use);1332}1333 1334#define UDMA_RESERVE_RESOURCE(res)					\1335static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud,	\1336					       enum udma_tp_level tpl,	\1337					       int id)			\1338{									\1339	if (id >= 0) {							\1340		if (test_bit(id, ud->res##_map)) {			\1341			dev_err(ud->dev, "res##%d is in use\n", id);	\1342			return ERR_PTR(-ENOENT);			\1343		}							\1344	} else {							\1345		int start;						\1346									\1347		if (tpl >= ud->res##_tpl.levels)			\1348			tpl = ud->res##_tpl.levels - 1;			\1349									\1350		start = ud->res##_tpl.start_idx[tpl];			\1351									\1352		id = find_next_zero_bit(ud->res##_map, ud->res##_cnt,	\1353					start);				\1354		if (id == ud->res##_cnt) {				\1355			return ERR_PTR(-ENOENT);			\1356		}							\1357	}								\1358									\1359	set_bit(id, ud->res##_map);					\1360	return &ud->res##s[id];						\1361}1362 1363UDMA_RESERVE_RESOURCE(bchan);1364UDMA_RESERVE_RESOURCE(tchan);1365UDMA_RESERVE_RESOURCE(rchan);1366 1367static int bcdma_get_bchan(struct udma_chan *uc)1368{1369	struct udma_dev *ud = uc->ud;1370	enum udma_tp_level tpl;1371	int ret;1372 1373	if (uc->bchan) {1374		dev_dbg(ud->dev, "chan%d: already have bchan%d allocated\n",1375			uc->id, uc->bchan->id);1376		return 0;1377	}1378 1379	/*1380	 * Use normal channels for peripherals, and highest TPL channel for1381	 * mem2mem1382	 */1383	if (uc->config.tr_trigger_type)1384		tpl = 0;1385	else1386		tpl = ud->bchan_tpl.levels - 1;1387 1388	uc->bchan = __udma_reserve_bchan(ud, tpl, -1);1389	if (IS_ERR(uc->bchan)) {1390		ret = PTR_ERR(uc->bchan);1391		uc->bchan = NULL;1392		return ret;1393	}1394 1395	uc->tchan = uc->bchan;1396 1397	return 0;1398}1399 1400static int udma_get_tchan(struct udma_chan *uc)1401{1402	struct udma_dev *ud = uc->ud;1403	int ret;1404 1405	if (uc->tchan) {1406		dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n",1407			uc->id, uc->tchan->id);1408		return 0;1409	}1410 1411	/*1412	 * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels.1413	 * For PKTDMA mapped channels it is configured to a channel which must1414	 * be used to service the peripheral.1415	 */1416	uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl,1417					 uc->config.mapped_channel_id);1418	if (IS_ERR(uc->tchan)) {1419		ret = PTR_ERR(uc->tchan);1420		uc->tchan = NULL;1421		return ret;1422	}1423 1424	if (ud->tflow_cnt) {1425		int tflow_id;1426 1427		/* Only PKTDMA have support for tx flows */1428		if (uc->config.default_flow_id >= 0)1429			tflow_id = uc->config.default_flow_id;1430		else1431			tflow_id = uc->tchan->id;1432 1433		if (test_bit(tflow_id, ud->tflow_map)) {1434			dev_err(ud->dev, "tflow%d is in use\n", tflow_id);1435			clear_bit(uc->tchan->id, ud->tchan_map);1436			uc->tchan = NULL;1437			return -ENOENT;1438		}1439 1440		uc->tchan->tflow_id = tflow_id;1441		set_bit(tflow_id, ud->tflow_map);1442	} else {1443		uc->tchan->tflow_id = -1;1444	}1445 1446	return 0;1447}1448 1449static int udma_get_rchan(struct udma_chan *uc)1450{1451	struct udma_dev *ud = uc->ud;1452	int ret;1453 1454	if (uc->rchan) {1455		dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n",1456			uc->id, uc->rchan->id);1457		return 0;1458	}1459 1460	/*1461	 * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels.1462	 * For PKTDMA mapped channels it is configured to a channel which must1463	 * be used to service the peripheral.1464	 */1465	uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl,1466					 uc->config.mapped_channel_id);1467	if (IS_ERR(uc->rchan)) {1468		ret = PTR_ERR(uc->rchan);1469		uc->rchan = NULL;1470		return ret;1471	}1472 1473	return 0;1474}1475 1476static int udma_get_chan_pair(struct udma_chan *uc)1477{1478	struct udma_dev *ud = uc->ud;1479	int chan_id, end;1480 1481	if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) {1482		dev_info(ud->dev, "chan%d: already have %d pair allocated\n",1483			 uc->id, uc->tchan->id);1484		return 0;1485	}1486 1487	if (uc->tchan) {1488		dev_err(ud->dev, "chan%d: already have tchan%d allocated\n",1489			uc->id, uc->tchan->id);1490		return -EBUSY;1491	} else if (uc->rchan) {1492		dev_err(ud->dev, "chan%d: already have rchan%d allocated\n",1493			uc->id, uc->rchan->id);1494		return -EBUSY;1495	}1496 1497	/* Can be optimized, but let's have it like this for now */1498	end = min(ud->tchan_cnt, ud->rchan_cnt);1499	/*1500	 * Try to use the highest TPL channel pair for MEM_TO_MEM channels1501	 * Note: in UDMAP the channel TPL is symmetric between tchan and rchan1502	 */1503	chan_id = ud->tchan_tpl.start_idx[ud->tchan_tpl.levels - 1];1504	for (; chan_id < end; chan_id++) {1505		if (!test_bit(chan_id, ud->tchan_map) &&1506		    !test_bit(chan_id, ud->rchan_map))1507			break;1508	}1509 1510	if (chan_id == end)1511		return -ENOENT;1512 1513	set_bit(chan_id, ud->tchan_map);1514	set_bit(chan_id, ud->rchan_map);1515	uc->tchan = &ud->tchans[chan_id];1516	uc->rchan = &ud->rchans[chan_id];1517 1518	/* UDMA does not use tx flows */1519	uc->tchan->tflow_id = -1;1520 1521	return 0;1522}1523 1524static int udma_get_rflow(struct udma_chan *uc, int flow_id)1525{1526	struct udma_dev *ud = uc->ud;1527	int ret;1528 1529	if (!uc->rchan) {1530		dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id);1531		return -EINVAL;1532	}1533 1534	if (uc->rflow) {1535		dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n",1536			uc->id, uc->rflow->id);1537		return 0;1538	}1539 1540	uc->rflow = __udma_get_rflow(ud, flow_id);1541	if (IS_ERR(uc->rflow)) {1542		ret = PTR_ERR(uc->rflow);1543		uc->rflow = NULL;1544		return ret;1545	}1546 1547	return 0;1548}1549 1550static void bcdma_put_bchan(struct udma_chan *uc)1551{1552	struct udma_dev *ud = uc->ud;1553 1554	if (uc->bchan) {1555		dev_dbg(ud->dev, "chan%d: put bchan%d\n", uc->id,1556			uc->bchan->id);1557		clear_bit(uc->bchan->id, ud->bchan_map);1558		uc->bchan = NULL;1559		uc->tchan = NULL;1560	}1561}1562 1563static void udma_put_rchan(struct udma_chan *uc)1564{1565	struct udma_dev *ud = uc->ud;1566 1567	if (uc->rchan) {1568		dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id,1569			uc->rchan->id);1570		clear_bit(uc->rchan->id, ud->rchan_map);1571		uc->rchan = NULL;1572	}1573}1574 1575static void udma_put_tchan(struct udma_chan *uc)1576{1577	struct udma_dev *ud = uc->ud;1578 1579	if (uc->tchan) {1580		dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id,1581			uc->tchan->id);1582		clear_bit(uc->tchan->id, ud->tchan_map);1583 1584		if (uc->tchan->tflow_id >= 0)1585			clear_bit(uc->tchan->tflow_id, ud->tflow_map);1586 1587		uc->tchan = NULL;1588	}1589}1590 1591static void udma_put_rflow(struct udma_chan *uc)1592{1593	struct udma_dev *ud = uc->ud;1594 1595	if (uc->rflow) {1596		dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id,1597			uc->rflow->id);1598		__udma_put_rflow(ud, uc->rflow);1599		uc->rflow = NULL;1600	}1601}1602 1603static void bcdma_free_bchan_resources(struct udma_chan *uc)1604{1605	if (!uc->bchan)1606		return;1607 1608	k3_ringacc_ring_free(uc->bchan->tc_ring);1609	k3_ringacc_ring_free(uc->bchan->t_ring);1610	uc->bchan->tc_ring = NULL;1611	uc->bchan->t_ring = NULL;1612	k3_configure_chan_coherency(&uc->vc.chan, 0);1613 1614	bcdma_put_bchan(uc);1615}1616 1617static int bcdma_alloc_bchan_resources(struct udma_chan *uc)1618{1619	struct k3_ring_cfg ring_cfg;1620	struct udma_dev *ud = uc->ud;1621	int ret;1622 1623	ret = bcdma_get_bchan(uc);1624	if (ret)1625		return ret;1626 1627	ret = k3_ringacc_request_rings_pair(ud->ringacc, uc->bchan->id, -1,1628					    &uc->bchan->t_ring,1629					    &uc->bchan->tc_ring);1630	if (ret) {1631		ret = -EBUSY;1632		goto err_ring;1633	}1634 1635	memset(&ring_cfg, 0, sizeof(ring_cfg));1636	ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;1637	ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;1638	ring_cfg.mode = K3_RINGACC_RING_MODE_RING;1639 1640	k3_configure_chan_coherency(&uc->vc.chan, ud->asel);1641	ring_cfg.asel = ud->asel;1642	ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan);1643 1644	ret = k3_ringacc_ring_cfg(uc->bchan->t_ring, &ring_cfg);1645	if (ret)1646		goto err_ringcfg;1647 1648	return 0;1649 1650err_ringcfg:1651	k3_ringacc_ring_free(uc->bchan->tc_ring);1652	uc->bchan->tc_ring = NULL;1653	k3_ringacc_ring_free(uc->bchan->t_ring);1654	uc->bchan->t_ring = NULL;1655	k3_configure_chan_coherency(&uc->vc.chan, 0);1656err_ring:1657	bcdma_put_bchan(uc);1658 1659	return ret;1660}1661 1662static void udma_free_tx_resources(struct udma_chan *uc)1663{1664	if (!uc->tchan)1665		return;1666 1667	k3_ringacc_ring_free(uc->tchan->t_ring);1668	k3_ringacc_ring_free(uc->tchan->tc_ring);1669	uc->tchan->t_ring = NULL;1670	uc->tchan->tc_ring = NULL;1671 1672	udma_put_tchan(uc);1673}1674 1675static int udma_alloc_tx_resources(struct udma_chan *uc)1676{1677	struct k3_ring_cfg ring_cfg;1678	struct udma_dev *ud = uc->ud;1679	struct udma_tchan *tchan;1680	int ring_idx, ret;1681 1682	ret = udma_get_tchan(uc);1683	if (ret)1684		return ret;1685 1686	tchan = uc->tchan;1687	if (tchan->tflow_id >= 0)1688		ring_idx = tchan->tflow_id;1689	else1690		ring_idx = ud->bchan_cnt + tchan->id;1691 1692	ret = k3_ringacc_request_rings_pair(ud->ringacc, ring_idx, -1,1693					    &tchan->t_ring,1694					    &tchan->tc_ring);1695	if (ret) {1696		ret = -EBUSY;1697		goto err_ring;1698	}1699 1700	memset(&ring_cfg, 0, sizeof(ring_cfg));1701	ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;1702	ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;1703	if (ud->match_data->type == DMA_TYPE_UDMA) {1704		ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE;1705	} else {1706		ring_cfg.mode = K3_RINGACC_RING_MODE_RING;1707 1708		k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel);1709		ring_cfg.asel = uc->config.asel;1710		ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan);1711	}1712 1713	ret = k3_ringacc_ring_cfg(tchan->t_ring, &ring_cfg);1714	ret |= k3_ringacc_ring_cfg(tchan->tc_ring, &ring_cfg);1715 1716	if (ret)1717		goto err_ringcfg;1718 1719	return 0;1720 1721err_ringcfg:1722	k3_ringacc_ring_free(uc->tchan->tc_ring);1723	uc->tchan->tc_ring = NULL;1724	k3_ringacc_ring_free(uc->tchan->t_ring);1725	uc->tchan->t_ring = NULL;1726err_ring:1727	udma_put_tchan(uc);1728 1729	return ret;1730}1731 1732static void udma_free_rx_resources(struct udma_chan *uc)1733{1734	if (!uc->rchan)1735		return;1736 1737	if (uc->rflow) {1738		struct udma_rflow *rflow = uc->rflow;1739 1740		k3_ringacc_ring_free(rflow->fd_ring);1741		k3_ringacc_ring_free(rflow->r_ring);1742		rflow->fd_ring = NULL;1743		rflow->r_ring = NULL;1744 1745		udma_put_rflow(uc);1746	}1747 1748	udma_put_rchan(uc);1749}1750 1751static int udma_alloc_rx_resources(struct udma_chan *uc)1752{1753	struct udma_dev *ud = uc->ud;1754	struct k3_ring_cfg ring_cfg;1755	struct udma_rflow *rflow;1756	int fd_ring_id;1757	int ret;1758 1759	ret = udma_get_rchan(uc);1760	if (ret)1761		return ret;1762 1763	/* For MEM_TO_MEM we don't need rflow or rings */1764	if (uc->config.dir == DMA_MEM_TO_MEM)1765		return 0;1766 1767	if (uc->config.default_flow_id >= 0)1768		ret = udma_get_rflow(uc, uc->config.default_flow_id);1769	else1770		ret = udma_get_rflow(uc, uc->rchan->id);1771 1772	if (ret) {1773		ret = -EBUSY;1774		goto err_rflow;1775	}1776 1777	rflow = uc->rflow;1778	if (ud->tflow_cnt)1779		fd_ring_id = ud->tflow_cnt + rflow->id;1780	else1781		fd_ring_id = ud->bchan_cnt + ud->tchan_cnt + ud->echan_cnt +1782			     uc->rchan->id;1783 1784	ret = k3_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1,1785					    &rflow->fd_ring, &rflow->r_ring);1786	if (ret) {1787		ret = -EBUSY;1788		goto err_ring;1789	}1790 1791	memset(&ring_cfg, 0, sizeof(ring_cfg));1792 1793	ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;1794	if (ud->match_data->type == DMA_TYPE_UDMA) {1795		if (uc->config.pkt_mode)1796			ring_cfg.size = SG_MAX_SEGMENTS;1797		else1798			ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;1799 1800		ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE;1801	} else {1802		ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;1803		ring_cfg.mode = K3_RINGACC_RING_MODE_RING;1804 1805		k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel);1806		ring_cfg.asel = uc->config.asel;1807		ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan);1808	}1809 1810	ret = k3_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg);1811 1812	ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;1813	ret |= k3_ringacc_ring_cfg(rflow->r_ring, &ring_cfg);1814 1815	if (ret)1816		goto err_ringcfg;1817 1818	return 0;1819 1820err_ringcfg:1821	k3_ringacc_ring_free(rflow->r_ring);1822	rflow->r_ring = NULL;1823	k3_ringacc_ring_free(rflow->fd_ring);1824	rflow->fd_ring = NULL;1825err_ring:1826	udma_put_rflow(uc);1827err_rflow:1828	udma_put_rchan(uc);1829 1830	return ret;1831}1832 1833#define TISCI_BCDMA_BCHAN_VALID_PARAMS (			\1834	TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |	\1835	TI_SCI_MSG_VALUE_RM_UDMAP_CH_EXTENDED_CH_TYPE_VALID)1836 1837#define TISCI_BCDMA_TCHAN_VALID_PARAMS (			\1838	TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |	\1839	TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID)1840 1841#define TISCI_BCDMA_RCHAN_VALID_PARAMS (			\1842	TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID)1843 1844#define TISCI_UDMA_TCHAN_VALID_PARAMS (				\1845	TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |	\1846	TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID |	\1847	TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID |	\1848	TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |		\1849	TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID |	\1850	TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |		\1851	TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |		\1852	TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)1853 1854#define TISCI_UDMA_RCHAN_VALID_PARAMS (				\1855	TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |	\1856	TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |		\1857	TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |		\1858	TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |		\1859	TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID |	\1860	TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID |	\1861	TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |	\1862	TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID |	\1863	TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)1864 1865static int udma_tisci_m2m_channel_config(struct udma_chan *uc)1866{1867	struct udma_dev *ud = uc->ud;1868	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;1869	const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;1870	struct udma_tchan *tchan = uc->tchan;1871	struct udma_rchan *rchan = uc->rchan;1872	u8 burst_size = 0;1873	int ret;1874	u8 tpl;1875 1876	/* Non synchronized - mem to mem type of transfer */1877	int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);1878	struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };1879	struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };1880 1881	if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) {1882		tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, tchan->id);1883 1884		burst_size = ud->match_data->burst_size[tpl];1885	}1886 1887	req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS;1888	req_tx.nav_id = tisci_rm->tisci_dev_id;1889	req_tx.index = tchan->id;1890	req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;1891	req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;1892	req_tx.txcq_qnum = tc_ring;1893	req_tx.tx_atype = ud->atype;1894	if (burst_size) {1895		req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID;1896		req_tx.tx_burst_size = burst_size;1897	}1898 1899	ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);1900	if (ret) {1901		dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);1902		return ret;1903	}1904 1905	req_rx.valid_params = TISCI_UDMA_RCHAN_VALID_PARAMS;1906	req_rx.nav_id = tisci_rm->tisci_dev_id;1907	req_rx.index = rchan->id;1908	req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;1909	req_rx.rxcq_qnum = tc_ring;1910	req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;1911	req_rx.rx_atype = ud->atype;1912	if (burst_size) {1913		req_rx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID;1914		req_rx.rx_burst_size = burst_size;1915	}1916 1917	ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);1918	if (ret)1919		dev_err(ud->dev, "rchan%d alloc failed %d\n", rchan->id, ret);1920 1921	return ret;1922}1923 1924static int bcdma_tisci_m2m_channel_config(struct udma_chan *uc)1925{1926	struct udma_dev *ud = uc->ud;1927	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;1928	const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;1929	struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };1930	struct udma_bchan *bchan = uc->bchan;1931	u8 burst_size = 0;1932	int ret;1933	u8 tpl;1934 1935	if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) {1936		tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, bchan->id);1937 1938		burst_size = ud->match_data->burst_size[tpl];1939	}1940 1941	req_tx.valid_params = TISCI_BCDMA_BCHAN_VALID_PARAMS;1942	req_tx.nav_id = tisci_rm->tisci_dev_id;1943	req_tx.extended_ch_type = TI_SCI_RM_BCDMA_EXTENDED_CH_TYPE_BCHAN;1944	req_tx.index = bchan->id;1945	if (burst_size) {1946		req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID;1947		req_tx.tx_burst_size = burst_size;1948	}1949 1950	ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);1951	if (ret)1952		dev_err(ud->dev, "bchan%d cfg failed %d\n", bchan->id, ret);1953 1954	return ret;1955}1956 1957static int udma_tisci_tx_channel_config(struct udma_chan *uc)1958{1959	struct udma_dev *ud = uc->ud;1960	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;1961	const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;1962	struct udma_tchan *tchan = uc->tchan;1963	int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);1964	struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };1965	u32 mode, fetch_size;1966	int ret;1967 1968	if (uc->config.pkt_mode) {1969		mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;1970		fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,1971						   uc->config.psd_size, 0);1972	} else {1973		mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR;1974		fetch_size = sizeof(struct cppi5_desc_hdr_t);1975	}1976 1977	req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS;1978	req_tx.nav_id = tisci_rm->tisci_dev_id;1979	req_tx.index = tchan->id;1980	req_tx.tx_chan_type = mode;1981	req_tx.tx_supr_tdpkt = uc->config.notdpkt;1982	req_tx.tx_fetch_size = fetch_size >> 2;1983	req_tx.txcq_qnum = tc_ring;1984	req_tx.tx_atype = uc->config.atype;1985	if (uc->config.ep_type == PSIL_EP_PDMA_XY &&1986	    ud->match_data->flags & UDMA_FLAG_TDTYPE) {1987		/* wait for peer to complete the teardown for PDMAs */1988		req_tx.valid_params |=1989				TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID;1990		req_tx.tx_tdtype = 1;1991	}1992 1993	ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);1994	if (ret)1995		dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);1996 1997	return ret;1998}1999 2000static int bcdma_tisci_tx_channel_config(struct udma_chan *uc)2001{2002	struct udma_dev *ud = uc->ud;2003	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;2004	const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;2005	struct udma_tchan *tchan = uc->tchan;2006	struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };2007	int ret;2008 2009	req_tx.valid_params = TISCI_BCDMA_TCHAN_VALID_PARAMS;2010	req_tx.nav_id = tisci_rm->tisci_dev_id;2011	req_tx.index = tchan->id;2012	req_tx.tx_supr_tdpkt = uc->config.notdpkt;2013	if (ud->match_data->flags & UDMA_FLAG_TDTYPE) {2014		/* wait for peer to complete the teardown for PDMAs */2015		req_tx.valid_params |=2016				TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID;2017		req_tx.tx_tdtype = 1;2018	}2019 2020	ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);2021	if (ret)2022		dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);2023 2024	return ret;2025}2026 2027#define pktdma_tisci_tx_channel_config bcdma_tisci_tx_channel_config2028 2029static int udma_tisci_rx_channel_config(struct udma_chan *uc)2030{2031	struct udma_dev *ud = uc->ud;2032	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;2033	const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;2034	struct udma_rchan *rchan = uc->rchan;2035	int fd_ring = k3_ringacc_get_ring_id(uc->rflow->fd_ring);2036	int rx_ring = k3_ringacc_get_ring_id(uc->rflow->r_ring);2037	struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };2038	struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };2039	u32 mode, fetch_size;2040	int ret;2041 2042	if (uc->config.pkt_mode) {2043		mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;2044		fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,2045						   uc->config.psd_size, 0);2046	} else {2047		mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR;2048		fetch_size = sizeof(struct cppi5_desc_hdr_t);2049	}2050 2051	req_rx.valid_params = TISCI_UDMA_RCHAN_VALID_PARAMS;2052	req_rx.nav_id = tisci_rm->tisci_dev_id;2053	req_rx.index = rchan->id;2054	req_rx.rx_fetch_size =  fetch_size >> 2;2055	req_rx.rxcq_qnum = rx_ring;2056	req_rx.rx_chan_type = mode;2057	req_rx.rx_atype = uc->config.atype;2058 2059	ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);2060	if (ret) {2061		dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret);2062		return ret;2063	}2064 2065	flow_req.valid_params =2066		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |2067		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |2068		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |2069		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |2070		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |2071		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |2072		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |2073		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |2074		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |2075		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |2076		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |2077		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |2078		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;2079 2080	flow_req.nav_id = tisci_rm->tisci_dev_id;2081	flow_req.flow_index = rchan->id;2082 2083	if (uc->config.needs_epib)2084		flow_req.rx_einfo_present = 1;2085	else2086		flow_req.rx_einfo_present = 0;2087	if (uc->config.psd_size)2088		flow_req.rx_psinfo_present = 1;2089	else2090		flow_req.rx_psinfo_present = 0;2091	flow_req.rx_error_handling = 1;2092	flow_req.rx_dest_qnum = rx_ring;2093	flow_req.rx_src_tag_hi_sel = UDMA_RFLOW_SRCTAG_NONE;2094	flow_req.rx_src_tag_lo_sel = UDMA_RFLOW_SRCTAG_SRC_TAG;2095	flow_req.rx_dest_tag_hi_sel = UDMA_RFLOW_DSTTAG_DST_TAG_HI;2096	flow_req.rx_dest_tag_lo_sel = UDMA_RFLOW_DSTTAG_DST_TAG_LO;2097	flow_req.rx_fdq0_sz0_qnum = fd_ring;2098	flow_req.rx_fdq1_qnum = fd_ring;2099	flow_req.rx_fdq2_qnum = fd_ring;2100	flow_req.rx_fdq3_qnum = fd_ring;2101 2102	ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req);2103 2104	if (ret)2105		dev_err(ud->dev, "flow%d config failed: %d\n", rchan->id, ret);2106 2107	return 0;2108}2109 2110static int bcdma_tisci_rx_channel_config(struct udma_chan *uc)2111{2112	struct udma_dev *ud = uc->ud;2113	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;2114	const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;2115	struct udma_rchan *rchan = uc->rchan;2116	struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };2117	int ret;2118 2119	req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS;2120	req_rx.nav_id = tisci_rm->tisci_dev_id;2121	req_rx.index = rchan->id;2122 2123	ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);2124	if (ret)2125		dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret);2126 2127	return ret;2128}2129 2130static int pktdma_tisci_rx_channel_config(struct udma_chan *uc)2131{2132	struct udma_dev *ud = uc->ud;2133	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;2134	const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;2135	struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };2136	struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };2137	int ret;2138 2139	req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS;2140	req_rx.nav_id = tisci_rm->tisci_dev_id;2141	req_rx.index = uc->rchan->id;2142 2143	ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);2144	if (ret) {2145		dev_err(ud->dev, "rchan%d cfg failed %d\n", uc->rchan->id, ret);2146		return ret;2147	}2148 2149	flow_req.valid_params =2150		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |2151		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |2152		TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID;2153 2154	flow_req.nav_id = tisci_rm->tisci_dev_id;2155	flow_req.flow_index = uc->rflow->id;2156 2157	if (uc->config.needs_epib)2158		flow_req.rx_einfo_present = 1;2159	else2160		flow_req.rx_einfo_present = 0;2161	if (uc->config.psd_size)2162		flow_req.rx_psinfo_present = 1;2163	else2164		flow_req.rx_psinfo_present = 0;2165	flow_req.rx_error_handling = 1;2166 2167	ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req);2168 2169	if (ret)2170		dev_err(ud->dev, "flow%d config failed: %d\n", uc->rflow->id,2171			ret);2172 2173	return ret;2174}2175 2176static int udma_alloc_chan_resources(struct dma_chan *chan)2177{2178	struct udma_chan *uc = to_udma_chan(chan);2179	struct udma_dev *ud = to_udma_dev(chan->device);2180	const struct udma_soc_data *soc_data = ud->soc_data;2181	struct k3_ring *irq_ring;2182	u32 irq_udma_idx;2183	int ret;2184 2185	uc->dma_dev = ud->dev;2186 2187	if (uc->config.pkt_mode || uc->config.dir == DMA_MEM_TO_MEM) {2188		uc->use_dma_pool = true;2189		/* in case of MEM_TO_MEM we have maximum of two TRs */2190		if (uc->config.dir == DMA_MEM_TO_MEM) {2191			uc->config.hdesc_size = cppi5_trdesc_calc_size(2192					sizeof(struct cppi5_tr_type15_t), 2);2193			uc->config.pkt_mode = false;2194		}2195	}2196 2197	if (uc->use_dma_pool) {2198		uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev,2199						 uc->config.hdesc_size,2200						 ud->desc_align,2201						 0);2202		if (!uc->hdesc_pool) {2203			dev_err(ud->ddev.dev,2204				"Descriptor pool allocation failed\n");2205			uc->use_dma_pool = false;2206			ret = -ENOMEM;2207			goto err_cleanup;2208		}2209	}2210 2211	/*2212	 * Make sure that the completion is in a known state:2213	 * No teardown, the channel is idle2214	 */2215	reinit_completion(&uc->teardown_completed);2216	complete_all(&uc->teardown_completed);2217	uc->state = UDMA_CHAN_IS_IDLE;2218 2219	switch (uc->config.dir) {2220	case DMA_MEM_TO_MEM:2221		/* Non synchronized - mem to mem type of transfer */2222		dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,2223			uc->id);2224 2225		ret = udma_get_chan_pair(uc);2226		if (ret)2227			goto err_cleanup;2228 2229		ret = udma_alloc_tx_resources(uc);2230		if (ret) {2231			udma_put_rchan(uc);2232			goto err_cleanup;2233		}2234 2235		ret = udma_alloc_rx_resources(uc);2236		if (ret) {2237			udma_free_tx_resources(uc);2238			goto err_cleanup;2239		}2240 2241		uc->config.src_thread = ud->psil_base + uc->tchan->id;2242		uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |2243					K3_PSIL_DST_THREAD_ID_OFFSET;2244 2245		irq_ring = uc->tchan->tc_ring;2246		irq_udma_idx = uc->tchan->id;2247 2248		ret = udma_tisci_m2m_channel_config(uc);2249		break;2250	case DMA_MEM_TO_DEV:2251		/* Slave transfer synchronized - mem to dev (TX) trasnfer */2252		dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,2253			uc->id);2254 2255		ret = udma_alloc_tx_resources(uc);2256		if (ret)2257			goto err_cleanup;2258 2259		uc->config.src_thread = ud->psil_base + uc->tchan->id;2260		uc->config.dst_thread = uc->config.remote_thread_id;2261		uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;2262 2263		irq_ring = uc->tchan->tc_ring;2264		irq_udma_idx = uc->tchan->id;2265 2266		ret = udma_tisci_tx_channel_config(uc);2267		break;2268	case DMA_DEV_TO_MEM:2269		/* Slave transfer synchronized - dev to mem (RX) trasnfer */2270		dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,2271			uc->id);2272 2273		ret = udma_alloc_rx_resources(uc);2274		if (ret)2275			goto err_cleanup;2276 2277		uc->config.src_thread = uc->config.remote_thread_id;2278		uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |2279					K3_PSIL_DST_THREAD_ID_OFFSET;2280 2281		irq_ring = uc->rflow->r_ring;2282		irq_udma_idx = soc_data->oes.udma_rchan + uc->rchan->id;2283 2284		ret = udma_tisci_rx_channel_config(uc);2285		break;2286	default:2287		/* Can not happen */2288		dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",2289			__func__, uc->id, uc->config.dir);2290		ret = -EINVAL;2291		goto err_cleanup;2292 2293	}2294 2295	/* check if the channel configuration was successful */2296	if (ret)2297		goto err_res_free;2298 2299	if (udma_is_chan_running(uc)) {2300		dev_warn(ud->dev, "chan%d: is running!\n", uc->id);2301		udma_reset_chan(uc, false);2302		if (udma_is_chan_running(uc)) {2303			dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);2304			ret = -EBUSY;2305			goto err_res_free;2306		}2307	}2308 2309	/* PSI-L pairing */2310	ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);2311	if (ret) {2312		dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",2313			uc->config.src_thread, uc->config.dst_thread);2314		goto err_res_free;2315	}2316 2317	uc->psil_paired = true;2318 2319	uc->irq_num_ring = k3_ringacc_get_ring_irq_num(irq_ring);2320	if (uc->irq_num_ring <= 0) {2321		dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",2322			k3_ringacc_get_ring_id(irq_ring));2323		ret = -EINVAL;2324		goto err_psi_free;2325	}2326 2327	ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,2328			  IRQF_TRIGGER_HIGH, uc->name, uc);2329	if (ret) {2330		dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);2331		goto err_irq_free;2332	}2333 2334	/* Event from UDMA (TR events) only needed for slave TR mode channels */2335	if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) {2336		uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx);2337		if (uc->irq_num_udma <= 0) {2338			dev_err(ud->dev, "Failed to get udma irq (index: %u)\n",2339				irq_udma_idx);2340			free_irq(uc->irq_num_ring, uc);2341			ret = -EINVAL;2342			goto err_irq_free;2343		}2344 2345		ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0,2346				  uc->name, uc);2347		if (ret) {2348			dev_err(ud->dev, "chan%d: UDMA irq request failed\n",2349				uc->id);2350			free_irq(uc->irq_num_ring, uc);2351			goto err_irq_free;2352		}2353	} else {2354		uc->irq_num_udma = 0;2355	}2356 2357	udma_reset_rings(uc);2358 2359	return 0;2360 2361err_irq_free:2362	uc->irq_num_ring = 0;2363	uc->irq_num_udma = 0;2364err_psi_free:2365	navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread);2366	uc->psil_paired = false;2367err_res_free:2368	udma_free_tx_resources(uc);2369	udma_free_rx_resources(uc);2370err_cleanup:2371	udma_reset_uchan(uc);2372 2373	if (uc->use_dma_pool) {2374		dma_pool_destroy(uc->hdesc_pool);2375		uc->use_dma_pool = false;2376	}2377 2378	return ret;2379}2380 2381static int bcdma_alloc_chan_resources(struct dma_chan *chan)2382{2383	struct udma_chan *uc = to_udma_chan(chan);2384	struct udma_dev *ud = to_udma_dev(chan->device);2385	const struct udma_oes_offsets *oes = &ud->soc_data->oes;2386	u32 irq_udma_idx, irq_ring_idx;2387	int ret;2388 2389	/* Only TR mode is supported */2390	uc->config.pkt_mode = false;2391 2392	/*2393	 * Make sure that the completion is in a known state:2394	 * No teardown, the channel is idle2395	 */2396	reinit_completion(&uc->teardown_completed);2397	complete_all(&uc->teardown_completed);2398	uc->state = UDMA_CHAN_IS_IDLE;2399 2400	switch (uc->config.dir) {2401	case DMA_MEM_TO_MEM:2402		/* Non synchronized - mem to mem type of transfer */2403		dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,2404			uc->id);2405 2406		ret = bcdma_alloc_bchan_resources(uc);2407		if (ret)2408			return ret;2409 2410		irq_ring_idx = uc->bchan->id + oes->bcdma_bchan_ring;2411		irq_udma_idx = uc->bchan->id + oes->bcdma_bchan_data;2412 2413		ret = bcdma_tisci_m2m_channel_config(uc);2414		break;2415	case DMA_MEM_TO_DEV:2416		/* Slave transfer synchronized - mem to dev (TX) trasnfer */2417		dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,2418			uc->id);2419 2420		ret = udma_alloc_tx_resources(uc);2421		if (ret) {2422			uc->config.remote_thread_id = -1;2423			return ret;2424		}2425 2426		uc->config.src_thread = ud->psil_base + uc->tchan->id;2427		uc->config.dst_thread = uc->config.remote_thread_id;2428		uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;2429 2430		irq_ring_idx = uc->tchan->id + oes->bcdma_tchan_ring;2431		irq_udma_idx = uc->tchan->id + oes->bcdma_tchan_data;2432 2433		ret = bcdma_tisci_tx_channel_config(uc);2434		break;2435	case DMA_DEV_TO_MEM:2436		/* Slave transfer synchronized - dev to mem (RX) trasnfer */2437		dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,2438			uc->id);2439 2440		ret = udma_alloc_rx_resources(uc);2441		if (ret) {2442			uc->config.remote_thread_id = -1;2443			return ret;2444		}2445 2446		uc->config.src_thread = uc->config.remote_thread_id;2447		uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |2448					K3_PSIL_DST_THREAD_ID_OFFSET;2449 2450		irq_ring_idx = uc->rchan->id + oes->bcdma_rchan_ring;2451		irq_udma_idx = uc->rchan->id + oes->bcdma_rchan_data;2452 2453		ret = bcdma_tisci_rx_channel_config(uc);2454		break;2455	default:2456		/* Can not happen */2457		dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",2458			__func__, uc->id, uc->config.dir);2459		return -EINVAL;2460	}2461 2462	/* check if the channel configuration was successful */2463	if (ret)2464		goto err_res_free;2465 2466	if (udma_is_chan_running(uc)) {2467		dev_warn(ud->dev, "chan%d: is running!\n", uc->id);2468		udma_reset_chan(uc, false);2469		if (udma_is_chan_running(uc)) {2470			dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);2471			ret = -EBUSY;2472			goto err_res_free;2473		}2474	}2475 2476	uc->dma_dev = dmaengine_get_dma_device(chan);2477	if (uc->config.dir == DMA_MEM_TO_MEM  && !uc->config.tr_trigger_type) {2478		uc->config.hdesc_size = cppi5_trdesc_calc_size(2479					sizeof(struct cppi5_tr_type15_t), 2);2480 2481		uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev,2482						 uc->config.hdesc_size,2483						 ud->desc_align,2484						 0);2485		if (!uc->hdesc_pool) {2486			dev_err(ud->ddev.dev,2487				"Descriptor pool allocation failed\n");2488			uc->use_dma_pool = false;2489			ret = -ENOMEM;2490			goto err_res_free;2491		}2492 2493		uc->use_dma_pool = true;2494	} else if (uc->config.dir != DMA_MEM_TO_MEM) {2495		/* PSI-L pairing */2496		ret = navss_psil_pair(ud, uc->config.src_thread,2497				      uc->config.dst_thread);2498		if (ret) {2499			dev_err(ud->dev,2500				"PSI-L pairing failed: 0x%04x -> 0x%04x\n",2501				uc->config.src_thread, uc->config.dst_thread);2502			goto err_res_free;2503		}2504 2505		uc->psil_paired = true;2506	}2507 2508	uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx);2509	if (uc->irq_num_ring <= 0) {2510		dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",2511			irq_ring_idx);2512		ret = -EINVAL;2513		goto err_psi_free;2514	}2515 2516	ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,2517			  IRQF_TRIGGER_HIGH, uc->name, uc);2518	if (ret) {2519		dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);2520		goto err_irq_free;2521	}2522 2523	/* Event from BCDMA (TR events) only needed for slave channels */2524	if (is_slave_direction(uc->config.dir)) {2525		uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx);2526		if (uc->irq_num_udma <= 0) {2527			dev_err(ud->dev, "Failed to get bcdma irq (index: %u)\n",2528				irq_udma_idx);2529			free_irq(uc->irq_num_ring, uc);2530			ret = -EINVAL;2531			goto err_irq_free;2532		}2533 2534		ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0,2535				  uc->name, uc);2536		if (ret) {2537			dev_err(ud->dev, "chan%d: BCDMA irq request failed\n",2538				uc->id);2539			free_irq(uc->irq_num_ring, uc);2540			goto err_irq_free;2541		}2542	} else {2543		uc->irq_num_udma = 0;2544	}2545 2546	udma_reset_rings(uc);2547 2548	INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work,2549				  udma_check_tx_completion);2550	return 0;2551 2552err_irq_free:2553	uc->irq_num_ring = 0;2554	uc->irq_num_udma = 0;2555err_psi_free:2556	if (uc->psil_paired)2557		navss_psil_unpair(ud, uc->config.src_thread,2558				  uc->config.dst_thread);2559	uc->psil_paired = false;2560err_res_free:2561	bcdma_free_bchan_resources(uc);2562	udma_free_tx_resources(uc);2563	udma_free_rx_resources(uc);2564 2565	udma_reset_uchan(uc);2566 2567	if (uc->use_dma_pool) {2568		dma_pool_destroy(uc->hdesc_pool);2569		uc->use_dma_pool = false;2570	}2571 2572	return ret;2573}2574 2575static int bcdma_router_config(struct dma_chan *chan)2576{2577	struct k3_event_route_data *router_data = chan->route_data;2578	struct udma_chan *uc = to_udma_chan(chan);2579	u32 trigger_event;2580 2581	if (!uc->bchan)2582		return -EINVAL;2583 2584	if (uc->config.tr_trigger_type != 1 && uc->config.tr_trigger_type != 2)2585		return -EINVAL;2586 2587	trigger_event = uc->ud->soc_data->bcdma_trigger_event_offset;2588	trigger_event += (uc->bchan->id * 2) + uc->config.tr_trigger_type - 1;2589 2590	return router_data->set_event(router_data->priv, trigger_event);2591}2592 2593static int pktdma_alloc_chan_resources(struct dma_chan *chan)2594{2595	struct udma_chan *uc = to_udma_chan(chan);2596	struct udma_dev *ud = to_udma_dev(chan->device);2597	const struct udma_oes_offsets *oes = &ud->soc_data->oes;2598	u32 irq_ring_idx;2599	int ret;2600 2601	/*2602	 * Make sure that the completion is in a known state:2603	 * No teardown, the channel is idle2604	 */2605	reinit_completion(&uc->teardown_completed);2606	complete_all(&uc->teardown_completed);2607	uc->state = UDMA_CHAN_IS_IDLE;2608 2609	switch (uc->config.dir) {2610	case DMA_MEM_TO_DEV:2611		/* Slave transfer synchronized - mem to dev (TX) trasnfer */2612		dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,2613			uc->id);2614 2615		ret = udma_alloc_tx_resources(uc);2616		if (ret) {2617			uc->config.remote_thread_id = -1;2618			return ret;2619		}2620 2621		uc->config.src_thread = ud->psil_base + uc->tchan->id;2622		uc->config.dst_thread = uc->config.remote_thread_id;2623		uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;2624 2625		irq_ring_idx = uc->tchan->tflow_id + oes->pktdma_tchan_flow;2626 2627		ret = pktdma_tisci_tx_channel_config(uc);2628		break;2629	case DMA_DEV_TO_MEM:2630		/* Slave transfer synchronized - dev to mem (RX) trasnfer */2631		dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,2632			uc->id);2633 2634		ret = udma_alloc_rx_resources(uc);2635		if (ret) {2636			uc->config.remote_thread_id = -1;2637			return ret;2638		}2639 2640		uc->config.src_thread = uc->config.remote_thread_id;2641		uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |2642					K3_PSIL_DST_THREAD_ID_OFFSET;2643 2644		irq_ring_idx = uc->rflow->id + oes->pktdma_rchan_flow;2645 2646		ret = pktdma_tisci_rx_channel_config(uc);2647		break;2648	default:2649		/* Can not happen */2650		dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",2651			__func__, uc->id, uc->config.dir);2652		return -EINVAL;2653	}2654 2655	/* check if the channel configuration was successful */2656	if (ret)2657		goto err_res_free;2658 2659	if (udma_is_chan_running(uc)) {2660		dev_warn(ud->dev, "chan%d: is running!\n", uc->id);2661		udma_reset_chan(uc, false);2662		if (udma_is_chan_running(uc)) {2663			dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);2664			ret = -EBUSY;2665			goto err_res_free;2666		}2667	}2668 2669	uc->dma_dev = dmaengine_get_dma_device(chan);2670	uc->hdesc_pool = dma_pool_create(uc->name, uc->dma_dev,2671					 uc->config.hdesc_size, ud->desc_align,2672					 0);2673	if (!uc->hdesc_pool) {2674		dev_err(ud->ddev.dev,2675			"Descriptor pool allocation failed\n");2676		uc->use_dma_pool = false;2677		ret = -ENOMEM;2678		goto err_res_free;2679	}2680 2681	uc->use_dma_pool = true;2682 2683	/* PSI-L pairing */2684	ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);2685	if (ret) {2686		dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",2687			uc->config.src_thread, uc->config.dst_thread);2688		goto err_res_free;2689	}2690 2691	uc->psil_paired = true;2692 2693	uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx);2694	if (uc->irq_num_ring <= 0) {2695		dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",2696			irq_ring_idx);2697		ret = -EINVAL;2698		goto err_psi_free;2699	}2700 2701	ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,2702			  IRQF_TRIGGER_HIGH, uc->name, uc);2703	if (ret) {2704		dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);2705		goto err_irq_free;2706	}2707 2708	uc->irq_num_udma = 0;2709 2710	udma_reset_rings(uc);2711 2712	INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work,2713				  udma_check_tx_completion);2714 2715	if (uc->tchan)2716		dev_dbg(ud->dev,2717			"chan%d: tchan%d, tflow%d, Remote thread: 0x%04x\n",2718			uc->id, uc->tchan->id, uc->tchan->tflow_id,2719			uc->config.remote_thread_id);2720	else if (uc->rchan)2721		dev_dbg(ud->dev,2722			"chan%d: rchan%d, rflow%d, Remote thread: 0x%04x\n",2723			uc->id, uc->rchan->id, uc->rflow->id,2724			uc->config.remote_thread_id);2725	return 0;2726 2727err_irq_free:2728	uc->irq_num_ring = 0;2729err_psi_free:2730	navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread);2731	uc->psil_paired = false;2732err_res_free:2733	udma_free_tx_resources(uc);2734	udma_free_rx_resources(uc);2735 2736	udma_reset_uchan(uc);2737 2738	dma_pool_destroy(uc->hdesc_pool);2739	uc->use_dma_pool = false;2740 2741	return ret;2742}2743 2744static int udma_slave_config(struct dma_chan *chan,2745			     struct dma_slave_config *cfg)2746{2747	struct udma_chan *uc = to_udma_chan(chan);2748 2749	memcpy(&uc->cfg, cfg, sizeof(uc->cfg));2750 2751	return 0;2752}2753 2754static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc,2755					    size_t tr_size, int tr_count,2756					    enum dma_transfer_direction dir)2757{2758	struct udma_hwdesc *hwdesc;2759	struct cppi5_desc_hdr_t *tr_desc;2760	struct udma_desc *d;2761	u32 reload_count = 0;2762	u32 ring_id;2763 2764	switch (tr_size) {2765	case 16:2766	case 32:2767	case 64:2768	case 128:2769		break;2770	default:2771		dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size);2772		return NULL;2773	}2774 2775	/* We have only one descriptor containing multiple TRs */2776	d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT);2777	if (!d)2778		return NULL;2779 2780	d->sglen = tr_count;2781 2782	d->hwdesc_count = 1;2783	hwdesc = &d->hwdesc[0];2784 2785	/* Allocate memory for DMA ring descriptor */2786	if (uc->use_dma_pool) {2787		hwdesc->cppi5_desc_size = uc->config.hdesc_size;2788		hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,2789						GFP_NOWAIT,2790						&hwdesc->cppi5_desc_paddr);2791	} else {2792		hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size,2793								 tr_count);2794		hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,2795						uc->ud->desc_align);2796		hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev,2797						hwdesc->cppi5_desc_size,2798						&hwdesc->cppi5_desc_paddr,2799						GFP_NOWAIT);2800	}2801 2802	if (!hwdesc->cppi5_desc_vaddr) {2803		kfree(d);2804		return NULL;2805	}2806 2807	/* Start of the TR req records */2808	hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;2809	/* Start address of the TR response array */2810	hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count;2811 2812	tr_desc = hwdesc->cppi5_desc_vaddr;2813 2814	if (uc->cyclic)2815		reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE;2816 2817	if (dir == DMA_DEV_TO_MEM)2818		ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);2819	else2820		ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);2821 2822	cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count);2823	cppi5_desc_set_pktids(tr_desc, uc->id,2824			      CPPI5_INFO1_DESC_FLOWID_DEFAULT);2825	cppi5_desc_set_retpolicy(tr_desc, 0, ring_id);2826 2827	return d;2828}2829 2830/**2831 * udma_get_tr_counters - calculate TR counters for a given length2832 * @len: Length of the trasnfer2833 * @align_to: Preferred alignment2834 * @tr0_cnt0: First TR icnt02835 * @tr0_cnt1: First TR icnt12836 * @tr1_cnt0: Second (if used) TR icnt02837 *2838 * For len < SZ_64K only one TR is enough, tr1_cnt0 is not updated2839 * For len >= SZ_64K two TRs are used in a simple way:2840 * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1)2841 * Second TR: the remaining length (tr1_cnt0)2842 *2843 * Returns the number of TRs the length needs (1 or 2)2844 * -EINVAL if the length can not be supported2845 */2846static int udma_get_tr_counters(size_t len, unsigned long align_to,2847				u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0)2848{2849	if (len < SZ_64K) {2850		*tr0_cnt0 = len;2851		*tr0_cnt1 = 1;2852 2853		return 1;2854	}2855 2856	if (align_to > 3)2857		align_to = 3;2858 2859realign:2860	*tr0_cnt0 = SZ_64K - BIT(align_to);2861	if (len / *tr0_cnt0 >= SZ_64K) {2862		if (align_to) {2863			align_to--;2864			goto realign;2865		}2866		return -EINVAL;2867	}2868 2869	*tr0_cnt1 = len / *tr0_cnt0;2870	*tr1_cnt0 = len % *tr0_cnt0;2871 2872	return 2;2873}2874 2875static struct udma_desc *2876udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl,2877		      unsigned int sglen, enum dma_transfer_direction dir,2878		      unsigned long tx_flags, void *context)2879{2880	struct scatterlist *sgent;2881	struct udma_desc *d;2882	struct cppi5_tr_type1_t *tr_req = NULL;2883	u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;2884	unsigned int i;2885	size_t tr_size;2886	int num_tr = 0;2887	int tr_idx = 0;2888	u64 asel;2889 2890	/* estimate the number of TRs we will need */2891	for_each_sg(sgl, sgent, sglen, i) {2892		if (sg_dma_len(sgent) < SZ_64K)2893			num_tr++;2894		else2895			num_tr += 2;2896	}2897 2898	/* Now allocate and setup the descriptor. */2899	tr_size = sizeof(struct cppi5_tr_type1_t);2900	d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir);2901	if (!d)2902		return NULL;2903 2904	d->sglen = sglen;2905 2906	if (uc->ud->match_data->type == DMA_TYPE_UDMA)2907		asel = 0;2908	else2909		asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;2910 2911	tr_req = d->hwdesc[0].tr_req_base;2912	for_each_sg(sgl, sgent, sglen, i) {2913		dma_addr_t sg_addr = sg_dma_address(sgent);2914 2915		num_tr = udma_get_tr_counters(sg_dma_len(sgent), __ffs(sg_addr),2916					      &tr0_cnt0, &tr0_cnt1, &tr1_cnt0);2917		if (num_tr < 0) {2918			dev_err(uc->ud->dev, "size %u is not supported\n",2919				sg_dma_len(sgent));2920			udma_free_hwdesc(uc, d);2921			kfree(d);2922			return NULL;2923		}2924 2925		cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,2926			      false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);2927		cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT);2928 2929		sg_addr |= asel;2930		tr_req[tr_idx].addr = sg_addr;2931		tr_req[tr_idx].icnt0 = tr0_cnt0;2932		tr_req[tr_idx].icnt1 = tr0_cnt1;2933		tr_req[tr_idx].dim1 = tr0_cnt0;2934		tr_idx++;2935 2936		if (num_tr == 2) {2937			cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,2938				      false, false,2939				      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);2940			cppi5_tr_csf_set(&tr_req[tr_idx].flags,2941					 CPPI5_TR_CSF_SUPR_EVT);2942 2943			tr_req[tr_idx].addr = sg_addr + tr0_cnt1 * tr0_cnt0;2944			tr_req[tr_idx].icnt0 = tr1_cnt0;2945			tr_req[tr_idx].icnt1 = 1;2946			tr_req[tr_idx].dim1 = tr1_cnt0;2947			tr_idx++;2948		}2949 2950		d->residue += sg_dma_len(sgent);2951	}2952 2953	cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags,2954			 CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);2955 2956	return d;2957}2958 2959static struct udma_desc *2960udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl,2961				unsigned int sglen,2962				enum dma_transfer_direction dir,2963				unsigned long tx_flags, void *context)2964{2965	struct scatterlist *sgent;2966	struct cppi5_tr_type15_t *tr_req = NULL;2967	enum dma_slave_buswidth dev_width;2968	u32 csf = CPPI5_TR_CSF_SUPR_EVT;2969	u16 tr_cnt0, tr_cnt1;2970	dma_addr_t dev_addr;2971	struct udma_desc *d;2972	unsigned int i;2973	size_t tr_size, sg_len;2974	int num_tr = 0;2975	int tr_idx = 0;2976	u32 burst, trigger_size, port_window;2977	u64 asel;2978 2979	if (dir == DMA_DEV_TO_MEM) {2980		dev_addr = uc->cfg.src_addr;2981		dev_width = uc->cfg.src_addr_width;2982		burst = uc->cfg.src_maxburst;2983		port_window = uc->cfg.src_port_window_size;2984	} else if (dir == DMA_MEM_TO_DEV) {2985		dev_addr = uc->cfg.dst_addr;2986		dev_width = uc->cfg.dst_addr_width;2987		burst = uc->cfg.dst_maxburst;2988		port_window = uc->cfg.dst_port_window_size;2989	} else {2990		dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);2991		return NULL;2992	}2993 2994	if (!burst)2995		burst = 1;2996 2997	if (port_window) {2998		if (port_window != burst) {2999			dev_err(uc->ud->dev,3000				"The burst must be equal to port_window\n");3001			return NULL;3002		}3003 3004		tr_cnt0 = dev_width * port_window;3005		tr_cnt1 = 1;3006	} else {3007		tr_cnt0 = dev_width;3008		tr_cnt1 = burst;3009	}3010	trigger_size = tr_cnt0 * tr_cnt1;3011 3012	/* estimate the number of TRs we will need */3013	for_each_sg(sgl, sgent, sglen, i) {3014		sg_len = sg_dma_len(sgent);3015 3016		if (sg_len % trigger_size) {3017			dev_err(uc->ud->dev,3018				"Not aligned SG entry (%zu for %u)\n", sg_len,3019				trigger_size);3020			return NULL;3021		}3022 3023		if (sg_len / trigger_size < SZ_64K)3024			num_tr++;3025		else3026			num_tr += 2;3027	}3028 3029	/* Now allocate and setup the descriptor. */3030	tr_size = sizeof(struct cppi5_tr_type15_t);3031	d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir);3032	if (!d)3033		return NULL;3034 3035	d->sglen = sglen;3036 3037	if (uc->ud->match_data->type == DMA_TYPE_UDMA) {3038		asel = 0;3039		csf |= CPPI5_TR_CSF_EOL_ICNT0;3040	} else {3041		asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;3042		dev_addr |= asel;3043	}3044 3045	tr_req = d->hwdesc[0].tr_req_base;3046	for_each_sg(sgl, sgent, sglen, i) {3047		u16 tr0_cnt2, tr0_cnt3, tr1_cnt2;3048		dma_addr_t sg_addr = sg_dma_address(sgent);3049 3050		sg_len = sg_dma_len(sgent);3051		num_tr = udma_get_tr_counters(sg_len / trigger_size, 0,3052					      &tr0_cnt2, &tr0_cnt3, &tr1_cnt2);3053		if (num_tr < 0) {3054			dev_err(uc->ud->dev, "size %zu is not supported\n",3055				sg_len);3056			udma_free_hwdesc(uc, d);3057			kfree(d);3058			return NULL;3059		}3060 3061		cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, false,3062			      true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);3063		cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf);3064		cppi5_tr_set_trigger(&tr_req[tr_idx].flags,3065				     uc->config.tr_trigger_type,3066				     CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, 0, 0);3067 3068		sg_addr |= asel;3069		if (dir == DMA_DEV_TO_MEM) {3070			tr_req[tr_idx].addr = dev_addr;3071			tr_req[tr_idx].icnt0 = tr_cnt0;3072			tr_req[tr_idx].icnt1 = tr_cnt1;3073			tr_req[tr_idx].icnt2 = tr0_cnt2;3074			tr_req[tr_idx].icnt3 = tr0_cnt3;3075			tr_req[tr_idx].dim1 = (-1) * tr_cnt0;3076 3077			tr_req[tr_idx].daddr = sg_addr;3078			tr_req[tr_idx].dicnt0 = tr_cnt0;3079			tr_req[tr_idx].dicnt1 = tr_cnt1;3080			tr_req[tr_idx].dicnt2 = tr0_cnt2;3081			tr_req[tr_idx].dicnt3 = tr0_cnt3;3082			tr_req[tr_idx].ddim1 = tr_cnt0;3083			tr_req[tr_idx].ddim2 = trigger_size;3084			tr_req[tr_idx].ddim3 = trigger_size * tr0_cnt2;3085		} else {3086			tr_req[tr_idx].addr = sg_addr;3087			tr_req[tr_idx].icnt0 = tr_cnt0;3088			tr_req[tr_idx].icnt1 = tr_cnt1;3089			tr_req[tr_idx].icnt2 = tr0_cnt2;3090			tr_req[tr_idx].icnt3 = tr0_cnt3;3091			tr_req[tr_idx].dim1 = tr_cnt0;3092			tr_req[tr_idx].dim2 = trigger_size;3093			tr_req[tr_idx].dim3 = trigger_size * tr0_cnt2;3094 3095			tr_req[tr_idx].daddr = dev_addr;3096			tr_req[tr_idx].dicnt0 = tr_cnt0;3097			tr_req[tr_idx].dicnt1 = tr_cnt1;3098			tr_req[tr_idx].dicnt2 = tr0_cnt2;3099			tr_req[tr_idx].dicnt3 = tr0_cnt3;3100			tr_req[tr_idx].ddim1 = (-1) * tr_cnt0;3101		}3102 3103		tr_idx++;3104 3105		if (num_tr == 2) {3106			cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15,3107				      false, true,3108				      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);3109			cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf);3110			cppi5_tr_set_trigger(&tr_req[tr_idx].flags,3111					     uc->config.tr_trigger_type,3112					     CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC,3113					     0, 0);3114 3115			sg_addr += trigger_size * tr0_cnt2 * tr0_cnt3;3116			if (dir == DMA_DEV_TO_MEM) {3117				tr_req[tr_idx].addr = dev_addr;3118				tr_req[tr_idx].icnt0 = tr_cnt0;3119				tr_req[tr_idx].icnt1 = tr_cnt1;3120				tr_req[tr_idx].icnt2 = tr1_cnt2;3121				tr_req[tr_idx].icnt3 = 1;3122				tr_req[tr_idx].dim1 = (-1) * tr_cnt0;3123 3124				tr_req[tr_idx].daddr = sg_addr;3125				tr_req[tr_idx].dicnt0 = tr_cnt0;3126				tr_req[tr_idx].dicnt1 = tr_cnt1;3127				tr_req[tr_idx].dicnt2 = tr1_cnt2;3128				tr_req[tr_idx].dicnt3 = 1;3129				tr_req[tr_idx].ddim1 = tr_cnt0;3130				tr_req[tr_idx].ddim2 = trigger_size;3131			} else {3132				tr_req[tr_idx].addr = sg_addr;3133				tr_req[tr_idx].icnt0 = tr_cnt0;3134				tr_req[tr_idx].icnt1 = tr_cnt1;3135				tr_req[tr_idx].icnt2 = tr1_cnt2;3136				tr_req[tr_idx].icnt3 = 1;3137				tr_req[tr_idx].dim1 = tr_cnt0;3138				tr_req[tr_idx].dim2 = trigger_size;3139 3140				tr_req[tr_idx].daddr = dev_addr;3141				tr_req[tr_idx].dicnt0 = tr_cnt0;3142				tr_req[tr_idx].dicnt1 = tr_cnt1;3143				tr_req[tr_idx].dicnt2 = tr1_cnt2;3144				tr_req[tr_idx].dicnt3 = 1;3145				tr_req[tr_idx].ddim1 = (-1) * tr_cnt0;3146			}3147			tr_idx++;3148		}3149 3150		d->residue += sg_len;3151	}3152 3153	cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, csf | CPPI5_TR_CSF_EOP);3154 3155	return d;3156}3157 3158static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d,3159				   enum dma_slave_buswidth dev_width,3160				   u16 elcnt)3161{3162	if (uc->config.ep_type != PSIL_EP_PDMA_XY)3163		return 0;3164 3165	/* Bus width translates to the element size (ES) */3166	switch (dev_width) {3167	case DMA_SLAVE_BUSWIDTH_1_BYTE:3168		d->static_tr.elsize = 0;3169		break;3170	case DMA_SLAVE_BUSWIDTH_2_BYTES:3171		d->static_tr.elsize = 1;3172		break;3173	case DMA_SLAVE_BUSWIDTH_3_BYTES:3174		d->static_tr.elsize = 2;3175		break;3176	case DMA_SLAVE_BUSWIDTH_4_BYTES:3177		d->static_tr.elsize = 3;3178		break;3179	case DMA_SLAVE_BUSWIDTH_8_BYTES:3180		d->static_tr.elsize = 4;3181		break;3182	default: /* not reached */3183		return -EINVAL;3184	}3185 3186	d->static_tr.elcnt = elcnt;3187 3188	if (uc->config.pkt_mode || !uc->cyclic) {3189		/*3190		 * PDMA must close the packet when the channel is in packet mode.3191		 * For TR mode when the channel is not cyclic we also need PDMA3192		 * to close the packet otherwise the transfer will stall because3193		 * PDMA holds on the data it has received from the peripheral.3194		 */3195		unsigned int div = dev_width * elcnt;3196 3197		if (uc->cyclic)3198			d->static_tr.bstcnt = d->residue / d->sglen / div;3199		else3200			d->static_tr.bstcnt = d->residue / div;3201	} else if (uc->ud->match_data->type == DMA_TYPE_BCDMA &&3202		   uc->config.dir == DMA_DEV_TO_MEM &&3203		   uc->cyclic) {3204		/*3205		 * For cyclic mode with BCDMA we have to set EOP in each TR to3206		 * prevent short packet errors seen on channel teardown. So the3207		 * PDMA must close the packet after every TR transfer by setting3208		 * burst count equal to the number of bytes transferred.3209		 */3210		struct cppi5_tr_type1_t *tr_req = d->hwdesc[0].tr_req_base;3211 3212		d->static_tr.bstcnt =3213			(tr_req->icnt0 * tr_req->icnt1) / dev_width;3214	} else {3215		d->static_tr.bstcnt = 0;3216	}3217 3218	if (uc->config.dir == DMA_DEV_TO_MEM &&3219	    d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask)3220		return -EINVAL;3221 3222	return 0;3223}3224 3225static struct udma_desc *3226udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl,3227		       unsigned int sglen, enum dma_transfer_direction dir,3228		       unsigned long tx_flags, void *context)3229{3230	struct scatterlist *sgent;3231	struct cppi5_host_desc_t *h_desc = NULL;3232	struct udma_desc *d;3233	u32 ring_id;3234	unsigned int i;3235	u64 asel;3236 3237	d = kzalloc(struct_size(d, hwdesc, sglen), GFP_NOWAIT);3238	if (!d)3239		return NULL;3240 3241	d->sglen = sglen;3242	d->hwdesc_count = sglen;3243 3244	if (dir == DMA_DEV_TO_MEM)3245		ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);3246	else3247		ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);3248 3249	if (uc->ud->match_data->type == DMA_TYPE_UDMA)3250		asel = 0;3251	else3252		asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;3253 3254	for_each_sg(sgl, sgent, sglen, i) {3255		struct udma_hwdesc *hwdesc = &d->hwdesc[i];3256		dma_addr_t sg_addr = sg_dma_address(sgent);3257		struct cppi5_host_desc_t *desc;3258		size_t sg_len = sg_dma_len(sgent);3259 3260		hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,3261						GFP_NOWAIT,3262						&hwdesc->cppi5_desc_paddr);3263		if (!hwdesc->cppi5_desc_vaddr) {3264			dev_err(uc->ud->dev,3265				"descriptor%d allocation failed\n", i);3266 3267			udma_free_hwdesc(uc, d);3268			kfree(d);3269			return NULL;3270		}3271 3272		d->residue += sg_len;3273		hwdesc->cppi5_desc_size = uc->config.hdesc_size;3274		desc = hwdesc->cppi5_desc_vaddr;3275 3276		if (i == 0) {3277			cppi5_hdesc_init(desc, 0, 0);3278			/* Flow and Packed ID */3279			cppi5_desc_set_pktids(&desc->hdr, uc->id,3280					      CPPI5_INFO1_DESC_FLOWID_DEFAULT);3281			cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id);3282		} else {3283			cppi5_hdesc_reset_hbdesc(desc);3284			cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff);3285		}3286 3287		/* attach the sg buffer to the descriptor */3288		sg_addr |= asel;3289		cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len);3290 3291		/* Attach link as host buffer descriptor */3292		if (h_desc)3293			cppi5_hdesc_link_hbdesc(h_desc,3294						hwdesc->cppi5_desc_paddr | asel);3295 3296		if (uc->ud->match_data->type == DMA_TYPE_PKTDMA ||3297		    dir == DMA_MEM_TO_DEV)3298			h_desc = desc;3299	}3300 3301	if (d->residue >= SZ_4M) {3302		dev_err(uc->ud->dev,3303			"%s: Transfer size %u is over the supported 4M range\n",3304			__func__, d->residue);3305		udma_free_hwdesc(uc, d);3306		kfree(d);3307		return NULL;3308	}3309 3310	h_desc = d->hwdesc[0].cppi5_desc_vaddr;3311	cppi5_hdesc_set_pktlen(h_desc, d->residue);3312 3313	return d;3314}3315 3316static int udma_attach_metadata(struct dma_async_tx_descriptor *desc,3317				void *data, size_t len)3318{3319	struct udma_desc *d = to_udma_desc(desc);3320	struct udma_chan *uc = to_udma_chan(desc->chan);3321	struct cppi5_host_desc_t *h_desc;3322	u32 psd_size = len;3323	u32 flags = 0;3324 3325	if (!uc->config.pkt_mode || !uc->config.metadata_size)3326		return -ENOTSUPP;3327 3328	if (!data || len > uc->config.metadata_size)3329		return -EINVAL;3330 3331	if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE)3332		return -EINVAL;3333 3334	h_desc = d->hwdesc[0].cppi5_desc_vaddr;3335	if (d->dir == DMA_MEM_TO_DEV)3336		memcpy(h_desc->epib, data, len);3337 3338	if (uc->config.needs_epib)3339		psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;3340 3341	d->metadata = data;3342	d->metadata_size = len;3343	if (uc->config.needs_epib)3344		flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;3345 3346	cppi5_hdesc_update_flags(h_desc, flags);3347	cppi5_hdesc_update_psdata_size(h_desc, psd_size);3348 3349	return 0;3350}3351 3352static void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc,3353				   size_t *payload_len, size_t *max_len)3354{3355	struct udma_desc *d = to_udma_desc(desc);3356	struct udma_chan *uc = to_udma_chan(desc->chan);3357	struct cppi5_host_desc_t *h_desc;3358 3359	if (!uc->config.pkt_mode || !uc->config.metadata_size)3360		return ERR_PTR(-ENOTSUPP);3361 3362	h_desc = d->hwdesc[0].cppi5_desc_vaddr;3363 3364	*max_len = uc->config.metadata_size;3365 3366	*payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ?3367		       CPPI5_INFO0_HDESC_EPIB_SIZE : 0;3368	*payload_len += cppi5_hdesc_get_psdata_size(h_desc);3369 3370	return h_desc->epib;3371}3372 3373static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc,3374				 size_t payload_len)3375{3376	struct udma_desc *d = to_udma_desc(desc);3377	struct udma_chan *uc = to_udma_chan(desc->chan);3378	struct cppi5_host_desc_t *h_desc;3379	u32 psd_size = payload_len;3380	u32 flags = 0;3381 3382	if (!uc->config.pkt_mode || !uc->config.metadata_size)3383		return -ENOTSUPP;3384 3385	if (payload_len > uc->config.metadata_size)3386		return -EINVAL;3387 3388	if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE)3389		return -EINVAL;3390 3391	h_desc = d->hwdesc[0].cppi5_desc_vaddr;3392 3393	if (uc->config.needs_epib) {3394		psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;3395		flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;3396	}3397 3398	cppi5_hdesc_update_flags(h_desc, flags);3399	cppi5_hdesc_update_psdata_size(h_desc, psd_size);3400 3401	return 0;3402}3403 3404static struct dma_descriptor_metadata_ops metadata_ops = {3405	.attach = udma_attach_metadata,3406	.get_ptr = udma_get_metadata_ptr,3407	.set_len = udma_set_metadata_len,3408};3409 3410static struct dma_async_tx_descriptor *3411udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,3412		   unsigned int sglen, enum dma_transfer_direction dir,3413		   unsigned long tx_flags, void *context)3414{3415	struct udma_chan *uc = to_udma_chan(chan);3416	enum dma_slave_buswidth dev_width;3417	struct udma_desc *d;3418	u32 burst;3419 3420	if (dir != uc->config.dir &&3421	    (uc->config.dir == DMA_MEM_TO_MEM && !uc->config.tr_trigger_type)) {3422		dev_err(chan->device->dev,3423			"%s: chan%d is for %s, not supporting %s\n",3424			__func__, uc->id,3425			dmaengine_get_direction_text(uc->config.dir),3426			dmaengine_get_direction_text(dir));3427		return NULL;3428	}3429 3430	if (dir == DMA_DEV_TO_MEM) {3431		dev_width = uc->cfg.src_addr_width;3432		burst = uc->cfg.src_maxburst;3433	} else if (dir == DMA_MEM_TO_DEV) {3434		dev_width = uc->cfg.dst_addr_width;3435		burst = uc->cfg.dst_maxburst;3436	} else {3437		dev_err(chan->device->dev, "%s: bad direction?\n", __func__);3438		return NULL;3439	}3440 3441	if (!burst)3442		burst = 1;3443 3444	uc->config.tx_flags = tx_flags;3445 3446	if (uc->config.pkt_mode)3447		d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags,3448					   context);3449	else if (is_slave_direction(uc->config.dir))3450		d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags,3451					  context);3452	else3453		d = udma_prep_slave_sg_triggered_tr(uc, sgl, sglen, dir,3454						    tx_flags, context);3455 3456	if (!d)3457		return NULL;3458 3459	d->dir = dir;3460	d->desc_idx = 0;3461	d->tr_idx = 0;3462 3463	/* static TR for remote PDMA */3464	if (udma_configure_statictr(uc, d, dev_width, burst)) {3465		dev_err(uc->ud->dev,3466			"%s: StaticTR Z is limited to maximum %u (%u)\n",3467			__func__, uc->ud->match_data->statictr_z_mask,3468			d->static_tr.bstcnt);3469 3470		udma_free_hwdesc(uc, d);3471		kfree(d);3472		return NULL;3473	}3474 3475	if (uc->config.metadata_size)3476		d->vd.tx.metadata_ops = &metadata_ops;3477 3478	return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);3479}3480 3481static struct udma_desc *3482udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr,3483			size_t buf_len, size_t period_len,3484			enum dma_transfer_direction dir, unsigned long flags)3485{3486	struct udma_desc *d;3487	size_t tr_size, period_addr;3488	struct cppi5_tr_type1_t *tr_req;3489	unsigned int periods = buf_len / period_len;3490	u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;3491	unsigned int i;3492	int num_tr;3493	u32 period_csf = 0;3494 3495	num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0,3496				      &tr0_cnt1, &tr1_cnt0);3497	if (num_tr < 0) {3498		dev_err(uc->ud->dev, "size %zu is not supported\n",3499			period_len);3500		return NULL;3501	}3502 3503	/* Now allocate and setup the descriptor. */3504	tr_size = sizeof(struct cppi5_tr_type1_t);3505	d = udma_alloc_tr_desc(uc, tr_size, periods * num_tr, dir);3506	if (!d)3507		return NULL;3508 3509	tr_req = d->hwdesc[0].tr_req_base;3510	if (uc->ud->match_data->type == DMA_TYPE_UDMA)3511		period_addr = buf_addr;3512	else3513		period_addr = buf_addr |3514			((u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT);3515 3516	/*3517	 * For BCDMA <-> PDMA transfers, the EOP flag needs to be set on the3518	 * last TR of a descriptor, to mark the packet as complete.3519	 * This is required for getting the teardown completion message in case3520	 * of TX, and to avoid short-packet error in case of RX.3521	 *3522	 * As we are in cyclic mode, we do not know which period might be the3523	 * last one, so set the flag for each period.3524	 */3525	if (uc->config.ep_type == PSIL_EP_PDMA_XY &&3526	    uc->ud->match_data->type == DMA_TYPE_BCDMA) {3527		period_csf = CPPI5_TR_CSF_EOP;3528	}3529 3530	for (i = 0; i < periods; i++) {3531		int tr_idx = i * num_tr;3532 3533		cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,3534			      false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);3535 3536		tr_req[tr_idx].addr = period_addr;3537		tr_req[tr_idx].icnt0 = tr0_cnt0;3538		tr_req[tr_idx].icnt1 = tr0_cnt1;3539		tr_req[tr_idx].dim1 = tr0_cnt0;3540 3541		if (num_tr == 2) {3542			cppi5_tr_csf_set(&tr_req[tr_idx].flags,3543					 CPPI5_TR_CSF_SUPR_EVT);3544			tr_idx++;3545 3546			cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,3547				      false, false,3548				      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);3549 3550			tr_req[tr_idx].addr = period_addr + tr0_cnt1 * tr0_cnt0;3551			tr_req[tr_idx].icnt0 = tr1_cnt0;3552			tr_req[tr_idx].icnt1 = 1;3553			tr_req[tr_idx].dim1 = tr1_cnt0;3554		}3555 3556		if (!(flags & DMA_PREP_INTERRUPT))3557			period_csf |= CPPI5_TR_CSF_SUPR_EVT;3558 3559		if (period_csf)3560			cppi5_tr_csf_set(&tr_req[tr_idx].flags, period_csf);3561 3562		period_addr += period_len;3563	}3564 3565	return d;3566}3567 3568static struct udma_desc *3569udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr,3570			 size_t buf_len, size_t period_len,3571			 enum dma_transfer_direction dir, unsigned long flags)3572{3573	struct udma_desc *d;3574	u32 ring_id;3575	int i;3576	int periods = buf_len / period_len;3577 3578	if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1))3579		return NULL;3580 3581	if (period_len >= SZ_4M)3582		return NULL;3583 3584	d = kzalloc(struct_size(d, hwdesc, periods), GFP_NOWAIT);3585	if (!d)3586		return NULL;3587 3588	d->hwdesc_count = periods;3589 3590	/* TODO: re-check this... */3591	if (dir == DMA_DEV_TO_MEM)3592		ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);3593	else3594		ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);3595 3596	if (uc->ud->match_data->type != DMA_TYPE_UDMA)3597		buf_addr |= (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;3598 3599	for (i = 0; i < periods; i++) {3600		struct udma_hwdesc *hwdesc = &d->hwdesc[i];3601		dma_addr_t period_addr = buf_addr + (period_len * i);3602		struct cppi5_host_desc_t *h_desc;3603 3604		hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,3605						GFP_NOWAIT,3606						&hwdesc->cppi5_desc_paddr);3607		if (!hwdesc->cppi5_desc_vaddr) {3608			dev_err(uc->ud->dev,3609				"descriptor%d allocation failed\n", i);3610 3611			udma_free_hwdesc(uc, d);3612			kfree(d);3613			return NULL;3614		}3615 3616		hwdesc->cppi5_desc_size = uc->config.hdesc_size;3617		h_desc = hwdesc->cppi5_desc_vaddr;3618 3619		cppi5_hdesc_init(h_desc, 0, 0);3620		cppi5_hdesc_set_pktlen(h_desc, period_len);3621 3622		/* Flow and Packed ID */3623		cppi5_desc_set_pktids(&h_desc->hdr, uc->id,3624				      CPPI5_INFO1_DESC_FLOWID_DEFAULT);3625		cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id);3626 3627		/* attach each period to a new descriptor */3628		cppi5_hdesc_attach_buf(h_desc,3629				       period_addr, period_len,3630				       period_addr, period_len);3631	}3632 3633	return d;3634}3635 3636static struct dma_async_tx_descriptor *3637udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,3638		     size_t period_len, enum dma_transfer_direction dir,3639		     unsigned long flags)3640{3641	struct udma_chan *uc = to_udma_chan(chan);3642	enum dma_slave_buswidth dev_width;3643	struct udma_desc *d;3644	u32 burst;3645 3646	if (dir != uc->config.dir) {3647		dev_err(chan->device->dev,3648			"%s: chan%d is for %s, not supporting %s\n",3649			__func__, uc->id,3650			dmaengine_get_direction_text(uc->config.dir),3651			dmaengine_get_direction_text(dir));3652		return NULL;3653	}3654 3655	uc->cyclic = true;3656 3657	if (dir == DMA_DEV_TO_MEM) {3658		dev_width = uc->cfg.src_addr_width;3659		burst = uc->cfg.src_maxburst;3660	} else if (dir == DMA_MEM_TO_DEV) {3661		dev_width = uc->cfg.dst_addr_width;3662		burst = uc->cfg.dst_maxburst;3663	} else {3664		dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);3665		return NULL;3666	}3667 3668	if (!burst)3669		burst = 1;3670 3671	if (uc->config.pkt_mode)3672		d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len,3673					     dir, flags);3674	else3675		d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len,3676					    dir, flags);3677 3678	if (!d)3679		return NULL;3680 3681	d->sglen = buf_len / period_len;3682 3683	d->dir = dir;3684	d->residue = buf_len;3685 3686	/* static TR for remote PDMA */3687	if (udma_configure_statictr(uc, d, dev_width, burst)) {3688		dev_err(uc->ud->dev,3689			"%s: StaticTR Z is limited to maximum %u (%u)\n",3690			__func__, uc->ud->match_data->statictr_z_mask,3691			d->static_tr.bstcnt);3692 3693		udma_free_hwdesc(uc, d);3694		kfree(d);3695		return NULL;3696	}3697 3698	if (uc->config.metadata_size)3699		d->vd.tx.metadata_ops = &metadata_ops;3700 3701	return vchan_tx_prep(&uc->vc, &d->vd, flags);3702}3703 3704static struct dma_async_tx_descriptor *3705udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,3706		     size_t len, unsigned long tx_flags)3707{3708	struct udma_chan *uc = to_udma_chan(chan);3709	struct udma_desc *d;3710	struct cppi5_tr_type15_t *tr_req;3711	int num_tr;3712	size_t tr_size = sizeof(struct cppi5_tr_type15_t);3713	u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;3714	u32 csf = CPPI5_TR_CSF_SUPR_EVT;3715 3716	if (uc->config.dir != DMA_MEM_TO_MEM) {3717		dev_err(chan->device->dev,3718			"%s: chan%d is for %s, not supporting %s\n",3719			__func__, uc->id,3720			dmaengine_get_direction_text(uc->config.dir),3721			dmaengine_get_direction_text(DMA_MEM_TO_MEM));3722		return NULL;3723	}3724 3725	num_tr = udma_get_tr_counters(len, __ffs(src | dest), &tr0_cnt0,3726				      &tr0_cnt1, &tr1_cnt0);3727	if (num_tr < 0) {3728		dev_err(uc->ud->dev, "size %zu is not supported\n",3729			len);3730		return NULL;3731	}3732 3733	d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM);3734	if (!d)3735		return NULL;3736 3737	d->dir = DMA_MEM_TO_MEM;3738	d->desc_idx = 0;3739	d->tr_idx = 0;3740	d->residue = len;3741 3742	if (uc->ud->match_data->type != DMA_TYPE_UDMA) {3743		src |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT;3744		dest |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT;3745	} else {3746		csf |= CPPI5_TR_CSF_EOL_ICNT0;3747	}3748 3749	tr_req = d->hwdesc[0].tr_req_base;3750 3751	cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true,3752		      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);3753	cppi5_tr_csf_set(&tr_req[0].flags, csf);3754 3755	tr_req[0].addr = src;3756	tr_req[0].icnt0 = tr0_cnt0;3757	tr_req[0].icnt1 = tr0_cnt1;3758	tr_req[0].icnt2 = 1;3759	tr_req[0].icnt3 = 1;3760	tr_req[0].dim1 = tr0_cnt0;3761 3762	tr_req[0].daddr = dest;3763	tr_req[0].dicnt0 = tr0_cnt0;3764	tr_req[0].dicnt1 = tr0_cnt1;3765	tr_req[0].dicnt2 = 1;3766	tr_req[0].dicnt3 = 1;3767	tr_req[0].ddim1 = tr0_cnt0;3768 3769	if (num_tr == 2) {3770		cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true,3771			      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);3772		cppi5_tr_csf_set(&tr_req[1].flags, csf);3773 3774		tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0;3775		tr_req[1].icnt0 = tr1_cnt0;3776		tr_req[1].icnt1 = 1;3777		tr_req[1].icnt2 = 1;3778		tr_req[1].icnt3 = 1;3779 3780		tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0;3781		tr_req[1].dicnt0 = tr1_cnt0;3782		tr_req[1].dicnt1 = 1;3783		tr_req[1].dicnt2 = 1;3784		tr_req[1].dicnt3 = 1;3785	}3786 3787	cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, csf | CPPI5_TR_CSF_EOP);3788 3789	if (uc->config.metadata_size)3790		d->vd.tx.metadata_ops = &metadata_ops;3791 3792	return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);3793}3794 3795static void udma_issue_pending(struct dma_chan *chan)3796{3797	struct udma_chan *uc = to_udma_chan(chan);3798	unsigned long flags;3799 3800	spin_lock_irqsave(&uc->vc.lock, flags);3801 3802	/* If we have something pending and no active descriptor, then */3803	if (vchan_issue_pending(&uc->vc) && !uc->desc) {3804		/*3805		 * start a descriptor if the channel is NOT [marked as3806		 * terminating _and_ it is still running (teardown has not3807		 * completed yet)].3808		 */3809		if (!(uc->state == UDMA_CHAN_IS_TERMINATING &&3810		      udma_is_chan_running(uc)))3811			udma_start(uc);3812	}3813 3814	spin_unlock_irqrestore(&uc->vc.lock, flags);3815}3816 3817static enum dma_status udma_tx_status(struct dma_chan *chan,3818				      dma_cookie_t cookie,3819				      struct dma_tx_state *txstate)3820{3821	struct udma_chan *uc = to_udma_chan(chan);3822	enum dma_status ret;3823	unsigned long flags;3824 3825	spin_lock_irqsave(&uc->vc.lock, flags);3826 3827	ret = dma_cookie_status(chan, cookie, txstate);3828 3829	if (!udma_is_chan_running(uc))3830		ret = DMA_COMPLETE;3831 3832	if (ret == DMA_IN_PROGRESS && udma_is_chan_paused(uc))3833		ret = DMA_PAUSED;3834 3835	if (ret == DMA_COMPLETE || !txstate)3836		goto out;3837 3838	if (uc->desc && uc->desc->vd.tx.cookie == cookie) {3839		u32 peer_bcnt = 0;3840		u32 bcnt = 0;3841		u32 residue = uc->desc->residue;3842		u32 delay = 0;3843 3844		if (uc->desc->dir == DMA_MEM_TO_DEV) {3845			bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);3846 3847			if (uc->config.ep_type != PSIL_EP_NATIVE) {3848				peer_bcnt = udma_tchanrt_read(uc,3849						UDMA_CHAN_RT_PEER_BCNT_REG);3850 3851				if (bcnt > peer_bcnt)3852					delay = bcnt - peer_bcnt;3853			}3854		} else if (uc->desc->dir == DMA_DEV_TO_MEM) {3855			bcnt = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);3856 3857			if (uc->config.ep_type != PSIL_EP_NATIVE) {3858				peer_bcnt = udma_rchanrt_read(uc,3859						UDMA_CHAN_RT_PEER_BCNT_REG);3860 3861				if (peer_bcnt > bcnt)3862					delay = peer_bcnt - bcnt;3863			}3864		} else {3865			bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);3866		}3867 3868		if (bcnt && !(bcnt % uc->desc->residue))3869			residue = 0;3870		else3871			residue -= bcnt % uc->desc->residue;3872 3873		if (!residue && (uc->config.dir == DMA_DEV_TO_MEM || !delay)) {3874			ret = DMA_COMPLETE;3875			delay = 0;3876		}3877 3878		dma_set_residue(txstate, residue);3879		dma_set_in_flight_bytes(txstate, delay);3880 3881	} else {3882		ret = DMA_COMPLETE;3883	}3884 3885out:3886	spin_unlock_irqrestore(&uc->vc.lock, flags);3887	return ret;3888}3889 3890static int udma_pause(struct dma_chan *chan)3891{3892	struct udma_chan *uc = to_udma_chan(chan);3893 3894	/* pause the channel */3895	switch (uc->config.dir) {3896	case DMA_DEV_TO_MEM:3897		udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,3898					 UDMA_PEER_RT_EN_PAUSE,3899					 UDMA_PEER_RT_EN_PAUSE);3900		break;3901	case DMA_MEM_TO_DEV:3902		udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,3903					 UDMA_PEER_RT_EN_PAUSE,3904					 UDMA_PEER_RT_EN_PAUSE);3905		break;3906	case DMA_MEM_TO_MEM:3907		udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG,3908					 UDMA_CHAN_RT_CTL_PAUSE,3909					 UDMA_CHAN_RT_CTL_PAUSE);3910		break;3911	default:3912		return -EINVAL;3913	}3914 3915	return 0;3916}3917 3918static int udma_resume(struct dma_chan *chan)3919{3920	struct udma_chan *uc = to_udma_chan(chan);3921 3922	/* resume the channel */3923	switch (uc->config.dir) {3924	case DMA_DEV_TO_MEM:3925		udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,3926					 UDMA_PEER_RT_EN_PAUSE, 0);3927 3928		break;3929	case DMA_MEM_TO_DEV:3930		udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,3931					 UDMA_PEER_RT_EN_PAUSE, 0);3932		break;3933	case DMA_MEM_TO_MEM:3934		udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG,3935					 UDMA_CHAN_RT_CTL_PAUSE, 0);3936		break;3937	default:3938		return -EINVAL;3939	}3940 3941	return 0;3942}3943 3944static int udma_terminate_all(struct dma_chan *chan)3945{3946	struct udma_chan *uc = to_udma_chan(chan);3947	unsigned long flags;3948	LIST_HEAD(head);3949 3950	spin_lock_irqsave(&uc->vc.lock, flags);3951 3952	if (udma_is_chan_running(uc))3953		udma_stop(uc);3954 3955	if (uc->desc) {3956		uc->terminated_desc = uc->desc;3957		uc->desc = NULL;3958		uc->terminated_desc->terminated = true;3959		cancel_delayed_work(&uc->tx_drain.work);3960	}3961 3962	uc->paused = false;3963 3964	vchan_get_all_descriptors(&uc->vc, &head);3965	spin_unlock_irqrestore(&uc->vc.lock, flags);3966	vchan_dma_desc_free_list(&uc->vc, &head);3967 3968	return 0;3969}3970 3971static void udma_synchronize(struct dma_chan *chan)3972{3973	struct udma_chan *uc = to_udma_chan(chan);3974	unsigned long timeout = msecs_to_jiffies(1000);3975 3976	vchan_synchronize(&uc->vc);3977 3978	if (uc->state == UDMA_CHAN_IS_TERMINATING) {3979		timeout = wait_for_completion_timeout(&uc->teardown_completed,3980						      timeout);3981		if (!timeout) {3982			dev_warn(uc->ud->dev, "chan%d teardown timeout!\n",3983				 uc->id);3984			udma_dump_chan_stdata(uc);3985			udma_reset_chan(uc, true);3986		}3987	}3988 3989	udma_reset_chan(uc, false);3990	if (udma_is_chan_running(uc))3991		dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id);3992 3993	cancel_delayed_work_sync(&uc->tx_drain.work);3994	udma_reset_rings(uc);3995}3996 3997static void udma_desc_pre_callback(struct virt_dma_chan *vc,3998				   struct virt_dma_desc *vd,3999				   struct dmaengine_result *result)4000{4001	struct udma_chan *uc = to_udma_chan(&vc->chan);4002	struct udma_desc *d;4003	u8 status;4004 4005	if (!vd)4006		return;4007 4008	d = to_udma_desc(&vd->tx);4009 4010	if (d->metadata_size)4011		udma_fetch_epib(uc, d);4012 4013	if (result) {4014		void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx);4015 4016		if (cppi5_desc_get_type(desc_vaddr) ==4017		    CPPI5_INFO0_DESC_TYPE_VAL_HOST) {4018			/* Provide residue information for the client */4019			result->residue = d->residue -4020					  cppi5_hdesc_get_pktlen(desc_vaddr);4021			if (result->residue)4022				result->result = DMA_TRANS_ABORTED;4023			else4024				result->result = DMA_TRANS_NOERROR;4025		} else {4026			result->residue = 0;4027			/* Propagate TR Response errors to the client */4028			status = d->hwdesc[0].tr_resp_base->status;4029			if (status)4030				result->result = DMA_TRANS_ABORTED;4031			else4032				result->result = DMA_TRANS_NOERROR;4033		}4034	}4035}4036 4037/*4038 * This tasklet handles the completion of a DMA descriptor by4039 * calling its callback and freeing it.4040 */4041static void udma_vchan_complete(struct tasklet_struct *t)4042{4043	struct virt_dma_chan *vc = from_tasklet(vc, t, task);4044	struct virt_dma_desc *vd, *_vd;4045	struct dmaengine_desc_callback cb;4046	LIST_HEAD(head);4047 4048	spin_lock_irq(&vc->lock);4049	list_splice_tail_init(&vc->desc_completed, &head);4050	vd = vc->cyclic;4051	if (vd) {4052		vc->cyclic = NULL;4053		dmaengine_desc_get_callback(&vd->tx, &cb);4054	} else {4055		memset(&cb, 0, sizeof(cb));4056	}4057	spin_unlock_irq(&vc->lock);4058 4059	udma_desc_pre_callback(vc, vd, NULL);4060	dmaengine_desc_callback_invoke(&cb, NULL);4061 4062	list_for_each_entry_safe(vd, _vd, &head, node) {4063		struct dmaengine_result result;4064 4065		dmaengine_desc_get_callback(&vd->tx, &cb);4066 4067		list_del(&vd->node);4068 4069		udma_desc_pre_callback(vc, vd, &result);4070		dmaengine_desc_callback_invoke(&cb, &result);4071 4072		vchan_vdesc_fini(vd);4073	}4074}4075 4076static void udma_free_chan_resources(struct dma_chan *chan)4077{4078	struct udma_chan *uc = to_udma_chan(chan);4079	struct udma_dev *ud = to_udma_dev(chan->device);4080 4081	udma_terminate_all(chan);4082	if (uc->terminated_desc) {4083		udma_reset_chan(uc, false);4084		udma_reset_rings(uc);4085	}4086 4087	cancel_delayed_work_sync(&uc->tx_drain.work);4088 4089	if (uc->irq_num_ring > 0) {4090		free_irq(uc->irq_num_ring, uc);4091 4092		uc->irq_num_ring = 0;4093	}4094	if (uc->irq_num_udma > 0) {4095		free_irq(uc->irq_num_udma, uc);4096 4097		uc->irq_num_udma = 0;4098	}4099 4100	/* Release PSI-L pairing */4101	if (uc->psil_paired) {4102		navss_psil_unpair(ud, uc->config.src_thread,4103				  uc->config.dst_thread);4104		uc->psil_paired = false;4105	}4106 4107	vchan_free_chan_resources(&uc->vc);4108	tasklet_kill(&uc->vc.task);4109 4110	bcdma_free_bchan_resources(uc);4111	udma_free_tx_resources(uc);4112	udma_free_rx_resources(uc);4113	udma_reset_uchan(uc);4114 4115	if (uc->use_dma_pool) {4116		dma_pool_destroy(uc->hdesc_pool);4117		uc->use_dma_pool = false;4118	}4119}4120 4121static struct platform_driver udma_driver;4122static struct platform_driver bcdma_driver;4123static struct platform_driver pktdma_driver;4124 4125struct udma_filter_param {4126	int remote_thread_id;4127	u32 atype;4128	u32 asel;4129	u32 tr_trigger_type;4130};4131 4132static bool udma_dma_filter_fn(struct dma_chan *chan, void *param)4133{4134	struct udma_chan_config *ucc;4135	struct psil_endpoint_config *ep_config;4136	struct udma_filter_param *filter_param;4137	struct udma_chan *uc;4138	struct udma_dev *ud;4139 4140	if (chan->device->dev->driver != &udma_driver.driver &&4141	    chan->device->dev->driver != &bcdma_driver.driver &&4142	    chan->device->dev->driver != &pktdma_driver.driver)4143		return false;4144 4145	uc = to_udma_chan(chan);4146	ucc = &uc->config;4147	ud = uc->ud;4148	filter_param = param;4149 4150	if (filter_param->atype > 2) {4151		dev_err(ud->dev, "Invalid channel atype: %u\n",4152			filter_param->atype);4153		return false;4154	}4155 4156	if (filter_param->asel > 15) {4157		dev_err(ud->dev, "Invalid channel asel: %u\n",4158			filter_param->asel);4159		return false;4160	}4161 4162	ucc->remote_thread_id = filter_param->remote_thread_id;4163	ucc->atype = filter_param->atype;4164	ucc->asel = filter_param->asel;4165	ucc->tr_trigger_type = filter_param->tr_trigger_type;4166 4167	if (ucc->tr_trigger_type) {4168		ucc->dir = DMA_MEM_TO_MEM;4169		goto triggered_bchan;4170	} else if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) {4171		ucc->dir = DMA_MEM_TO_DEV;4172	} else {4173		ucc->dir = DMA_DEV_TO_MEM;4174	}4175 4176	ep_config = psil_get_ep_config(ucc->remote_thread_id);4177	if (IS_ERR(ep_config)) {4178		dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n",4179			ucc->remote_thread_id);4180		ucc->dir = DMA_MEM_TO_MEM;4181		ucc->remote_thread_id = -1;4182		ucc->atype = 0;4183		ucc->asel = 0;4184		return false;4185	}4186 4187	if (ud->match_data->type == DMA_TYPE_BCDMA &&4188	    ep_config->pkt_mode) {4189		dev_err(ud->dev,4190			"Only TR mode is supported (psi-l thread 0x%04x)\n",4191			ucc->remote_thread_id);4192		ucc->dir = DMA_MEM_TO_MEM;4193		ucc->remote_thread_id = -1;4194		ucc->atype = 0;4195		ucc->asel = 0;4196		return false;4197	}4198 4199	ucc->pkt_mode = ep_config->pkt_mode;4200	ucc->channel_tpl = ep_config->channel_tpl;4201	ucc->notdpkt = ep_config->notdpkt;4202	ucc->ep_type = ep_config->ep_type;4203 4204	if (ud->match_data->type == DMA_TYPE_PKTDMA &&4205	    ep_config->mapped_channel_id >= 0) {4206		ucc->mapped_channel_id = ep_config->mapped_channel_id;4207		ucc->default_flow_id = ep_config->default_flow_id;4208	} else {4209		ucc->mapped_channel_id = -1;4210		ucc->default_flow_id = -1;4211	}4212 4213	if (ucc->ep_type != PSIL_EP_NATIVE) {4214		const struct udma_match_data *match_data = ud->match_data;4215 4216		if (match_data->flags & UDMA_FLAG_PDMA_ACC32)4217			ucc->enable_acc32 = ep_config->pdma_acc32;4218		if (match_data->flags & UDMA_FLAG_PDMA_BURST)4219			ucc->enable_burst = ep_config->pdma_burst;4220	}4221 4222	ucc->needs_epib = ep_config->needs_epib;4223	ucc->psd_size = ep_config->psd_size;4224	ucc->metadata_size =4225			(ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) +4226			ucc->psd_size;4227 4228	if (ucc->pkt_mode)4229		ucc->hdesc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +4230				 ucc->metadata_size, ud->desc_align);4231 4232	dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id,4233		ucc->remote_thread_id, dmaengine_get_direction_text(ucc->dir));4234 4235	return true;4236 4237triggered_bchan:4238	dev_dbg(ud->dev, "chan%d: triggered channel (type: %u)\n", uc->id,4239		ucc->tr_trigger_type);4240 4241	return true;4242 4243}4244 4245static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec,4246				      struct of_dma *ofdma)4247{4248	struct udma_dev *ud = ofdma->of_dma_data;4249	dma_cap_mask_t mask = ud->ddev.cap_mask;4250	struct udma_filter_param filter_param;4251	struct dma_chan *chan;4252 4253	if (ud->match_data->type == DMA_TYPE_BCDMA) {4254		if (dma_spec->args_count != 3)4255			return NULL;4256 4257		filter_param.tr_trigger_type = dma_spec->args[0];4258		filter_param.remote_thread_id = dma_spec->args[1];4259		filter_param.asel = dma_spec->args[2];4260		filter_param.atype = 0;4261	} else {4262		if (dma_spec->args_count != 1 && dma_spec->args_count != 2)4263			return NULL;4264 4265		filter_param.remote_thread_id = dma_spec->args[0];4266		filter_param.tr_trigger_type = 0;4267		if (dma_spec->args_count == 2) {4268			if (ud->match_data->type == DMA_TYPE_UDMA) {4269				filter_param.atype = dma_spec->args[1];4270				filter_param.asel = 0;4271			} else {4272				filter_param.atype = 0;4273				filter_param.asel = dma_spec->args[1];4274			}4275		} else {4276			filter_param.atype = 0;4277			filter_param.asel = 0;4278		}4279	}4280 4281	chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param,4282				     ofdma->of_node);4283	if (!chan) {4284		dev_err(ud->dev, "get channel fail in %s.\n", __func__);4285		return ERR_PTR(-EINVAL);4286	}4287 4288	return chan;4289}4290 4291static struct udma_match_data am654_main_data = {4292	.type = DMA_TYPE_UDMA,4293	.psil_base = 0x1000,4294	.enable_memcpy_support = true,4295	.statictr_z_mask = GENMASK(11, 0),4296	.burst_size = {4297		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */4298		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */4299		0, /* No UH Channels */4300	},4301};4302 4303static struct udma_match_data am654_mcu_data = {4304	.type = DMA_TYPE_UDMA,4305	.psil_base = 0x6000,4306	.enable_memcpy_support = false,4307	.statictr_z_mask = GENMASK(11, 0),4308	.burst_size = {4309		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */4310		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */4311		0, /* No UH Channels */4312	},4313};4314 4315static struct udma_match_data j721e_main_data = {4316	.type = DMA_TYPE_UDMA,4317	.psil_base = 0x1000,4318	.enable_memcpy_support = true,4319	.flags = UDMA_FLAGS_J7_CLASS,4320	.statictr_z_mask = GENMASK(23, 0),4321	.burst_size = {4322		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */4323		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* H Channels */4324		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* UH Channels */4325	},4326};4327 4328static struct udma_match_data j721e_mcu_data = {4329	.type = DMA_TYPE_UDMA,4330	.psil_base = 0x6000,4331	.enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */4332	.flags = UDMA_FLAGS_J7_CLASS,4333	.statictr_z_mask = GENMASK(23, 0),4334	.burst_size = {4335		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */4336		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES, /* H Channels */4337		0, /* No UH Channels */4338	},4339};4340 4341static struct udma_soc_data am62a_dmss_csi_soc_data = {4342	.oes = {4343		.bcdma_rchan_data = 0xe00,4344		.bcdma_rchan_ring = 0x1000,4345	},4346};4347 4348static struct udma_soc_data j721s2_bcdma_csi_soc_data = {4349	.oes = {4350		.bcdma_tchan_data = 0x800,4351		.bcdma_tchan_ring = 0xa00,4352		.bcdma_rchan_data = 0xe00,4353		.bcdma_rchan_ring = 0x1000,4354	},4355};4356 4357static struct udma_match_data am62a_bcdma_csirx_data = {4358	.type = DMA_TYPE_BCDMA,4359	.psil_base = 0x3100,4360	.enable_memcpy_support = false,4361	.burst_size = {4362		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */4363		0, /* No H Channels */4364		0, /* No UH Channels */4365	},4366	.soc_data = &am62a_dmss_csi_soc_data,4367};4368 4369static struct udma_match_data am64_bcdma_data = {4370	.type = DMA_TYPE_BCDMA,4371	.psil_base = 0x2000, /* for tchan and rchan, not applicable to bchan */4372	.enable_memcpy_support = true, /* Supported via bchan */4373	.flags = UDMA_FLAGS_J7_CLASS,4374	.statictr_z_mask = GENMASK(23, 0),4375	.burst_size = {4376		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */4377		0, /* No H Channels */4378		0, /* No UH Channels */4379	},4380};4381 4382static struct udma_match_data am64_pktdma_data = {4383	.type = DMA_TYPE_PKTDMA,4384	.psil_base = 0x1000,4385	.enable_memcpy_support = false, /* PKTDMA does not support MEM_TO_MEM */4386	.flags = UDMA_FLAGS_J7_CLASS,4387	.statictr_z_mask = GENMASK(23, 0),4388	.burst_size = {4389		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */4390		0, /* No H Channels */4391		0, /* No UH Channels */4392	},4393};4394 4395static struct udma_match_data j721s2_bcdma_csi_data = {4396	.type = DMA_TYPE_BCDMA,4397	.psil_base = 0x2000,4398	.enable_memcpy_support = false,4399	.burst_size = {4400		TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */4401		0, /* No H Channels */4402		0, /* No UH Channels */4403	},4404	.soc_data = &j721s2_bcdma_csi_soc_data,4405};4406 4407static const struct of_device_id udma_of_match[] = {4408	{4409		.compatible = "ti,am654-navss-main-udmap",4410		.data = &am654_main_data,4411	},4412	{4413		.compatible = "ti,am654-navss-mcu-udmap",4414		.data = &am654_mcu_data,4415	}, {4416		.compatible = "ti,j721e-navss-main-udmap",4417		.data = &j721e_main_data,4418	}, {4419		.compatible = "ti,j721e-navss-mcu-udmap",4420		.data = &j721e_mcu_data,4421	},4422	{4423		.compatible = "ti,am64-dmss-bcdma",4424		.data = &am64_bcdma_data,4425	},4426	{4427		.compatible = "ti,am64-dmss-pktdma",4428		.data = &am64_pktdma_data,4429	},4430	{4431		.compatible = "ti,am62a-dmss-bcdma-csirx",4432		.data = &am62a_bcdma_csirx_data,4433	},4434	{4435		.compatible = "ti,j721s2-dmss-bcdma-csi",4436		.data = &j721s2_bcdma_csi_data,4437	},4438	{ /* Sentinel */ },4439};4440MODULE_DEVICE_TABLE(of, udma_of_match);4441 4442static struct udma_soc_data am654_soc_data = {4443	.oes = {4444		.udma_rchan = 0x200,4445	},4446};4447 4448static struct udma_soc_data j721e_soc_data = {4449	.oes = {4450		.udma_rchan = 0x400,4451	},4452};4453 4454static struct udma_soc_data j7200_soc_data = {4455	.oes = {4456		.udma_rchan = 0x80,4457	},4458};4459 4460static struct udma_soc_data am64_soc_data = {4461	.oes = {4462		.bcdma_bchan_data = 0x2200,4463		.bcdma_bchan_ring = 0x2400,4464		.bcdma_tchan_data = 0x2800,4465		.bcdma_tchan_ring = 0x2a00,4466		.bcdma_rchan_data = 0x2e00,4467		.bcdma_rchan_ring = 0x3000,4468		.pktdma_tchan_flow = 0x1200,4469		.pktdma_rchan_flow = 0x1600,4470	},4471	.bcdma_trigger_event_offset = 0xc400,4472};4473 4474static const struct soc_device_attribute k3_soc_devices[] = {4475	{ .family = "AM65X", .data = &am654_soc_data },4476	{ .family = "J721E", .data = &j721e_soc_data },4477	{ .family = "J7200", .data = &j7200_soc_data },4478	{ .family = "AM64X", .data = &am64_soc_data },4479	{ .family = "J721S2", .data = &j721e_soc_data},4480	{ .family = "AM62X", .data = &am64_soc_data },4481	{ .family = "AM62AX", .data = &am64_soc_data },4482	{ .family = "J784S4", .data = &j721e_soc_data },4483	{ .family = "AM62PX", .data = &am64_soc_data },4484	{ .family = "J722S", .data = &am64_soc_data },4485	{ /* sentinel */ }4486};4487 4488static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud)4489{4490	u32 cap2, cap3, cap4;4491	int i;4492 4493	ud->mmrs[MMR_GCFG] = devm_platform_ioremap_resource_byname(pdev, mmr_names[MMR_GCFG]);4494	if (IS_ERR(ud->mmrs[MMR_GCFG]))4495		return PTR_ERR(ud->mmrs[MMR_GCFG]);4496 4497	cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28);4498	cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);4499 4500	switch (ud->match_data->type) {4501	case DMA_TYPE_UDMA:4502		ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3);4503		ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2);4504		ud->echan_cnt = UDMA_CAP2_ECHAN_CNT(cap2);4505		ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2);4506		break;4507	case DMA_TYPE_BCDMA:4508		ud->bchan_cnt = BCDMA_CAP2_BCHAN_CNT(cap2) +4509				BCDMA_CAP3_HBCHAN_CNT(cap3) +4510				BCDMA_CAP3_UBCHAN_CNT(cap3);4511		ud->tchan_cnt = BCDMA_CAP2_TCHAN_CNT(cap2);4512		ud->rchan_cnt = BCDMA_CAP2_RCHAN_CNT(cap2);4513		ud->rflow_cnt = ud->rchan_cnt;4514		break;4515	case DMA_TYPE_PKTDMA:4516		cap4 = udma_read(ud->mmrs[MMR_GCFG], 0x30);4517		ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2);4518		ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2);4519		ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3);4520		ud->tflow_cnt = PKTDMA_CAP4_TFLOW_CNT(cap4);4521		break;4522	default:4523		return -EINVAL;4524	}4525 4526	for (i = 1; i < MMR_LAST; i++) {4527		if (i == MMR_BCHANRT && ud->bchan_cnt == 0)4528			continue;4529		if (i == MMR_TCHANRT && ud->tchan_cnt == 0)4530			continue;4531		if (i == MMR_RCHANRT && ud->rchan_cnt == 0)4532			continue;4533 4534		ud->mmrs[i] = devm_platform_ioremap_resource_byname(pdev, mmr_names[i]);4535		if (IS_ERR(ud->mmrs[i]))4536			return PTR_ERR(ud->mmrs[i]);4537	}4538 4539	return 0;4540}4541 4542static void udma_mark_resource_ranges(struct udma_dev *ud, unsigned long *map,4543				      struct ti_sci_resource_desc *rm_desc,4544				      char *name)4545{4546	bitmap_clear(map, rm_desc->start, rm_desc->num);4547	bitmap_clear(map, rm_desc->start_sec, rm_desc->num_sec);4548	dev_dbg(ud->dev, "ti_sci resource range for %s: %d:%d | %d:%d\n", name,4549		rm_desc->start, rm_desc->num, rm_desc->start_sec,4550		rm_desc->num_sec);4551}4552 4553static const char * const range_names[] = {4554	[RM_RANGE_BCHAN] = "ti,sci-rm-range-bchan",4555	[RM_RANGE_TCHAN] = "ti,sci-rm-range-tchan",4556	[RM_RANGE_RCHAN] = "ti,sci-rm-range-rchan",4557	[RM_RANGE_RFLOW] = "ti,sci-rm-range-rflow",4558	[RM_RANGE_TFLOW] = "ti,sci-rm-range-tflow",4559};4560 4561static int udma_setup_resources(struct udma_dev *ud)4562{4563	int ret, i, j;4564	struct device *dev = ud->dev;4565	struct ti_sci_resource *rm_res, irq_res;4566	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;4567	u32 cap3;4568 4569	/* Set up the throughput level start indexes */4570	cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);4571	if (of_device_is_compatible(dev->of_node,4572				    "ti,am654-navss-main-udmap")) {4573		ud->tchan_tpl.levels = 2;4574		ud->tchan_tpl.start_idx[0] = 8;4575	} else if (of_device_is_compatible(dev->of_node,4576					   "ti,am654-navss-mcu-udmap")) {4577		ud->tchan_tpl.levels = 2;4578		ud->tchan_tpl.start_idx[0] = 2;4579	} else if (UDMA_CAP3_UCHAN_CNT(cap3)) {4580		ud->tchan_tpl.levels = 3;4581		ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3);4582		ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);4583	} else if (UDMA_CAP3_HCHAN_CNT(cap3)) {4584		ud->tchan_tpl.levels = 2;4585		ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);4586	} else {4587		ud->tchan_tpl.levels = 1;4588	}4589 4590	ud->rchan_tpl.levels = ud->tchan_tpl.levels;4591	ud->rchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0];4592	ud->rchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1];4593 4594	ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),4595					   sizeof(unsigned long), GFP_KERNEL);4596	ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),4597				  GFP_KERNEL);4598	ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),4599					   sizeof(unsigned long), GFP_KERNEL);4600	ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),4601				  GFP_KERNEL);4602	ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt),4603					      sizeof(unsigned long),4604					      GFP_KERNEL);4605	ud->rflow_gp_map_allocated = devm_kcalloc(dev,4606						  BITS_TO_LONGS(ud->rflow_cnt),4607						  sizeof(unsigned long),4608						  GFP_KERNEL);4609	ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),4610					sizeof(unsigned long),4611					GFP_KERNEL);4612	ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),4613				  GFP_KERNEL);4614 4615	if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map ||4616	    !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans ||4617	    !ud->rflows || !ud->rflow_in_use)4618		return -ENOMEM;4619 4620	/*4621	 * RX flows with the same Ids as RX channels are reserved to be used4622	 * as default flows if remote HW can't generate flow_ids. Those4623	 * RX flows can be requested only explicitly by id.4624	 */4625	bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt);4626 4627	/* by default no GP rflows are assigned to Linux */4628	bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt);4629 4630	/* Get resource ranges from tisci */4631	for (i = 0; i < RM_RANGE_LAST; i++) {4632		if (i == RM_RANGE_BCHAN || i == RM_RANGE_TFLOW)4633			continue;4634 4635		tisci_rm->rm_ranges[i] =4636			devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,4637						    tisci_rm->tisci_dev_id,4638						    (char *)range_names[i]);4639	}4640 4641	/* tchan ranges */4642	rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];4643	if (IS_ERR(rm_res)) {4644		bitmap_zero(ud->tchan_map, ud->tchan_cnt);4645		irq_res.sets = 1;4646	} else {4647		bitmap_fill(ud->tchan_map, ud->tchan_cnt);4648		for (i = 0; i < rm_res->sets; i++)4649			udma_mark_resource_ranges(ud, ud->tchan_map,4650						  &rm_res->desc[i], "tchan");4651		irq_res.sets = rm_res->sets;4652	}4653 4654	/* rchan and matching default flow ranges */4655	rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];4656	if (IS_ERR(rm_res)) {4657		bitmap_zero(ud->rchan_map, ud->rchan_cnt);4658		irq_res.sets++;4659	} else {4660		bitmap_fill(ud->rchan_map, ud->rchan_cnt);4661		for (i = 0; i < rm_res->sets; i++)4662			udma_mark_resource_ranges(ud, ud->rchan_map,4663						  &rm_res->desc[i], "rchan");4664		irq_res.sets += rm_res->sets;4665	}4666 4667	irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL);4668	if (!irq_res.desc)4669		return -ENOMEM;4670	rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];4671	if (IS_ERR(rm_res)) {4672		irq_res.desc[0].start = 0;4673		irq_res.desc[0].num = ud->tchan_cnt;4674		i = 1;4675	} else {4676		for (i = 0; i < rm_res->sets; i++) {4677			irq_res.desc[i].start = rm_res->desc[i].start;4678			irq_res.desc[i].num = rm_res->desc[i].num;4679			irq_res.desc[i].start_sec = rm_res->desc[i].start_sec;4680			irq_res.desc[i].num_sec = rm_res->desc[i].num_sec;4681		}4682	}4683	rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];4684	if (IS_ERR(rm_res)) {4685		irq_res.desc[i].start = 0;4686		irq_res.desc[i].num = ud->rchan_cnt;4687	} else {4688		for (j = 0; j < rm_res->sets; j++, i++) {4689			if (rm_res->desc[j].num) {4690				irq_res.desc[i].start = rm_res->desc[j].start +4691						ud->soc_data->oes.udma_rchan;4692				irq_res.desc[i].num = rm_res->desc[j].num;4693			}4694			if (rm_res->desc[j].num_sec) {4695				irq_res.desc[i].start_sec = rm_res->desc[j].start_sec +4696						ud->soc_data->oes.udma_rchan;4697				irq_res.desc[i].num_sec = rm_res->desc[j].num_sec;4698			}4699		}4700	}4701	ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);4702	kfree(irq_res.desc);4703	if (ret) {4704		dev_err(ud->dev, "Failed to allocate MSI interrupts\n");4705		return ret;4706	}4707 4708	/* GP rflow ranges */4709	rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];4710	if (IS_ERR(rm_res)) {4711		/* all gp flows are assigned exclusively to Linux */4712		bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt,4713			     ud->rflow_cnt - ud->rchan_cnt);4714	} else {4715		for (i = 0; i < rm_res->sets; i++)4716			udma_mark_resource_ranges(ud, ud->rflow_gp_map,4717						  &rm_res->desc[i], "gp-rflow");4718	}4719 4720	return 0;4721}4722 4723static int bcdma_setup_resources(struct udma_dev *ud)4724{4725	int ret, i, j;4726	struct device *dev = ud->dev;4727	struct ti_sci_resource *rm_res, irq_res;4728	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;4729	const struct udma_oes_offsets *oes = &ud->soc_data->oes;4730	u32 cap;4731 4732	/* Set up the throughput level start indexes */4733	cap = udma_read(ud->mmrs[MMR_GCFG], 0x2c);4734	if (BCDMA_CAP3_UBCHAN_CNT(cap)) {4735		ud->bchan_tpl.levels = 3;4736		ud->bchan_tpl.start_idx[1] = BCDMA_CAP3_UBCHAN_CNT(cap);4737		ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap);4738	} else if (BCDMA_CAP3_HBCHAN_CNT(cap)) {4739		ud->bchan_tpl.levels = 2;4740		ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap);4741	} else {4742		ud->bchan_tpl.levels = 1;4743	}4744 4745	cap = udma_read(ud->mmrs[MMR_GCFG], 0x30);4746	if (BCDMA_CAP4_URCHAN_CNT(cap)) {4747		ud->rchan_tpl.levels = 3;4748		ud->rchan_tpl.start_idx[1] = BCDMA_CAP4_URCHAN_CNT(cap);4749		ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap);4750	} else if (BCDMA_CAP4_HRCHAN_CNT(cap)) {4751		ud->rchan_tpl.levels = 2;4752		ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap);4753	} else {4754		ud->rchan_tpl.levels = 1;4755	}4756 4757	if (BCDMA_CAP4_UTCHAN_CNT(cap)) {4758		ud->tchan_tpl.levels = 3;4759		ud->tchan_tpl.start_idx[1] = BCDMA_CAP4_UTCHAN_CNT(cap);4760		ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap);4761	} else if (BCDMA_CAP4_HTCHAN_CNT(cap)) {4762		ud->tchan_tpl.levels = 2;4763		ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap);4764	} else {4765		ud->tchan_tpl.levels = 1;4766	}4767 4768	ud->bchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->bchan_cnt),4769					   sizeof(unsigned long), GFP_KERNEL);4770	ud->bchans = devm_kcalloc(dev, ud->bchan_cnt, sizeof(*ud->bchans),4771				  GFP_KERNEL);4772	ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),4773					   sizeof(unsigned long), GFP_KERNEL);4774	ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),4775				  GFP_KERNEL);4776	ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),4777					   sizeof(unsigned long), GFP_KERNEL);4778	ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),4779				  GFP_KERNEL);4780	/* BCDMA do not really have flows, but the driver expect it */4781	ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rchan_cnt),4782					sizeof(unsigned long),4783					GFP_KERNEL);4784	ud->rflows = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rflows),4785				  GFP_KERNEL);4786 4787	if (!ud->bchan_map || !ud->tchan_map || !ud->rchan_map ||4788	    !ud->rflow_in_use || !ud->bchans || !ud->tchans || !ud->rchans ||4789	    !ud->rflows)4790		return -ENOMEM;4791 4792	/* Get resource ranges from tisci */4793	for (i = 0; i < RM_RANGE_LAST; i++) {4794		if (i == RM_RANGE_RFLOW || i == RM_RANGE_TFLOW)4795			continue;4796		if (i == RM_RANGE_BCHAN && ud->bchan_cnt == 0)4797			continue;4798		if (i == RM_RANGE_TCHAN && ud->tchan_cnt == 0)4799			continue;4800		if (i == RM_RANGE_RCHAN && ud->rchan_cnt == 0)4801			continue;4802 4803		tisci_rm->rm_ranges[i] =4804			devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,4805						    tisci_rm->tisci_dev_id,4806						    (char *)range_names[i]);4807	}4808 4809	irq_res.sets = 0;4810 4811	/* bchan ranges */4812	if (ud->bchan_cnt) {4813		rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN];4814		if (IS_ERR(rm_res)) {4815			bitmap_zero(ud->bchan_map, ud->bchan_cnt);4816			irq_res.sets++;4817		} else {4818			bitmap_fill(ud->bchan_map, ud->bchan_cnt);4819			for (i = 0; i < rm_res->sets; i++)4820				udma_mark_resource_ranges(ud, ud->bchan_map,4821							  &rm_res->desc[i],4822							  "bchan");4823			irq_res.sets += rm_res->sets;4824		}4825	}4826 4827	/* tchan ranges */4828	if (ud->tchan_cnt) {4829		rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];4830		if (IS_ERR(rm_res)) {4831			bitmap_zero(ud->tchan_map, ud->tchan_cnt);4832			irq_res.sets += 2;4833		} else {4834			bitmap_fill(ud->tchan_map, ud->tchan_cnt);4835			for (i = 0; i < rm_res->sets; i++)4836				udma_mark_resource_ranges(ud, ud->tchan_map,4837							  &rm_res->desc[i],4838							  "tchan");4839			irq_res.sets += rm_res->sets * 2;4840		}4841	}4842 4843	/* rchan ranges */4844	if (ud->rchan_cnt) {4845		rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];4846		if (IS_ERR(rm_res)) {4847			bitmap_zero(ud->rchan_map, ud->rchan_cnt);4848			irq_res.sets += 2;4849		} else {4850			bitmap_fill(ud->rchan_map, ud->rchan_cnt);4851			for (i = 0; i < rm_res->sets; i++)4852				udma_mark_resource_ranges(ud, ud->rchan_map,4853							  &rm_res->desc[i],4854							  "rchan");4855			irq_res.sets += rm_res->sets * 2;4856		}4857	}4858 4859	irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL);4860	if (!irq_res.desc)4861		return -ENOMEM;4862	if (ud->bchan_cnt) {4863		rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN];4864		if (IS_ERR(rm_res)) {4865			irq_res.desc[0].start = oes->bcdma_bchan_ring;4866			irq_res.desc[0].num = ud->bchan_cnt;4867			i = 1;4868		} else {4869			for (i = 0; i < rm_res->sets; i++) {4870				irq_res.desc[i].start = rm_res->desc[i].start +4871							oes->bcdma_bchan_ring;4872				irq_res.desc[i].num = rm_res->desc[i].num;4873			}4874		}4875	} else {4876		i = 0;4877	}4878 4879	if (ud->tchan_cnt) {4880		rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];4881		if (IS_ERR(rm_res)) {4882			irq_res.desc[i].start = oes->bcdma_tchan_data;4883			irq_res.desc[i].num = ud->tchan_cnt;4884			irq_res.desc[i + 1].start = oes->bcdma_tchan_ring;4885			irq_res.desc[i + 1].num = ud->tchan_cnt;4886			i += 2;4887		} else {4888			for (j = 0; j < rm_res->sets; j++, i += 2) {4889				irq_res.desc[i].start = rm_res->desc[j].start +4890							oes->bcdma_tchan_data;4891				irq_res.desc[i].num = rm_res->desc[j].num;4892 4893				irq_res.desc[i + 1].start = rm_res->desc[j].start +4894							oes->bcdma_tchan_ring;4895				irq_res.desc[i + 1].num = rm_res->desc[j].num;4896			}4897		}4898	}4899	if (ud->rchan_cnt) {4900		rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];4901		if (IS_ERR(rm_res)) {4902			irq_res.desc[i].start = oes->bcdma_rchan_data;4903			irq_res.desc[i].num = ud->rchan_cnt;4904			irq_res.desc[i + 1].start = oes->bcdma_rchan_ring;4905			irq_res.desc[i + 1].num = ud->rchan_cnt;4906			i += 2;4907		} else {4908			for (j = 0; j < rm_res->sets; j++, i += 2) {4909				irq_res.desc[i].start = rm_res->desc[j].start +4910							oes->bcdma_rchan_data;4911				irq_res.desc[i].num = rm_res->desc[j].num;4912 4913				irq_res.desc[i + 1].start = rm_res->desc[j].start +4914							oes->bcdma_rchan_ring;4915				irq_res.desc[i + 1].num = rm_res->desc[j].num;4916			}4917		}4918	}4919 4920	ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);4921	kfree(irq_res.desc);4922	if (ret) {4923		dev_err(ud->dev, "Failed to allocate MSI interrupts\n");4924		return ret;4925	}4926 4927	return 0;4928}4929 4930static int pktdma_setup_resources(struct udma_dev *ud)4931{4932	int ret, i, j;4933	struct device *dev = ud->dev;4934	struct ti_sci_resource *rm_res, irq_res;4935	struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;4936	const struct udma_oes_offsets *oes = &ud->soc_data->oes;4937	u32 cap3;4938 4939	/* Set up the throughput level start indexes */4940	cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);4941	if (UDMA_CAP3_UCHAN_CNT(cap3)) {4942		ud->tchan_tpl.levels = 3;4943		ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3);4944		ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);4945	} else if (UDMA_CAP3_HCHAN_CNT(cap3)) {4946		ud->tchan_tpl.levels = 2;4947		ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);4948	} else {4949		ud->tchan_tpl.levels = 1;4950	}4951 4952	ud->rchan_tpl.levels = ud->tchan_tpl.levels;4953	ud->rchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0];4954	ud->rchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1];4955 4956	ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),4957					   sizeof(unsigned long), GFP_KERNEL);4958	ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),4959				  GFP_KERNEL);4960	ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),4961					   sizeof(unsigned long), GFP_KERNEL);4962	ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),4963				  GFP_KERNEL);4964	ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),4965					sizeof(unsigned long),4966					GFP_KERNEL);4967	ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),4968				  GFP_KERNEL);4969	ud->tflow_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tflow_cnt),4970					   sizeof(unsigned long), GFP_KERNEL);4971 4972	if (!ud->tchan_map || !ud->rchan_map || !ud->tflow_map || !ud->tchans ||4973	    !ud->rchans || !ud->rflows || !ud->rflow_in_use)4974		return -ENOMEM;4975 4976	/* Get resource ranges from tisci */4977	for (i = 0; i < RM_RANGE_LAST; i++) {4978		if (i == RM_RANGE_BCHAN)4979			continue;4980 4981		tisci_rm->rm_ranges[i] =4982			devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,4983						    tisci_rm->tisci_dev_id,4984						    (char *)range_names[i]);4985	}4986 4987	/* tchan ranges */4988	rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];4989	if (IS_ERR(rm_res)) {4990		bitmap_zero(ud->tchan_map, ud->tchan_cnt);4991	} else {4992		bitmap_fill(ud->tchan_map, ud->tchan_cnt);4993		for (i = 0; i < rm_res->sets; i++)4994			udma_mark_resource_ranges(ud, ud->tchan_map,4995						  &rm_res->desc[i], "tchan");4996	}4997 4998	/* rchan ranges */4999	rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];5000	if (IS_ERR(rm_res)) {5001		bitmap_zero(ud->rchan_map, ud->rchan_cnt);5002	} else {5003		bitmap_fill(ud->rchan_map, ud->rchan_cnt);5004		for (i = 0; i < rm_res->sets; i++)5005			udma_mark_resource_ranges(ud, ud->rchan_map,5006						  &rm_res->desc[i], "rchan");5007	}5008 5009	/* rflow ranges */5010	rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];5011	if (IS_ERR(rm_res)) {5012		/* all rflows are assigned exclusively to Linux */5013		bitmap_zero(ud->rflow_in_use, ud->rflow_cnt);5014		irq_res.sets = 1;5015	} else {5016		bitmap_fill(ud->rflow_in_use, ud->rflow_cnt);5017		for (i = 0; i < rm_res->sets; i++)5018			udma_mark_resource_ranges(ud, ud->rflow_in_use,5019						  &rm_res->desc[i], "rflow");5020		irq_res.sets = rm_res->sets;5021	}5022 5023	/* tflow ranges */5024	rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW];5025	if (IS_ERR(rm_res)) {5026		/* all tflows are assigned exclusively to Linux */5027		bitmap_zero(ud->tflow_map, ud->tflow_cnt);5028		irq_res.sets++;5029	} else {5030		bitmap_fill(ud->tflow_map, ud->tflow_cnt);5031		for (i = 0; i < rm_res->sets; i++)5032			udma_mark_resource_ranges(ud, ud->tflow_map,5033						  &rm_res->desc[i], "tflow");5034		irq_res.sets += rm_res->sets;5035	}5036 5037	irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL);5038	if (!irq_res.desc)5039		return -ENOMEM;5040	rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW];5041	if (IS_ERR(rm_res)) {5042		irq_res.desc[0].start = oes->pktdma_tchan_flow;5043		irq_res.desc[0].num = ud->tflow_cnt;5044		i = 1;5045	} else {5046		for (i = 0; i < rm_res->sets; i++) {5047			irq_res.desc[i].start = rm_res->desc[i].start +5048						oes->pktdma_tchan_flow;5049			irq_res.desc[i].num = rm_res->desc[i].num;5050		}5051	}5052	rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];5053	if (IS_ERR(rm_res)) {5054		irq_res.desc[i].start = oes->pktdma_rchan_flow;5055		irq_res.desc[i].num = ud->rflow_cnt;5056	} else {5057		for (j = 0; j < rm_res->sets; j++, i++) {5058			irq_res.desc[i].start = rm_res->desc[j].start +5059						oes->pktdma_rchan_flow;5060			irq_res.desc[i].num = rm_res->desc[j].num;5061		}5062	}5063	ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);5064	kfree(irq_res.desc);5065	if (ret) {5066		dev_err(ud->dev, "Failed to allocate MSI interrupts\n");5067		return ret;5068	}5069 5070	return 0;5071}5072 5073static int setup_resources(struct udma_dev *ud)5074{5075	struct device *dev = ud->dev;5076	int ch_count, ret;5077 5078	switch (ud->match_data->type) {5079	case DMA_TYPE_UDMA:5080		ret = udma_setup_resources(ud);5081		break;5082	case DMA_TYPE_BCDMA:5083		ret = bcdma_setup_resources(ud);5084		break;5085	case DMA_TYPE_PKTDMA:5086		ret = pktdma_setup_resources(ud);5087		break;5088	default:5089		return -EINVAL;5090	}5091 5092	if (ret)5093		return ret;5094 5095	ch_count  = ud->bchan_cnt + ud->tchan_cnt + ud->rchan_cnt;5096	if (ud->bchan_cnt)5097		ch_count -= bitmap_weight(ud->bchan_map, ud->bchan_cnt);5098	ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt);5099	ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt);5100	if (!ch_count)5101		return -ENODEV;5102 5103	ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels),5104				    GFP_KERNEL);5105	if (!ud->channels)5106		return -ENOMEM;5107 5108	switch (ud->match_data->type) {5109	case DMA_TYPE_UDMA:5110		dev_info(dev,5111			 "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n",5112			 ch_count,5113			 ud->tchan_cnt - bitmap_weight(ud->tchan_map,5114						       ud->tchan_cnt),5115			 ud->rchan_cnt - bitmap_weight(ud->rchan_map,5116						       ud->rchan_cnt),5117			 ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map,5118						       ud->rflow_cnt));5119		break;5120	case DMA_TYPE_BCDMA:5121		dev_info(dev,5122			 "Channels: %d (bchan: %u, tchan: %u, rchan: %u)\n",5123			 ch_count,5124			 ud->bchan_cnt - bitmap_weight(ud->bchan_map,5125						       ud->bchan_cnt),5126			 ud->tchan_cnt - bitmap_weight(ud->tchan_map,5127						       ud->tchan_cnt),5128			 ud->rchan_cnt - bitmap_weight(ud->rchan_map,5129						       ud->rchan_cnt));5130		break;5131	case DMA_TYPE_PKTDMA:5132		dev_info(dev,5133			 "Channels: %d (tchan: %u, rchan: %u)\n",5134			 ch_count,5135			 ud->tchan_cnt - bitmap_weight(ud->tchan_map,5136						       ud->tchan_cnt),5137			 ud->rchan_cnt - bitmap_weight(ud->rchan_map,5138						       ud->rchan_cnt));5139		break;5140	default:5141		break;5142	}5143 5144	return ch_count;5145}5146 5147static int udma_setup_rx_flush(struct udma_dev *ud)5148{5149	struct udma_rx_flush *rx_flush = &ud->rx_flush;5150	struct cppi5_desc_hdr_t *tr_desc;5151	struct cppi5_tr_type1_t *tr_req;5152	struct cppi5_host_desc_t *desc;5153	struct device *dev = ud->dev;5154	struct udma_hwdesc *hwdesc;5155	size_t tr_size;5156 5157	/* Allocate 1K buffer for discarded data on RX channel teardown */5158	rx_flush->buffer_size = SZ_1K;5159	rx_flush->buffer_vaddr = devm_kzalloc(dev, rx_flush->buffer_size,5160					      GFP_KERNEL);5161	if (!rx_flush->buffer_vaddr)5162		return -ENOMEM;5163 5164	rx_flush->buffer_paddr = dma_map_single(dev, rx_flush->buffer_vaddr,5165						rx_flush->buffer_size,5166						DMA_TO_DEVICE);5167	if (dma_mapping_error(dev, rx_flush->buffer_paddr))5168		return -ENOMEM;5169 5170	/* Set up descriptor to be used for TR mode */5171	hwdesc = &rx_flush->hwdescs[0];5172	tr_size = sizeof(struct cppi5_tr_type1_t);5173	hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1);5174	hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,5175					ud->desc_align);5176 5177	hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,5178						GFP_KERNEL);5179	if (!hwdesc->cppi5_desc_vaddr)5180		return -ENOMEM;5181 5182	hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,5183						  hwdesc->cppi5_desc_size,5184						  DMA_TO_DEVICE);5185	if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))5186		return -ENOMEM;5187 5188	/* Start of the TR req records */5189	hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;5190	/* Start address of the TR response array */5191	hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size;5192 5193	tr_desc = hwdesc->cppi5_desc_vaddr;5194	cppi5_trdesc_init(tr_desc, 1, tr_size, 0, 0);5195	cppi5_desc_set_pktids(tr_desc, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT);5196	cppi5_desc_set_retpolicy(tr_desc, 0, 0);5197 5198	tr_req = hwdesc->tr_req_base;5199	cppi5_tr_init(&tr_req->flags, CPPI5_TR_TYPE1, false, false,5200		      CPPI5_TR_EVENT_SIZE_COMPLETION, 0);5201	cppi5_tr_csf_set(&tr_req->flags, CPPI5_TR_CSF_SUPR_EVT);5202 5203	tr_req->addr = rx_flush->buffer_paddr;5204	tr_req->icnt0 = rx_flush->buffer_size;5205	tr_req->icnt1 = 1;5206 5207	dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,5208				   hwdesc->cppi5_desc_size, DMA_TO_DEVICE);5209 5210	/* Set up descriptor to be used for packet mode */5211	hwdesc = &rx_flush->hwdescs[1];5212	hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +5213					CPPI5_INFO0_HDESC_EPIB_SIZE +5214					CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE,5215					ud->desc_align);5216 5217	hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,5218						GFP_KERNEL);5219	if (!hwdesc->cppi5_desc_vaddr)5220		return -ENOMEM;5221 5222	hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,5223						  hwdesc->cppi5_desc_size,5224						  DMA_TO_DEVICE);5225	if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))5226		return -ENOMEM;5227 5228	desc = hwdesc->cppi5_desc_vaddr;5229	cppi5_hdesc_init(desc, 0, 0);5230	cppi5_desc_set_pktids(&desc->hdr, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT);5231	cppi5_desc_set_retpolicy(&desc->hdr, 0, 0);5232 5233	cppi5_hdesc_attach_buf(desc,5234			       rx_flush->buffer_paddr, rx_flush->buffer_size,5235			       rx_flush->buffer_paddr, rx_flush->buffer_size);5236 5237	dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,5238				   hwdesc->cppi5_desc_size, DMA_TO_DEVICE);5239	return 0;5240}5241 5242#ifdef CONFIG_DEBUG_FS5243static void udma_dbg_summary_show_chan(struct seq_file *s,5244				       struct dma_chan *chan)5245{5246	struct udma_chan *uc = to_udma_chan(chan);5247	struct udma_chan_config *ucc = &uc->config;5248 5249	seq_printf(s, " %-13s| %s", dma_chan_name(chan),5250		   chan->dbg_client_name ?: "in-use");5251	if (ucc->tr_trigger_type)5252		seq_puts(s, " (triggered, ");5253	else5254		seq_printf(s, " (%s, ",5255			   dmaengine_get_direction_text(uc->config.dir));5256 5257	switch (uc->config.dir) {5258	case DMA_MEM_TO_MEM:5259		if (uc->ud->match_data->type == DMA_TYPE_BCDMA) {5260			seq_printf(s, "bchan%d)\n", uc->bchan->id);5261			return;5262		}5263 5264		seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id,5265			   ucc->src_thread, ucc->dst_thread);5266		break;5267	case DMA_DEV_TO_MEM:5268		seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id,5269			   ucc->src_thread, ucc->dst_thread);5270		if (uc->ud->match_data->type == DMA_TYPE_PKTDMA)5271			seq_printf(s, "rflow%d, ", uc->rflow->id);5272		break;5273	case DMA_MEM_TO_DEV:5274		seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id,5275			   ucc->src_thread, ucc->dst_thread);5276		if (uc->ud->match_data->type == DMA_TYPE_PKTDMA)5277			seq_printf(s, "tflow%d, ", uc->tchan->tflow_id);5278		break;5279	default:5280		seq_printf(s, ")\n");5281		return;5282	}5283 5284	if (ucc->ep_type == PSIL_EP_NATIVE) {5285		seq_printf(s, "PSI-L Native");5286		if (ucc->metadata_size) {5287			seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : "");5288			if (ucc->psd_size)5289				seq_printf(s, " PSDsize:%u", ucc->psd_size);5290			seq_printf(s, " ]");5291		}5292	} else {5293		seq_printf(s, "PDMA");5294		if (ucc->enable_acc32 || ucc->enable_burst)5295			seq_printf(s, "[%s%s ]",5296				   ucc->enable_acc32 ? " ACC32" : "",5297				   ucc->enable_burst ? " BURST" : "");5298	}5299 5300	seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode");5301}5302 5303static void udma_dbg_summary_show(struct seq_file *s,5304				  struct dma_device *dma_dev)5305{5306	struct dma_chan *chan;5307 5308	list_for_each_entry(chan, &dma_dev->channels, device_node) {5309		if (chan->client_count)5310			udma_dbg_summary_show_chan(s, chan);5311	}5312}5313#endif /* CONFIG_DEBUG_FS */5314 5315static enum dmaengine_alignment udma_get_copy_align(struct udma_dev *ud)5316{5317	const struct udma_match_data *match_data = ud->match_data;5318	u8 tpl;5319 5320	if (!match_data->enable_memcpy_support)5321		return DMAENGINE_ALIGN_8_BYTES;5322 5323	/* Get the highest TPL level the device supports for memcpy */5324	if (ud->bchan_cnt)5325		tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, 0);5326	else if (ud->tchan_cnt)5327		tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, 0);5328	else5329		return DMAENGINE_ALIGN_8_BYTES;5330 5331	switch (match_data->burst_size[tpl]) {5332	case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES:5333		return DMAENGINE_ALIGN_256_BYTES;5334	case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES:5335		return DMAENGINE_ALIGN_128_BYTES;5336	case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES:5337	fallthrough;5338	default:5339		return DMAENGINE_ALIGN_64_BYTES;5340	}5341}5342 5343#define TI_UDMAC_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \5344				 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \5345				 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \5346				 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \5347				 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))5348 5349static int udma_probe(struct platform_device *pdev)5350{5351	struct device_node *navss_node = pdev->dev.parent->of_node;5352	const struct soc_device_attribute *soc;5353	struct device *dev = &pdev->dev;5354	struct udma_dev *ud;5355	const struct of_device_id *match;5356	int i, ret;5357	int ch_count;5358 5359	ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48));5360	if (ret)5361		dev_err(dev, "failed to set dma mask stuff\n");5362 5363	ud = devm_kzalloc(dev, sizeof(*ud), GFP_KERNEL);5364	if (!ud)5365		return -ENOMEM;5366 5367	match = of_match_node(udma_of_match, dev->of_node);5368	if (!match) {5369		dev_err(dev, "No compatible match found\n");5370		return -ENODEV;5371	}5372	ud->match_data = match->data;5373 5374	ud->soc_data = ud->match_data->soc_data;5375	if (!ud->soc_data) {5376		soc = soc_device_match(k3_soc_devices);5377		if (!soc) {5378			dev_err(dev, "No compatible SoC found\n");5379			return -ENODEV;5380		}5381		ud->soc_data = soc->data;5382	}5383 5384	ret = udma_get_mmrs(pdev, ud);5385	if (ret)5386		return ret;5387 5388	ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci");5389	if (IS_ERR(ud->tisci_rm.tisci))5390		return PTR_ERR(ud->tisci_rm.tisci);5391 5392	ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id",5393				   &ud->tisci_rm.tisci_dev_id);5394	if (ret) {5395		dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);5396		return ret;5397	}5398	pdev->id = ud->tisci_rm.tisci_dev_id;5399 5400	ret = of_property_read_u32(navss_node, "ti,sci-dev-id",5401				   &ud->tisci_rm.tisci_navss_dev_id);5402	if (ret) {5403		dev_err(dev, "NAVSS ti,sci-dev-id read failure %d\n", ret);5404		return ret;5405	}5406 5407	if (ud->match_data->type == DMA_TYPE_UDMA) {5408		ret = of_property_read_u32(dev->of_node, "ti,udma-atype",5409					   &ud->atype);5410		if (!ret && ud->atype > 2) {5411			dev_err(dev, "Invalid atype: %u\n", ud->atype);5412			return -EINVAL;5413		}5414	} else {5415		ret = of_property_read_u32(dev->of_node, "ti,asel",5416					   &ud->asel);5417		if (!ret && ud->asel > 15) {5418			dev_err(dev, "Invalid asel: %u\n", ud->asel);5419			return -EINVAL;5420		}5421	}5422 5423	ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops;5424	ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops;5425 5426	if (ud->match_data->type == DMA_TYPE_UDMA) {5427		ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc");5428	} else {5429		struct k3_ringacc_init_data ring_init_data;5430 5431		ring_init_data.tisci = ud->tisci_rm.tisci;5432		ring_init_data.tisci_dev_id = ud->tisci_rm.tisci_dev_id;5433		if (ud->match_data->type == DMA_TYPE_BCDMA) {5434			ring_init_data.num_rings = ud->bchan_cnt +5435						   ud->tchan_cnt +5436						   ud->rchan_cnt;5437		} else {5438			ring_init_data.num_rings = ud->rflow_cnt +5439						   ud->tflow_cnt;5440		}5441 5442		ud->ringacc = k3_ringacc_dmarings_init(pdev, &ring_init_data);5443	}5444 5445	if (IS_ERR(ud->ringacc))5446		return PTR_ERR(ud->ringacc);5447 5448	dev->msi.domain = of_msi_get_domain(dev, dev->of_node,5449					    DOMAIN_BUS_TI_SCI_INTA_MSI);5450	if (!dev->msi.domain) {5451		return -EPROBE_DEFER;5452	}5453 5454	dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask);5455	/* cyclic operation is not supported via PKTDMA */5456	if (ud->match_data->type != DMA_TYPE_PKTDMA) {5457		dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask);5458		ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic;5459	}5460 5461	ud->ddev.device_config = udma_slave_config;5462	ud->ddev.device_prep_slave_sg = udma_prep_slave_sg;5463	ud->ddev.device_issue_pending = udma_issue_pending;5464	ud->ddev.device_tx_status = udma_tx_status;5465	ud->ddev.device_pause = udma_pause;5466	ud->ddev.device_resume = udma_resume;5467	ud->ddev.device_terminate_all = udma_terminate_all;5468	ud->ddev.device_synchronize = udma_synchronize;5469#ifdef CONFIG_DEBUG_FS5470	ud->ddev.dbg_summary_show = udma_dbg_summary_show;5471#endif5472 5473	switch (ud->match_data->type) {5474	case DMA_TYPE_UDMA:5475		ud->ddev.device_alloc_chan_resources =5476					udma_alloc_chan_resources;5477		break;5478	case DMA_TYPE_BCDMA:5479		ud->ddev.device_alloc_chan_resources =5480					bcdma_alloc_chan_resources;5481		ud->ddev.device_router_config = bcdma_router_config;5482		break;5483	case DMA_TYPE_PKTDMA:5484		ud->ddev.device_alloc_chan_resources =5485					pktdma_alloc_chan_resources;5486		break;5487	default:5488		return -EINVAL;5489	}5490	ud->ddev.device_free_chan_resources = udma_free_chan_resources;5491 5492	ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS;5493	ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS;5494	ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);5495	ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;5496	ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT |5497				       DESC_METADATA_ENGINE;5498	if (ud->match_data->enable_memcpy_support &&5499	    !(ud->match_data->type == DMA_TYPE_BCDMA && ud->bchan_cnt == 0)) {5500		dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask);5501		ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy;5502		ud->ddev.directions |= BIT(DMA_MEM_TO_MEM);5503	}5504 5505	ud->ddev.dev = dev;5506	ud->dev = dev;5507	ud->psil_base = ud->match_data->psil_base;5508 5509	INIT_LIST_HEAD(&ud->ddev.channels);5510	INIT_LIST_HEAD(&ud->desc_to_purge);5511 5512	ch_count = setup_resources(ud);5513	if (ch_count <= 0)5514		return ch_count;5515 5516	spin_lock_init(&ud->lock);5517	INIT_WORK(&ud->purge_work, udma_purge_desc_work);5518 5519	ud->desc_align = 64;5520	if (ud->desc_align < dma_get_cache_alignment())5521		ud->desc_align = dma_get_cache_alignment();5522 5523	ret = udma_setup_rx_flush(ud);5524	if (ret)5525		return ret;5526 5527	for (i = 0; i < ud->bchan_cnt; i++) {5528		struct udma_bchan *bchan = &ud->bchans[i];5529 5530		bchan->id = i;5531		bchan->reg_rt = ud->mmrs[MMR_BCHANRT] + i * 0x1000;5532	}5533 5534	for (i = 0; i < ud->tchan_cnt; i++) {5535		struct udma_tchan *tchan = &ud->tchans[i];5536 5537		tchan->id = i;5538		tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000;5539	}5540 5541	for (i = 0; i < ud->rchan_cnt; i++) {5542		struct udma_rchan *rchan = &ud->rchans[i];5543 5544		rchan->id = i;5545		rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000;5546	}5547 5548	for (i = 0; i < ud->rflow_cnt; i++) {5549		struct udma_rflow *rflow = &ud->rflows[i];5550 5551		rflow->id = i;5552	}5553 5554	for (i = 0; i < ch_count; i++) {5555		struct udma_chan *uc = &ud->channels[i];5556 5557		uc->ud = ud;5558		uc->vc.desc_free = udma_desc_free;5559		uc->id = i;5560		uc->bchan = NULL;5561		uc->tchan = NULL;5562		uc->rchan = NULL;5563		uc->config.remote_thread_id = -1;5564		uc->config.mapped_channel_id = -1;5565		uc->config.default_flow_id = -1;5566		uc->config.dir = DMA_MEM_TO_MEM;5567		uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d",5568					  dev_name(dev), i);5569 5570		vchan_init(&uc->vc, &ud->ddev);5571		/* Use custom vchan completion handling */5572		tasklet_setup(&uc->vc.task, udma_vchan_complete);5573		init_completion(&uc->teardown_completed);5574		INIT_DELAYED_WORK(&uc->tx_drain.work, udma_check_tx_completion);5575	}5576 5577	/* Configure the copy_align to the maximum burst size the device supports */5578	ud->ddev.copy_align = udma_get_copy_align(ud);5579 5580	ret = dma_async_device_register(&ud->ddev);5581	if (ret) {5582		dev_err(dev, "failed to register slave DMA engine: %d\n", ret);5583		return ret;5584	}5585 5586	platform_set_drvdata(pdev, ud);5587 5588	ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud);5589	if (ret) {5590		dev_err(dev, "failed to register of_dma controller\n");5591		dma_async_device_unregister(&ud->ddev);5592	}5593 5594	return ret;5595}5596 5597static int __maybe_unused udma_pm_suspend(struct device *dev)5598{5599	struct udma_dev *ud = dev_get_drvdata(dev);5600	struct dma_device *dma_dev = &ud->ddev;5601	struct dma_chan *chan;5602	struct udma_chan *uc;5603 5604	list_for_each_entry(chan, &dma_dev->channels, device_node) {5605		if (chan->client_count) {5606			uc = to_udma_chan(chan);5607			/* backup the channel configuration */5608			memcpy(&uc->backup_config, &uc->config,5609			       sizeof(struct udma_chan_config));5610			dev_dbg(dev, "Suspending channel %s\n",5611				dma_chan_name(chan));5612			ud->ddev.device_free_chan_resources(chan);5613		}5614	}5615 5616	return 0;5617}5618 5619static int __maybe_unused udma_pm_resume(struct device *dev)5620{5621	struct udma_dev *ud = dev_get_drvdata(dev);5622	struct dma_device *dma_dev = &ud->ddev;5623	struct dma_chan *chan;5624	struct udma_chan *uc;5625	int ret;5626 5627	list_for_each_entry(chan, &dma_dev->channels, device_node) {5628		if (chan->client_count) {5629			uc = to_udma_chan(chan);5630			/* restore the channel configuration */5631			memcpy(&uc->config, &uc->backup_config,5632			       sizeof(struct udma_chan_config));5633			dev_dbg(dev, "Resuming channel %s\n",5634				dma_chan_name(chan));5635			ret = ud->ddev.device_alloc_chan_resources(chan);5636			if (ret)5637				return ret;5638		}5639	}5640 5641	return 0;5642}5643 5644static const struct dev_pm_ops udma_pm_ops = {5645	SET_LATE_SYSTEM_SLEEP_PM_OPS(udma_pm_suspend, udma_pm_resume)5646};5647 5648static struct platform_driver udma_driver = {5649	.driver = {5650		.name	= "ti-udma",5651		.of_match_table = udma_of_match,5652		.suppress_bind_attrs = true,5653		.pm = &udma_pm_ops,5654	},5655	.probe		= udma_probe,5656};5657 5658module_platform_driver(udma_driver);5659MODULE_DESCRIPTION("Texas Instruments UDMA support");5660MODULE_LICENSE("GPL v2");5661 5662/* Private interfaces to UDMA */5663#include "k3-udma-private.c"5664