brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Copyright (c) 2006 ARM Ltd.4 * Copyright (c) 2010 ST-Ericsson SA5 * Copyright (c) 2017 Linaro Ltd.6 *7 * Author: Peter Pearse <peter.pearse@arm.com>8 * Author: Linus Walleij <linus.walleij@linaro.org>9 *10 * Documentation: ARM DDI 0196G == PL08011 * Documentation: ARM DDI 0218E == PL08112 * Documentation: S3C6410 User's Manual == PL080S13 *14 * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any15 * channel.16 *17 * The PL080 has 8 channels available for simultaneous use, and the PL08118 * has only two channels. So on these DMA controllers the number of channels19 * and the number of incoming DMA signals are two totally different things.20 * It is usually not possible to theoretically handle all physical signals,21 * so a multiplexing scheme with possible denial of use is necessary.22 *23 * The PL080 has a dual bus master, PL081 has a single master.24 *25 * PL080S is a version modified by Samsung and used in S3C64xx SoCs.26 * It differs in following aspects:27 * - CH_CONFIG register at different offset,28 * - separate CH_CONTROL2 register for transfer size,29 * - bigger maximum transfer size,30 * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word,31 * - no support for peripheral flow control.32 *33 * Memory to peripheral transfer may be visualized as34 *	Get data from memory to DMAC35 *	Until no data left36 *		On burst request from peripheral37 *			Destination burst from DMAC to peripheral38 *			Clear burst request39 *	Raise terminal count interrupt40 *41 * For peripherals with a FIFO:42 * Source      burst size == half the depth of the peripheral FIFO43 * Destination burst size == the depth of the peripheral FIFO44 *45 * (Bursts are irrelevant for mem to mem transfers - there are no burst46 * signals, the DMA controller will simply facilitate its AHB master.)47 *48 * ASSUMES default (little) endianness for DMA transfers49 *50 * The PL08x has two flow control settings:51 *  - DMAC flow control: the transfer size defines the number of transfers52 *    which occur for the current LLI entry, and the DMAC raises TC at the53 *    end of every LLI entry.  Observed behaviour shows the DMAC listening54 *    to both the BREQ and SREQ signals (contrary to documented),55 *    transferring data if either is active.  The LBREQ and LSREQ signals56 *    are ignored.57 *58 *  - Peripheral flow control: the transfer size is ignored (and should be59 *    zero).  The data is transferred from the current LLI entry, until60 *    after the final transfer signalled by LBREQ or LSREQ.  The DMAC61 *    will then move to the next LLI entry. Unsupported by PL080S.62 */63#include <linux/amba/bus.h>64#include <linux/amba/pl08x.h>65#include <linux/debugfs.h>66#include <linux/delay.h>67#include <linux/device.h>68#include <linux/dmaengine.h>69#include <linux/dmapool.h>70#include <linux/dma-mapping.h>71#include <linux/export.h>72#include <linux/init.h>73#include <linux/interrupt.h>74#include <linux/module.h>75#include <linux/of.h>76#include <linux/of_dma.h>77#include <linux/pm_runtime.h>78#include <linux/seq_file.h>79#include <linux/slab.h>80#include <linux/amba/pl080.h>81 82#include "dmaengine.h"83#include "virt-dma.h"84 85#define DRIVER_NAME	"pl08xdmac"86 87#define PL80X_DMA_BUSWIDTHS \88	BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \89	BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \90	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \91	BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)92 93static struct amba_driver pl08x_amba_driver;94struct pl08x_driver_data;95 96/**97 * struct vendor_data - vendor-specific config parameters for PL08x derivatives98 * @config_offset: offset to the configuration register99 * @channels: the number of channels available in this variant100 * @signals: the number of request signals available from the hardware101 * @dualmaster: whether this version supports dual AHB masters or not.102 * @nomadik: whether this variant is a ST Microelectronics Nomadik, where the103 *	channels have Nomadik security extension bits that need to be checked104 *	for permission before use and some registers are missing105 * @pl080s: whether this variant is a Samsung PL080S, which has separate106 *	register and LLI word for transfer size.107 * @ftdmac020: whether this variant is a Faraday Technology FTDMAC020108 * @max_transfer_size: the maximum single element transfer size for this109 *	PL08x variant.110 */111struct vendor_data {112	u8 config_offset;113	u8 channels;114	u8 signals;115	bool dualmaster;116	bool nomadik;117	bool pl080s;118	bool ftdmac020;119	u32 max_transfer_size;120};121 122/**123 * struct pl08x_bus_data - information of source or destination124 * busses for a transfer125 * @addr: current address126 * @maxwidth: the maximum width of a transfer on this bus127 * @buswidth: the width of this bus in bytes: 1, 2 or 4128 */129struct pl08x_bus_data {130	dma_addr_t addr;131	u8 maxwidth;132	u8 buswidth;133};134 135#define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth)136 137/**138 * struct pl08x_phy_chan - holder for the physical channels139 * @id: physical index to this channel140 * @base: memory base address for this physical channel141 * @reg_config: configuration address for this physical channel142 * @reg_control: control address for this physical channel143 * @reg_src: transfer source address register144 * @reg_dst: transfer destination address register145 * @reg_lli: transfer LLI address register146 * @reg_busy: if the variant has a special per-channel busy register,147 * this contains a pointer to it148 * @lock: a lock to use when altering an instance of this struct149 * @serving: the virtual channel currently being served by this physical150 * channel151 * @locked: channel unavailable for the system, e.g. dedicated to secure152 * world153 * @ftdmac020: channel is on a FTDMAC020154 * @pl080s: channel is on a PL08s155 */156struct pl08x_phy_chan {157	unsigned int id;158	void __iomem *base;159	void __iomem *reg_config;160	void __iomem *reg_control;161	void __iomem *reg_src;162	void __iomem *reg_dst;163	void __iomem *reg_lli;164	void __iomem *reg_busy;165	spinlock_t lock;166	struct pl08x_dma_chan *serving;167	bool locked;168	bool ftdmac020;169	bool pl080s;170};171 172/**173 * struct pl08x_sg - structure containing data per sg174 * @src_addr: src address of sg175 * @dst_addr: dst address of sg176 * @len: transfer len in bytes177 * @node: node for txd's dsg_list178 */179struct pl08x_sg {180	dma_addr_t src_addr;181	dma_addr_t dst_addr;182	size_t len;183	struct list_head node;184};185 186/**187 * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor188 * @vd: virtual DMA descriptor189 * @dsg_list: list of children sg's190 * @llis_bus: DMA memory address (physical) start for the LLIs191 * @llis_va: virtual memory address start for the LLIs192 * @cctl: control reg values for current txd193 * @ccfg: config reg values for current txd194 * @done: this marks completed descriptors, which should not have their195 *   mux released.196 * @cyclic: indicate cyclic transfers197 */198struct pl08x_txd {199	struct virt_dma_desc vd;200	struct list_head dsg_list;201	dma_addr_t llis_bus;202	u32 *llis_va;203	/* Default cctl value for LLIs */204	u32 cctl;205	/*206	 * Settings to be put into the physical channel when we207	 * trigger this txd.  Other registers are in llis_va[0].208	 */209	u32 ccfg;210	bool done;211	bool cyclic;212};213 214/**215 * enum pl08x_dma_chan_state - holds the PL08x specific virtual channel216 * states217 * @PL08X_CHAN_IDLE: the channel is idle218 * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport219 * channel and is running a transfer on it220 * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport221 * channel, but the transfer is currently paused222 * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport223 * channel to become available (only pertains to memcpy channels)224 */225enum pl08x_dma_chan_state {226	PL08X_CHAN_IDLE,227	PL08X_CHAN_RUNNING,228	PL08X_CHAN_PAUSED,229	PL08X_CHAN_WAITING,230};231 232/**233 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel234 * @vc: wrapped virtual channel235 * @phychan: the physical channel utilized by this channel, if there is one236 * @name: name of channel237 * @cd: channel platform data238 * @cfg: slave configuration239 * @at: active transaction on this channel240 * @host: a pointer to the host (internal use)241 * @state: whether the channel is idle, paused, running etc242 * @slave: whether this channel is a device (slave) or for memcpy243 * @signal: the physical DMA request signal which this channel is using244 * @mux_use: count of descriptors using this DMA request signal setting245 * @waiting_at: time in jiffies when this channel moved to waiting state246 */247struct pl08x_dma_chan {248	struct virt_dma_chan vc;249	struct pl08x_phy_chan *phychan;250	const char *name;251	struct pl08x_channel_data *cd;252	struct dma_slave_config cfg;253	struct pl08x_txd *at;254	struct pl08x_driver_data *host;255	enum pl08x_dma_chan_state state;256	bool slave;257	int signal;258	unsigned mux_use;259	unsigned long waiting_at;260};261 262/**263 * struct pl08x_driver_data - the local state holder for the PL08x264 * @slave: optional slave engine for this instance265 * @memcpy: memcpy engine for this instance266 * @has_slave: the PL08x has a slave engine (routed signals)267 * @base: virtual memory base (remapped) for the PL08x268 * @adev: the corresponding AMBA (PrimeCell) bus entry269 * @vd: vendor data for this PL08x variant270 * @pd: platform data passed in from the platform/machine271 * @phy_chans: array of data for the physical channels272 * @pool: a pool for the LLI descriptors273 * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI274 * fetches275 * @mem_buses: set to indicate memory transfers on AHB2.276 * @lli_words: how many words are used in each LLI item for this variant277 */278struct pl08x_driver_data {279	struct dma_device slave;280	struct dma_device memcpy;281	bool has_slave;282	void __iomem *base;283	struct amba_device *adev;284	const struct vendor_data *vd;285	struct pl08x_platform_data *pd;286	struct pl08x_phy_chan *phy_chans;287	struct dma_pool *pool;288	u8 lli_buses;289	u8 mem_buses;290	u8 lli_words;291};292 293/*294 * PL08X specific defines295 */296 297/* The order of words in an LLI. */298#define PL080_LLI_SRC		0299#define PL080_LLI_DST		1300#define PL080_LLI_LLI		2301#define PL080_LLI_CCTL		3302#define PL080S_LLI_CCTL2	4303 304/* Total words in an LLI. */305#define PL080_LLI_WORDS		4306#define PL080S_LLI_WORDS	8307 308/*309 * Number of LLIs in each LLI buffer allocated for one transfer310 * (maximum times we call dma_pool_alloc on this pool without freeing)311 */312#define MAX_NUM_TSFR_LLIS	512313#define PL08X_ALIGN		8314 315static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)316{317	return container_of(chan, struct pl08x_dma_chan, vc.chan);318}319 320static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx)321{322	return container_of(tx, struct pl08x_txd, vd.tx);323}324 325/*326 * Mux handling.327 *328 * This gives us the DMA request input to the PL08x primecell which the329 * peripheral described by the channel data will be routed to, possibly330 * via a board/SoC specific external MUX.  One important point to note331 * here is that this does not depend on the physical channel.332 */333static int pl08x_request_mux(struct pl08x_dma_chan *plchan)334{335	const struct pl08x_platform_data *pd = plchan->host->pd;336	int ret;337 338	if (plchan->mux_use++ == 0 && pd->get_xfer_signal) {339		ret = pd->get_xfer_signal(plchan->cd);340		if (ret < 0) {341			plchan->mux_use = 0;342			return ret;343		}344 345		plchan->signal = ret;346	}347	return 0;348}349 350static void pl08x_release_mux(struct pl08x_dma_chan *plchan)351{352	const struct pl08x_platform_data *pd = plchan->host->pd;353 354	if (plchan->signal >= 0) {355		WARN_ON(plchan->mux_use == 0);356 357		if (--plchan->mux_use == 0 && pd->put_xfer_signal) {358			pd->put_xfer_signal(plchan->cd, plchan->signal);359			plchan->signal = -1;360		}361	}362}363 364/*365 * Physical channel handling366 */367 368/* Whether a certain channel is busy or not */369static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch)370{371	unsigned int val;372 373	/* If we have a special busy register, take a shortcut */374	if (ch->reg_busy) {375		val = readl(ch->reg_busy);376		return !!(val & BIT(ch->id));377	}378	val = readl(ch->reg_config);379	return val & PL080_CONFIG_ACTIVE;380}381 382/*383 * pl08x_write_lli() - Write an LLI into the DMA controller.384 *385 * The PL08x derivatives support linked lists, but the first item of the386 * list containing the source, destination, control word and next LLI is387 * ignored. Instead the driver has to write those values directly into the388 * SRC, DST, LLI and control registers. On FTDMAC020 also the SIZE389 * register need to be set up for the first transfer.390 */391static void pl08x_write_lli(struct pl08x_driver_data *pl08x,392		struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg)393{394	if (pl08x->vd->pl080s)395		dev_vdbg(&pl08x->adev->dev,396			"WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "397			"clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n",398			phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],399			lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL],400			lli[PL080S_LLI_CCTL2], ccfg);401	else402		dev_vdbg(&pl08x->adev->dev,403			"WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "404			"clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n",405			phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],406			lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg);407 408	writel_relaxed(lli[PL080_LLI_SRC], phychan->reg_src);409	writel_relaxed(lli[PL080_LLI_DST], phychan->reg_dst);410	writel_relaxed(lli[PL080_LLI_LLI], phychan->reg_lli);411 412	/*413	 * The FTMAC020 has a different layout in the CCTL word of the LLI414	 * and the CCTL register which is split in CSR and SIZE registers.415	 * Convert the LLI item CCTL into the proper values to write into416	 * the CSR and SIZE registers.417	 */418	if (phychan->ftdmac020) {419		u32 llictl = lli[PL080_LLI_CCTL];420		u32 val = 0;421 422		/* Write the transfer size (12 bits) to the size register */423		writel_relaxed(llictl & FTDMAC020_LLI_TRANSFER_SIZE_MASK,424			       phychan->base + FTDMAC020_CH_SIZE);425		/*426		 * Then write the control bits 28..16 to the control register427		 * by shuffleing the bits around to where they are in the428		 * main register. The mapping is as follows:429		 * Bit 28: TC_MSK - mask on all except last LLI430		 * Bit 27..25: SRC_WIDTH431		 * Bit 24..22: DST_WIDTH432		 * Bit 21..20: SRCAD_CTRL433		 * Bit 19..17: DSTAD_CTRL434		 * Bit 17: SRC_SEL435		 * Bit 16: DST_SEL436		 */437		if (llictl & FTDMAC020_LLI_TC_MSK)438			val |= FTDMAC020_CH_CSR_TC_MSK;439		val |= ((llictl  & FTDMAC020_LLI_SRC_WIDTH_MSK) >>440			(FTDMAC020_LLI_SRC_WIDTH_SHIFT -441			 FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT));442		val |= ((llictl  & FTDMAC020_LLI_DST_WIDTH_MSK) >>443			(FTDMAC020_LLI_DST_WIDTH_SHIFT -444			 FTDMAC020_CH_CSR_DST_WIDTH_SHIFT));445		val |= ((llictl  & FTDMAC020_LLI_SRCAD_CTL_MSK) >>446			(FTDMAC020_LLI_SRCAD_CTL_SHIFT -447			 FTDMAC020_CH_CSR_SRCAD_CTL_SHIFT));448		val |= ((llictl  & FTDMAC020_LLI_DSTAD_CTL_MSK) >>449			(FTDMAC020_LLI_DSTAD_CTL_SHIFT -450			 FTDMAC020_CH_CSR_DSTAD_CTL_SHIFT));451		if (llictl & FTDMAC020_LLI_SRC_SEL)452			val |= FTDMAC020_CH_CSR_SRC_SEL;453		if (llictl & FTDMAC020_LLI_DST_SEL)454			val |= FTDMAC020_CH_CSR_DST_SEL;455 456		/*457		 * Set up the bits that exist in the CSR but are not458		 * part the LLI, i.e. only gets written to the control459		 * register right here.460		 *461		 * FIXME: do not just handle memcpy, also handle slave DMA.462		 */463		switch (pl08x->pd->memcpy_burst_size) {464		default:465		case PL08X_BURST_SZ_1:466			val |= PL080_BSIZE_1 <<467				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;468			break;469		case PL08X_BURST_SZ_4:470			val |= PL080_BSIZE_4 <<471				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;472			break;473		case PL08X_BURST_SZ_8:474			val |= PL080_BSIZE_8 <<475				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;476			break;477		case PL08X_BURST_SZ_16:478			val |= PL080_BSIZE_16 <<479				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;480			break;481		case PL08X_BURST_SZ_32:482			val |= PL080_BSIZE_32 <<483				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;484			break;485		case PL08X_BURST_SZ_64:486			val |= PL080_BSIZE_64 <<487				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;488			break;489		case PL08X_BURST_SZ_128:490			val |= PL080_BSIZE_128 <<491				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;492			break;493		case PL08X_BURST_SZ_256:494			val |= PL080_BSIZE_256 <<495				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;496			break;497		}498 499		/* Protection flags */500		if (pl08x->pd->memcpy_prot_buff)501			val |= FTDMAC020_CH_CSR_PROT2;502		if (pl08x->pd->memcpy_prot_cache)503			val |= FTDMAC020_CH_CSR_PROT3;504		/* We are the kernel, so we are in privileged mode */505		val |= FTDMAC020_CH_CSR_PROT1;506 507		writel_relaxed(val, phychan->reg_control);508	} else {509		/* Bits are just identical */510		writel_relaxed(lli[PL080_LLI_CCTL], phychan->reg_control);511	}512 513	/* Second control word on the PL080s */514	if (pl08x->vd->pl080s)515		writel_relaxed(lli[PL080S_LLI_CCTL2],516				phychan->base + PL080S_CH_CONTROL2);517 518	writel(ccfg, phychan->reg_config);519}520 521/*522 * Set the initial DMA register values i.e. those for the first LLI523 * The next LLI pointer and the configuration interrupt bit have524 * been set when the LLIs were constructed.  Poke them into the hardware525 * and start the transfer.526 */527static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan)528{529	struct pl08x_driver_data *pl08x = plchan->host;530	struct pl08x_phy_chan *phychan = plchan->phychan;531	struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc);532	struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);533	u32 val;534 535	list_del(&txd->vd.node);536 537	plchan->at = txd;538 539	/* Wait for channel inactive */540	while (pl08x_phy_channel_busy(phychan))541		cpu_relax();542 543	pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg);544 545	/* Enable the DMA channel */546	/* Do not access config register until channel shows as disabled */547	while (readl(pl08x->base + PL080_EN_CHAN) & BIT(phychan->id))548		cpu_relax();549 550	/* Do not access config register until channel shows as inactive */551	if (phychan->ftdmac020) {552		val = readl(phychan->reg_config);553		while (val & FTDMAC020_CH_CFG_BUSY)554			val = readl(phychan->reg_config);555 556		val = readl(phychan->reg_control);557		while (val & FTDMAC020_CH_CSR_EN)558			val = readl(phychan->reg_control);559 560		writel(val | FTDMAC020_CH_CSR_EN,561		       phychan->reg_control);562	} else {563		val = readl(phychan->reg_config);564		while ((val & PL080_CONFIG_ACTIVE) ||565		       (val & PL080_CONFIG_ENABLE))566			val = readl(phychan->reg_config);567 568		writel(val | PL080_CONFIG_ENABLE, phychan->reg_config);569	}570}571 572/*573 * Pause the channel by setting the HALT bit.574 *575 * For M->P transfers, pause the DMAC first and then stop the peripheral -576 * the FIFO can only drain if the peripheral is still requesting data.577 * (note: this can still timeout if the DMAC FIFO never drains of data.)578 *579 * For P->M transfers, disable the peripheral first to stop it filling580 * the DMAC FIFO, and then pause the DMAC.581 */582static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch)583{584	u32 val;585	int timeout;586 587	if (ch->ftdmac020) {588		/* Use the enable bit on the FTDMAC020 */589		val = readl(ch->reg_control);590		val &= ~FTDMAC020_CH_CSR_EN;591		writel(val, ch->reg_control);592		return;593	}594 595	/* Set the HALT bit and wait for the FIFO to drain */596	val = readl(ch->reg_config);597	val |= PL080_CONFIG_HALT;598	writel(val, ch->reg_config);599 600	/* Wait for channel inactive */601	for (timeout = 1000; timeout; timeout--) {602		if (!pl08x_phy_channel_busy(ch))603			break;604		udelay(1);605	}606	if (pl08x_phy_channel_busy(ch))607		pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id);608}609 610static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch)611{612	u32 val;613 614	/* Use the enable bit on the FTDMAC020 */615	if (ch->ftdmac020) {616		val = readl(ch->reg_control);617		val |= FTDMAC020_CH_CSR_EN;618		writel(val, ch->reg_control);619		return;620	}621 622	/* Clear the HALT bit */623	val = readl(ch->reg_config);624	val &= ~PL080_CONFIG_HALT;625	writel(val, ch->reg_config);626}627 628/*629 * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and630 * clears any pending interrupt status.  This should not be used for631 * an on-going transfer, but as a method of shutting down a channel632 * (eg, when it's no longer used) or terminating a transfer.633 */634static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x,635	struct pl08x_phy_chan *ch)636{637	u32 val;638 639	/* The layout for the FTDMAC020 is different */640	if (ch->ftdmac020) {641		/* Disable all interrupts */642		val = readl(ch->reg_config);643		val |= (FTDMAC020_CH_CFG_INT_ABT_MASK |644			FTDMAC020_CH_CFG_INT_ERR_MASK |645			FTDMAC020_CH_CFG_INT_TC_MASK);646		writel(val, ch->reg_config);647 648		/* Abort and disable channel */649		val = readl(ch->reg_control);650		val &= ~FTDMAC020_CH_CSR_EN;651		val |= FTDMAC020_CH_CSR_ABT;652		writel(val, ch->reg_control);653 654		/* Clear ABT and ERR interrupt flags */655		writel(BIT(ch->id) | BIT(ch->id + 16),656		       pl08x->base + PL080_ERR_CLEAR);657		writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR);658 659		return;660	}661 662	val = readl(ch->reg_config);663	val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK |664		 PL080_CONFIG_TC_IRQ_MASK);665	writel(val, ch->reg_config);666 667	writel(BIT(ch->id), pl08x->base + PL080_ERR_CLEAR);668	writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR);669}670 671static u32 get_bytes_in_phy_channel(struct pl08x_phy_chan *ch)672{673	u32 val;674	u32 bytes;675 676	if (ch->ftdmac020) {677		bytes = readl(ch->base + FTDMAC020_CH_SIZE);678 679		val = readl(ch->reg_control);680		val &= FTDMAC020_CH_CSR_SRC_WIDTH_MSK;681		val >>= FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT;682	} else if (ch->pl080s) {683		val = readl(ch->base + PL080S_CH_CONTROL2);684		bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK;685 686		val = readl(ch->reg_control);687		val &= PL080_CONTROL_SWIDTH_MASK;688		val >>= PL080_CONTROL_SWIDTH_SHIFT;689	} else {690		/* Plain PL08x */691		val = readl(ch->reg_control);692		bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK;693 694		val &= PL080_CONTROL_SWIDTH_MASK;695		val >>= PL080_CONTROL_SWIDTH_SHIFT;696	}697 698	switch (val) {699	case PL080_WIDTH_8BIT:700		break;701	case PL080_WIDTH_16BIT:702		bytes *= 2;703		break;704	case PL080_WIDTH_32BIT:705		bytes *= 4;706		break;707	}708	return bytes;709}710 711static u32 get_bytes_in_lli(struct pl08x_phy_chan *ch, const u32 *llis_va)712{713	u32 val;714	u32 bytes;715 716	if (ch->ftdmac020) {717		val = llis_va[PL080_LLI_CCTL];718		bytes = val & FTDMAC020_LLI_TRANSFER_SIZE_MASK;719 720		val = llis_va[PL080_LLI_CCTL];721		val &= FTDMAC020_LLI_SRC_WIDTH_MSK;722		val >>= FTDMAC020_LLI_SRC_WIDTH_SHIFT;723	} else if (ch->pl080s) {724		val = llis_va[PL080S_LLI_CCTL2];725		bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK;726 727		val = llis_va[PL080_LLI_CCTL];728		val &= PL080_CONTROL_SWIDTH_MASK;729		val >>= PL080_CONTROL_SWIDTH_SHIFT;730	} else {731		/* Plain PL08x */732		val = llis_va[PL080_LLI_CCTL];733		bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK;734 735		val &= PL080_CONTROL_SWIDTH_MASK;736		val >>= PL080_CONTROL_SWIDTH_SHIFT;737	}738 739	switch (val) {740	case PL080_WIDTH_8BIT:741		break;742	case PL080_WIDTH_16BIT:743		bytes *= 2;744		break;745	case PL080_WIDTH_32BIT:746		bytes *= 4;747		break;748	}749	return bytes;750}751 752/* The channel should be paused when calling this */753static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan)754{755	struct pl08x_driver_data *pl08x = plchan->host;756	const u32 *llis_va, *llis_va_limit;757	struct pl08x_phy_chan *ch;758	dma_addr_t llis_bus;759	struct pl08x_txd *txd;760	u32 llis_max_words;761	size_t bytes;762	u32 clli;763 764	ch = plchan->phychan;765	txd = plchan->at;766 767	if (!ch || !txd)768		return 0;769 770	/*771	 * Follow the LLIs to get the number of remaining772	 * bytes in the currently active transaction.773	 */774	clli = readl(ch->reg_lli) & ~PL080_LLI_LM_AHB2;775 776	/* First get the remaining bytes in the active transfer */777	bytes = get_bytes_in_phy_channel(ch);778 779	if (!clli)780		return bytes;781 782	llis_va = txd->llis_va;783	llis_bus = txd->llis_bus;784 785	llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS;786	BUG_ON(clli < llis_bus || clli >= llis_bus +787						sizeof(u32) * llis_max_words);788 789	/*790	 * Locate the next LLI - as this is an array,791	 * it's simple maths to find.792	 */793	llis_va += (clli - llis_bus) / sizeof(u32);794 795	llis_va_limit = llis_va + llis_max_words;796 797	for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) {798		bytes += get_bytes_in_lli(ch, llis_va);799 800		/*801		 * A LLI pointer going backward terminates the LLI list802		 */803		if (llis_va[PL080_LLI_LLI] <= clli)804			break;805	}806 807	return bytes;808}809 810/*811 * Allocate a physical channel for a virtual channel812 *813 * Try to locate a physical channel to be used for this transfer. If all814 * are taken return NULL and the requester will have to cope by using815 * some fallback PIO mode or retrying later.816 */817static struct pl08x_phy_chan *818pl08x_get_phy_channel(struct pl08x_driver_data *pl08x,819		      struct pl08x_dma_chan *virt_chan)820{821	struct pl08x_phy_chan *ch = NULL;822	unsigned long flags;823	int i;824 825	for (i = 0; i < pl08x->vd->channels; i++) {826		ch = &pl08x->phy_chans[i];827 828		spin_lock_irqsave(&ch->lock, flags);829 830		if (!ch->locked && !ch->serving) {831			ch->serving = virt_chan;832			spin_unlock_irqrestore(&ch->lock, flags);833			break;834		}835 836		spin_unlock_irqrestore(&ch->lock, flags);837	}838 839	if (i == pl08x->vd->channels) {840		/* No physical channel available, cope with it */841		return NULL;842	}843 844	return ch;845}846 847/* Mark the physical channel as free.  Note, this write is atomic. */848static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x,849					 struct pl08x_phy_chan *ch)850{851	ch->serving = NULL;852}853 854/*855 * Try to allocate a physical channel.  When successful, assign it to856 * this virtual channel, and initiate the next descriptor.  The857 * virtual channel lock must be held at this point.858 */859static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan)860{861	struct pl08x_driver_data *pl08x = plchan->host;862	struct pl08x_phy_chan *ch;863 864	ch = pl08x_get_phy_channel(pl08x, plchan);865	if (!ch) {866		dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name);867		plchan->state = PL08X_CHAN_WAITING;868		plchan->waiting_at = jiffies;869		return;870	}871 872	dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n",873		ch->id, plchan->name);874 875	plchan->phychan = ch;876	plchan->state = PL08X_CHAN_RUNNING;877	pl08x_start_next_txd(plchan);878}879 880static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch,881	struct pl08x_dma_chan *plchan)882{883	struct pl08x_driver_data *pl08x = plchan->host;884 885	dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n",886		ch->id, plchan->name);887 888	/*889	 * We do this without taking the lock; we're really only concerned890	 * about whether this pointer is NULL or not, and we're guaranteed891	 * that this will only be called when it _already_ is non-NULL.892	 */893	ch->serving = plchan;894	plchan->phychan = ch;895	plchan->state = PL08X_CHAN_RUNNING;896	pl08x_start_next_txd(plchan);897}898 899/*900 * Free a physical DMA channel, potentially reallocating it to another901 * virtual channel if we have any pending.902 */903static void pl08x_phy_free(struct pl08x_dma_chan *plchan)904{905	struct pl08x_driver_data *pl08x = plchan->host;906	struct pl08x_dma_chan *p, *next;907	unsigned long waiting_at;908 retry:909	next = NULL;910	waiting_at = jiffies;911 912	/*913	 * Find a waiting virtual channel for the next transfer.914	 * To be fair, time when each channel reached waiting state is compared915	 * to select channel that is waiting for the longest time.916	 */917	list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node)918		if (p->state == PL08X_CHAN_WAITING &&919		    p->waiting_at <= waiting_at) {920			next = p;921			waiting_at = p->waiting_at;922		}923 924	if (!next && pl08x->has_slave) {925		list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node)926			if (p->state == PL08X_CHAN_WAITING &&927			    p->waiting_at <= waiting_at) {928				next = p;929				waiting_at = p->waiting_at;930			}931	}932 933	/* Ensure that the physical channel is stopped */934	pl08x_terminate_phy_chan(pl08x, plchan->phychan);935 936	if (next) {937		bool success;938 939		/*940		 * Eww.  We know this isn't going to deadlock941		 * but lockdep probably doesn't.942		 */943		spin_lock(&next->vc.lock);944		/* Re-check the state now that we have the lock */945		success = next->state == PL08X_CHAN_WAITING;946		if (success)947			pl08x_phy_reassign_start(plchan->phychan, next);948		spin_unlock(&next->vc.lock);949 950		/* If the state changed, try to find another channel */951		if (!success)952			goto retry;953	} else {954		/* No more jobs, so free up the physical channel */955		pl08x_put_phy_channel(pl08x, plchan->phychan);956	}957 958	plchan->phychan = NULL;959	plchan->state = PL08X_CHAN_IDLE;960}961 962/*963 * LLI handling964 */965 966static inline unsigned int967pl08x_get_bytes_for_lli(struct pl08x_driver_data *pl08x,968			u32 cctl,969			bool source)970{971	u32 val;972 973	if (pl08x->vd->ftdmac020) {974		if (source)975			val = (cctl & FTDMAC020_LLI_SRC_WIDTH_MSK) >>976				FTDMAC020_LLI_SRC_WIDTH_SHIFT;977		else978			val = (cctl & FTDMAC020_LLI_DST_WIDTH_MSK) >>979				FTDMAC020_LLI_DST_WIDTH_SHIFT;980	} else {981		if (source)982			val = (cctl & PL080_CONTROL_SWIDTH_MASK) >>983				PL080_CONTROL_SWIDTH_SHIFT;984		else985			val = (cctl & PL080_CONTROL_DWIDTH_MASK) >>986				PL080_CONTROL_DWIDTH_SHIFT;987	}988 989	switch (val) {990	case PL080_WIDTH_8BIT:991		return 1;992	case PL080_WIDTH_16BIT:993		return 2;994	case PL080_WIDTH_32BIT:995		return 4;996	default:997		break;998	}999	BUG();1000	return 0;1001}1002 1003static inline u32 pl08x_lli_control_bits(struct pl08x_driver_data *pl08x,1004					 u32 cctl,1005					 u8 srcwidth, u8 dstwidth,1006					 size_t tsize)1007{1008	u32 retbits = cctl;1009 1010	/*1011	 * Remove all src, dst and transfer size bits, then set the1012	 * width and size according to the parameters. The bit offsets1013	 * are different in the FTDMAC020 so we need to accound for this.1014	 */1015	if (pl08x->vd->ftdmac020) {1016		retbits &= ~FTDMAC020_LLI_DST_WIDTH_MSK;1017		retbits &= ~FTDMAC020_LLI_SRC_WIDTH_MSK;1018		retbits &= ~FTDMAC020_LLI_TRANSFER_SIZE_MASK;1019 1020		switch (srcwidth) {1021		case 1:1022			retbits |= PL080_WIDTH_8BIT <<1023				FTDMAC020_LLI_SRC_WIDTH_SHIFT;1024			break;1025		case 2:1026			retbits |= PL080_WIDTH_16BIT <<1027				FTDMAC020_LLI_SRC_WIDTH_SHIFT;1028			break;1029		case 4:1030			retbits |= PL080_WIDTH_32BIT <<1031				FTDMAC020_LLI_SRC_WIDTH_SHIFT;1032			break;1033		default:1034			BUG();1035			break;1036		}1037 1038		switch (dstwidth) {1039		case 1:1040			retbits |= PL080_WIDTH_8BIT <<1041				FTDMAC020_LLI_DST_WIDTH_SHIFT;1042			break;1043		case 2:1044			retbits |= PL080_WIDTH_16BIT <<1045				FTDMAC020_LLI_DST_WIDTH_SHIFT;1046			break;1047		case 4:1048			retbits |= PL080_WIDTH_32BIT <<1049				FTDMAC020_LLI_DST_WIDTH_SHIFT;1050			break;1051		default:1052			BUG();1053			break;1054		}1055 1056		tsize &= FTDMAC020_LLI_TRANSFER_SIZE_MASK;1057		retbits |= tsize << FTDMAC020_LLI_TRANSFER_SIZE_SHIFT;1058	} else {1059		retbits &= ~PL080_CONTROL_DWIDTH_MASK;1060		retbits &= ~PL080_CONTROL_SWIDTH_MASK;1061		retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK;1062 1063		switch (srcwidth) {1064		case 1:1065			retbits |= PL080_WIDTH_8BIT <<1066				PL080_CONTROL_SWIDTH_SHIFT;1067			break;1068		case 2:1069			retbits |= PL080_WIDTH_16BIT <<1070				PL080_CONTROL_SWIDTH_SHIFT;1071			break;1072		case 4:1073			retbits |= PL080_WIDTH_32BIT <<1074				PL080_CONTROL_SWIDTH_SHIFT;1075			break;1076		default:1077			BUG();1078			break;1079		}1080 1081		switch (dstwidth) {1082		case 1:1083			retbits |= PL080_WIDTH_8BIT <<1084				PL080_CONTROL_DWIDTH_SHIFT;1085			break;1086		case 2:1087			retbits |= PL080_WIDTH_16BIT <<1088				PL080_CONTROL_DWIDTH_SHIFT;1089			break;1090		case 4:1091			retbits |= PL080_WIDTH_32BIT <<1092				PL080_CONTROL_DWIDTH_SHIFT;1093			break;1094		default:1095			BUG();1096			break;1097		}1098 1099		tsize &= PL080_CONTROL_TRANSFER_SIZE_MASK;1100		retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT;1101	}1102 1103	return retbits;1104}1105 1106struct pl08x_lli_build_data {1107	struct pl08x_txd *txd;1108	struct pl08x_bus_data srcbus;1109	struct pl08x_bus_data dstbus;1110	size_t remainder;1111	u32 lli_bus;1112};1113 1114/*1115 * Autoselect a master bus to use for the transfer. Slave will be the chosen as1116 * victim in case src & dest are not similarly aligned. i.e. If after aligning1117 * masters address with width requirements of transfer (by sending few byte by1118 * byte data), slave is still not aligned, then its width will be reduced to1119 * BYTE.1120 * - prefers the destination bus if both available1121 * - prefers bus with fixed address (i.e. peripheral)1122 */1123static void pl08x_choose_master_bus(struct pl08x_driver_data *pl08x,1124				    struct pl08x_lli_build_data *bd,1125				    struct pl08x_bus_data **mbus,1126				    struct pl08x_bus_data **sbus,1127				    u32 cctl)1128{1129	bool dst_incr;1130	bool src_incr;1131 1132	/*1133	 * The FTDMAC020 only supports memory-to-memory transfer, so1134	 * source and destination always increase.1135	 */1136	if (pl08x->vd->ftdmac020) {1137		dst_incr = true;1138		src_incr = true;1139	} else {1140		dst_incr = !!(cctl & PL080_CONTROL_DST_INCR);1141		src_incr = !!(cctl & PL080_CONTROL_SRC_INCR);1142	}1143 1144	/*1145	 * If either bus is not advancing, i.e. it is a peripheral, that1146	 * one becomes master1147	 */1148	if (!dst_incr) {1149		*mbus = &bd->dstbus;1150		*sbus = &bd->srcbus;1151	} else if (!src_incr) {1152		*mbus = &bd->srcbus;1153		*sbus = &bd->dstbus;1154	} else {1155		if (bd->dstbus.buswidth >= bd->srcbus.buswidth) {1156			*mbus = &bd->dstbus;1157			*sbus = &bd->srcbus;1158		} else {1159			*mbus = &bd->srcbus;1160			*sbus = &bd->dstbus;1161		}1162	}1163}1164 1165/*1166 * Fills in one LLI for a certain transfer descriptor and advance the counter1167 */1168static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x,1169				    struct pl08x_lli_build_data *bd,1170				    int num_llis, int len, u32 cctl, u32 cctl2)1171{1172	u32 offset = num_llis * pl08x->lli_words;1173	u32 *llis_va = bd->txd->llis_va + offset;1174	dma_addr_t llis_bus = bd->txd->llis_bus;1175 1176	BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS);1177 1178	/* Advance the offset to next LLI. */1179	offset += pl08x->lli_words;1180 1181	llis_va[PL080_LLI_SRC] = bd->srcbus.addr;1182	llis_va[PL080_LLI_DST] = bd->dstbus.addr;1183	llis_va[PL080_LLI_LLI] = (llis_bus + sizeof(u32) * offset);1184	llis_va[PL080_LLI_LLI] |= bd->lli_bus;1185	llis_va[PL080_LLI_CCTL] = cctl;1186	if (pl08x->vd->pl080s)1187		llis_va[PL080S_LLI_CCTL2] = cctl2;1188 1189	if (pl08x->vd->ftdmac020) {1190		/* FIXME: only memcpy so far so both increase */1191		bd->srcbus.addr += len;1192		bd->dstbus.addr += len;1193	} else {1194		if (cctl & PL080_CONTROL_SRC_INCR)1195			bd->srcbus.addr += len;1196		if (cctl & PL080_CONTROL_DST_INCR)1197			bd->dstbus.addr += len;1198	}1199 1200	BUG_ON(bd->remainder < len);1201 1202	bd->remainder -= len;1203}1204 1205static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x,1206			struct pl08x_lli_build_data *bd, u32 *cctl, u32 len,1207			int num_llis, size_t *total_bytes)1208{1209	*cctl = pl08x_lli_control_bits(pl08x, *cctl, 1, 1, len);1210	pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len);1211	(*total_bytes) += len;1212}1213 1214#if 11215static void pl08x_dump_lli(struct pl08x_driver_data *pl08x,1216			   const u32 *llis_va, int num_llis)1217{1218	int i;1219 1220	if (pl08x->vd->pl080s) {1221		dev_vdbg(&pl08x->adev->dev,1222			"%-3s %-9s  %-10s %-10s %-10s %-10s %s\n",1223			"lli", "", "csrc", "cdst", "clli", "cctl", "cctl2");1224		for (i = 0; i < num_llis; i++) {1225			dev_vdbg(&pl08x->adev->dev,1226				"%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",1227				i, llis_va, llis_va[PL080_LLI_SRC],1228				llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],1229				llis_va[PL080_LLI_CCTL],1230				llis_va[PL080S_LLI_CCTL2]);1231			llis_va += pl08x->lli_words;1232		}1233	} else {1234		dev_vdbg(&pl08x->adev->dev,1235			"%-3s %-9s  %-10s %-10s %-10s %s\n",1236			"lli", "", "csrc", "cdst", "clli", "cctl");1237		for (i = 0; i < num_llis; i++) {1238			dev_vdbg(&pl08x->adev->dev,1239				"%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n",1240				i, llis_va, llis_va[PL080_LLI_SRC],1241				llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],1242				llis_va[PL080_LLI_CCTL]);1243			llis_va += pl08x->lli_words;1244		}1245	}1246}1247#else1248static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x,1249				  const u32 *llis_va, int num_llis) {}1250#endif1251 1252/*1253 * This fills in the table of LLIs for the transfer descriptor1254 * Note that we assume we never have to change the burst sizes1255 * Return 0 for error1256 */1257static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,1258			      struct pl08x_txd *txd)1259{1260	struct pl08x_bus_data *mbus, *sbus;1261	struct pl08x_lli_build_data bd;1262	int num_llis = 0;1263	u32 cctl, early_bytes = 0;1264	size_t max_bytes_per_lli, total_bytes;1265	u32 *llis_va, *last_lli;1266	struct pl08x_sg *dsg;1267 1268	txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus);1269	if (!txd->llis_va) {1270		dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__);1271		return 0;1272	}1273 1274	bd.txd = txd;1275	bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0;1276	cctl = txd->cctl;1277 1278	/* Find maximum width of the source bus */1279	bd.srcbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, true);1280 1281	/* Find maximum width of the destination bus */1282	bd.dstbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, false);1283 1284	list_for_each_entry(dsg, &txd->dsg_list, node) {1285		total_bytes = 0;1286		cctl = txd->cctl;1287 1288		bd.srcbus.addr = dsg->src_addr;1289		bd.dstbus.addr = dsg->dst_addr;1290		bd.remainder = dsg->len;1291		bd.srcbus.buswidth = bd.srcbus.maxwidth;1292		bd.dstbus.buswidth = bd.dstbus.maxwidth;1293 1294		pl08x_choose_master_bus(pl08x, &bd, &mbus, &sbus, cctl);1295 1296		dev_vdbg(&pl08x->adev->dev,1297			"src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n",1298			(u64)bd.srcbus.addr,1299			cctl & PL080_CONTROL_SRC_INCR ? "+" : "",1300			bd.srcbus.buswidth,1301			(u64)bd.dstbus.addr,1302			cctl & PL080_CONTROL_DST_INCR ? "+" : "",1303			bd.dstbus.buswidth,1304			bd.remainder);1305		dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n",1306			mbus == &bd.srcbus ? "src" : "dst",1307			sbus == &bd.srcbus ? "src" : "dst");1308 1309		/*1310		 * Zero length is only allowed if all these requirements are1311		 * met:1312		 * - flow controller is peripheral.1313		 * - src.addr is aligned to src.width1314		 * - dst.addr is aligned to dst.width1315		 *1316		 * sg_len == 1 should be true, as there can be two cases here:1317		 *1318		 * - Memory addresses are contiguous and are not scattered.1319		 *   Here, Only one sg will be passed by user driver, with1320		 *   memory address and zero length. We pass this to controller1321		 *   and after the transfer it will receive the last burst1322		 *   request from peripheral and so transfer finishes.1323		 *1324		 * - Memory addresses are scattered and are not contiguous.1325		 *   Here, Obviously as DMA controller doesn't know when a lli's1326		 *   transfer gets over, it can't load next lli. So in this1327		 *   case, there has to be an assumption that only one lli is1328		 *   supported. Thus, we can't have scattered addresses.1329		 */1330		if (!bd.remainder) {1331			u32 fc;1332 1333			/* FTDMAC020 only does memory-to-memory */1334			if (pl08x->vd->ftdmac020)1335				fc = PL080_FLOW_MEM2MEM;1336			else1337				fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >>1338					PL080_CONFIG_FLOW_CONTROL_SHIFT;1339			if (!((fc >= PL080_FLOW_SRC2DST_DST) &&1340					(fc <= PL080_FLOW_SRC2DST_SRC))) {1341				dev_err(&pl08x->adev->dev, "%s sg len can't be zero",1342					__func__);1343				return 0;1344			}1345 1346			if (!IS_BUS_ALIGNED(&bd.srcbus) ||1347				!IS_BUS_ALIGNED(&bd.dstbus)) {1348				dev_err(&pl08x->adev->dev,1349					"%s src & dst address must be aligned to src"1350					" & dst width if peripheral is flow controller",1351					__func__);1352				return 0;1353			}1354 1355			cctl = pl08x_lli_control_bits(pl08x, cctl,1356					bd.srcbus.buswidth, bd.dstbus.buswidth,1357					0);1358			pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,1359					0, cctl, 0);1360			break;1361		}1362 1363		/*1364		 * Send byte by byte for following cases1365		 * - Less than a bus width available1366		 * - until master bus is aligned1367		 */1368		if (bd.remainder < mbus->buswidth)1369			early_bytes = bd.remainder;1370		else if (!IS_BUS_ALIGNED(mbus)) {1371			early_bytes = mbus->buswidth -1372				(mbus->addr & (mbus->buswidth - 1));1373			if ((bd.remainder - early_bytes) < mbus->buswidth)1374				early_bytes = bd.remainder;1375		}1376 1377		if (early_bytes) {1378			dev_vdbg(&pl08x->adev->dev,1379				"%s byte width LLIs (remain 0x%08zx)\n",1380				__func__, bd.remainder);1381			prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes,1382				num_llis++, &total_bytes);1383		}1384 1385		if (bd.remainder) {1386			/*1387			 * Master now aligned1388			 * - if slave is not then we must set its width down1389			 */1390			if (!IS_BUS_ALIGNED(sbus)) {1391				dev_dbg(&pl08x->adev->dev,1392					"%s set down bus width to one byte\n",1393					__func__);1394 1395				sbus->buswidth = 1;1396			}1397 1398			/*1399			 * Bytes transferred = tsize * src width, not1400			 * MIN(buswidths)1401			 */1402			max_bytes_per_lli = bd.srcbus.buswidth *1403						pl08x->vd->max_transfer_size;1404			dev_vdbg(&pl08x->adev->dev,1405				"%s max bytes per lli = %zu\n",1406				__func__, max_bytes_per_lli);1407 1408			/*1409			 * Make largest possible LLIs until less than one bus1410			 * width left1411			 */1412			while (bd.remainder > (mbus->buswidth - 1)) {1413				size_t lli_len, tsize, width;1414 1415				/*1416				 * If enough left try to send max possible,1417				 * otherwise try to send the remainder1418				 */1419				lli_len = min(bd.remainder, max_bytes_per_lli);1420 1421				/*1422				 * Check against maximum bus alignment:1423				 * Calculate actual transfer size in relation to1424				 * bus width an get a maximum remainder of the1425				 * highest bus width - 11426				 */1427				width = max(mbus->buswidth, sbus->buswidth);1428				lli_len = (lli_len / width) * width;1429				tsize = lli_len / bd.srcbus.buswidth;1430 1431				dev_vdbg(&pl08x->adev->dev,1432					"%s fill lli with single lli chunk of "1433					"size 0x%08zx (remainder 0x%08zx)\n",1434					__func__, lli_len, bd.remainder);1435 1436				cctl = pl08x_lli_control_bits(pl08x, cctl,1437					bd.srcbus.buswidth, bd.dstbus.buswidth,1438					tsize);1439				pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,1440						lli_len, cctl, tsize);1441				total_bytes += lli_len;1442			}1443 1444			/*1445			 * Send any odd bytes1446			 */1447			if (bd.remainder) {1448				dev_vdbg(&pl08x->adev->dev,1449					"%s align with boundary, send odd bytes (remain %zu)\n",1450					__func__, bd.remainder);1451				prep_byte_width_lli(pl08x, &bd, &cctl,1452					bd.remainder, num_llis++, &total_bytes);1453			}1454		}1455 1456		if (total_bytes != dsg->len) {1457			dev_err(&pl08x->adev->dev,1458				"%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n",1459				__func__, total_bytes, dsg->len);1460			return 0;1461		}1462 1463		if (num_llis >= MAX_NUM_TSFR_LLIS) {1464			dev_err(&pl08x->adev->dev,1465				"%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n",1466				__func__, MAX_NUM_TSFR_LLIS);1467			return 0;1468		}1469	}1470 1471	llis_va = txd->llis_va;1472	last_lli = llis_va + (num_llis - 1) * pl08x->lli_words;1473 1474	if (txd->cyclic) {1475		/* Link back to the first LLI. */1476		last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus;1477	} else {1478		/* The final LLI terminates the LLI. */1479		last_lli[PL080_LLI_LLI] = 0;1480		/* The final LLI element shall also fire an interrupt. */1481		if (pl08x->vd->ftdmac020)1482			last_lli[PL080_LLI_CCTL] &= ~FTDMAC020_LLI_TC_MSK;1483		else1484			last_lli[PL080_LLI_CCTL] |= PL080_CONTROL_TC_IRQ_EN;1485	}1486 1487	pl08x_dump_lli(pl08x, llis_va, num_llis);1488 1489	return num_llis;1490}1491 1492static void pl08x_free_txd(struct pl08x_driver_data *pl08x,1493			   struct pl08x_txd *txd)1494{1495	struct pl08x_sg *dsg, *_dsg;1496 1497	if (txd->llis_va)1498		dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus);1499 1500	list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {1501		list_del(&dsg->node);1502		kfree(dsg);1503	}1504 1505	kfree(txd);1506}1507 1508static void pl08x_desc_free(struct virt_dma_desc *vd)1509{1510	struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);1511	struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan);1512 1513	dma_descriptor_unmap(&vd->tx);1514	if (!txd->done)1515		pl08x_release_mux(plchan);1516 1517	pl08x_free_txd(plchan->host, txd);1518}1519 1520static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x,1521				struct pl08x_dma_chan *plchan)1522{1523	LIST_HEAD(head);1524 1525	vchan_get_all_descriptors(&plchan->vc, &head);1526	vchan_dma_desc_free_list(&plchan->vc, &head);1527}1528 1529/*1530 * The DMA ENGINE API1531 */1532static void pl08x_free_chan_resources(struct dma_chan *chan)1533{1534	/* Ensure all queued descriptors are freed */1535	vchan_free_chan_resources(to_virt_chan(chan));1536}1537 1538/*1539 * Code accessing dma_async_is_complete() in a tight loop may give problems.1540 * If slaves are relying on interrupts to signal completion this function1541 * must not be called with interrupts disabled.1542 */1543static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan,1544		dma_cookie_t cookie, struct dma_tx_state *txstate)1545{1546	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);1547	struct virt_dma_desc *vd;1548	unsigned long flags;1549	enum dma_status ret;1550	size_t bytes = 0;1551 1552	ret = dma_cookie_status(chan, cookie, txstate);1553	if (ret == DMA_COMPLETE)1554		return ret;1555 1556	/*1557	 * There's no point calculating the residue if there's1558	 * no txstate to store the value.1559	 */1560	if (!txstate) {1561		if (plchan->state == PL08X_CHAN_PAUSED)1562			ret = DMA_PAUSED;1563		return ret;1564	}1565 1566	spin_lock_irqsave(&plchan->vc.lock, flags);1567	ret = dma_cookie_status(chan, cookie, txstate);1568	if (ret != DMA_COMPLETE) {1569		vd = vchan_find_desc(&plchan->vc, cookie);1570		if (vd) {1571			/* On the issued list, so hasn't been processed yet */1572			struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);1573			struct pl08x_sg *dsg;1574 1575			list_for_each_entry(dsg, &txd->dsg_list, node)1576				bytes += dsg->len;1577		} else {1578			bytes = pl08x_getbytes_chan(plchan);1579		}1580	}1581	spin_unlock_irqrestore(&plchan->vc.lock, flags);1582 1583	/*1584	 * This cookie not complete yet1585	 * Get number of bytes left in the active transactions and queue1586	 */1587	dma_set_residue(txstate, bytes);1588 1589	if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS)1590		ret = DMA_PAUSED;1591 1592	/* Whether waiting or running, we're in progress */1593	return ret;1594}1595 1596/* PrimeCell DMA extension */1597struct burst_table {1598	u32 burstwords;1599	u32 reg;1600};1601 1602static const struct burst_table burst_sizes[] = {1603	{1604		.burstwords = 256,1605		.reg = PL080_BSIZE_256,1606	},1607	{1608		.burstwords = 128,1609		.reg = PL080_BSIZE_128,1610	},1611	{1612		.burstwords = 64,1613		.reg = PL080_BSIZE_64,1614	},1615	{1616		.burstwords = 32,1617		.reg = PL080_BSIZE_32,1618	},1619	{1620		.burstwords = 16,1621		.reg = PL080_BSIZE_16,1622	},1623	{1624		.burstwords = 8,1625		.reg = PL080_BSIZE_8,1626	},1627	{1628		.burstwords = 4,1629		.reg = PL080_BSIZE_4,1630	},1631	{1632		.burstwords = 0,1633		.reg = PL080_BSIZE_1,1634	},1635};1636 1637/*1638 * Given the source and destination available bus masks, select which1639 * will be routed to each port.  We try to have source and destination1640 * on separate ports, but always respect the allowable settings.1641 */1642static u32 pl08x_select_bus(bool ftdmac020, u8 src, u8 dst)1643{1644	u32 cctl = 0;1645	u32 dst_ahb2;1646	u32 src_ahb2;1647 1648	/* The FTDMAC020 use different bits to indicate src/dst bus */1649	if (ftdmac020) {1650		dst_ahb2 = FTDMAC020_LLI_DST_SEL;1651		src_ahb2 = FTDMAC020_LLI_SRC_SEL;1652	} else {1653		dst_ahb2 = PL080_CONTROL_DST_AHB2;1654		src_ahb2 = PL080_CONTROL_SRC_AHB2;1655	}1656 1657	if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1)))1658		cctl |= dst_ahb2;1659	if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2)))1660		cctl |= src_ahb2;1661 1662	return cctl;1663}1664 1665static u32 pl08x_cctl(u32 cctl)1666{1667	cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 |1668		  PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR |1669		  PL080_CONTROL_PROT_MASK);1670 1671	/* Access the cell in privileged mode, non-bufferable, non-cacheable */1672	return cctl | PL080_CONTROL_PROT_SYS;1673}1674 1675static u32 pl08x_width(enum dma_slave_buswidth width)1676{1677	switch (width) {1678	case DMA_SLAVE_BUSWIDTH_1_BYTE:1679		return PL080_WIDTH_8BIT;1680	case DMA_SLAVE_BUSWIDTH_2_BYTES:1681		return PL080_WIDTH_16BIT;1682	case DMA_SLAVE_BUSWIDTH_4_BYTES:1683		return PL080_WIDTH_32BIT;1684	default:1685		return ~0;1686	}1687}1688 1689static u32 pl08x_burst(u32 maxburst)1690{1691	int i;1692 1693	for (i = 0; i < ARRAY_SIZE(burst_sizes); i++)1694		if (burst_sizes[i].burstwords <= maxburst)1695			break;1696 1697	return burst_sizes[i].reg;1698}1699 1700static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan,1701	enum dma_slave_buswidth addr_width, u32 maxburst)1702{1703	u32 width, burst, cctl = 0;1704 1705	width = pl08x_width(addr_width);1706	if (width == ~0)1707		return ~0;1708 1709	cctl |= width << PL080_CONTROL_SWIDTH_SHIFT;1710	cctl |= width << PL080_CONTROL_DWIDTH_SHIFT;1711 1712	/*1713	 * If this channel will only request single transfers, set this1714	 * down to ONE element.  Also select one element if no maxburst1715	 * is specified.1716	 */1717	if (plchan->cd->single)1718		maxburst = 1;1719 1720	burst = pl08x_burst(maxburst);1721	cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT;1722	cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT;1723 1724	return pl08x_cctl(cctl);1725}1726 1727/*1728 * Slave transactions callback to the slave device to allow1729 * synchronization of slave DMA signals with the DMAC enable1730 */1731static void pl08x_issue_pending(struct dma_chan *chan)1732{1733	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);1734	unsigned long flags;1735 1736	spin_lock_irqsave(&plchan->vc.lock, flags);1737	if (vchan_issue_pending(&plchan->vc)) {1738		if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING)1739			pl08x_phy_alloc_and_start(plchan);1740	}1741	spin_unlock_irqrestore(&plchan->vc.lock, flags);1742}1743 1744static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan)1745{1746	struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);1747 1748	if (txd)1749		INIT_LIST_HEAD(&txd->dsg_list);1750	return txd;1751}1752 1753static u32 pl08x_memcpy_cctl(struct pl08x_driver_data *pl08x)1754{1755	u32 cctl = 0;1756 1757	/* Conjure cctl */1758	switch (pl08x->pd->memcpy_burst_size) {1759	default:1760		dev_err(&pl08x->adev->dev,1761			"illegal burst size for memcpy, set to 1\n");1762		fallthrough;1763	case PL08X_BURST_SZ_1:1764		cctl |= PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT |1765			PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT;1766		break;1767	case PL08X_BURST_SZ_4:1768		cctl |= PL080_BSIZE_4 << PL080_CONTROL_SB_SIZE_SHIFT |1769			PL080_BSIZE_4 << PL080_CONTROL_DB_SIZE_SHIFT;1770		break;1771	case PL08X_BURST_SZ_8:1772		cctl |= PL080_BSIZE_8 << PL080_CONTROL_SB_SIZE_SHIFT |1773			PL080_BSIZE_8 << PL080_CONTROL_DB_SIZE_SHIFT;1774		break;1775	case PL08X_BURST_SZ_16:1776		cctl |= PL080_BSIZE_16 << PL080_CONTROL_SB_SIZE_SHIFT |1777			PL080_BSIZE_16 << PL080_CONTROL_DB_SIZE_SHIFT;1778		break;1779	case PL08X_BURST_SZ_32:1780		cctl |= PL080_BSIZE_32 << PL080_CONTROL_SB_SIZE_SHIFT |1781			PL080_BSIZE_32 << PL080_CONTROL_DB_SIZE_SHIFT;1782		break;1783	case PL08X_BURST_SZ_64:1784		cctl |= PL080_BSIZE_64 << PL080_CONTROL_SB_SIZE_SHIFT |1785			PL080_BSIZE_64 << PL080_CONTROL_DB_SIZE_SHIFT;1786		break;1787	case PL08X_BURST_SZ_128:1788		cctl |= PL080_BSIZE_128 << PL080_CONTROL_SB_SIZE_SHIFT |1789			PL080_BSIZE_128 << PL080_CONTROL_DB_SIZE_SHIFT;1790		break;1791	case PL08X_BURST_SZ_256:1792		cctl |= PL080_BSIZE_256 << PL080_CONTROL_SB_SIZE_SHIFT |1793			PL080_BSIZE_256 << PL080_CONTROL_DB_SIZE_SHIFT;1794		break;1795	}1796 1797	switch (pl08x->pd->memcpy_bus_width) {1798	default:1799		dev_err(&pl08x->adev->dev,1800			"illegal bus width for memcpy, set to 8 bits\n");1801		fallthrough;1802	case PL08X_BUS_WIDTH_8_BITS:1803		cctl |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT |1804			PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT;1805		break;1806	case PL08X_BUS_WIDTH_16_BITS:1807		cctl |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT |1808			PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT;1809		break;1810	case PL08X_BUS_WIDTH_32_BITS:1811		cctl |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT |1812			PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT;1813		break;1814	}1815 1816	/* Protection flags */1817	if (pl08x->pd->memcpy_prot_buff)1818		cctl |= PL080_CONTROL_PROT_BUFF;1819	if (pl08x->pd->memcpy_prot_cache)1820		cctl |= PL080_CONTROL_PROT_CACHE;1821 1822	/* We are the kernel, so we are in privileged mode */1823	cctl |= PL080_CONTROL_PROT_SYS;1824 1825	/* Both to be incremented or the code will break */1826	cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR;1827 1828	if (pl08x->vd->dualmaster)1829		cctl |= pl08x_select_bus(false,1830					 pl08x->mem_buses,1831					 pl08x->mem_buses);1832 1833	return cctl;1834}1835 1836static u32 pl08x_ftdmac020_memcpy_cctl(struct pl08x_driver_data *pl08x)1837{1838	u32 cctl = 0;1839 1840	/* Conjure cctl */1841	switch (pl08x->pd->memcpy_bus_width) {1842	default:1843		dev_err(&pl08x->adev->dev,1844			"illegal bus width for memcpy, set to 8 bits\n");1845		fallthrough;1846	case PL08X_BUS_WIDTH_8_BITS:1847		cctl |= PL080_WIDTH_8BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |1848			PL080_WIDTH_8BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;1849		break;1850	case PL08X_BUS_WIDTH_16_BITS:1851		cctl |= PL080_WIDTH_16BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |1852			PL080_WIDTH_16BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;1853		break;1854	case PL08X_BUS_WIDTH_32_BITS:1855		cctl |= PL080_WIDTH_32BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |1856			PL080_WIDTH_32BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;1857		break;1858	}1859 1860	/*1861	 * By default mask the TC IRQ on all LLIs, it will be unmasked on1862	 * the last LLI item by other code.1863	 */1864	cctl |= FTDMAC020_LLI_TC_MSK;1865 1866	/*1867	 * Both to be incremented so leave bits FTDMAC020_LLI_SRCAD_CTL1868	 * and FTDMAC020_LLI_DSTAD_CTL as zero1869	 */1870	if (pl08x->vd->dualmaster)1871		cctl |= pl08x_select_bus(true,1872					 pl08x->mem_buses,1873					 pl08x->mem_buses);1874 1875	return cctl;1876}1877 1878/*1879 * Initialize a descriptor to be used by memcpy submit1880 */1881static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy(1882		struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,1883		size_t len, unsigned long flags)1884{1885	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);1886	struct pl08x_driver_data *pl08x = plchan->host;1887	struct pl08x_txd *txd;1888	struct pl08x_sg *dsg;1889	int ret;1890 1891	txd = pl08x_get_txd(plchan);1892	if (!txd) {1893		dev_err(&pl08x->adev->dev,1894			"%s no memory for descriptor\n", __func__);1895		return NULL;1896	}1897 1898	dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);1899	if (!dsg) {1900		pl08x_free_txd(pl08x, txd);1901		return NULL;1902	}1903	list_add_tail(&dsg->node, &txd->dsg_list);1904 1905	dsg->src_addr = src;1906	dsg->dst_addr = dest;1907	dsg->len = len;1908	if (pl08x->vd->ftdmac020) {1909		/* Writing CCFG zero ENABLES all interrupts */1910		txd->ccfg = 0;1911		txd->cctl = pl08x_ftdmac020_memcpy_cctl(pl08x);1912	} else {1913		txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |1914			PL080_CONFIG_TC_IRQ_MASK |1915			PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;1916		txd->cctl = pl08x_memcpy_cctl(pl08x);1917	}1918 1919	ret = pl08x_fill_llis_for_desc(plchan->host, txd);1920	if (!ret) {1921		pl08x_free_txd(pl08x, txd);1922		return NULL;1923	}1924 1925	return vchan_tx_prep(&plchan->vc, &txd->vd, flags);1926}1927 1928static struct pl08x_txd *pl08x_init_txd(1929		struct dma_chan *chan,1930		enum dma_transfer_direction direction,1931		dma_addr_t *slave_addr)1932{1933	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);1934	struct pl08x_driver_data *pl08x = plchan->host;1935	struct pl08x_txd *txd;1936	enum dma_slave_buswidth addr_width;1937	int ret, tmp;1938	u8 src_buses, dst_buses;1939	u32 maxburst, cctl;1940 1941	txd = pl08x_get_txd(plchan);1942	if (!txd) {1943		dev_err(&pl08x->adev->dev, "%s no txd\n", __func__);1944		return NULL;1945	}1946 1947	/*1948	 * Set up addresses, the PrimeCell configured address1949	 * will take precedence since this may configure the1950	 * channel target address dynamically at runtime.1951	 */1952	if (direction == DMA_MEM_TO_DEV) {1953		cctl = PL080_CONTROL_SRC_INCR;1954		*slave_addr = plchan->cfg.dst_addr;1955		addr_width = plchan->cfg.dst_addr_width;1956		maxburst = plchan->cfg.dst_maxburst;1957		src_buses = pl08x->mem_buses;1958		dst_buses = plchan->cd->periph_buses;1959	} else if (direction == DMA_DEV_TO_MEM) {1960		cctl = PL080_CONTROL_DST_INCR;1961		*slave_addr = plchan->cfg.src_addr;1962		addr_width = plchan->cfg.src_addr_width;1963		maxburst = plchan->cfg.src_maxburst;1964		src_buses = plchan->cd->periph_buses;1965		dst_buses = pl08x->mem_buses;1966	} else {1967		pl08x_free_txd(pl08x, txd);1968		dev_err(&pl08x->adev->dev,1969			"%s direction unsupported\n", __func__);1970		return NULL;1971	}1972 1973	cctl |= pl08x_get_cctl(plchan, addr_width, maxburst);1974	if (cctl == ~0) {1975		pl08x_free_txd(pl08x, txd);1976		dev_err(&pl08x->adev->dev,1977			"DMA slave configuration botched?\n");1978		return NULL;1979	}1980 1981	txd->cctl = cctl | pl08x_select_bus(false, src_buses, dst_buses);1982 1983	if (plchan->cfg.device_fc)1984		tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER :1985			PL080_FLOW_PER2MEM_PER;1986	else1987		tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER :1988			PL080_FLOW_PER2MEM;1989 1990	txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |1991		PL080_CONFIG_TC_IRQ_MASK |1992		tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT;1993 1994	ret = pl08x_request_mux(plchan);1995	if (ret < 0) {1996		pl08x_free_txd(pl08x, txd);1997		dev_dbg(&pl08x->adev->dev,1998			"unable to mux for transfer on %s due to platform restrictions\n",1999			plchan->name);2000		return NULL;2001	}2002 2003	dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n",2004		 plchan->signal, plchan->name);2005 2006	/* Assign the flow control signal to this channel */2007	if (direction == DMA_MEM_TO_DEV)2008		txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT;2009	else2010		txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT;2011 2012	return txd;2013}2014 2015static int pl08x_tx_add_sg(struct pl08x_txd *txd,2016			   enum dma_transfer_direction direction,2017			   dma_addr_t slave_addr,2018			   dma_addr_t buf_addr,2019			   unsigned int len)2020{2021	struct pl08x_sg *dsg;2022 2023	dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);2024	if (!dsg)2025		return -ENOMEM;2026 2027	list_add_tail(&dsg->node, &txd->dsg_list);2028 2029	dsg->len = len;2030	if (direction == DMA_MEM_TO_DEV) {2031		dsg->src_addr = buf_addr;2032		dsg->dst_addr = slave_addr;2033	} else {2034		dsg->src_addr = slave_addr;2035		dsg->dst_addr = buf_addr;2036	}2037 2038	return 0;2039}2040 2041static struct dma_async_tx_descriptor *pl08x_prep_slave_sg(2042		struct dma_chan *chan, struct scatterlist *sgl,2043		unsigned int sg_len, enum dma_transfer_direction direction,2044		unsigned long flags, void *context)2045{2046	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);2047	struct pl08x_driver_data *pl08x = plchan->host;2048	struct pl08x_txd *txd;2049	struct scatterlist *sg;2050	int ret, tmp;2051	dma_addr_t slave_addr;2052 2053	dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n",2054			__func__, sg_dma_len(sgl), plchan->name);2055 2056	txd = pl08x_init_txd(chan, direction, &slave_addr);2057	if (!txd)2058		return NULL;2059 2060	for_each_sg(sgl, sg, sg_len, tmp) {2061		ret = pl08x_tx_add_sg(txd, direction, slave_addr,2062				      sg_dma_address(sg),2063				      sg_dma_len(sg));2064		if (ret) {2065			pl08x_release_mux(plchan);2066			pl08x_free_txd(pl08x, txd);2067			dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n",2068					__func__);2069			return NULL;2070		}2071	}2072 2073	ret = pl08x_fill_llis_for_desc(plchan->host, txd);2074	if (!ret) {2075		pl08x_release_mux(plchan);2076		pl08x_free_txd(pl08x, txd);2077		return NULL;2078	}2079 2080	return vchan_tx_prep(&plchan->vc, &txd->vd, flags);2081}2082 2083static struct dma_async_tx_descriptor *pl08x_prep_dma_cyclic(2084		struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,2085		size_t period_len, enum dma_transfer_direction direction,2086		unsigned long flags)2087{2088	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);2089	struct pl08x_driver_data *pl08x = plchan->host;2090	struct pl08x_txd *txd;2091	int ret, tmp;2092	dma_addr_t slave_addr;2093 2094	dev_dbg(&pl08x->adev->dev,2095		"%s prepare cyclic transaction of %zd/%zd bytes %s %s\n",2096		__func__, period_len, buf_len,2097		direction == DMA_MEM_TO_DEV ? "to" : "from",2098		plchan->name);2099 2100	txd = pl08x_init_txd(chan, direction, &slave_addr);2101	if (!txd)2102		return NULL;2103 2104	txd->cyclic = true;2105	txd->cctl |= PL080_CONTROL_TC_IRQ_EN;2106	for (tmp = 0; tmp < buf_len; tmp += period_len) {2107		ret = pl08x_tx_add_sg(txd, direction, slave_addr,2108				      buf_addr + tmp, period_len);2109		if (ret) {2110			pl08x_release_mux(plchan);2111			pl08x_free_txd(pl08x, txd);2112			return NULL;2113		}2114	}2115 2116	ret = pl08x_fill_llis_for_desc(plchan->host, txd);2117	if (!ret) {2118		pl08x_release_mux(plchan);2119		pl08x_free_txd(pl08x, txd);2120		return NULL;2121	}2122 2123	return vchan_tx_prep(&plchan->vc, &txd->vd, flags);2124}2125 2126static int pl08x_config(struct dma_chan *chan,2127			struct dma_slave_config *config)2128{2129	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);2130	struct pl08x_driver_data *pl08x = plchan->host;2131 2132	if (!plchan->slave)2133		return -EINVAL;2134 2135	/* Reject definitely invalid configurations */2136	if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||2137	    config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)2138		return -EINVAL;2139 2140	if (config->device_fc && pl08x->vd->pl080s) {2141		dev_err(&pl08x->adev->dev,2142			"%s: PL080S does not support peripheral flow control\n",2143			__func__);2144		return -EINVAL;2145	}2146 2147	plchan->cfg = *config;2148 2149	return 0;2150}2151 2152static int pl08x_terminate_all(struct dma_chan *chan)2153{2154	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);2155	struct pl08x_driver_data *pl08x = plchan->host;2156	unsigned long flags;2157 2158	spin_lock_irqsave(&plchan->vc.lock, flags);2159	if (!plchan->phychan && !plchan->at) {2160		spin_unlock_irqrestore(&plchan->vc.lock, flags);2161		return 0;2162	}2163 2164	plchan->state = PL08X_CHAN_IDLE;2165 2166	if (plchan->phychan) {2167		/*2168		 * Mark physical channel as free and free any slave2169		 * signal2170		 */2171		pl08x_phy_free(plchan);2172	}2173	/* Dequeue jobs and free LLIs */2174	if (plchan->at) {2175		vchan_terminate_vdesc(&plchan->at->vd);2176		plchan->at = NULL;2177	}2178	/* Dequeue jobs not yet fired as well */2179	pl08x_free_txd_list(pl08x, plchan);2180 2181	spin_unlock_irqrestore(&plchan->vc.lock, flags);2182 2183	return 0;2184}2185 2186static void pl08x_synchronize(struct dma_chan *chan)2187{2188	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);2189 2190	vchan_synchronize(&plchan->vc);2191}2192 2193static int pl08x_pause(struct dma_chan *chan)2194{2195	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);2196	unsigned long flags;2197 2198	/*2199	 * Anything succeeds on channels with no physical allocation and2200	 * no queued transfers.2201	 */2202	spin_lock_irqsave(&plchan->vc.lock, flags);2203	if (!plchan->phychan && !plchan->at) {2204		spin_unlock_irqrestore(&plchan->vc.lock, flags);2205		return 0;2206	}2207 2208	pl08x_pause_phy_chan(plchan->phychan);2209	plchan->state = PL08X_CHAN_PAUSED;2210 2211	spin_unlock_irqrestore(&plchan->vc.lock, flags);2212 2213	return 0;2214}2215 2216static int pl08x_resume(struct dma_chan *chan)2217{2218	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);2219	unsigned long flags;2220 2221	/*2222	 * Anything succeeds on channels with no physical allocation and2223	 * no queued transfers.2224	 */2225	spin_lock_irqsave(&plchan->vc.lock, flags);2226	if (!plchan->phychan && !plchan->at) {2227		spin_unlock_irqrestore(&plchan->vc.lock, flags);2228		return 0;2229	}2230 2231	pl08x_resume_phy_chan(plchan->phychan);2232	plchan->state = PL08X_CHAN_RUNNING;2233 2234	spin_unlock_irqrestore(&plchan->vc.lock, flags);2235 2236	return 0;2237}2238 2239bool pl08x_filter_id(struct dma_chan *chan, void *chan_id)2240{2241	struct pl08x_dma_chan *plchan;2242	const char *name = chan_id;2243 2244	/* Reject channels for devices not bound to this driver */2245	if (chan->device->dev->driver != &pl08x_amba_driver.drv)2246		return false;2247 2248	plchan = to_pl08x_chan(chan);2249 2250	/* Check that the channel is not taken! */2251	if (!strcmp(plchan->name, name))2252		return true;2253 2254	return false;2255}2256EXPORT_SYMBOL_GPL(pl08x_filter_id);2257 2258static bool pl08x_filter_fn(struct dma_chan *chan, void *chan_id)2259{2260	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);2261 2262	return plchan->cd == chan_id;2263}2264 2265/*2266 * Just check that the device is there and active2267 * TODO: turn this bit on/off depending on the number of physical channels2268 * actually used, if it is zero... well shut it off. That will save some2269 * power. Cut the clock at the same time.2270 */2271static void pl08x_ensure_on(struct pl08x_driver_data *pl08x)2272{2273	/* The Nomadik variant does not have the config register */2274	if (pl08x->vd->nomadik)2275		return;2276	/* The FTDMAC020 variant does this in another register */2277	if (pl08x->vd->ftdmac020) {2278		writel(PL080_CONFIG_ENABLE, pl08x->base + FTDMAC020_CSR);2279		return;2280	}2281	writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG);2282}2283 2284static irqreturn_t pl08x_irq(int irq, void *dev)2285{2286	struct pl08x_driver_data *pl08x = dev;2287	u32 mask = 0, err, tc, i;2288 2289	/* check & clear - ERR & TC interrupts */2290	err = readl(pl08x->base + PL080_ERR_STATUS);2291	if (err) {2292		dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n",2293			__func__, err);2294		writel(err, pl08x->base + PL080_ERR_CLEAR);2295	}2296	tc = readl(pl08x->base + PL080_TC_STATUS);2297	if (tc)2298		writel(tc, pl08x->base + PL080_TC_CLEAR);2299 2300	if (!err && !tc)2301		return IRQ_NONE;2302 2303	for (i = 0; i < pl08x->vd->channels; i++) {2304		if ((BIT(i) & err) || (BIT(i) & tc)) {2305			/* Locate physical channel */2306			struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i];2307			struct pl08x_dma_chan *plchan = phychan->serving;2308			struct pl08x_txd *tx;2309 2310			if (!plchan) {2311				dev_err(&pl08x->adev->dev,2312					"%s Error TC interrupt on unused channel: 0x%08x\n",2313					__func__, i);2314				continue;2315			}2316 2317			spin_lock(&plchan->vc.lock);2318			tx = plchan->at;2319			if (tx && tx->cyclic) {2320				vchan_cyclic_callback(&tx->vd);2321			} else if (tx) {2322				plchan->at = NULL;2323				/*2324				 * This descriptor is done, release its mux2325				 * reservation.2326				 */2327				pl08x_release_mux(plchan);2328				tx->done = true;2329				vchan_cookie_complete(&tx->vd);2330 2331				/*2332				 * And start the next descriptor (if any),2333				 * otherwise free this channel.2334				 */2335				if (vchan_next_desc(&plchan->vc))2336					pl08x_start_next_txd(plchan);2337				else2338					pl08x_phy_free(plchan);2339			}2340			spin_unlock(&plchan->vc.lock);2341 2342			mask |= BIT(i);2343		}2344	}2345 2346	return mask ? IRQ_HANDLED : IRQ_NONE;2347}2348 2349static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan)2350{2351	chan->slave = true;2352	chan->name = chan->cd->bus_id;2353	chan->cfg.src_addr = chan->cd->addr;2354	chan->cfg.dst_addr = chan->cd->addr;2355}2356 2357/*2358 * Initialise the DMAC memcpy/slave channels.2359 * Make a local wrapper to hold required data2360 */2361static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,2362		struct dma_device *dmadev, unsigned int channels, bool slave)2363{2364	struct pl08x_dma_chan *chan;2365	int i;2366 2367	INIT_LIST_HEAD(&dmadev->channels);2368 2369	/*2370	 * Register as many memcpy as we have physical channels,2371	 * we won't always be able to use all but the code will have2372	 * to cope with that situation.2373	 */2374	for (i = 0; i < channels; i++) {2375		chan = kzalloc(sizeof(*chan), GFP_KERNEL);2376		if (!chan)2377			return -ENOMEM;2378 2379		chan->host = pl08x;2380		chan->state = PL08X_CHAN_IDLE;2381		chan->signal = -1;2382 2383		if (slave) {2384			chan->cd = &pl08x->pd->slave_channels[i];2385			/*2386			 * Some implementations have muxed signals, whereas some2387			 * use a mux in front of the signals and need dynamic2388			 * assignment of signals.2389			 */2390			chan->signal = i;2391			pl08x_dma_slave_init(chan);2392		} else {2393			chan->cd = kzalloc(sizeof(*chan->cd), GFP_KERNEL);2394			if (!chan->cd) {2395				kfree(chan);2396				return -ENOMEM;2397			}2398			chan->cd->bus_id = "memcpy";2399			chan->cd->periph_buses = pl08x->pd->mem_buses;2400			chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);2401			if (!chan->name) {2402				kfree(chan->cd);2403				kfree(chan);2404				return -ENOMEM;2405			}2406		}2407		dev_dbg(&pl08x->adev->dev,2408			 "initialize virtual channel \"%s\"\n",2409			 chan->name);2410 2411		chan->vc.desc_free = pl08x_desc_free;2412		vchan_init(&chan->vc, dmadev);2413	}2414	dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n",2415		 i, slave ? "slave" : "memcpy");2416	return i;2417}2418 2419static void pl08x_free_virtual_channels(struct dma_device *dmadev)2420{2421	struct pl08x_dma_chan *chan = NULL;2422	struct pl08x_dma_chan *next;2423 2424	list_for_each_entry_safe(chan,2425				 next, &dmadev->channels, vc.chan.device_node) {2426		list_del(&chan->vc.chan.device_node);2427		kfree(chan);2428	}2429}2430 2431#ifdef CONFIG_DEBUG_FS2432static const char *pl08x_state_str(enum pl08x_dma_chan_state state)2433{2434	switch (state) {2435	case PL08X_CHAN_IDLE:2436		return "idle";2437	case PL08X_CHAN_RUNNING:2438		return "running";2439	case PL08X_CHAN_PAUSED:2440		return "paused";2441	case PL08X_CHAN_WAITING:2442		return "waiting";2443	default:2444		break;2445	}2446	return "UNKNOWN STATE";2447}2448 2449static int pl08x_debugfs_show(struct seq_file *s, void *data)2450{2451	struct pl08x_driver_data *pl08x = s->private;2452	struct pl08x_dma_chan *chan;2453	struct pl08x_phy_chan *ch;2454	unsigned long flags;2455	int i;2456 2457	seq_printf(s, "PL08x physical channels:\n");2458	seq_printf(s, "CHANNEL:\tUSER:\n");2459	seq_printf(s, "--------\t-----\n");2460	for (i = 0; i < pl08x->vd->channels; i++) {2461		struct pl08x_dma_chan *virt_chan;2462 2463		ch = &pl08x->phy_chans[i];2464 2465		spin_lock_irqsave(&ch->lock, flags);2466		virt_chan = ch->serving;2467 2468		seq_printf(s, "%d\t\t%s%s\n",2469			   ch->id,2470			   virt_chan ? virt_chan->name : "(none)",2471			   ch->locked ? " LOCKED" : "");2472 2473		spin_unlock_irqrestore(&ch->lock, flags);2474	}2475 2476	seq_printf(s, "\nPL08x virtual memcpy channels:\n");2477	seq_printf(s, "CHANNEL:\tSTATE:\n");2478	seq_printf(s, "--------\t------\n");2479	list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) {2480		seq_printf(s, "%s\t\t%s\n", chan->name,2481			   pl08x_state_str(chan->state));2482	}2483 2484	if (pl08x->has_slave) {2485		seq_printf(s, "\nPL08x virtual slave channels:\n");2486		seq_printf(s, "CHANNEL:\tSTATE:\n");2487		seq_printf(s, "--------\t------\n");2488		list_for_each_entry(chan, &pl08x->slave.channels,2489				    vc.chan.device_node) {2490			seq_printf(s, "%s\t\t%s\n", chan->name,2491				   pl08x_state_str(chan->state));2492		}2493	}2494 2495	return 0;2496}2497 2498DEFINE_SHOW_ATTRIBUTE(pl08x_debugfs);2499 2500static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)2501{2502	/* Expose a simple debugfs interface to view all clocks */2503	debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO,2504			    NULL, pl08x, &pl08x_debugfs_fops);2505}2506 2507#else2508static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)2509{2510}2511#endif2512 2513#ifdef CONFIG_OF2514static struct dma_chan *pl08x_find_chan_id(struct pl08x_driver_data *pl08x,2515					 u32 id)2516{2517	struct pl08x_dma_chan *chan;2518 2519	/* Trying to get a slave channel from something with no slave support */2520	if (!pl08x->has_slave)2521		return NULL;2522 2523	list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) {2524		if (chan->signal == id)2525			return &chan->vc.chan;2526	}2527 2528	return NULL;2529}2530 2531static struct dma_chan *pl08x_of_xlate(struct of_phandle_args *dma_spec,2532				       struct of_dma *ofdma)2533{2534	struct pl08x_driver_data *pl08x = ofdma->of_dma_data;2535	struct dma_chan *dma_chan;2536	struct pl08x_dma_chan *plchan;2537 2538	if (!pl08x)2539		return NULL;2540 2541	if (dma_spec->args_count != 2) {2542		dev_err(&pl08x->adev->dev,2543			"DMA channel translation requires two cells\n");2544		return NULL;2545	}2546 2547	dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]);2548	if (!dma_chan) {2549		dev_err(&pl08x->adev->dev,2550			"DMA slave channel not found\n");2551		return NULL;2552	}2553 2554	plchan = to_pl08x_chan(dma_chan);2555	dev_dbg(&pl08x->adev->dev,2556		"translated channel for signal %d\n",2557		dma_spec->args[0]);2558 2559	/* Augment channel data for applicable AHB buses */2560	plchan->cd->periph_buses = dma_spec->args[1];2561	return dma_get_slave_channel(dma_chan);2562}2563 2564static int pl08x_of_probe(struct amba_device *adev,2565			  struct pl08x_driver_data *pl08x,2566			  struct device_node *np)2567{2568	struct pl08x_platform_data *pd;2569	struct pl08x_channel_data *chanp = NULL;2570	u32 val;2571	int ret;2572	int i;2573 2574	pd = devm_kzalloc(&adev->dev, sizeof(*pd), GFP_KERNEL);2575	if (!pd)2576		return -ENOMEM;2577 2578	/* Eligible bus masters for fetching LLIs */2579	if (of_property_read_bool(np, "lli-bus-interface-ahb1"))2580		pd->lli_buses |= PL08X_AHB1;2581	if (of_property_read_bool(np, "lli-bus-interface-ahb2"))2582		pd->lli_buses |= PL08X_AHB2;2583	if (!pd->lli_buses) {2584		dev_info(&adev->dev, "no bus masters for LLIs stated, assume all\n");2585		pd->lli_buses |= PL08X_AHB1 | PL08X_AHB2;2586	}2587 2588	/* Eligible bus masters for memory access */2589	if (of_property_read_bool(np, "mem-bus-interface-ahb1"))2590		pd->mem_buses |= PL08X_AHB1;2591	if (of_property_read_bool(np, "mem-bus-interface-ahb2"))2592		pd->mem_buses |= PL08X_AHB2;2593	if (!pd->mem_buses) {2594		dev_info(&adev->dev, "no bus masters for memory stated, assume all\n");2595		pd->mem_buses |= PL08X_AHB1 | PL08X_AHB2;2596	}2597 2598	/* Parse the memcpy channel properties */2599	ret = of_property_read_u32(np, "memcpy-burst-size", &val);2600	if (ret) {2601		dev_info(&adev->dev, "no memcpy burst size specified, using 1 byte\n");2602		val = 1;2603	}2604	switch (val) {2605	default:2606		dev_err(&adev->dev, "illegal burst size for memcpy, set to 1\n");2607		fallthrough;2608	case 1:2609		pd->memcpy_burst_size = PL08X_BURST_SZ_1;2610		break;2611	case 4:2612		pd->memcpy_burst_size = PL08X_BURST_SZ_4;2613		break;2614	case 8:2615		pd->memcpy_burst_size = PL08X_BURST_SZ_8;2616		break;2617	case 16:2618		pd->memcpy_burst_size = PL08X_BURST_SZ_16;2619		break;2620	case 32:2621		pd->memcpy_burst_size = PL08X_BURST_SZ_32;2622		break;2623	case 64:2624		pd->memcpy_burst_size = PL08X_BURST_SZ_64;2625		break;2626	case 128:2627		pd->memcpy_burst_size = PL08X_BURST_SZ_128;2628		break;2629	case 256:2630		pd->memcpy_burst_size = PL08X_BURST_SZ_256;2631		break;2632	}2633 2634	ret = of_property_read_u32(np, "memcpy-bus-width", &val);2635	if (ret) {2636		dev_info(&adev->dev, "no memcpy bus width specified, using 8 bits\n");2637		val = 8;2638	}2639	switch (val) {2640	default:2641		dev_err(&adev->dev, "illegal bus width for memcpy, set to 8 bits\n");2642		fallthrough;2643	case 8:2644		pd->memcpy_bus_width = PL08X_BUS_WIDTH_8_BITS;2645		break;2646	case 16:2647		pd->memcpy_bus_width = PL08X_BUS_WIDTH_16_BITS;2648		break;2649	case 32:2650		pd->memcpy_bus_width = PL08X_BUS_WIDTH_32_BITS;2651		break;2652	}2653 2654	/*2655	 * Allocate channel data for all possible slave channels (one2656	 * for each possible signal), channels will then be allocated2657	 * for a device and have it's AHB interfaces set up at2658	 * translation time.2659	 */2660	if (pl08x->vd->signals) {2661		chanp = devm_kcalloc(&adev->dev,2662				     pl08x->vd->signals,2663				     sizeof(struct pl08x_channel_data),2664				     GFP_KERNEL);2665		if (!chanp)2666			return -ENOMEM;2667 2668		pd->slave_channels = chanp;2669		for (i = 0; i < pl08x->vd->signals; i++) {2670			/*2671			 * chanp->periph_buses will be assigned at translation2672			 */2673			chanp->bus_id = kasprintf(GFP_KERNEL, "slave%d", i);2674			chanp++;2675		}2676		pd->num_slave_channels = pl08x->vd->signals;2677	}2678 2679	pl08x->pd = pd;2680 2681	return of_dma_controller_register(adev->dev.of_node, pl08x_of_xlate,2682					  pl08x);2683}2684#else2685static inline int pl08x_of_probe(struct amba_device *adev,2686				 struct pl08x_driver_data *pl08x,2687				 struct device_node *np)2688{2689	return -EINVAL;2690}2691#endif2692 2693static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)2694{2695	struct pl08x_driver_data *pl08x;2696	struct vendor_data *vd = id->data;2697	struct device_node *np = adev->dev.of_node;2698	u32 tsfr_size;2699	int ret = 0;2700	int i;2701 2702	ret = amba_request_regions(adev, NULL);2703	if (ret)2704		return ret;2705 2706	/* Ensure that we can do DMA */2707	ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32));2708	if (ret)2709		goto out_no_pl08x;2710 2711	/* Create the driver state holder */2712	pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL);2713	if (!pl08x) {2714		ret = -ENOMEM;2715		goto out_no_pl08x;2716	}2717 2718	/* Assign useful pointers to the driver state */2719	pl08x->adev = adev;2720	pl08x->vd = vd;2721 2722	pl08x->base = ioremap(adev->res.start, resource_size(&adev->res));2723	if (!pl08x->base) {2724		ret = -ENOMEM;2725		goto out_no_ioremap;2726	}2727 2728	if (vd->ftdmac020) {2729		u32 val;2730 2731		val = readl(pl08x->base + FTDMAC020_REVISION);2732		dev_info(&pl08x->adev->dev, "FTDMAC020 %d.%d rel %d\n",2733			 (val >> 16) & 0xff, (val >> 8) & 0xff, val & 0xff);2734		val = readl(pl08x->base + FTDMAC020_FEATURE);2735		dev_info(&pl08x->adev->dev, "FTDMAC020 %d channels, "2736			 "%s built-in bridge, %s, %s linked lists\n",2737			 (val >> 12) & 0x0f,2738			 (val & BIT(10)) ? "no" : "has",2739			 (val & BIT(9)) ? "AHB0 and AHB1" : "AHB0",2740			 (val & BIT(8)) ? "supports" : "does not support");2741 2742		/* Vendor data from feature register */2743		if (!(val & BIT(8)))2744			dev_warn(&pl08x->adev->dev,2745				 "linked lists not supported, required\n");2746		vd->channels = (val >> 12) & 0x0f;2747		vd->dualmaster = !!(val & BIT(9));2748	}2749 2750	/* Initialize memcpy engine */2751	dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask);2752	pl08x->memcpy.dev = &adev->dev;2753	pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources;2754	pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy;2755	pl08x->memcpy.device_tx_status = pl08x_dma_tx_status;2756	pl08x->memcpy.device_issue_pending = pl08x_issue_pending;2757	pl08x->memcpy.device_config = pl08x_config;2758	pl08x->memcpy.device_pause = pl08x_pause;2759	pl08x->memcpy.device_resume = pl08x_resume;2760	pl08x->memcpy.device_terminate_all = pl08x_terminate_all;2761	pl08x->memcpy.device_synchronize = pl08x_synchronize;2762	pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS;2763	pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS;2764	pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM);2765	pl08x->memcpy.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;2766	if (vd->ftdmac020)2767		pl08x->memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES;2768 2769 2770	/*2771	 * Initialize slave engine, if the block has no signals, that means2772	 * we have no slave support.2773	 */2774	if (vd->signals) {2775		pl08x->has_slave = true;2776		dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask);2777		dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask);2778		pl08x->slave.dev = &adev->dev;2779		pl08x->slave.device_free_chan_resources =2780			pl08x_free_chan_resources;2781		pl08x->slave.device_tx_status = pl08x_dma_tx_status;2782		pl08x->slave.device_issue_pending = pl08x_issue_pending;2783		pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg;2784		pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic;2785		pl08x->slave.device_config = pl08x_config;2786		pl08x->slave.device_pause = pl08x_pause;2787		pl08x->slave.device_resume = pl08x_resume;2788		pl08x->slave.device_terminate_all = pl08x_terminate_all;2789		pl08x->slave.device_synchronize = pl08x_synchronize;2790		pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS;2791		pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS;2792		pl08x->slave.directions =2793			BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);2794		pl08x->slave.residue_granularity =2795			DMA_RESIDUE_GRANULARITY_SEGMENT;2796	}2797 2798	/* Get the platform data */2799	pl08x->pd = dev_get_platdata(&adev->dev);2800	if (!pl08x->pd) {2801		if (np) {2802			ret = pl08x_of_probe(adev, pl08x, np);2803			if (ret)2804				goto out_no_platdata;2805		} else {2806			dev_err(&adev->dev, "no platform data supplied\n");2807			ret = -EINVAL;2808			goto out_no_platdata;2809		}2810	} else {2811		pl08x->slave.filter.map = pl08x->pd->slave_map;2812		pl08x->slave.filter.mapcnt = pl08x->pd->slave_map_len;2813		pl08x->slave.filter.fn = pl08x_filter_fn;2814	}2815 2816	/* By default, AHB1 only.  If dualmaster, from platform */2817	pl08x->lli_buses = PL08X_AHB1;2818	pl08x->mem_buses = PL08X_AHB1;2819	if (pl08x->vd->dualmaster) {2820		pl08x->lli_buses = pl08x->pd->lli_buses;2821		pl08x->mem_buses = pl08x->pd->mem_buses;2822	}2823 2824	if (vd->pl080s)2825		pl08x->lli_words = PL080S_LLI_WORDS;2826	else2827		pl08x->lli_words = PL080_LLI_WORDS;2828	tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32);2829 2830	/* A DMA memory pool for LLIs, align on 1-byte boundary */2831	pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev,2832						tsfr_size, PL08X_ALIGN, 0);2833	if (!pl08x->pool) {2834		ret = -ENOMEM;2835		goto out_no_lli_pool;2836	}2837 2838	/* Turn on the PL08x */2839	pl08x_ensure_on(pl08x);2840 2841	/* Clear any pending interrupts */2842	if (vd->ftdmac020)2843		/* This variant has error IRQs in bits 16-19 */2844		writel(0x0000FFFF, pl08x->base + PL080_ERR_CLEAR);2845	else2846		writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR);2847	writel(0x000000FF, pl08x->base + PL080_TC_CLEAR);2848 2849	/* Attach the interrupt handler */2850	ret = request_irq(adev->irq[0], pl08x_irq, 0, DRIVER_NAME, pl08x);2851	if (ret) {2852		dev_err(&adev->dev, "%s failed to request interrupt %d\n",2853			__func__, adev->irq[0]);2854		goto out_no_irq;2855	}2856 2857	/* Initialize physical channels */2858	pl08x->phy_chans = kcalloc(vd->channels, sizeof(*pl08x->phy_chans),2859				   GFP_KERNEL);2860	if (!pl08x->phy_chans) {2861		ret = -ENOMEM;2862		goto out_no_phychans;2863	}2864 2865	for (i = 0; i < vd->channels; i++) {2866		struct pl08x_phy_chan *ch = &pl08x->phy_chans[i];2867 2868		ch->id = i;2869		ch->base = pl08x->base + PL080_Cx_BASE(i);2870		if (vd->ftdmac020) {2871			/* FTDMA020 has a special channel busy register */2872			ch->reg_busy = ch->base + FTDMAC020_CH_BUSY;2873			ch->reg_config = ch->base + FTDMAC020_CH_CFG;2874			ch->reg_control = ch->base + FTDMAC020_CH_CSR;2875			ch->reg_src = ch->base + FTDMAC020_CH_SRC_ADDR;2876			ch->reg_dst = ch->base + FTDMAC020_CH_DST_ADDR;2877			ch->reg_lli = ch->base + FTDMAC020_CH_LLP;2878			ch->ftdmac020 = true;2879		} else {2880			ch->reg_config = ch->base + vd->config_offset;2881			ch->reg_control = ch->base + PL080_CH_CONTROL;2882			ch->reg_src = ch->base + PL080_CH_SRC_ADDR;2883			ch->reg_dst = ch->base + PL080_CH_DST_ADDR;2884			ch->reg_lli = ch->base + PL080_CH_LLI;2885		}2886		if (vd->pl080s)2887			ch->pl080s = true;2888 2889		spin_lock_init(&ch->lock);2890 2891		/*2892		 * Nomadik variants can have channels that are locked2893		 * down for the secure world only. Lock up these channels2894		 * by perpetually serving a dummy virtual channel.2895		 */2896		if (vd->nomadik) {2897			u32 val;2898 2899			val = readl(ch->reg_config);2900			if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) {2901				dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i);2902				ch->locked = true;2903			}2904		}2905 2906		dev_dbg(&adev->dev, "physical channel %d is %s\n",2907			i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE");2908	}2909 2910	/* Register as many memcpy channels as there are physical channels */2911	ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy,2912					      pl08x->vd->channels, false);2913	if (ret <= 0) {2914		dev_warn(&pl08x->adev->dev,2915			 "%s failed to enumerate memcpy channels - %d\n",2916			 __func__, ret);2917		goto out_no_memcpy;2918	}2919 2920	/* Register slave channels */2921	if (pl08x->has_slave) {2922		ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave,2923					pl08x->pd->num_slave_channels, true);2924		if (ret < 0) {2925			dev_warn(&pl08x->adev->dev,2926				 "%s failed to enumerate slave channels - %d\n",2927				 __func__, ret);2928			goto out_no_slave;2929		}2930	}2931 2932	ret = dma_async_device_register(&pl08x->memcpy);2933	if (ret) {2934		dev_warn(&pl08x->adev->dev,2935			"%s failed to register memcpy as an async device - %d\n",2936			__func__, ret);2937		goto out_no_memcpy_reg;2938	}2939 2940	if (pl08x->has_slave) {2941		ret = dma_async_device_register(&pl08x->slave);2942		if (ret) {2943			dev_warn(&pl08x->adev->dev,2944			"%s failed to register slave as an async device - %d\n",2945			__func__, ret);2946			goto out_no_slave_reg;2947		}2948	}2949 2950	amba_set_drvdata(adev, pl08x);2951	init_pl08x_debugfs(pl08x);2952	dev_info(&pl08x->adev->dev, "DMA: PL%03x%s rev%u at 0x%08llx irq %d\n",2953		 amba_part(adev), pl08x->vd->pl080s ? "s" : "", amba_rev(adev),2954		 (unsigned long long)adev->res.start, adev->irq[0]);2955 2956	return 0;2957 2958out_no_slave_reg:2959	dma_async_device_unregister(&pl08x->memcpy);2960out_no_memcpy_reg:2961	if (pl08x->has_slave)2962		pl08x_free_virtual_channels(&pl08x->slave);2963out_no_slave:2964	pl08x_free_virtual_channels(&pl08x->memcpy);2965out_no_memcpy:2966	kfree(pl08x->phy_chans);2967out_no_phychans:2968	free_irq(adev->irq[0], pl08x);2969out_no_irq:2970	dma_pool_destroy(pl08x->pool);2971out_no_lli_pool:2972out_no_platdata:2973	iounmap(pl08x->base);2974out_no_ioremap:2975	kfree(pl08x);2976out_no_pl08x:2977	amba_release_regions(adev);2978	return ret;2979}2980 2981/* PL080 has 8 channels and the PL080 have just 2 */2982static struct vendor_data vendor_pl080 = {2983	.config_offset = PL080_CH_CONFIG,2984	.channels = 8,2985	.signals = 16,2986	.dualmaster = true,2987	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,2988};2989 2990static struct vendor_data vendor_nomadik = {2991	.config_offset = PL080_CH_CONFIG,2992	.channels = 8,2993	.signals = 32,2994	.dualmaster = true,2995	.nomadik = true,2996	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,2997};2998 2999static struct vendor_data vendor_pl080s = {3000	.config_offset = PL080S_CH_CONFIG,3001	.channels = 8,3002	.signals = 32,3003	.pl080s = true,3004	.max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK,3005};3006 3007static struct vendor_data vendor_pl081 = {3008	.config_offset = PL080_CH_CONFIG,3009	.channels = 2,3010	.signals = 16,3011	.dualmaster = false,3012	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,3013};3014 3015static struct vendor_data vendor_ftdmac020 = {3016	.config_offset = PL080_CH_CONFIG,3017	.ftdmac020 = true,3018	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,3019};3020 3021static const struct amba_id pl08x_ids[] = {3022	/* Samsung PL080S variant */3023	{3024		.id	= 0x0a141080,3025		.mask	= 0xffffffff,3026		.data	= &vendor_pl080s,3027	},3028	/* PL080 */3029	{3030		.id	= 0x00041080,3031		.mask	= 0x000fffff,3032		.data	= &vendor_pl080,3033	},3034	/* PL081 */3035	{3036		.id	= 0x00041081,3037		.mask	= 0x000fffff,3038		.data	= &vendor_pl081,3039	},3040	/* Nomadik 8815 PL080 variant */3041	{3042		.id	= 0x00280080,3043		.mask	= 0x00ffffff,3044		.data	= &vendor_nomadik,3045	},3046	/* Faraday Technology FTDMAC020 */3047	{3048		.id	= 0x0003b080,3049		.mask	= 0x000fffff,3050		.data	= &vendor_ftdmac020,3051	},3052	{ 0, 0 },3053};3054 3055MODULE_DEVICE_TABLE(amba, pl08x_ids);3056 3057static struct amba_driver pl08x_amba_driver = {3058	.drv.name	= DRIVER_NAME,3059	.id_table	= pl08x_ids,3060	.probe		= pl08x_probe,3061};3062 3063static int __init pl08x_init(void)3064{3065	int retval;3066	retval = amba_driver_register(&pl08x_amba_driver);3067	if (retval)3068		printk(KERN_WARNING DRIVER_NAME3069		       "failed to register as an AMBA device (%d)\n",3070		       retval);3071	return retval;3072}3073subsys_initcall(pl08x_init);3074