brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) Ericsson AB 2007-20084 * Copyright (C) ST-Ericsson SA 2008-20105 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson6 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson7 */8 9#include <linux/dma-mapping.h>10#include <linux/kernel.h>11#include <linux/slab.h>12#include <linux/export.h>13#include <linux/dmaengine.h>14#include <linux/platform_device.h>15#include <linux/clk.h>16#include <linux/delay.h>17#include <linux/log2.h>18#include <linux/pm.h>19#include <linux/pm_runtime.h>20#include <linux/err.h>21#include <linux/of.h>22#include <linux/of_address.h>23#include <linux/of_dma.h>24#include <linux/amba/bus.h>25#include <linux/regulator/consumer.h>26 27#include "dmaengine.h"28#include "ste_dma40.h"29#include "ste_dma40_ll.h"30 31/**32 * struct stedma40_platform_data - Configuration struct for the dma device.33 *34 * @disabled_channels: A vector, ending with -1, that marks physical channels35 * that are for different reasons not available for the driver.36 * @soft_lli_chans: A vector, that marks physical channels will use LLI by SW37 * which avoids HW bug that exists in some versions of the controller.38 * SoftLLI introduces relink overhead that could impact performance for39 * certain use cases.40 * @num_of_soft_lli_chans: The number of channels that needs to be configured41 * to use SoftLLI.42 * @use_esram_lcla: flag for mapping the lcla into esram region43 * @num_of_memcpy_chans: The number of channels reserved for memcpy.44 * @num_of_phy_chans: The number of physical channels implemented in HW.45 * 0 means reading the number of channels from DMA HW but this is only valid46 * for 'multiple of 4' channels, like 8.47 */48struct stedma40_platform_data {49	int				 disabled_channels[STEDMA40_MAX_PHYS];50	int				*soft_lli_chans;51	int				 num_of_soft_lli_chans;52	bool				 use_esram_lcla;53	int				 num_of_memcpy_chans;54	int				 num_of_phy_chans;55};56 57#define D40_NAME "dma40"58 59#define D40_PHY_CHAN -160 61/* For masking out/in 2 bit channel positions */62#define D40_CHAN_POS(chan)  (2 * (chan / 2))63#define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))64 65/* Maximum iterations taken before giving up suspending a channel */66#define D40_SUSPEND_MAX_IT 50067 68/* Milliseconds */69#define DMA40_AUTOSUSPEND_DELAY	10070 71/* Hardware requirement on LCLA alignment */72#define LCLA_ALIGNMENT 0x4000073 74/* Max number of links per event group */75#define D40_LCLA_LINK_PER_EVENT_GRP 12876#define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP77 78/* Max number of logical channels per physical channel */79#define D40_MAX_LOG_CHAN_PER_PHY 3280 81/* Attempts before giving up to trying to get pages that are aligned */82#define MAX_LCLA_ALLOC_ATTEMPTS 25683 84/* Bit markings for allocation map */85#define D40_ALLOC_FREE		BIT(31)86#define D40_ALLOC_PHY		BIT(30)87#define D40_ALLOC_LOG_FREE	088 89#define D40_MEMCPY_MAX_CHANS	890 91/* Reserved event lines for memcpy only. */92#define DB8500_DMA_MEMCPY_EV_0	5193#define DB8500_DMA_MEMCPY_EV_1	5694#define DB8500_DMA_MEMCPY_EV_2	5795#define DB8500_DMA_MEMCPY_EV_3	5896#define DB8500_DMA_MEMCPY_EV_4	5997#define DB8500_DMA_MEMCPY_EV_5	6098 99static int dma40_memcpy_channels[] = {100	DB8500_DMA_MEMCPY_EV_0,101	DB8500_DMA_MEMCPY_EV_1,102	DB8500_DMA_MEMCPY_EV_2,103	DB8500_DMA_MEMCPY_EV_3,104	DB8500_DMA_MEMCPY_EV_4,105	DB8500_DMA_MEMCPY_EV_5,106};107 108/* Default configuration for physical memcpy */109static const struct stedma40_chan_cfg dma40_memcpy_conf_phy = {110	.mode = STEDMA40_MODE_PHYSICAL,111	.dir = DMA_MEM_TO_MEM,112 113	.src_info.data_width = DMA_SLAVE_BUSWIDTH_1_BYTE,114	.src_info.psize = STEDMA40_PSIZE_PHY_1,115	.src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,116 117	.dst_info.data_width = DMA_SLAVE_BUSWIDTH_1_BYTE,118	.dst_info.psize = STEDMA40_PSIZE_PHY_1,119	.dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,120};121 122/* Default configuration for logical memcpy */123static const struct stedma40_chan_cfg dma40_memcpy_conf_log = {124	.mode = STEDMA40_MODE_LOGICAL,125	.dir = DMA_MEM_TO_MEM,126 127	.src_info.data_width = DMA_SLAVE_BUSWIDTH_1_BYTE,128	.src_info.psize = STEDMA40_PSIZE_LOG_1,129	.src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,130 131	.dst_info.data_width = DMA_SLAVE_BUSWIDTH_1_BYTE,132	.dst_info.psize = STEDMA40_PSIZE_LOG_1,133	.dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,134};135 136/**137 * enum d40_command - The different commands and/or statuses.138 *139 * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,140 * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.141 * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.142 * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.143 */144enum d40_command {145	D40_DMA_STOP		= 0,146	D40_DMA_RUN		= 1,147	D40_DMA_SUSPEND_REQ	= 2,148	D40_DMA_SUSPENDED	= 3149};150 151/*152 * enum d40_events - The different Event Enables for the event lines.153 *154 * @D40_DEACTIVATE_EVENTLINE: De-activate Event line, stopping the logical chan.155 * @D40_ACTIVATE_EVENTLINE: Activate the Event line, to start a logical chan.156 * @D40_SUSPEND_REQ_EVENTLINE: Requesting for suspending a event line.157 * @D40_ROUND_EVENTLINE: Status check for event line.158 */159 160enum d40_events {161	D40_DEACTIVATE_EVENTLINE	= 0,162	D40_ACTIVATE_EVENTLINE		= 1,163	D40_SUSPEND_REQ_EVENTLINE	= 2,164	D40_ROUND_EVENTLINE		= 3165};166 167/*168 * These are the registers that has to be saved and later restored169 * when the DMA hw is powered off.170 * TODO: Add save/restore of D40_DREG_GCC on dma40 v3 or later, if that works.171 */172static __maybe_unused u32 d40_backup_regs[] = {173	D40_DREG_LCPA,174	D40_DREG_LCLA,175	D40_DREG_PRMSE,176	D40_DREG_PRMSO,177	D40_DREG_PRMOE,178	D40_DREG_PRMOO,179};180 181#define BACKUP_REGS_SZ ARRAY_SIZE(d40_backup_regs)182 183/*184 * since 9540 and 8540 has the same HW revision185 * use v4a for 9540 or earlier186 * use v4b for 8540 or later187 * HW revision:188 * DB8500ed has revision 0189 * DB8500v1 has revision 2190 * DB8500v2 has revision 3191 * AP9540v1 has revision 4192 * DB8540v1 has revision 4193 * TODO: Check if all these registers have to be saved/restored on dma40 v4a194 */195static u32 d40_backup_regs_v4a[] = {196	D40_DREG_PSEG1,197	D40_DREG_PSEG2,198	D40_DREG_PSEG3,199	D40_DREG_PSEG4,200	D40_DREG_PCEG1,201	D40_DREG_PCEG2,202	D40_DREG_PCEG3,203	D40_DREG_PCEG4,204	D40_DREG_RSEG1,205	D40_DREG_RSEG2,206	D40_DREG_RSEG3,207	D40_DREG_RSEG4,208	D40_DREG_RCEG1,209	D40_DREG_RCEG2,210	D40_DREG_RCEG3,211	D40_DREG_RCEG4,212};213 214#define BACKUP_REGS_SZ_V4A ARRAY_SIZE(d40_backup_regs_v4a)215 216static u32 d40_backup_regs_v4b[] = {217	D40_DREG_CPSEG1,218	D40_DREG_CPSEG2,219	D40_DREG_CPSEG3,220	D40_DREG_CPSEG4,221	D40_DREG_CPSEG5,222	D40_DREG_CPCEG1,223	D40_DREG_CPCEG2,224	D40_DREG_CPCEG3,225	D40_DREG_CPCEG4,226	D40_DREG_CPCEG5,227	D40_DREG_CRSEG1,228	D40_DREG_CRSEG2,229	D40_DREG_CRSEG3,230	D40_DREG_CRSEG4,231	D40_DREG_CRSEG5,232	D40_DREG_CRCEG1,233	D40_DREG_CRCEG2,234	D40_DREG_CRCEG3,235	D40_DREG_CRCEG4,236	D40_DREG_CRCEG5,237};238 239#define BACKUP_REGS_SZ_V4B ARRAY_SIZE(d40_backup_regs_v4b)240 241static __maybe_unused u32 d40_backup_regs_chan[] = {242	D40_CHAN_REG_SSCFG,243	D40_CHAN_REG_SSELT,244	D40_CHAN_REG_SSPTR,245	D40_CHAN_REG_SSLNK,246	D40_CHAN_REG_SDCFG,247	D40_CHAN_REG_SDELT,248	D40_CHAN_REG_SDPTR,249	D40_CHAN_REG_SDLNK,250};251 252#define BACKUP_REGS_SZ_MAX ((BACKUP_REGS_SZ_V4A > BACKUP_REGS_SZ_V4B) ? \253			     BACKUP_REGS_SZ_V4A : BACKUP_REGS_SZ_V4B)254 255/**256 * struct d40_interrupt_lookup - lookup table for interrupt handler257 *258 * @src: Interrupt mask register.259 * @clr: Interrupt clear register.260 * @is_error: true if this is an error interrupt.261 * @offset: start delta in the lookup_log_chans in d40_base. If equals to262 * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.263 */264struct d40_interrupt_lookup {265	u32 src;266	u32 clr;267	bool is_error;268	int offset;269};270 271 272static struct d40_interrupt_lookup il_v4a[] = {273	{D40_DREG_LCTIS0, D40_DREG_LCICR0, false,  0},274	{D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},275	{D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},276	{D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},277	{D40_DREG_LCEIS0, D40_DREG_LCICR0, true,   0},278	{D40_DREG_LCEIS1, D40_DREG_LCICR1, true,  32},279	{D40_DREG_LCEIS2, D40_DREG_LCICR2, true,  64},280	{D40_DREG_LCEIS3, D40_DREG_LCICR3, true,  96},281	{D40_DREG_PCTIS,  D40_DREG_PCICR,  false, D40_PHY_CHAN},282	{D40_DREG_PCEIS,  D40_DREG_PCICR,  true,  D40_PHY_CHAN},283};284 285static struct d40_interrupt_lookup il_v4b[] = {286	{D40_DREG_CLCTIS1, D40_DREG_CLCICR1, false,  0},287	{D40_DREG_CLCTIS2, D40_DREG_CLCICR2, false, 32},288	{D40_DREG_CLCTIS3, D40_DREG_CLCICR3, false, 64},289	{D40_DREG_CLCTIS4, D40_DREG_CLCICR4, false, 96},290	{D40_DREG_CLCTIS5, D40_DREG_CLCICR5, false, 128},291	{D40_DREG_CLCEIS1, D40_DREG_CLCICR1, true,   0},292	{D40_DREG_CLCEIS2, D40_DREG_CLCICR2, true,  32},293	{D40_DREG_CLCEIS3, D40_DREG_CLCICR3, true,  64},294	{D40_DREG_CLCEIS4, D40_DREG_CLCICR4, true,  96},295	{D40_DREG_CLCEIS5, D40_DREG_CLCICR5, true,  128},296	{D40_DREG_CPCTIS,  D40_DREG_CPCICR,  false, D40_PHY_CHAN},297	{D40_DREG_CPCEIS,  D40_DREG_CPCICR,  true,  D40_PHY_CHAN},298};299 300/**301 * struct d40_reg_val - simple lookup struct302 *303 * @reg: The register.304 * @val: The value that belongs to the register in reg.305 */306struct d40_reg_val {307	unsigned int reg;308	unsigned int val;309};310 311static __initdata struct d40_reg_val dma_init_reg_v4a[] = {312	/* Clock every part of the DMA block from start */313	{ .reg = D40_DREG_GCC,    .val = D40_DREG_GCC_ENABLE_ALL},314 315	/* Interrupts on all logical channels */316	{ .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},317	{ .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},318	{ .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},319	{ .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},320	{ .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},321	{ .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},322	{ .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},323	{ .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},324	{ .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},325	{ .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},326	{ .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},327	{ .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}328};329static __initdata struct d40_reg_val dma_init_reg_v4b[] = {330	/* Clock every part of the DMA block from start */331	{ .reg = D40_DREG_GCC,    .val = D40_DREG_GCC_ENABLE_ALL},332 333	/* Interrupts on all logical channels */334	{ .reg = D40_DREG_CLCMIS1, .val = 0xFFFFFFFF},335	{ .reg = D40_DREG_CLCMIS2, .val = 0xFFFFFFFF},336	{ .reg = D40_DREG_CLCMIS3, .val = 0xFFFFFFFF},337	{ .reg = D40_DREG_CLCMIS4, .val = 0xFFFFFFFF},338	{ .reg = D40_DREG_CLCMIS5, .val = 0xFFFFFFFF},339	{ .reg = D40_DREG_CLCICR1, .val = 0xFFFFFFFF},340	{ .reg = D40_DREG_CLCICR2, .val = 0xFFFFFFFF},341	{ .reg = D40_DREG_CLCICR3, .val = 0xFFFFFFFF},342	{ .reg = D40_DREG_CLCICR4, .val = 0xFFFFFFFF},343	{ .reg = D40_DREG_CLCICR5, .val = 0xFFFFFFFF},344	{ .reg = D40_DREG_CLCTIS1, .val = 0xFFFFFFFF},345	{ .reg = D40_DREG_CLCTIS2, .val = 0xFFFFFFFF},346	{ .reg = D40_DREG_CLCTIS3, .val = 0xFFFFFFFF},347	{ .reg = D40_DREG_CLCTIS4, .val = 0xFFFFFFFF},348	{ .reg = D40_DREG_CLCTIS5, .val = 0xFFFFFFFF}349};350 351/**352 * struct d40_lli_pool - Structure for keeping LLIs in memory353 *354 * @base: Pointer to memory area when the pre_alloc_lli's are not large355 * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if356 * pre_alloc_lli is used.357 * @dma_addr: DMA address, if mapped358 * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.359 * @pre_alloc_lli: Pre allocated area for the most common case of transfers,360 * one buffer to one buffer.361 */362struct d40_lli_pool {363	void	*base;364	int	 size;365	dma_addr_t	dma_addr;366	/* Space for dst and src, plus an extra for padding */367	u8	 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];368};369 370/**371 * struct d40_desc - A descriptor is one DMA job.372 *373 * @lli_phy: LLI settings for physical channel. Both src and dst=374 * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if375 * lli_len equals one.376 * @lli_log: Same as above but for logical channels.377 * @lli_pool: The pool with two entries pre-allocated.378 * @lli_len: Number of llis of current descriptor.379 * @lli_current: Number of transferred llis.380 * @lcla_alloc: Number of LCLA entries allocated.381 * @txd: DMA engine struct. Used for among other things for communication382 * during a transfer.383 * @node: List entry.384 * @is_in_client_list: true if the client owns this descriptor.385 * @cyclic: true if this is a cyclic job386 *387 * This descriptor is used for both logical and physical transfers.388 */389struct d40_desc {390	/* LLI physical */391	struct d40_phy_lli_bidir	 lli_phy;392	/* LLI logical */393	struct d40_log_lli_bidir	 lli_log;394 395	struct d40_lli_pool		 lli_pool;396	int				 lli_len;397	int				 lli_current;398	int				 lcla_alloc;399 400	struct dma_async_tx_descriptor	 txd;401	struct list_head		 node;402 403	bool				 is_in_client_list;404	bool				 cyclic;405};406 407/**408 * struct d40_lcla_pool - LCLA pool settings and data.409 *410 * @base: The virtual address of LCLA. 18 bit aligned.411 * @dma_addr: DMA address, if mapped412 * @base_unaligned: The original kmalloc pointer, if kmalloc is used.413 * This pointer is only there for clean-up on error.414 * @pages: The number of pages needed for all physical channels.415 * Only used later for clean-up on error416 * @lock: Lock to protect the content in this struct.417 * @alloc_map: big map over which LCLA entry is own by which job.418 */419struct d40_lcla_pool {420	void		*base;421	dma_addr_t	dma_addr;422	void		*base_unaligned;423	int		 pages;424	spinlock_t	 lock;425	struct d40_desc	**alloc_map;426};427 428/**429 * struct d40_phy_res - struct for handling eventlines mapped to physical430 * channels.431 *432 * @lock: A lock protection this entity.433 * @reserved: True if used by secure world or otherwise.434 * @num: The physical channel number of this entity.435 * @allocated_src: Bit mapped to show which src event line's are mapped to436 * this physical channel. Can also be free or physically allocated.437 * @allocated_dst: Same as for src but is dst.438 * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as439 * event line number.440 * @use_soft_lli: To mark if the linked lists of channel are managed by SW.441 */442struct d40_phy_res {443	spinlock_t lock;444	bool	   reserved;445	int	   num;446	u32	   allocated_src;447	u32	   allocated_dst;448	bool	   use_soft_lli;449};450 451struct d40_base;452 453/**454 * struct d40_chan - Struct that describes a channel.455 *456 * @lock: A spinlock to protect this struct.457 * @log_num: The logical number, if any of this channel.458 * @pending_tx: The number of pending transfers. Used between interrupt handler459 * and tasklet.460 * @busy: Set to true when transfer is ongoing on this channel.461 * @phy_chan: Pointer to physical channel which this instance runs on. If this462 * point is NULL, then the channel is not allocated.463 * @chan: DMA engine handle.464 * @tasklet: Tasklet that gets scheduled from interrupt context to complete a465 * transfer and call client callback.466 * @client: Cliented owned descriptor list.467 * @pending_queue: Submitted jobs, to be issued by issue_pending()468 * @active: Active descriptor.469 * @done: Completed jobs470 * @queue: Queued jobs.471 * @prepare_queue: Prepared jobs.472 * @dma_cfg: The client configuration of this dma channel.473 * @slave_config: DMA slave configuration.474 * @configured: whether the dma_cfg configuration is valid475 * @base: Pointer to the device instance struct.476 * @src_def_cfg: Default cfg register setting for src.477 * @dst_def_cfg: Default cfg register setting for dst.478 * @log_def: Default logical channel settings.479 * @lcpa: Pointer to dst and src lcpa settings.480 * @runtime_addr: runtime configured address.481 * @runtime_direction: runtime configured direction.482 *483 * This struct can either "be" a logical or a physical channel.484 */485struct d40_chan {486	spinlock_t			 lock;487	int				 log_num;488	int				 pending_tx;489	bool				 busy;490	struct d40_phy_res		*phy_chan;491	struct dma_chan			 chan;492	struct tasklet_struct		 tasklet;493	struct list_head		 client;494	struct list_head		 pending_queue;495	struct list_head		 active;496	struct list_head		 done;497	struct list_head		 queue;498	struct list_head		 prepare_queue;499	struct stedma40_chan_cfg	 dma_cfg;500	struct dma_slave_config		 slave_config;501	bool				 configured;502	struct d40_base			*base;503	/* Default register configurations */504	u32				 src_def_cfg;505	u32				 dst_def_cfg;506	struct d40_def_lcsp		 log_def;507	struct d40_log_lli_full		*lcpa;508	/* Runtime reconfiguration */509	dma_addr_t			runtime_addr;510	enum dma_transfer_direction	runtime_direction;511};512 513/**514 * struct d40_gen_dmac - generic values to represent u8500/u8540 DMA515 * controller516 *517 * @backup: the pointer to the registers address array for backup518 * @backup_size: the size of the registers address array for backup519 * @realtime_en: the realtime enable register520 * @realtime_clear: the realtime clear register521 * @high_prio_en: the high priority enable register522 * @high_prio_clear: the high priority clear register523 * @interrupt_en: the interrupt enable register524 * @interrupt_clear: the interrupt clear register525 * @il: the pointer to struct d40_interrupt_lookup526 * @il_size: the size of d40_interrupt_lookup array527 * @init_reg: the pointer to the struct d40_reg_val528 * @init_reg_size: the size of d40_reg_val array529 */530struct d40_gen_dmac {531	u32				*backup;532	u32				 backup_size;533	u32				 realtime_en;534	u32				 realtime_clear;535	u32				 high_prio_en;536	u32				 high_prio_clear;537	u32				 interrupt_en;538	u32				 interrupt_clear;539	struct d40_interrupt_lookup	*il;540	u32				 il_size;541	struct d40_reg_val		*init_reg;542	u32				 init_reg_size;543};544 545/**546 * struct d40_base - The big global struct, one for each probe'd instance.547 *548 * @interrupt_lock: Lock used to make sure one interrupt is handle a time.549 * @execmd_lock: Lock for execute command usage since several channels share550 * the same physical register.551 * @dev: The device structure.552 * @virtbase: The virtual base address of the DMA's register.553 * @rev: silicon revision detected.554 * @clk: Pointer to the DMA clock structure.555 * @irq: The IRQ number.556 * @num_memcpy_chans: The number of channels used for memcpy (mem-to-mem557 * transfers).558 * @num_phy_chans: The number of physical channels. Read from HW. This559 * is the number of available channels for this driver, not counting "Secure560 * mode" allocated physical channels.561 * @num_log_chans: The number of logical channels. Calculated from562 * num_phy_chans.563 * @dma_both: dma_device channels that can do both memcpy and slave transfers.564 * @dma_slave: dma_device channels that can do only do slave transfers.565 * @dma_memcpy: dma_device channels that can do only do memcpy transfers.566 * @phy_chans: Room for all possible physical channels in system.567 * @log_chans: Room for all possible logical channels in system.568 * @lookup_log_chans: Used to map interrupt number to logical channel. Points569 * to log_chans entries.570 * @lookup_phy_chans: Used to map interrupt number to physical channel. Points571 * to phy_chans entries.572 * @plat_data: Pointer to provided platform_data which is the driver573 * configuration.574 * @lcpa_regulator: Pointer to hold the regulator for the esram bank for lcla.575 * @phy_res: Vector containing all physical channels.576 * @lcla_pool: lcla pool settings and data.577 * @lcpa_base: The virtual mapped address of LCPA.578 * @phy_lcpa: The physical address of the LCPA.579 * @lcpa_size: The size of the LCPA area.580 * @desc_slab: cache for descriptors.581 * @reg_val_backup: Here the values of some hardware registers are stored582 * before the DMA is powered off. They are restored when the power is back on.583 * @reg_val_backup_v4: Backup of registers that only exits on dma40 v3 and584 * later585 * @reg_val_backup_chan: Backup data for standard channel parameter registers.586 * @regs_interrupt: Scratch space for registers during interrupt.587 * @gcc_pwr_off_mask: Mask to maintain the channels that can be turned off.588 * @gen_dmac: the struct for generic registers values to represent u8500/8540589 * DMA controller590 */591struct d40_base {592	spinlock_t			 interrupt_lock;593	spinlock_t			 execmd_lock;594	struct device			 *dev;595	void __iomem			 *virtbase;596	u8				  rev:4;597	struct clk			 *clk;598	int				  irq;599	int				  num_memcpy_chans;600	int				  num_phy_chans;601	int				  num_log_chans;602	struct dma_device		  dma_both;603	struct dma_device		  dma_slave;604	struct dma_device		  dma_memcpy;605	struct d40_chan			 *phy_chans;606	struct d40_chan			 *log_chans;607	struct d40_chan			**lookup_log_chans;608	struct d40_chan			**lookup_phy_chans;609	struct stedma40_platform_data	 *plat_data;610	struct regulator		 *lcpa_regulator;611	/* Physical half channels */612	struct d40_phy_res		 *phy_res;613	struct d40_lcla_pool		  lcla_pool;614	void				 *lcpa_base;615	dma_addr_t			  phy_lcpa;616	resource_size_t			  lcpa_size;617	struct kmem_cache		 *desc_slab;618	u32				  reg_val_backup[BACKUP_REGS_SZ];619	u32				  reg_val_backup_v4[BACKUP_REGS_SZ_MAX];620	u32				 *reg_val_backup_chan;621	u32				 *regs_interrupt;622	u16				  gcc_pwr_off_mask;623	struct d40_gen_dmac		  gen_dmac;624};625 626static struct device *chan2dev(struct d40_chan *d40c)627{628	return &d40c->chan.dev->device;629}630 631static bool chan_is_physical(struct d40_chan *chan)632{633	return chan->log_num == D40_PHY_CHAN;634}635 636static bool chan_is_logical(struct d40_chan *chan)637{638	return !chan_is_physical(chan);639}640 641static void __iomem *chan_base(struct d40_chan *chan)642{643	return chan->base->virtbase + D40_DREG_PCBASE +644	       chan->phy_chan->num * D40_DREG_PCDELTA;645}646 647#define d40_err(dev, format, arg...)		\648	dev_err(dev, "[%s] " format, __func__, ## arg)649 650#define chan_err(d40c, format, arg...)		\651	d40_err(chan2dev(d40c), format, ## arg)652 653static int d40_set_runtime_config_write(struct dma_chan *chan,654				  struct dma_slave_config *config,655				  enum dma_transfer_direction direction);656 657static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,658			      int lli_len)659{660	bool is_log = chan_is_logical(d40c);661	u32 align;662	void *base;663 664	if (is_log)665		align = sizeof(struct d40_log_lli);666	else667		align = sizeof(struct d40_phy_lli);668 669	if (lli_len == 1) {670		base = d40d->lli_pool.pre_alloc_lli;671		d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);672		d40d->lli_pool.base = NULL;673	} else {674		d40d->lli_pool.size = lli_len * 2 * align;675 676		base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);677		d40d->lli_pool.base = base;678 679		if (d40d->lli_pool.base == NULL)680			return -ENOMEM;681	}682 683	if (is_log) {684		d40d->lli_log.src = PTR_ALIGN(base, align);685		d40d->lli_log.dst = d40d->lli_log.src + lli_len;686 687		d40d->lli_pool.dma_addr = 0;688	} else {689		d40d->lli_phy.src = PTR_ALIGN(base, align);690		d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;691 692		d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,693							 d40d->lli_phy.src,694							 d40d->lli_pool.size,695							 DMA_TO_DEVICE);696 697		if (dma_mapping_error(d40c->base->dev,698				      d40d->lli_pool.dma_addr)) {699			kfree(d40d->lli_pool.base);700			d40d->lli_pool.base = NULL;701			d40d->lli_pool.dma_addr = 0;702			return -ENOMEM;703		}704	}705 706	return 0;707}708 709static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)710{711	if (d40d->lli_pool.dma_addr)712		dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,713				 d40d->lli_pool.size, DMA_TO_DEVICE);714 715	kfree(d40d->lli_pool.base);716	d40d->lli_pool.base = NULL;717	d40d->lli_pool.size = 0;718	d40d->lli_log.src = NULL;719	d40d->lli_log.dst = NULL;720	d40d->lli_phy.src = NULL;721	d40d->lli_phy.dst = NULL;722}723 724static int d40_lcla_alloc_one(struct d40_chan *d40c,725			      struct d40_desc *d40d)726{727	unsigned long flags;728	int i;729	int ret = -EINVAL;730 731	spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);732 733	/*734	 * Allocate both src and dst at the same time, therefore the half735	 * start on 1 since 0 can't be used since zero is used as end marker.736	 */737	for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {738		int idx = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i;739 740		if (!d40c->base->lcla_pool.alloc_map[idx]) {741			d40c->base->lcla_pool.alloc_map[idx] = d40d;742			d40d->lcla_alloc++;743			ret = i;744			break;745		}746	}747 748	spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);749 750	return ret;751}752 753static int d40_lcla_free_all(struct d40_chan *d40c,754			     struct d40_desc *d40d)755{756	unsigned long flags;757	int i;758	int ret = -EINVAL;759 760	if (chan_is_physical(d40c))761		return 0;762 763	spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);764 765	for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {766		int idx = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i;767 768		if (d40c->base->lcla_pool.alloc_map[idx] == d40d) {769			d40c->base->lcla_pool.alloc_map[idx] = NULL;770			d40d->lcla_alloc--;771			if (d40d->lcla_alloc == 0) {772				ret = 0;773				break;774			}775		}776	}777 778	spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);779 780	return ret;781 782}783 784static void d40_desc_remove(struct d40_desc *d40d)785{786	list_del(&d40d->node);787}788 789static struct d40_desc *d40_desc_get(struct d40_chan *d40c)790{791	struct d40_desc *desc = NULL;792 793	if (!list_empty(&d40c->client)) {794		struct d40_desc *d;795		struct d40_desc *_d;796 797		list_for_each_entry_safe(d, _d, &d40c->client, node) {798			if (async_tx_test_ack(&d->txd)) {799				d40_desc_remove(d);800				desc = d;801				memset(desc, 0, sizeof(*desc));802				break;803			}804		}805	}806 807	if (!desc)808		desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT);809 810	if (desc)811		INIT_LIST_HEAD(&desc->node);812 813	return desc;814}815 816static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)817{818 819	d40_pool_lli_free(d40c, d40d);820	d40_lcla_free_all(d40c, d40d);821	kmem_cache_free(d40c->base->desc_slab, d40d);822}823 824static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)825{826	list_add_tail(&desc->node, &d40c->active);827}828 829static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc)830{831	struct d40_phy_lli *lli_dst = desc->lli_phy.dst;832	struct d40_phy_lli *lli_src = desc->lli_phy.src;833	void __iomem *base = chan_base(chan);834 835	writel(lli_src->reg_cfg, base + D40_CHAN_REG_SSCFG);836	writel(lli_src->reg_elt, base + D40_CHAN_REG_SSELT);837	writel(lli_src->reg_ptr, base + D40_CHAN_REG_SSPTR);838	writel(lli_src->reg_lnk, base + D40_CHAN_REG_SSLNK);839 840	writel(lli_dst->reg_cfg, base + D40_CHAN_REG_SDCFG);841	writel(lli_dst->reg_elt, base + D40_CHAN_REG_SDELT);842	writel(lli_dst->reg_ptr, base + D40_CHAN_REG_SDPTR);843	writel(lli_dst->reg_lnk, base + D40_CHAN_REG_SDLNK);844}845 846static void d40_desc_done(struct d40_chan *d40c, struct d40_desc *desc)847{848	list_add_tail(&desc->node, &d40c->done);849}850 851static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc)852{853	struct d40_lcla_pool *pool = &chan->base->lcla_pool;854	struct d40_log_lli_bidir *lli = &desc->lli_log;855	int lli_current = desc->lli_current;856	int lli_len = desc->lli_len;857	bool cyclic = desc->cyclic;858	int curr_lcla = -EINVAL;859	int first_lcla = 0;860	bool use_esram_lcla = chan->base->plat_data->use_esram_lcla;861	bool linkback;862 863	/*864	 * We may have partially running cyclic transfers, in case we did't get865	 * enough LCLA entries.866	 */867	linkback = cyclic && lli_current == 0;868 869	/*870	 * For linkback, we need one LCLA even with only one link, because we871	 * can't link back to the one in LCPA space872	 */873	if (linkback || (lli_len - lli_current > 1)) {874		/*875		 * If the channel is expected to use only soft_lli don't876		 * allocate a lcla. This is to avoid a HW issue that exists877		 * in some controller during a peripheral to memory transfer878		 * that uses linked lists.879		 */880		if (!(chan->phy_chan->use_soft_lli &&881			chan->dma_cfg.dir == DMA_DEV_TO_MEM))882			curr_lcla = d40_lcla_alloc_one(chan, desc);883 884		first_lcla = curr_lcla;885	}886 887	/*888	 * For linkback, we normally load the LCPA in the loop since we need to889	 * link it to the second LCLA and not the first.  However, if we890	 * couldn't even get a first LCLA, then we have to run in LCPA and891	 * reload manually.892	 */893	if (!linkback || curr_lcla == -EINVAL) {894		unsigned int flags = 0;895 896		if (curr_lcla == -EINVAL)897			flags |= LLI_TERM_INT;898 899		d40_log_lli_lcpa_write(chan->lcpa,900				       &lli->dst[lli_current],901				       &lli->src[lli_current],902				       curr_lcla,903				       flags);904		lli_current++;905	}906 907	if (curr_lcla < 0)908		goto set_current;909 910	for (; lli_current < lli_len; lli_current++) {911		unsigned int lcla_offset = chan->phy_chan->num * 1024 +912					   8 * curr_lcla * 2;913		struct d40_log_lli *lcla = pool->base + lcla_offset;914		unsigned int flags = 0;915		int next_lcla;916 917		if (lli_current + 1 < lli_len)918			next_lcla = d40_lcla_alloc_one(chan, desc);919		else920			next_lcla = linkback ? first_lcla : -EINVAL;921 922		if (cyclic || next_lcla == -EINVAL)923			flags |= LLI_TERM_INT;924 925		if (linkback && curr_lcla == first_lcla) {926			/* First link goes in both LCPA and LCLA */927			d40_log_lli_lcpa_write(chan->lcpa,928					       &lli->dst[lli_current],929					       &lli->src[lli_current],930					       next_lcla, flags);931		}932 933		/*934		 * One unused LCLA in the cyclic case if the very first935		 * next_lcla fails...936		 */937		d40_log_lli_lcla_write(lcla,938				       &lli->dst[lli_current],939				       &lli->src[lli_current],940				       next_lcla, flags);941 942		/*943		 * Cache maintenance is not needed if lcla is944		 * mapped in esram945		 */946		if (!use_esram_lcla) {947			dma_sync_single_range_for_device(chan->base->dev,948						pool->dma_addr, lcla_offset,949						2 * sizeof(struct d40_log_lli),950						DMA_TO_DEVICE);951		}952		curr_lcla = next_lcla;953 954		if (curr_lcla == -EINVAL || curr_lcla == first_lcla) {955			lli_current++;956			break;957		}958	}959 set_current:960	desc->lli_current = lli_current;961}962 963static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)964{965	if (chan_is_physical(d40c)) {966		d40_phy_lli_load(d40c, d40d);967		d40d->lli_current = d40d->lli_len;968	} else969		d40_log_lli_to_lcxa(d40c, d40d);970}971 972static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)973{974	return list_first_entry_or_null(&d40c->active, struct d40_desc, node);975}976 977/* remove desc from current queue and add it to the pending_queue */978static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)979{980	d40_desc_remove(desc);981	desc->is_in_client_list = false;982	list_add_tail(&desc->node, &d40c->pending_queue);983}984 985static struct d40_desc *d40_first_pending(struct d40_chan *d40c)986{987	return list_first_entry_or_null(&d40c->pending_queue, struct d40_desc,988					node);989}990 991static struct d40_desc *d40_first_queued(struct d40_chan *d40c)992{993	return list_first_entry_or_null(&d40c->queue, struct d40_desc, node);994}995 996static struct d40_desc *d40_first_done(struct d40_chan *d40c)997{998	return list_first_entry_or_null(&d40c->done, struct d40_desc, node);999}1000 1001static int d40_psize_2_burst_size(bool is_log, int psize)1002{1003	if (is_log) {1004		if (psize == STEDMA40_PSIZE_LOG_1)1005			return 1;1006	} else {1007		if (psize == STEDMA40_PSIZE_PHY_1)1008			return 1;1009	}1010 1011	return 2 << psize;1012}1013 1014/*1015 * The dma only supports transmitting packages up to1016 * STEDMA40_MAX_SEG_SIZE * data_width, where data_width is stored in Bytes.1017 *1018 * Calculate the total number of dma elements required to send the entire sg list.1019 */1020static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)1021{1022	int dmalen;1023	u32 max_w = max(data_width1, data_width2);1024	u32 min_w = min(data_width1, data_width2);1025	u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE * min_w, max_w);1026 1027	if (seg_max > STEDMA40_MAX_SEG_SIZE)1028		seg_max -= max_w;1029 1030	if (!IS_ALIGNED(size, max_w))1031		return -EINVAL;1032 1033	if (size <= seg_max)1034		dmalen = 1;1035	else {1036		dmalen = size / seg_max;1037		if (dmalen * seg_max < size)1038			dmalen++;1039	}1040	return dmalen;1041}1042 1043static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,1044			   u32 data_width1, u32 data_width2)1045{1046	struct scatterlist *sg;1047	int i;1048	int len = 0;1049	int ret;1050 1051	for_each_sg(sgl, sg, sg_len, i) {1052		ret = d40_size_2_dmalen(sg_dma_len(sg),1053					data_width1, data_width2);1054		if (ret < 0)1055			return ret;1056		len += ret;1057	}1058	return len;1059}1060 1061static int __d40_execute_command_phy(struct d40_chan *d40c,1062				     enum d40_command command)1063{1064	u32 status;1065	int i;1066	void __iomem *active_reg;1067	int ret = 0;1068	unsigned long flags;1069	u32 wmask;1070 1071	if (command == D40_DMA_STOP) {1072		ret = __d40_execute_command_phy(d40c, D40_DMA_SUSPEND_REQ);1073		if (ret)1074			return ret;1075	}1076 1077	spin_lock_irqsave(&d40c->base->execmd_lock, flags);1078 1079	if (d40c->phy_chan->num % 2 == 0)1080		active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;1081	else1082		active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;1083 1084	if (command == D40_DMA_SUSPEND_REQ) {1085		status = (readl(active_reg) &1086			  D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>1087			D40_CHAN_POS(d40c->phy_chan->num);1088 1089		if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)1090			goto unlock;1091	}1092 1093	wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));1094	writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),1095	       active_reg);1096 1097	if (command == D40_DMA_SUSPEND_REQ) {1098 1099		for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {1100			status = (readl(active_reg) &1101				  D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>1102				D40_CHAN_POS(d40c->phy_chan->num);1103 1104			cpu_relax();1105			/*1106			 * Reduce the number of bus accesses while1107			 * waiting for the DMA to suspend.1108			 */1109			udelay(3);1110 1111			if (status == D40_DMA_STOP ||1112			    status == D40_DMA_SUSPENDED)1113				break;1114		}1115 1116		if (i == D40_SUSPEND_MAX_IT) {1117			chan_err(d40c,1118				"unable to suspend the chl %d (log: %d) status %x\n",1119				d40c->phy_chan->num, d40c->log_num,1120				status);1121			dump_stack();1122			ret = -EBUSY;1123		}1124 1125	}1126 unlock:1127	spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);1128	return ret;1129}1130 1131static void d40_term_all(struct d40_chan *d40c)1132{1133	struct d40_desc *d40d;1134	struct d40_desc *_d;1135 1136	/* Release completed descriptors */1137	while ((d40d = d40_first_done(d40c))) {1138		d40_desc_remove(d40d);1139		d40_desc_free(d40c, d40d);1140	}1141 1142	/* Release active descriptors */1143	while ((d40d = d40_first_active_get(d40c))) {1144		d40_desc_remove(d40d);1145		d40_desc_free(d40c, d40d);1146	}1147 1148	/* Release queued descriptors waiting for transfer */1149	while ((d40d = d40_first_queued(d40c))) {1150		d40_desc_remove(d40d);1151		d40_desc_free(d40c, d40d);1152	}1153 1154	/* Release pending descriptors */1155	while ((d40d = d40_first_pending(d40c))) {1156		d40_desc_remove(d40d);1157		d40_desc_free(d40c, d40d);1158	}1159 1160	/* Release client owned descriptors */1161	if (!list_empty(&d40c->client))1162		list_for_each_entry_safe(d40d, _d, &d40c->client, node) {1163			d40_desc_remove(d40d);1164			d40_desc_free(d40c, d40d);1165		}1166 1167	/* Release descriptors in prepare queue */1168	if (!list_empty(&d40c->prepare_queue))1169		list_for_each_entry_safe(d40d, _d,1170					 &d40c->prepare_queue, node) {1171			d40_desc_remove(d40d);1172			d40_desc_free(d40c, d40d);1173		}1174 1175	d40c->pending_tx = 0;1176}1177 1178static void __d40_config_set_event(struct d40_chan *d40c,1179				   enum d40_events event_type, u32 event,1180				   int reg)1181{1182	void __iomem *addr = chan_base(d40c) + reg;1183	int tries;1184	u32 status;1185 1186	switch (event_type) {1187 1188	case D40_DEACTIVATE_EVENTLINE:1189 1190		writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))1191		       | ~D40_EVENTLINE_MASK(event), addr);1192		break;1193 1194	case D40_SUSPEND_REQ_EVENTLINE:1195		status = (readl(addr) & D40_EVENTLINE_MASK(event)) >>1196			  D40_EVENTLINE_POS(event);1197 1198		if (status == D40_DEACTIVATE_EVENTLINE ||1199		    status == D40_SUSPEND_REQ_EVENTLINE)1200			break;1201 1202		writel((D40_SUSPEND_REQ_EVENTLINE << D40_EVENTLINE_POS(event))1203		       | ~D40_EVENTLINE_MASK(event), addr);1204 1205		for (tries = 0 ; tries < D40_SUSPEND_MAX_IT; tries++) {1206 1207			status = (readl(addr) & D40_EVENTLINE_MASK(event)) >>1208				  D40_EVENTLINE_POS(event);1209 1210			cpu_relax();1211			/*1212			 * Reduce the number of bus accesses while1213			 * waiting for the DMA to suspend.1214			 */1215			udelay(3);1216 1217			if (status == D40_DEACTIVATE_EVENTLINE)1218				break;1219		}1220 1221		if (tries == D40_SUSPEND_MAX_IT) {1222			chan_err(d40c,1223				"unable to stop the event_line chl %d (log: %d)"1224				"status %x\n", d40c->phy_chan->num,1225				 d40c->log_num, status);1226		}1227		break;1228 1229	case D40_ACTIVATE_EVENTLINE:1230	/*1231	 * The hardware sometimes doesn't register the enable when src and dst1232	 * event lines are active on the same logical channel.  Retry to ensure1233	 * it does.  Usually only one retry is sufficient.1234	 */1235		tries = 100;1236		while (--tries) {1237			writel((D40_ACTIVATE_EVENTLINE <<1238				D40_EVENTLINE_POS(event)) |1239				~D40_EVENTLINE_MASK(event), addr);1240 1241			if (readl(addr) & D40_EVENTLINE_MASK(event))1242				break;1243		}1244 1245		if (tries != 99)1246			dev_dbg(chan2dev(d40c),1247				"[%s] workaround enable S%cLNK (%d tries)\n",1248				__func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',1249				100 - tries);1250 1251		WARN_ON(!tries);1252		break;1253 1254	case D40_ROUND_EVENTLINE:1255		BUG();1256		break;1257 1258	}1259}1260 1261static void d40_config_set_event(struct d40_chan *d40c,1262				 enum d40_events event_type)1263{1264	u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dev_type);1265 1266	/* Enable event line connected to device (or memcpy) */1267	if ((d40c->dma_cfg.dir == DMA_DEV_TO_MEM) ||1268	    (d40c->dma_cfg.dir == DMA_DEV_TO_DEV))1269		__d40_config_set_event(d40c, event_type, event,1270				       D40_CHAN_REG_SSLNK);1271 1272	if (d40c->dma_cfg.dir !=  DMA_DEV_TO_MEM)1273		__d40_config_set_event(d40c, event_type, event,1274				       D40_CHAN_REG_SDLNK);1275}1276 1277static u32 d40_chan_has_events(struct d40_chan *d40c)1278{1279	void __iomem *chanbase = chan_base(d40c);1280	u32 val;1281 1282	val = readl(chanbase + D40_CHAN_REG_SSLNK);1283	val |= readl(chanbase + D40_CHAN_REG_SDLNK);1284 1285	return val;1286}1287 1288static int1289__d40_execute_command_log(struct d40_chan *d40c, enum d40_command command)1290{1291	unsigned long flags;1292	int ret = 0;1293	u32 active_status;1294	void __iomem *active_reg;1295 1296	if (d40c->phy_chan->num % 2 == 0)1297		active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;1298	else1299		active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;1300 1301 1302	spin_lock_irqsave(&d40c->phy_chan->lock, flags);1303 1304	switch (command) {1305	case D40_DMA_STOP:1306	case D40_DMA_SUSPEND_REQ:1307 1308		active_status = (readl(active_reg) &1309				 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>1310				 D40_CHAN_POS(d40c->phy_chan->num);1311 1312		if (active_status == D40_DMA_RUN)1313			d40_config_set_event(d40c, D40_SUSPEND_REQ_EVENTLINE);1314		else1315			d40_config_set_event(d40c, D40_DEACTIVATE_EVENTLINE);1316 1317		if (!d40_chan_has_events(d40c) && (command == D40_DMA_STOP))1318			ret = __d40_execute_command_phy(d40c, command);1319 1320		break;1321 1322	case D40_DMA_RUN:1323 1324		d40_config_set_event(d40c, D40_ACTIVATE_EVENTLINE);1325		ret = __d40_execute_command_phy(d40c, command);1326		break;1327 1328	case D40_DMA_SUSPENDED:1329		BUG();1330		break;1331	}1332 1333	spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);1334	return ret;1335}1336 1337static int d40_channel_execute_command(struct d40_chan *d40c,1338				       enum d40_command command)1339{1340	if (chan_is_logical(d40c))1341		return __d40_execute_command_log(d40c, command);1342	else1343		return __d40_execute_command_phy(d40c, command);1344}1345 1346static u32 d40_get_prmo(struct d40_chan *d40c)1347{1348	static const unsigned int phy_map[] = {1349		[STEDMA40_PCHAN_BASIC_MODE]1350			= D40_DREG_PRMO_PCHAN_BASIC,1351		[STEDMA40_PCHAN_MODULO_MODE]1352			= D40_DREG_PRMO_PCHAN_MODULO,1353		[STEDMA40_PCHAN_DOUBLE_DST_MODE]1354			= D40_DREG_PRMO_PCHAN_DOUBLE_DST,1355	};1356	static const unsigned int log_map[] = {1357		[STEDMA40_LCHAN_SRC_PHY_DST_LOG]1358			= D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG,1359		[STEDMA40_LCHAN_SRC_LOG_DST_PHY]1360			= D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY,1361		[STEDMA40_LCHAN_SRC_LOG_DST_LOG]1362			= D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,1363	};1364 1365	if (chan_is_physical(d40c))1366		return phy_map[d40c->dma_cfg.mode_opt];1367	else1368		return log_map[d40c->dma_cfg.mode_opt];1369}1370 1371static void d40_config_write(struct d40_chan *d40c)1372{1373	u32 addr_base;1374	u32 var;1375 1376	/* Odd addresses are even addresses + 4 */1377	addr_base = (d40c->phy_chan->num % 2) * 4;1378	/* Setup channel mode to logical or physical */1379	var = ((u32)(chan_is_logical(d40c)) + 1) <<1380		D40_CHAN_POS(d40c->phy_chan->num);1381	writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);1382 1383	/* Setup operational mode option register */1384	var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num);1385 1386	writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);1387 1388	if (chan_is_logical(d40c)) {1389		int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS)1390			   & D40_SREG_ELEM_LOG_LIDX_MASK;1391		void __iomem *chanbase = chan_base(d40c);1392 1393		/* Set default config for CFG reg */1394		writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG);1395		writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG);1396 1397		/* Set LIDX for lcla */1398		writel(lidx, chanbase + D40_CHAN_REG_SSELT);1399		writel(lidx, chanbase + D40_CHAN_REG_SDELT);1400 1401		/* Clear LNK which will be used by d40_chan_has_events() */1402		writel(0, chanbase + D40_CHAN_REG_SSLNK);1403		writel(0, chanbase + D40_CHAN_REG_SDLNK);1404	}1405}1406 1407static u32 d40_residue(struct d40_chan *d40c)1408{1409	u32 num_elt;1410 1411	if (chan_is_logical(d40c))1412		num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)1413			>> D40_MEM_LCSP2_ECNT_POS;1414	else {1415		u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT);1416		num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK)1417			  >> D40_SREG_ELEM_PHY_ECNT_POS;1418	}1419 1420	return num_elt * d40c->dma_cfg.dst_info.data_width;1421}1422 1423static bool d40_tx_is_linked(struct d40_chan *d40c)1424{1425	bool is_link;1426 1427	if (chan_is_logical(d40c))1428		is_link = readl(&d40c->lcpa->lcsp3) &  D40_MEM_LCSP3_DLOS_MASK;1429	else1430		is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK)1431			  & D40_SREG_LNK_PHYS_LNK_MASK;1432 1433	return is_link;1434}1435 1436static int d40_pause(struct dma_chan *chan)1437{1438	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);1439	int res = 0;1440	unsigned long flags;1441 1442	if (d40c->phy_chan == NULL) {1443		chan_err(d40c, "Channel is not allocated!\n");1444		return -EINVAL;1445	}1446 1447	if (!d40c->busy)1448		return 0;1449 1450	spin_lock_irqsave(&d40c->lock, flags);1451	pm_runtime_get_sync(d40c->base->dev);1452 1453	res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);1454 1455	pm_runtime_mark_last_busy(d40c->base->dev);1456	pm_runtime_put_autosuspend(d40c->base->dev);1457	spin_unlock_irqrestore(&d40c->lock, flags);1458	return res;1459}1460 1461static int d40_resume(struct dma_chan *chan)1462{1463	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);1464	int res = 0;1465	unsigned long flags;1466 1467	if (d40c->phy_chan == NULL) {1468		chan_err(d40c, "Channel is not allocated!\n");1469		return -EINVAL;1470	}1471 1472	if (!d40c->busy)1473		return 0;1474 1475	spin_lock_irqsave(&d40c->lock, flags);1476	pm_runtime_get_sync(d40c->base->dev);1477 1478	/* If bytes left to transfer or linked tx resume job */1479	if (d40_residue(d40c) || d40_tx_is_linked(d40c))1480		res = d40_channel_execute_command(d40c, D40_DMA_RUN);1481 1482	pm_runtime_mark_last_busy(d40c->base->dev);1483	pm_runtime_put_autosuspend(d40c->base->dev);1484	spin_unlock_irqrestore(&d40c->lock, flags);1485	return res;1486}1487 1488static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)1489{1490	struct d40_chan *d40c = container_of(tx->chan,1491					     struct d40_chan,1492					     chan);1493	struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);1494	unsigned long flags;1495	dma_cookie_t cookie;1496 1497	spin_lock_irqsave(&d40c->lock, flags);1498	cookie = dma_cookie_assign(tx);1499	d40_desc_queue(d40c, d40d);1500	spin_unlock_irqrestore(&d40c->lock, flags);1501 1502	return cookie;1503}1504 1505static int d40_start(struct d40_chan *d40c)1506{1507	return d40_channel_execute_command(d40c, D40_DMA_RUN);1508}1509 1510static struct d40_desc *d40_queue_start(struct d40_chan *d40c)1511{1512	struct d40_desc *d40d;1513	int err;1514 1515	/* Start queued jobs, if any */1516	d40d = d40_first_queued(d40c);1517 1518	if (d40d != NULL) {1519		if (!d40c->busy) {1520			d40c->busy = true;1521			pm_runtime_get_sync(d40c->base->dev);1522		}1523 1524		/* Remove from queue */1525		d40_desc_remove(d40d);1526 1527		/* Add to active queue */1528		d40_desc_submit(d40c, d40d);1529 1530		/* Initiate DMA job */1531		d40_desc_load(d40c, d40d);1532 1533		/* Start dma job */1534		err = d40_start(d40c);1535 1536		if (err)1537			return NULL;1538	}1539 1540	return d40d;1541}1542 1543/* called from interrupt context */1544static void dma_tc_handle(struct d40_chan *d40c)1545{1546	struct d40_desc *d40d;1547 1548	/* Get first active entry from list */1549	d40d = d40_first_active_get(d40c);1550 1551	if (d40d == NULL)1552		return;1553 1554	if (d40d->cyclic) {1555		/*1556		 * If this was a paritially loaded list, we need to reloaded1557		 * it, and only when the list is completed.  We need to check1558		 * for done because the interrupt will hit for every link, and1559		 * not just the last one.1560		 */1561		if (d40d->lli_current < d40d->lli_len1562		    && !d40_tx_is_linked(d40c)1563		    && !d40_residue(d40c)) {1564			d40_lcla_free_all(d40c, d40d);1565			d40_desc_load(d40c, d40d);1566			(void) d40_start(d40c);1567 1568			if (d40d->lli_current == d40d->lli_len)1569				d40d->lli_current = 0;1570		}1571	} else {1572		d40_lcla_free_all(d40c, d40d);1573 1574		if (d40d->lli_current < d40d->lli_len) {1575			d40_desc_load(d40c, d40d);1576			/* Start dma job */1577			(void) d40_start(d40c);1578			return;1579		}1580 1581		if (d40_queue_start(d40c) == NULL) {1582			d40c->busy = false;1583 1584			pm_runtime_mark_last_busy(d40c->base->dev);1585			pm_runtime_put_autosuspend(d40c->base->dev);1586		}1587 1588		d40_desc_remove(d40d);1589		d40_desc_done(d40c, d40d);1590	}1591 1592	d40c->pending_tx++;1593	tasklet_schedule(&d40c->tasklet);1594 1595}1596 1597static void dma_tasklet(struct tasklet_struct *t)1598{1599	struct d40_chan *d40c = from_tasklet(d40c, t, tasklet);1600	struct d40_desc *d40d;1601	unsigned long flags;1602	bool callback_active;1603	struct dmaengine_desc_callback cb;1604 1605	spin_lock_irqsave(&d40c->lock, flags);1606 1607	/* Get first entry from the done list */1608	d40d = d40_first_done(d40c);1609	if (d40d == NULL) {1610		/* Check if we have reached here for cyclic job */1611		d40d = d40_first_active_get(d40c);1612		if (d40d == NULL || !d40d->cyclic)1613			goto check_pending_tx;1614	}1615 1616	if (!d40d->cyclic)1617		dma_cookie_complete(&d40d->txd);1618 1619	/*1620	 * If terminating a channel pending_tx is set to zero.1621	 * This prevents any finished active jobs to return to the client.1622	 */1623	if (d40c->pending_tx == 0) {1624		spin_unlock_irqrestore(&d40c->lock, flags);1625		return;1626	}1627 1628	/* Callback to client */1629	callback_active = !!(d40d->txd.flags & DMA_PREP_INTERRUPT);1630	dmaengine_desc_get_callback(&d40d->txd, &cb);1631 1632	if (!d40d->cyclic) {1633		if (async_tx_test_ack(&d40d->txd)) {1634			d40_desc_remove(d40d);1635			d40_desc_free(d40c, d40d);1636		} else if (!d40d->is_in_client_list) {1637			d40_desc_remove(d40d);1638			d40_lcla_free_all(d40c, d40d);1639			list_add_tail(&d40d->node, &d40c->client);1640			d40d->is_in_client_list = true;1641		}1642	}1643 1644	d40c->pending_tx--;1645 1646	if (d40c->pending_tx)1647		tasklet_schedule(&d40c->tasklet);1648 1649	spin_unlock_irqrestore(&d40c->lock, flags);1650 1651	if (callback_active)1652		dmaengine_desc_callback_invoke(&cb, NULL);1653 1654	return;1655 check_pending_tx:1656	/* Rescue maneuver if receiving double interrupts */1657	if (d40c->pending_tx > 0)1658		d40c->pending_tx--;1659	spin_unlock_irqrestore(&d40c->lock, flags);1660}1661 1662static irqreturn_t d40_handle_interrupt(int irq, void *data)1663{1664	int i;1665	u32 idx;1666	u32 row;1667	long chan = -1;1668	struct d40_chan *d40c;1669	struct d40_base *base = data;1670	u32 *regs = base->regs_interrupt;1671	struct d40_interrupt_lookup *il = base->gen_dmac.il;1672	u32 il_size = base->gen_dmac.il_size;1673 1674	spin_lock(&base->interrupt_lock);1675 1676	/* Read interrupt status of both logical and physical channels */1677	for (i = 0; i < il_size; i++)1678		regs[i] = readl(base->virtbase + il[i].src);1679 1680	for (;;) {1681 1682		chan = find_next_bit((unsigned long *)regs,1683				     BITS_PER_LONG * il_size, chan + 1);1684 1685		/* No more set bits found? */1686		if (chan == BITS_PER_LONG * il_size)1687			break;1688 1689		row = chan / BITS_PER_LONG;1690		idx = chan & (BITS_PER_LONG - 1);1691 1692		if (il[row].offset == D40_PHY_CHAN)1693			d40c = base->lookup_phy_chans[idx];1694		else1695			d40c = base->lookup_log_chans[il[row].offset + idx];1696 1697		if (!d40c) {1698			/*1699			 * No error because this can happen if something else1700			 * in the system is using the channel.1701			 */1702			continue;1703		}1704 1705		/* ACK interrupt */1706		writel(BIT(idx), base->virtbase + il[row].clr);1707 1708		spin_lock(&d40c->lock);1709 1710		if (!il[row].is_error)1711			dma_tc_handle(d40c);1712		else1713			d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n",1714				chan, il[row].offset, idx);1715 1716		spin_unlock(&d40c->lock);1717	}1718 1719	spin_unlock(&base->interrupt_lock);1720 1721	return IRQ_HANDLED;1722}1723 1724static int d40_validate_conf(struct d40_chan *d40c,1725			     struct stedma40_chan_cfg *conf)1726{1727	int res = 0;1728	bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;1729 1730	if (!conf->dir) {1731		chan_err(d40c, "Invalid direction.\n");1732		res = -EINVAL;1733	}1734 1735	if ((is_log && conf->dev_type > d40c->base->num_log_chans)  ||1736	    (!is_log && conf->dev_type > d40c->base->num_phy_chans) ||1737	    (conf->dev_type < 0)) {1738		chan_err(d40c, "Invalid device type (%d)\n", conf->dev_type);1739		res = -EINVAL;1740	}1741 1742	if (conf->dir == DMA_DEV_TO_DEV) {1743		/*1744		 * DMAC HW supports it. Will be added to this driver,1745		 * in case any dma client requires it.1746		 */1747		chan_err(d40c, "periph to periph not supported\n");1748		res = -EINVAL;1749	}1750 1751	if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *1752	    conf->src_info.data_width !=1753	    d40_psize_2_burst_size(is_log, conf->dst_info.psize) *1754	    conf->dst_info.data_width) {1755		/*1756		 * The DMAC hardware only supports1757		 * src (burst x width) == dst (burst x width)1758		 */1759 1760		chan_err(d40c, "src (burst x width) != dst (burst x width)\n");1761		res = -EINVAL;1762	}1763 1764	return res;1765}1766 1767static bool d40_alloc_mask_set(struct d40_phy_res *phy,1768			       bool is_src, int log_event_line, bool is_log,1769			       bool *first_user)1770{1771	unsigned long flags;1772	spin_lock_irqsave(&phy->lock, flags);1773 1774	*first_user = ((phy->allocated_src | phy->allocated_dst)1775			== D40_ALLOC_FREE);1776 1777	if (!is_log) {1778		/* Physical interrupts are masked per physical full channel */1779		if (phy->allocated_src == D40_ALLOC_FREE &&1780		    phy->allocated_dst == D40_ALLOC_FREE) {1781			phy->allocated_dst = D40_ALLOC_PHY;1782			phy->allocated_src = D40_ALLOC_PHY;1783			goto found_unlock;1784		} else1785			goto not_found_unlock;1786	}1787 1788	/* Logical channel */1789	if (is_src) {1790		if (phy->allocated_src == D40_ALLOC_PHY)1791			goto not_found_unlock;1792 1793		if (phy->allocated_src == D40_ALLOC_FREE)1794			phy->allocated_src = D40_ALLOC_LOG_FREE;1795 1796		if (!(phy->allocated_src & BIT(log_event_line))) {1797			phy->allocated_src |= BIT(log_event_line);1798			goto found_unlock;1799		} else1800			goto not_found_unlock;1801	} else {1802		if (phy->allocated_dst == D40_ALLOC_PHY)1803			goto not_found_unlock;1804 1805		if (phy->allocated_dst == D40_ALLOC_FREE)1806			phy->allocated_dst = D40_ALLOC_LOG_FREE;1807 1808		if (!(phy->allocated_dst & BIT(log_event_line))) {1809			phy->allocated_dst |= BIT(log_event_line);1810			goto found_unlock;1811		}1812	}1813 not_found_unlock:1814	spin_unlock_irqrestore(&phy->lock, flags);1815	return false;1816 found_unlock:1817	spin_unlock_irqrestore(&phy->lock, flags);1818	return true;1819}1820 1821static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,1822			       int log_event_line)1823{1824	unsigned long flags;1825	bool is_free = false;1826 1827	spin_lock_irqsave(&phy->lock, flags);1828	if (!log_event_line) {1829		phy->allocated_dst = D40_ALLOC_FREE;1830		phy->allocated_src = D40_ALLOC_FREE;1831		is_free = true;1832		goto unlock;1833	}1834 1835	/* Logical channel */1836	if (is_src) {1837		phy->allocated_src &= ~BIT(log_event_line);1838		if (phy->allocated_src == D40_ALLOC_LOG_FREE)1839			phy->allocated_src = D40_ALLOC_FREE;1840	} else {1841		phy->allocated_dst &= ~BIT(log_event_line);1842		if (phy->allocated_dst == D40_ALLOC_LOG_FREE)1843			phy->allocated_dst = D40_ALLOC_FREE;1844	}1845 1846	is_free = ((phy->allocated_src | phy->allocated_dst) ==1847		   D40_ALLOC_FREE);1848 unlock:1849	spin_unlock_irqrestore(&phy->lock, flags);1850 1851	return is_free;1852}1853 1854static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user)1855{1856	int dev_type = d40c->dma_cfg.dev_type;1857	int event_group;1858	int event_line;1859	struct d40_phy_res *phys;1860	int i;1861	int j;1862	int log_num;1863	int num_phy_chans;1864	bool is_src;1865	bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;1866 1867	phys = d40c->base->phy_res;1868	num_phy_chans = d40c->base->num_phy_chans;1869 1870	if (d40c->dma_cfg.dir == DMA_DEV_TO_MEM) {1871		log_num = 2 * dev_type;1872		is_src = true;1873	} else if (d40c->dma_cfg.dir == DMA_MEM_TO_DEV ||1874		   d40c->dma_cfg.dir == DMA_MEM_TO_MEM) {1875		/* dst event lines are used for logical memcpy */1876		log_num = 2 * dev_type + 1;1877		is_src = false;1878	} else1879		return -EINVAL;1880 1881	event_group = D40_TYPE_TO_GROUP(dev_type);1882	event_line = D40_TYPE_TO_EVENT(dev_type);1883 1884	if (!is_log) {1885		if (d40c->dma_cfg.dir == DMA_MEM_TO_MEM) {1886			/* Find physical half channel */1887			if (d40c->dma_cfg.use_fixed_channel) {1888				i = d40c->dma_cfg.phy_channel;1889				if (d40_alloc_mask_set(&phys[i], is_src,1890						       0, is_log,1891						       first_phy_user))1892					goto found_phy;1893			} else {1894				for (i = 0; i < num_phy_chans; i++) {1895					if (d40_alloc_mask_set(&phys[i], is_src,1896						       0, is_log,1897						       first_phy_user))1898						goto found_phy;1899				}1900			}1901		} else1902			for (j = 0; j < d40c->base->num_phy_chans; j += 8) {1903				int phy_num = j  + event_group * 2;1904				for (i = phy_num; i < phy_num + 2; i++) {1905					if (d40_alloc_mask_set(&phys[i],1906							       is_src,1907							       0,1908							       is_log,1909							       first_phy_user))1910						goto found_phy;1911				}1912			}1913		return -EINVAL;1914found_phy:1915		d40c->phy_chan = &phys[i];1916		d40c->log_num = D40_PHY_CHAN;1917		goto out;1918	}1919	if (dev_type == -1)1920		return -EINVAL;1921 1922	/* Find logical channel */1923	for (j = 0; j < d40c->base->num_phy_chans; j += 8) {1924		int phy_num = j + event_group * 2;1925 1926		if (d40c->dma_cfg.use_fixed_channel) {1927			i = d40c->dma_cfg.phy_channel;1928 1929			if ((i != phy_num) && (i != phy_num + 1)) {1930				dev_err(chan2dev(d40c),1931					"invalid fixed phy channel %d\n", i);1932				return -EINVAL;1933			}1934 1935			if (d40_alloc_mask_set(&phys[i], is_src, event_line,1936					       is_log, first_phy_user))1937				goto found_log;1938 1939			dev_err(chan2dev(d40c),1940				"could not allocate fixed phy channel %d\n", i);1941			return -EINVAL;1942		}1943 1944		/*1945		 * Spread logical channels across all available physical rather1946		 * than pack every logical channel at the first available phy1947		 * channels.1948		 */1949		if (is_src) {1950			for (i = phy_num; i < phy_num + 2; i++) {1951				if (d40_alloc_mask_set(&phys[i], is_src,1952						       event_line, is_log,1953						       first_phy_user))1954					goto found_log;1955			}1956		} else {1957			for (i = phy_num + 1; i >= phy_num; i--) {1958				if (d40_alloc_mask_set(&phys[i], is_src,1959						       event_line, is_log,1960						       first_phy_user))1961					goto found_log;1962			}1963		}1964	}1965	return -EINVAL;1966 1967found_log:1968	d40c->phy_chan = &phys[i];1969	d40c->log_num = log_num;1970out:1971 1972	if (is_log)1973		d40c->base->lookup_log_chans[d40c->log_num] = d40c;1974	else1975		d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;1976 1977	return 0;1978 1979}1980 1981static int d40_config_memcpy(struct d40_chan *d40c)1982{1983	dma_cap_mask_t cap = d40c->chan.device->cap_mask;1984 1985	if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {1986		d40c->dma_cfg = dma40_memcpy_conf_log;1987		d40c->dma_cfg.dev_type = dma40_memcpy_channels[d40c->chan.chan_id];1988 1989		d40_log_cfg(&d40c->dma_cfg,1990			    &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);1991 1992	} else if (dma_has_cap(DMA_MEMCPY, cap) &&1993		   dma_has_cap(DMA_SLAVE, cap)) {1994		d40c->dma_cfg = dma40_memcpy_conf_phy;1995 1996		/* Generate interrupt at end of transfer or relink. */1997		d40c->dst_def_cfg |= BIT(D40_SREG_CFG_TIM_POS);1998 1999		/* Generate interrupt on error. */2000		d40c->src_def_cfg |= BIT(D40_SREG_CFG_EIM_POS);2001		d40c->dst_def_cfg |= BIT(D40_SREG_CFG_EIM_POS);2002 2003	} else {2004		chan_err(d40c, "No memcpy\n");2005		return -EINVAL;2006	}2007 2008	return 0;2009}2010 2011static int d40_free_dma(struct d40_chan *d40c)2012{2013 2014	int res = 0;2015	u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dev_type);2016	struct d40_phy_res *phy = d40c->phy_chan;2017	bool is_src;2018 2019	/* Terminate all queued and active transfers */2020	d40_term_all(d40c);2021 2022	if (phy == NULL) {2023		chan_err(d40c, "phy == null\n");2024		return -EINVAL;2025	}2026 2027	if (phy->allocated_src == D40_ALLOC_FREE &&2028	    phy->allocated_dst == D40_ALLOC_FREE) {2029		chan_err(d40c, "channel already free\n");2030		return -EINVAL;2031	}2032 2033	if (d40c->dma_cfg.dir == DMA_MEM_TO_DEV ||2034	    d40c->dma_cfg.dir == DMA_MEM_TO_MEM)2035		is_src = false;2036	else if (d40c->dma_cfg.dir == DMA_DEV_TO_MEM)2037		is_src = true;2038	else {2039		chan_err(d40c, "Unknown direction\n");2040		return -EINVAL;2041	}2042 2043	pm_runtime_get_sync(d40c->base->dev);2044	res = d40_channel_execute_command(d40c, D40_DMA_STOP);2045	if (res) {2046		chan_err(d40c, "stop failed\n");2047		goto mark_last_busy;2048	}2049 2050	d40_alloc_mask_free(phy, is_src, chan_is_logical(d40c) ? event : 0);2051 2052	if (chan_is_logical(d40c))2053		d40c->base->lookup_log_chans[d40c->log_num] = NULL;2054	else2055		d40c->base->lookup_phy_chans[phy->num] = NULL;2056 2057	if (d40c->busy) {2058		pm_runtime_mark_last_busy(d40c->base->dev);2059		pm_runtime_put_autosuspend(d40c->base->dev);2060	}2061 2062	d40c->busy = false;2063	d40c->phy_chan = NULL;2064	d40c->configured = false;2065 mark_last_busy:2066	pm_runtime_mark_last_busy(d40c->base->dev);2067	pm_runtime_put_autosuspend(d40c->base->dev);2068	return res;2069}2070 2071static bool d40_is_paused(struct d40_chan *d40c)2072{2073	void __iomem *chanbase = chan_base(d40c);2074	bool is_paused = false;2075	unsigned long flags;2076	void __iomem *active_reg;2077	u32 status;2078	u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dev_type);2079 2080	spin_lock_irqsave(&d40c->lock, flags);2081 2082	if (chan_is_physical(d40c)) {2083		if (d40c->phy_chan->num % 2 == 0)2084			active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;2085		else2086			active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;2087 2088		status = (readl(active_reg) &2089			  D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>2090			D40_CHAN_POS(d40c->phy_chan->num);2091		if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)2092			is_paused = true;2093		goto unlock;2094	}2095 2096	if (d40c->dma_cfg.dir == DMA_MEM_TO_DEV ||2097	    d40c->dma_cfg.dir == DMA_MEM_TO_MEM) {2098		status = readl(chanbase + D40_CHAN_REG_SDLNK);2099	} else if (d40c->dma_cfg.dir == DMA_DEV_TO_MEM) {2100		status = readl(chanbase + D40_CHAN_REG_SSLNK);2101	} else {2102		chan_err(d40c, "Unknown direction\n");2103		goto unlock;2104	}2105 2106	status = (status & D40_EVENTLINE_MASK(event)) >>2107		D40_EVENTLINE_POS(event);2108 2109	if (status != D40_DMA_RUN)2110		is_paused = true;2111 unlock:2112	spin_unlock_irqrestore(&d40c->lock, flags);2113	return is_paused;2114 2115}2116 2117static u32 stedma40_residue(struct dma_chan *chan)2118{2119	struct d40_chan *d40c =2120		container_of(chan, struct d40_chan, chan);2121	u32 bytes_left;2122	unsigned long flags;2123 2124	spin_lock_irqsave(&d40c->lock, flags);2125	bytes_left = d40_residue(d40c);2126	spin_unlock_irqrestore(&d40c->lock, flags);2127 2128	return bytes_left;2129}2130 2131static int2132d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc,2133		struct scatterlist *sg_src, struct scatterlist *sg_dst,2134		unsigned int sg_len, dma_addr_t src_dev_addr,2135		dma_addr_t dst_dev_addr)2136{2137	struct stedma40_chan_cfg *cfg = &chan->dma_cfg;2138	struct stedma40_half_channel_info *src_info = &cfg->src_info;2139	struct stedma40_half_channel_info *dst_info = &cfg->dst_info;2140	int ret;2141 2142	ret = d40_log_sg_to_lli(sg_src, sg_len,2143				src_dev_addr,2144				desc->lli_log.src,2145				chan->log_def.lcsp1,2146				src_info->data_width,2147				dst_info->data_width);2148 2149	ret = d40_log_sg_to_lli(sg_dst, sg_len,2150				dst_dev_addr,2151				desc->lli_log.dst,2152				chan->log_def.lcsp3,2153				dst_info->data_width,2154				src_info->data_width);2155 2156	return ret < 0 ? ret : 0;2157}2158 2159static int2160d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc,2161		struct scatterlist *sg_src, struct scatterlist *sg_dst,2162		unsigned int sg_len, dma_addr_t src_dev_addr,2163		dma_addr_t dst_dev_addr)2164{2165	struct stedma40_chan_cfg *cfg = &chan->dma_cfg;2166	struct stedma40_half_channel_info *src_info = &cfg->src_info;2167	struct stedma40_half_channel_info *dst_info = &cfg->dst_info;2168	unsigned long flags = 0;2169	int ret;2170 2171	if (desc->cyclic)2172		flags |= LLI_CYCLIC | LLI_TERM_INT;2173 2174	ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr,2175				desc->lli_phy.src,2176				virt_to_phys(desc->lli_phy.src),2177				chan->src_def_cfg,2178				src_info, dst_info, flags);2179 2180	ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr,2181				desc->lli_phy.dst,2182				virt_to_phys(desc->lli_phy.dst),2183				chan->dst_def_cfg,2184				dst_info, src_info, flags);2185 2186	dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr,2187				   desc->lli_pool.size, DMA_TO_DEVICE);2188 2189	return ret < 0 ? ret : 0;2190}2191 2192static struct d40_desc *2193d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,2194	      unsigned int sg_len, unsigned long dma_flags)2195{2196	struct stedma40_chan_cfg *cfg;2197	struct d40_desc *desc;2198	int ret;2199 2200	desc = d40_desc_get(chan);2201	if (!desc)2202		return NULL;2203 2204	cfg = &chan->dma_cfg;2205	desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,2206					cfg->dst_info.data_width);2207	if (desc->lli_len < 0) {2208		chan_err(chan, "Unaligned size\n");2209		goto free_desc;2210	}2211 2212	ret = d40_pool_lli_alloc(chan, desc, desc->lli_len);2213	if (ret < 0) {2214		chan_err(chan, "Could not allocate lli\n");2215		goto free_desc;2216	}2217 2218	desc->lli_current = 0;2219	desc->txd.flags = dma_flags;2220	desc->txd.tx_submit = d40_tx_submit;2221 2222	dma_async_tx_descriptor_init(&desc->txd, &chan->chan);2223 2224	return desc;2225 free_desc:2226	d40_desc_free(chan, desc);2227	return NULL;2228}2229 2230static struct dma_async_tx_descriptor *2231d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,2232	    struct scatterlist *sg_dst, unsigned int sg_len,2233	    enum dma_transfer_direction direction, unsigned long dma_flags)2234{2235	struct d40_chan *chan = container_of(dchan, struct d40_chan, chan);2236	dma_addr_t src_dev_addr;2237	dma_addr_t dst_dev_addr;2238	struct d40_desc *desc;2239	unsigned long flags;2240	int ret;2241 2242	if (!chan->phy_chan) {2243		chan_err(chan, "Cannot prepare unallocated channel\n");2244		return NULL;2245	}2246 2247	d40_set_runtime_config_write(dchan, &chan->slave_config, direction);2248 2249	spin_lock_irqsave(&chan->lock, flags);2250 2251	desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags);2252	if (desc == NULL)2253		goto unlock;2254 2255	if (sg_next(&sg_src[sg_len - 1]) == sg_src)2256		desc->cyclic = true;2257 2258	src_dev_addr = 0;2259	dst_dev_addr = 0;2260	if (direction == DMA_DEV_TO_MEM)2261		src_dev_addr = chan->runtime_addr;2262	else if (direction == DMA_MEM_TO_DEV)2263		dst_dev_addr = chan->runtime_addr;2264 2265	if (chan_is_logical(chan))2266		ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst,2267				      sg_len, src_dev_addr, dst_dev_addr);2268	else2269		ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst,2270				      sg_len, src_dev_addr, dst_dev_addr);2271 2272	if (ret) {2273		chan_err(chan, "Failed to prepare %s sg job: %d\n",2274			 chan_is_logical(chan) ? "log" : "phy", ret);2275		goto free_desc;2276	}2277 2278	/*2279	 * add descriptor to the prepare queue in order to be able2280	 * to free them later in terminate_all2281	 */2282	list_add_tail(&desc->node, &chan->prepare_queue);2283 2284	spin_unlock_irqrestore(&chan->lock, flags);2285 2286	return &desc->txd;2287 free_desc:2288	d40_desc_free(chan, desc);2289 unlock:2290	spin_unlock_irqrestore(&chan->lock, flags);2291	return NULL;2292}2293 2294static bool stedma40_filter(struct dma_chan *chan, void *data)2295{2296	struct stedma40_chan_cfg *info = data;2297	struct d40_chan *d40c =2298		container_of(chan, struct d40_chan, chan);2299	int err;2300 2301	if (data) {2302		err = d40_validate_conf(d40c, info);2303		if (!err)2304			d40c->dma_cfg = *info;2305	} else2306		err = d40_config_memcpy(d40c);2307 2308	if (!err)2309		d40c->configured = true;2310 2311	return err == 0;2312}2313 2314static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src)2315{2316	bool realtime = d40c->dma_cfg.realtime;2317	bool highprio = d40c->dma_cfg.high_priority;2318	u32 rtreg;2319	u32 event = D40_TYPE_TO_EVENT(dev_type);2320	u32 group = D40_TYPE_TO_GROUP(dev_type);2321	u32 bit = BIT(event);2322	u32 prioreg;2323	struct d40_gen_dmac *dmac = &d40c->base->gen_dmac;2324 2325	rtreg = realtime ? dmac->realtime_en : dmac->realtime_clear;2326	/*2327	 * Due to a hardware bug, in some cases a logical channel triggered by2328	 * a high priority destination event line can generate extra packet2329	 * transactions.2330	 *2331	 * The workaround is to not set the high priority level for the2332	 * destination event lines that trigger logical channels.2333	 */2334	if (!src && chan_is_logical(d40c))2335		highprio = false;2336 2337	prioreg = highprio ? dmac->high_prio_en : dmac->high_prio_clear;2338 2339	/* Destination event lines are stored in the upper halfword */2340	if (!src)2341		bit <<= 16;2342 2343	writel(bit, d40c->base->virtbase + prioreg + group * 4);2344	writel(bit, d40c->base->virtbase + rtreg + group * 4);2345}2346 2347static void d40_set_prio_realtime(struct d40_chan *d40c)2348{2349	if (d40c->base->rev < 3)2350		return;2351 2352	if ((d40c->dma_cfg.dir ==  DMA_DEV_TO_MEM) ||2353	    (d40c->dma_cfg.dir == DMA_DEV_TO_DEV))2354		__d40_set_prio_rt(d40c, d40c->dma_cfg.dev_type, true);2355 2356	if ((d40c->dma_cfg.dir ==  DMA_MEM_TO_DEV) ||2357	    (d40c->dma_cfg.dir == DMA_DEV_TO_DEV))2358		__d40_set_prio_rt(d40c, d40c->dma_cfg.dev_type, false);2359}2360 2361#define D40_DT_FLAGS_MODE(flags)       ((flags >> 0) & 0x1)2362#define D40_DT_FLAGS_DIR(flags)        ((flags >> 1) & 0x1)2363#define D40_DT_FLAGS_BIG_ENDIAN(flags) ((flags >> 2) & 0x1)2364#define D40_DT_FLAGS_FIXED_CHAN(flags) ((flags >> 3) & 0x1)2365#define D40_DT_FLAGS_HIGH_PRIO(flags)  ((flags >> 4) & 0x1)2366 2367static struct dma_chan *d40_xlate(struct of_phandle_args *dma_spec,2368				  struct of_dma *ofdma)2369{2370	struct stedma40_chan_cfg cfg;2371	dma_cap_mask_t cap;2372	u32 flags;2373 2374	memset(&cfg, 0, sizeof(struct stedma40_chan_cfg));2375 2376	dma_cap_zero(cap);2377	dma_cap_set(DMA_SLAVE, cap);2378 2379	cfg.dev_type = dma_spec->args[0];2380	flags = dma_spec->args[2];2381 2382	switch (D40_DT_FLAGS_MODE(flags)) {2383	case 0: cfg.mode = STEDMA40_MODE_LOGICAL; break;2384	case 1: cfg.mode = STEDMA40_MODE_PHYSICAL; break;2385	}2386 2387	switch (D40_DT_FLAGS_DIR(flags)) {2388	case 0:2389		cfg.dir = DMA_MEM_TO_DEV;2390		cfg.dst_info.big_endian = D40_DT_FLAGS_BIG_ENDIAN(flags);2391		break;2392	case 1:2393		cfg.dir = DMA_DEV_TO_MEM;2394		cfg.src_info.big_endian = D40_DT_FLAGS_BIG_ENDIAN(flags);2395		break;2396	}2397 2398	if (D40_DT_FLAGS_FIXED_CHAN(flags)) {2399		cfg.phy_channel = dma_spec->args[1];2400		cfg.use_fixed_channel = true;2401	}2402 2403	if (D40_DT_FLAGS_HIGH_PRIO(flags))2404		cfg.high_priority = true;2405 2406	return dma_request_channel(cap, stedma40_filter, &cfg);2407}2408 2409/* DMA ENGINE functions */2410static int d40_alloc_chan_resources(struct dma_chan *chan)2411{2412	int err;2413	unsigned long flags;2414	struct d40_chan *d40c =2415		container_of(chan, struct d40_chan, chan);2416	bool is_free_phy;2417	spin_lock_irqsave(&d40c->lock, flags);2418 2419	dma_cookie_init(chan);2420 2421	/* If no dma configuration is set use default configuration (memcpy) */2422	if (!d40c->configured) {2423		err = d40_config_memcpy(d40c);2424		if (err) {2425			chan_err(d40c, "Failed to configure memcpy channel\n");2426			goto mark_last_busy;2427		}2428	}2429 2430	err = d40_allocate_channel(d40c, &is_free_phy);2431	if (err) {2432		chan_err(d40c, "Failed to allocate channel\n");2433		d40c->configured = false;2434		goto mark_last_busy;2435	}2436 2437	pm_runtime_get_sync(d40c->base->dev);2438 2439	d40_set_prio_realtime(d40c);2440 2441	if (chan_is_logical(d40c)) {2442		if (d40c->dma_cfg.dir == DMA_DEV_TO_MEM)2443			d40c->lcpa = d40c->base->lcpa_base +2444				d40c->dma_cfg.dev_type * D40_LCPA_CHAN_SIZE;2445		else2446			d40c->lcpa = d40c->base->lcpa_base +2447				d40c->dma_cfg.dev_type *2448				D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;2449 2450		/* Unmask the Global Interrupt Mask. */2451		d40c->src_def_cfg |= BIT(D40_SREG_CFG_LOG_GIM_POS);2452		d40c->dst_def_cfg |= BIT(D40_SREG_CFG_LOG_GIM_POS);2453	}2454 2455	dev_dbg(chan2dev(d40c), "allocated %s channel (phy %d%s)\n",2456		 chan_is_logical(d40c) ? "logical" : "physical",2457		 d40c->phy_chan->num,2458		 d40c->dma_cfg.use_fixed_channel ? ", fixed" : "");2459 2460 2461	/*2462	 * Only write channel configuration to the DMA if the physical2463	 * resource is free. In case of multiple logical channels2464	 * on the same physical resource, only the first write is necessary.2465	 */2466	if (is_free_phy)2467		d40_config_write(d40c);2468 mark_last_busy:2469	pm_runtime_mark_last_busy(d40c->base->dev);2470	pm_runtime_put_autosuspend(d40c->base->dev);2471	spin_unlock_irqrestore(&d40c->lock, flags);2472	return err;2473}2474 2475static void d40_free_chan_resources(struct dma_chan *chan)2476{2477	struct d40_chan *d40c =2478		container_of(chan, struct d40_chan, chan);2479	int err;2480	unsigned long flags;2481 2482	if (d40c->phy_chan == NULL) {2483		chan_err(d40c, "Cannot free unallocated channel\n");2484		return;2485	}2486 2487	spin_lock_irqsave(&d40c->lock, flags);2488 2489	err = d40_free_dma(d40c);2490 2491	if (err)2492		chan_err(d40c, "Failed to free channel\n");2493	spin_unlock_irqrestore(&d40c->lock, flags);2494}2495 2496static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,2497						       dma_addr_t dst,2498						       dma_addr_t src,2499						       size_t size,2500						       unsigned long dma_flags)2501{2502	struct scatterlist dst_sg;2503	struct scatterlist src_sg;2504 2505	sg_init_table(&dst_sg, 1);2506	sg_init_table(&src_sg, 1);2507 2508	sg_dma_address(&dst_sg) = dst;2509	sg_dma_address(&src_sg) = src;2510 2511	sg_dma_len(&dst_sg) = size;2512	sg_dma_len(&src_sg) = size;2513 2514	return d40_prep_sg(chan, &src_sg, &dst_sg, 1,2515			   DMA_MEM_TO_MEM, dma_flags);2516}2517 2518static struct dma_async_tx_descriptor *2519d40_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,2520		  unsigned int sg_len, enum dma_transfer_direction direction,2521		  unsigned long dma_flags, void *context)2522{2523	if (!is_slave_direction(direction))2524		return NULL;2525 2526	return d40_prep_sg(chan, sgl, sgl, sg_len, direction, dma_flags);2527}2528 2529static struct dma_async_tx_descriptor *2530dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,2531		     size_t buf_len, size_t period_len,2532		     enum dma_transfer_direction direction, unsigned long flags)2533{2534	unsigned int periods = buf_len / period_len;2535	struct dma_async_tx_descriptor *txd;2536	struct scatterlist *sg;2537	int i;2538 2539	sg = kcalloc(periods + 1, sizeof(struct scatterlist), GFP_NOWAIT);2540	if (!sg)2541		return NULL;2542 2543	for (i = 0; i < periods; i++) {2544		sg_dma_address(&sg[i]) = dma_addr;2545		sg_dma_len(&sg[i]) = period_len;2546		dma_addr += period_len;2547	}2548 2549	sg_chain(sg, periods + 1, sg);2550 2551	txd = d40_prep_sg(chan, sg, sg, periods, direction,2552			  DMA_PREP_INTERRUPT);2553 2554	kfree(sg);2555 2556	return txd;2557}2558 2559static enum dma_status d40_tx_status(struct dma_chan *chan,2560				     dma_cookie_t cookie,2561				     struct dma_tx_state *txstate)2562{2563	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);2564	enum dma_status ret;2565 2566	if (d40c->phy_chan == NULL) {2567		chan_err(d40c, "Cannot read status of unallocated channel\n");2568		return -EINVAL;2569	}2570 2571	ret = dma_cookie_status(chan, cookie, txstate);2572	if (ret != DMA_COMPLETE && txstate)2573		dma_set_residue(txstate, stedma40_residue(chan));2574 2575	if (d40_is_paused(d40c))2576		ret = DMA_PAUSED;2577 2578	return ret;2579}2580 2581static void d40_issue_pending(struct dma_chan *chan)2582{2583	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);2584	unsigned long flags;2585 2586	if (d40c->phy_chan == NULL) {2587		chan_err(d40c, "Channel is not allocated!\n");2588		return;2589	}2590 2591	spin_lock_irqsave(&d40c->lock, flags);2592 2593	list_splice_tail_init(&d40c->pending_queue, &d40c->queue);2594 2595	/* Busy means that queued jobs are already being processed */2596	if (!d40c->busy)2597		(void) d40_queue_start(d40c);2598 2599	spin_unlock_irqrestore(&d40c->lock, flags);2600}2601 2602static int d40_terminate_all(struct dma_chan *chan)2603{2604	unsigned long flags;2605	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);2606	int ret;2607 2608	if (d40c->phy_chan == NULL) {2609		chan_err(d40c, "Channel is not allocated!\n");2610		return -EINVAL;2611	}2612 2613	spin_lock_irqsave(&d40c->lock, flags);2614 2615	pm_runtime_get_sync(d40c->base->dev);2616	ret = d40_channel_execute_command(d40c, D40_DMA_STOP);2617	if (ret)2618		chan_err(d40c, "Failed to stop channel\n");2619 2620	d40_term_all(d40c);2621	pm_runtime_mark_last_busy(d40c->base->dev);2622	pm_runtime_put_autosuspend(d40c->base->dev);2623	if (d40c->busy) {2624		pm_runtime_mark_last_busy(d40c->base->dev);2625		pm_runtime_put_autosuspend(d40c->base->dev);2626	}2627	d40c->busy = false;2628 2629	spin_unlock_irqrestore(&d40c->lock, flags);2630	return 0;2631}2632 2633static int2634dma40_config_to_halfchannel(struct d40_chan *d40c,2635			    struct stedma40_half_channel_info *info,2636			    u32 maxburst)2637{2638	int psize;2639 2640	if (chan_is_logical(d40c)) {2641		if (maxburst >= 16)2642			psize = STEDMA40_PSIZE_LOG_16;2643		else if (maxburst >= 8)2644			psize = STEDMA40_PSIZE_LOG_8;2645		else if (maxburst >= 4)2646			psize = STEDMA40_PSIZE_LOG_4;2647		else2648			psize = STEDMA40_PSIZE_LOG_1;2649	} else {2650		if (maxburst >= 16)2651			psize = STEDMA40_PSIZE_PHY_16;2652		else if (maxburst >= 8)2653			psize = STEDMA40_PSIZE_PHY_8;2654		else if (maxburst >= 4)2655			psize = STEDMA40_PSIZE_PHY_4;2656		else2657			psize = STEDMA40_PSIZE_PHY_1;2658	}2659 2660	info->psize = psize;2661	info->flow_ctrl = STEDMA40_NO_FLOW_CTRL;2662 2663	return 0;2664}2665 2666static int d40_set_runtime_config(struct dma_chan *chan,2667				  struct dma_slave_config *config)2668{2669	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);2670 2671	memcpy(&d40c->slave_config, config, sizeof(*config));2672 2673	return 0;2674}2675 2676/* Runtime reconfiguration extension */2677static int d40_set_runtime_config_write(struct dma_chan *chan,2678				  struct dma_slave_config *config,2679				  enum dma_transfer_direction direction)2680{2681	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);2682	struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;2683	enum dma_slave_buswidth src_addr_width, dst_addr_width;2684	dma_addr_t config_addr;2685	u32 src_maxburst, dst_maxburst;2686	int ret;2687 2688	if (d40c->phy_chan == NULL) {2689		chan_err(d40c, "Channel is not allocated!\n");2690		return -EINVAL;2691	}2692 2693	src_addr_width = config->src_addr_width;2694	src_maxburst = config->src_maxburst;2695	dst_addr_width = config->dst_addr_width;2696	dst_maxburst = config->dst_maxburst;2697 2698	if (direction == DMA_DEV_TO_MEM) {2699		config_addr = config->src_addr;2700 2701		if (cfg->dir != DMA_DEV_TO_MEM)2702			dev_dbg(d40c->base->dev,2703				"channel was not configured for peripheral "2704				"to memory transfer (%d) overriding\n",2705				cfg->dir);2706		cfg->dir = DMA_DEV_TO_MEM;2707 2708		/* Configure the memory side */2709		if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)2710			dst_addr_width = src_addr_width;2711		if (dst_maxburst == 0)2712			dst_maxburst = src_maxburst;2713 2714	} else if (direction == DMA_MEM_TO_DEV) {2715		config_addr = config->dst_addr;2716 2717		if (cfg->dir != DMA_MEM_TO_DEV)2718			dev_dbg(d40c->base->dev,2719				"channel was not configured for memory "2720				"to peripheral transfer (%d) overriding\n",2721				cfg->dir);2722		cfg->dir = DMA_MEM_TO_DEV;2723 2724		/* Configure the memory side */2725		if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)2726			src_addr_width = dst_addr_width;2727		if (src_maxburst == 0)2728			src_maxburst = dst_maxburst;2729	} else {2730		dev_err(d40c->base->dev,2731			"unrecognized channel direction %d\n",2732			direction);2733		return -EINVAL;2734	}2735 2736	if (config_addr <= 0) {2737		dev_err(d40c->base->dev, "no address supplied\n");2738		return -EINVAL;2739	}2740 2741	if (src_maxburst * src_addr_width != dst_maxburst * dst_addr_width) {2742		dev_err(d40c->base->dev,2743			"src/dst width/maxburst mismatch: %d*%d != %d*%d\n",2744			src_maxburst,2745			src_addr_width,2746			dst_maxburst,2747			dst_addr_width);2748		return -EINVAL;2749	}2750 2751	if (src_maxburst > 16) {2752		src_maxburst = 16;2753		dst_maxburst = src_maxburst * src_addr_width / dst_addr_width;2754	} else if (dst_maxburst > 16) {2755		dst_maxburst = 16;2756		src_maxburst = dst_maxburst * dst_addr_width / src_addr_width;2757	}2758 2759	/* Only valid widths are; 1, 2, 4 and 8. */2760	if (src_addr_width <= DMA_SLAVE_BUSWIDTH_UNDEFINED ||2761	    src_addr_width >  DMA_SLAVE_BUSWIDTH_8_BYTES   ||2762	    dst_addr_width <= DMA_SLAVE_BUSWIDTH_UNDEFINED ||2763	    dst_addr_width >  DMA_SLAVE_BUSWIDTH_8_BYTES   ||2764	    !is_power_of_2(src_addr_width) ||2765	    !is_power_of_2(dst_addr_width))2766		return -EINVAL;2767 2768	cfg->src_info.data_width = src_addr_width;2769	cfg->dst_info.data_width = dst_addr_width;2770 2771	ret = dma40_config_to_halfchannel(d40c, &cfg->src_info,2772					  src_maxburst);2773	if (ret)2774		return ret;2775 2776	ret = dma40_config_to_halfchannel(d40c, &cfg->dst_info,2777					  dst_maxburst);2778	if (ret)2779		return ret;2780 2781	/* Fill in register values */2782	if (chan_is_logical(d40c))2783		d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);2784	else2785		d40_phy_cfg(cfg, &d40c->src_def_cfg, &d40c->dst_def_cfg);2786 2787	/* These settings will take precedence later */2788	d40c->runtime_addr = config_addr;2789	d40c->runtime_direction = direction;2790	dev_dbg(d40c->base->dev,2791		"configured channel %s for %s, data width %d/%d, "2792		"maxburst %d/%d elements, LE, no flow control\n",2793		dma_chan_name(chan),2794		(direction == DMA_DEV_TO_MEM) ? "RX" : "TX",2795		src_addr_width, dst_addr_width,2796		src_maxburst, dst_maxburst);2797 2798	return 0;2799}2800 2801/* Initialization functions */2802 2803static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,2804				 struct d40_chan *chans, int offset,2805				 int num_chans)2806{2807	int i = 0;2808	struct d40_chan *d40c;2809 2810	INIT_LIST_HEAD(&dma->channels);2811 2812	for (i = offset; i < offset + num_chans; i++) {2813		d40c = &chans[i];2814		d40c->base = base;2815		d40c->chan.device = dma;2816 2817		spin_lock_init(&d40c->lock);2818 2819		d40c->log_num = D40_PHY_CHAN;2820 2821		INIT_LIST_HEAD(&d40c->done);2822		INIT_LIST_HEAD(&d40c->active);2823		INIT_LIST_HEAD(&d40c->queue);2824		INIT_LIST_HEAD(&d40c->pending_queue);2825		INIT_LIST_HEAD(&d40c->client);2826		INIT_LIST_HEAD(&d40c->prepare_queue);2827 2828		tasklet_setup(&d40c->tasklet, dma_tasklet);2829 2830		list_add_tail(&d40c->chan.device_node,2831			      &dma->channels);2832	}2833}2834 2835static void d40_ops_init(struct d40_base *base, struct dma_device *dev)2836{2837	if (dma_has_cap(DMA_SLAVE, dev->cap_mask)) {2838		dev->device_prep_slave_sg = d40_prep_slave_sg;2839		dev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);2840	}2841 2842	if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) {2843		dev->device_prep_dma_memcpy = d40_prep_memcpy;2844		dev->directions = BIT(DMA_MEM_TO_MEM);2845		/*2846		 * This controller can only access address at even2847		 * 32bit boundaries, i.e. 2^22848		 */2849		dev->copy_align = DMAENGINE_ALIGN_4_BYTES;2850	}2851 2852	if (dma_has_cap(DMA_CYCLIC, dev->cap_mask))2853		dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic;2854 2855	dev->device_alloc_chan_resources = d40_alloc_chan_resources;2856	dev->device_free_chan_resources = d40_free_chan_resources;2857	dev->device_issue_pending = d40_issue_pending;2858	dev->device_tx_status = d40_tx_status;2859	dev->device_config = d40_set_runtime_config;2860	dev->device_pause = d40_pause;2861	dev->device_resume = d40_resume;2862	dev->device_terminate_all = d40_terminate_all;2863	dev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;2864	dev->dev = base->dev;2865}2866 2867static int __init d40_dmaengine_init(struct d40_base *base,2868				     int num_reserved_chans)2869{2870	int err ;2871 2872	d40_chan_init(base, &base->dma_slave, base->log_chans,2873		      0, base->num_log_chans);2874 2875	dma_cap_zero(base->dma_slave.cap_mask);2876	dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);2877	dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);2878 2879	d40_ops_init(base, &base->dma_slave);2880 2881	err = dmaenginem_async_device_register(&base->dma_slave);2882 2883	if (err) {2884		d40_err(base->dev, "Failed to register slave channels\n");2885		goto exit;2886	}2887 2888	d40_chan_init(base, &base->dma_memcpy, base->log_chans,2889		      base->num_log_chans, base->num_memcpy_chans);2890 2891	dma_cap_zero(base->dma_memcpy.cap_mask);2892	dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);2893 2894	d40_ops_init(base, &base->dma_memcpy);2895 2896	err = dmaenginem_async_device_register(&base->dma_memcpy);2897 2898	if (err) {2899		d40_err(base->dev,2900			"Failed to register memcpy only channels\n");2901		goto exit;2902	}2903 2904	d40_chan_init(base, &base->dma_both, base->phy_chans,2905		      0, num_reserved_chans);2906 2907	dma_cap_zero(base->dma_both.cap_mask);2908	dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);2909	dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);2910	dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);2911 2912	d40_ops_init(base, &base->dma_both);2913	err = dmaenginem_async_device_register(&base->dma_both);2914 2915	if (err) {2916		d40_err(base->dev,2917			"Failed to register logical and physical capable channels\n");2918		goto exit;2919	}2920	return 0;2921 exit:2922	return err;2923}2924 2925/* Suspend resume functionality */2926#ifdef CONFIG_PM_SLEEP2927static int dma40_suspend(struct device *dev)2928{2929	struct d40_base *base = dev_get_drvdata(dev);2930	int ret;2931 2932	ret = pm_runtime_force_suspend(dev);2933	if (ret)2934		return ret;2935 2936	if (base->lcpa_regulator)2937		ret = regulator_disable(base->lcpa_regulator);2938	return ret;2939}2940 2941static int dma40_resume(struct device *dev)2942{2943	struct d40_base *base = dev_get_drvdata(dev);2944	int ret = 0;2945 2946	if (base->lcpa_regulator) {2947		ret = regulator_enable(base->lcpa_regulator);2948		if (ret)2949			return ret;2950	}2951 2952	return pm_runtime_force_resume(dev);2953}2954#endif2955 2956#ifdef CONFIG_PM2957static void dma40_backup(void __iomem *baseaddr, u32 *backup,2958			 u32 *regaddr, int num, bool save)2959{2960	int i;2961 2962	for (i = 0; i < num; i++) {2963		void __iomem *addr = baseaddr + regaddr[i];2964 2965		if (save)2966			backup[i] = readl_relaxed(addr);2967		else2968			writel_relaxed(backup[i], addr);2969	}2970}2971 2972static void d40_save_restore_registers(struct d40_base *base, bool save)2973{2974	int i;2975 2976	/* Save/Restore channel specific registers */2977	for (i = 0; i < base->num_phy_chans; i++) {2978		void __iomem *addr;2979		int idx;2980 2981		if (base->phy_res[i].reserved)2982			continue;2983 2984		addr = base->virtbase + D40_DREG_PCBASE + i * D40_DREG_PCDELTA;2985		idx = i * ARRAY_SIZE(d40_backup_regs_chan);2986 2987		dma40_backup(addr, &base->reg_val_backup_chan[idx],2988			     d40_backup_regs_chan,2989			     ARRAY_SIZE(d40_backup_regs_chan),2990			     save);2991	}2992 2993	/* Save/Restore global registers */2994	dma40_backup(base->virtbase, base->reg_val_backup,2995		     d40_backup_regs, ARRAY_SIZE(d40_backup_regs),2996		     save);2997 2998	/* Save/Restore registers only existing on dma40 v3 and later */2999	if (base->gen_dmac.backup)3000		dma40_backup(base->virtbase, base->reg_val_backup_v4,3001			     base->gen_dmac.backup,3002			base->gen_dmac.backup_size,3003			save);3004}3005 3006static int dma40_runtime_suspend(struct device *dev)3007{3008	struct d40_base *base = dev_get_drvdata(dev);3009 3010	d40_save_restore_registers(base, true);3011 3012	/* Don't disable/enable clocks for v1 due to HW bugs */3013	if (base->rev != 1)3014		writel_relaxed(base->gcc_pwr_off_mask,3015			       base->virtbase + D40_DREG_GCC);3016 3017	return 0;3018}3019 3020static int dma40_runtime_resume(struct device *dev)3021{3022	struct d40_base *base = dev_get_drvdata(dev);3023 3024	d40_save_restore_registers(base, false);3025 3026	writel_relaxed(D40_DREG_GCC_ENABLE_ALL,3027		       base->virtbase + D40_DREG_GCC);3028	return 0;3029}3030#endif3031 3032static const struct dev_pm_ops dma40_pm_ops = {3033	SET_LATE_SYSTEM_SLEEP_PM_OPS(dma40_suspend, dma40_resume)3034	SET_RUNTIME_PM_OPS(dma40_runtime_suspend,3035				dma40_runtime_resume,3036				NULL)3037};3038 3039/* Initialization functions. */3040 3041static int __init d40_phy_res_init(struct d40_base *base)3042{3043	int i;3044	int num_phy_chans_avail = 0;3045	u32 val[2];3046	int odd_even_bit = -2;3047	int gcc = D40_DREG_GCC_ENA;3048 3049	val[0] = readl(base->virtbase + D40_DREG_PRSME);3050	val[1] = readl(base->virtbase + D40_DREG_PRSMO);3051 3052	for (i = 0; i < base->num_phy_chans; i++) {3053		base->phy_res[i].num = i;3054		odd_even_bit += 2 * ((i % 2) == 0);3055		if (((val[i % 2] >> odd_even_bit) & 3) == 1) {3056			/* Mark security only channels as occupied */3057			base->phy_res[i].allocated_src = D40_ALLOC_PHY;3058			base->phy_res[i].allocated_dst = D40_ALLOC_PHY;3059			base->phy_res[i].reserved = true;3060			gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),3061						       D40_DREG_GCC_SRC);3062			gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),3063						       D40_DREG_GCC_DST);3064 3065 3066		} else {3067			base->phy_res[i].allocated_src = D40_ALLOC_FREE;3068			base->phy_res[i].allocated_dst = D40_ALLOC_FREE;3069			base->phy_res[i].reserved = false;3070			num_phy_chans_avail++;3071		}3072		spin_lock_init(&base->phy_res[i].lock);3073	}3074 3075	/* Mark disabled channels as occupied */3076	for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {3077		int chan = base->plat_data->disabled_channels[i];3078 3079		base->phy_res[chan].allocated_src = D40_ALLOC_PHY;3080		base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;3081		base->phy_res[chan].reserved = true;3082		gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),3083					       D40_DREG_GCC_SRC);3084		gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),3085					       D40_DREG_GCC_DST);3086		num_phy_chans_avail--;3087	}3088 3089	/* Mark soft_lli channels */3090	for (i = 0; i < base->plat_data->num_of_soft_lli_chans; i++) {3091		int chan = base->plat_data->soft_lli_chans[i];3092 3093		base->phy_res[chan].use_soft_lli = true;3094	}3095 3096	dev_info(base->dev, "%d of %d physical DMA channels available\n",3097		 num_phy_chans_avail, base->num_phy_chans);3098 3099	/* Verify settings extended vs standard */3100	val[0] = readl(base->virtbase + D40_DREG_PRTYP);3101 3102	for (i = 0; i < base->num_phy_chans; i++) {3103 3104		if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&3105		    (val[0] & 0x3) != 1)3106			dev_info(base->dev,3107				 "[%s] INFO: channel %d is misconfigured (%d)\n",3108				 __func__, i, val[0] & 0x3);3109 3110		val[0] = val[0] >> 2;3111	}3112 3113	/*3114	 * To keep things simple, Enable all clocks initially.3115	 * The clocks will get managed later post channel allocation.3116	 * The clocks for the event lines on which reserved channels exists3117	 * are not managed here.3118	 */3119	writel(D40_DREG_GCC_ENABLE_ALL, base->virtbase + D40_DREG_GCC);3120	base->gcc_pwr_off_mask = gcc;3121 3122	return num_phy_chans_avail;3123}3124 3125/* Called from the registered devm action */3126static void d40_drop_kmem_cache_action(void *d)3127{3128	struct kmem_cache *desc_slab = d;3129 3130	kmem_cache_destroy(desc_slab);3131}3132 3133static int __init d40_hw_detect_init(struct platform_device *pdev,3134				     struct d40_base **retbase)3135{3136	struct stedma40_platform_data *plat_data = dev_get_platdata(&pdev->dev);3137	struct device *dev = &pdev->dev;3138	struct clk *clk;3139	void __iomem *virtbase;3140	struct d40_base *base;3141	int num_log_chans;3142	int num_phy_chans;3143	int num_memcpy_chans;3144	int i;3145	u32 pid;3146	u32 cid;3147	u8 rev;3148	int ret;3149 3150	clk = devm_clk_get_enabled(dev, NULL);3151	if (IS_ERR(clk))3152		return PTR_ERR(clk);3153 3154	/* Get IO for DMAC base address */3155	virtbase = devm_platform_ioremap_resource_byname(pdev, "base");3156	if (IS_ERR(virtbase))3157		return PTR_ERR(virtbase);3158 3159	/* This is just a regular AMBA PrimeCell ID actually */3160	for (pid = 0, i = 0; i < 4; i++)3161		pid |= (readl(virtbase + SZ_4K - 0x20 + 4 * i)3162			& 255) << (i * 8);3163	for (cid = 0, i = 0; i < 4; i++)3164		cid |= (readl(virtbase + SZ_4K - 0x10 + 4 * i)3165			& 255) << (i * 8);3166 3167	if (cid != AMBA_CID) {3168		d40_err(dev, "Unknown hardware! No PrimeCell ID\n");3169		return -EINVAL;3170	}3171	if (AMBA_MANF_BITS(pid) != AMBA_VENDOR_ST) {3172		d40_err(dev, "Unknown designer! Got %x wanted %x\n",3173			AMBA_MANF_BITS(pid),3174			AMBA_VENDOR_ST);3175		return -EINVAL;3176	}3177	/*3178	 * HW revision:3179	 * DB8500ed has revision 03180	 * ? has revision 13181	 * DB8500v1 has revision 23182	 * DB8500v2 has revision 33183	 * AP9540v1 has revision 43184	 * DB8540v1 has revision 43185	 */3186	rev = AMBA_REV_BITS(pid);3187	if (rev < 2) {3188		d40_err(dev, "hardware revision: %d is not supported", rev);3189		return -EINVAL;3190	}3191 3192	/* The number of physical channels on this HW */3193	if (plat_data->num_of_phy_chans)3194		num_phy_chans = plat_data->num_of_phy_chans;3195	else3196		num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;3197 3198	/* The number of channels used for memcpy */3199	if (plat_data->num_of_memcpy_chans)3200		num_memcpy_chans = plat_data->num_of_memcpy_chans;3201	else3202		num_memcpy_chans = ARRAY_SIZE(dma40_memcpy_channels);3203 3204	num_log_chans = num_phy_chans * D40_MAX_LOG_CHAN_PER_PHY;3205 3206	dev_info(dev,3207		 "hardware rev: %d with %d physical and %d logical channels\n",3208		 rev, num_phy_chans, num_log_chans);3209 3210	base = devm_kzalloc(dev,3211		ALIGN(sizeof(struct d40_base), 4) +3212		(num_phy_chans + num_log_chans + num_memcpy_chans) *3213		sizeof(struct d40_chan), GFP_KERNEL);3214 3215	if (!base)3216		return -ENOMEM;3217 3218	base->rev = rev;3219	base->clk = clk;3220	base->num_memcpy_chans = num_memcpy_chans;3221	base->num_phy_chans = num_phy_chans;3222	base->num_log_chans = num_log_chans;3223	base->virtbase = virtbase;3224	base->plat_data = plat_data;3225	base->dev = dev;3226	base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);3227	base->log_chans = &base->phy_chans[num_phy_chans];3228 3229	if (base->plat_data->num_of_phy_chans == 14) {3230		base->gen_dmac.backup = d40_backup_regs_v4b;3231		base->gen_dmac.backup_size = BACKUP_REGS_SZ_V4B;3232		base->gen_dmac.interrupt_en = D40_DREG_CPCMIS;3233		base->gen_dmac.interrupt_clear = D40_DREG_CPCICR;3234		base->gen_dmac.realtime_en = D40_DREG_CRSEG1;3235		base->gen_dmac.realtime_clear = D40_DREG_CRCEG1;3236		base->gen_dmac.high_prio_en = D40_DREG_CPSEG1;3237		base->gen_dmac.high_prio_clear = D40_DREG_CPCEG1;3238		base->gen_dmac.il = il_v4b;3239		base->gen_dmac.il_size = ARRAY_SIZE(il_v4b);3240		base->gen_dmac.init_reg = dma_init_reg_v4b;3241		base->gen_dmac.init_reg_size = ARRAY_SIZE(dma_init_reg_v4b);3242	} else {3243		if (base->rev >= 3) {3244			base->gen_dmac.backup = d40_backup_regs_v4a;3245			base->gen_dmac.backup_size = BACKUP_REGS_SZ_V4A;3246		}3247		base->gen_dmac.interrupt_en = D40_DREG_PCMIS;3248		base->gen_dmac.interrupt_clear = D40_DREG_PCICR;3249		base->gen_dmac.realtime_en = D40_DREG_RSEG1;3250		base->gen_dmac.realtime_clear = D40_DREG_RCEG1;3251		base->gen_dmac.high_prio_en = D40_DREG_PSEG1;3252		base->gen_dmac.high_prio_clear = D40_DREG_PCEG1;3253		base->gen_dmac.il = il_v4a;3254		base->gen_dmac.il_size = ARRAY_SIZE(il_v4a);3255		base->gen_dmac.init_reg = dma_init_reg_v4a;3256		base->gen_dmac.init_reg_size = ARRAY_SIZE(dma_init_reg_v4a);3257	}3258 3259	base->phy_res = devm_kcalloc(dev, num_phy_chans,3260				     sizeof(*base->phy_res),3261				     GFP_KERNEL);3262	if (!base->phy_res)3263		return -ENOMEM;3264 3265	base->lookup_phy_chans = devm_kcalloc(dev, num_phy_chans,3266					      sizeof(*base->lookup_phy_chans),3267					      GFP_KERNEL);3268	if (!base->lookup_phy_chans)3269		return -ENOMEM;3270 3271	base->lookup_log_chans = devm_kcalloc(dev, num_log_chans,3272					      sizeof(*base->lookup_log_chans),3273					      GFP_KERNEL);3274	if (!base->lookup_log_chans)3275		return -ENOMEM;3276 3277	base->reg_val_backup_chan = devm_kmalloc_array(dev, base->num_phy_chans,3278						  sizeof(d40_backup_regs_chan),3279						  GFP_KERNEL);3280	if (!base->reg_val_backup_chan)3281		return -ENOMEM;3282 3283	base->lcla_pool.alloc_map = devm_kcalloc(dev, num_phy_chans3284					    * D40_LCLA_LINK_PER_EVENT_GRP,3285					    sizeof(*base->lcla_pool.alloc_map),3286					    GFP_KERNEL);3287	if (!base->lcla_pool.alloc_map)3288		return -ENOMEM;3289 3290	base->regs_interrupt = devm_kmalloc_array(dev, base->gen_dmac.il_size,3291					     sizeof(*base->regs_interrupt),3292					     GFP_KERNEL);3293	if (!base->regs_interrupt)3294		return -ENOMEM;3295 3296	base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),3297					    0, SLAB_HWCACHE_ALIGN,3298					    NULL);3299	if (!base->desc_slab)3300		return -ENOMEM;3301 3302	ret = devm_add_action_or_reset(dev, d40_drop_kmem_cache_action,3303				       base->desc_slab);3304	if (ret)3305		return ret;3306 3307	*retbase = base;3308 3309	return 0;3310}3311 3312static void __init d40_hw_init(struct d40_base *base)3313{3314 3315	int i;3316	u32 prmseo[2] = {0, 0};3317	u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};3318	u32 pcmis = 0;3319	u32 pcicr = 0;3320	struct d40_reg_val *dma_init_reg = base->gen_dmac.init_reg;3321	u32 reg_size = base->gen_dmac.init_reg_size;3322 3323	for (i = 0; i < reg_size; i++)3324		writel(dma_init_reg[i].val,3325		       base->virtbase + dma_init_reg[i].reg);3326 3327	/* Configure all our dma channels to default settings */3328	for (i = 0; i < base->num_phy_chans; i++) {3329 3330		activeo[i % 2] = activeo[i % 2] << 2;3331 3332		if (base->phy_res[base->num_phy_chans - i - 1].allocated_src3333		    == D40_ALLOC_PHY) {3334			activeo[i % 2] |= 3;3335			continue;3336		}3337 3338		/* Enable interrupt # */3339		pcmis = (pcmis << 1) | 1;3340 3341		/* Clear interrupt # */3342		pcicr = (pcicr << 1) | 1;3343 3344		/* Set channel to physical mode */3345		prmseo[i % 2] = prmseo[i % 2] << 2;3346		prmseo[i % 2] |= 1;3347 3348	}3349 3350	writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);3351	writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);3352	writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);3353	writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);3354 3355	/* Write which interrupt to enable */3356	writel(pcmis, base->virtbase + base->gen_dmac.interrupt_en);3357 3358	/* Write which interrupt to clear */3359	writel(pcicr, base->virtbase + base->gen_dmac.interrupt_clear);3360 3361	/* These are __initdata and cannot be accessed after init */3362	base->gen_dmac.init_reg = NULL;3363	base->gen_dmac.init_reg_size = 0;3364}3365 3366static int __init d40_lcla_allocate(struct d40_base *base)3367{3368	struct d40_lcla_pool *pool = &base->lcla_pool;3369	unsigned long *page_list;3370	int i, j;3371	int ret;3372 3373	/*3374	 * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,3375	 * To full fill this hardware requirement without wasting 256 kb3376	 * we allocate pages until we get an aligned one.3377	 */3378	page_list = kmalloc_array(MAX_LCLA_ALLOC_ATTEMPTS,3379				  sizeof(*page_list),3380				  GFP_KERNEL);3381	if (!page_list)3382		return -ENOMEM;3383 3384	/* Calculating how many pages that are required */3385	base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;3386 3387	for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {3388		page_list[i] = __get_free_pages(GFP_KERNEL,3389						base->lcla_pool.pages);3390		if (!page_list[i]) {3391 3392			d40_err(base->dev, "Failed to allocate %d pages.\n",3393				base->lcla_pool.pages);3394			ret = -ENOMEM;3395 3396			for (j = 0; j < i; j++)3397				free_pages(page_list[j], base->lcla_pool.pages);3398			goto free_page_list;3399		}3400 3401		if ((virt_to_phys((void *)page_list[i]) &3402		     (LCLA_ALIGNMENT - 1)) == 0)3403			break;3404	}3405 3406	for (j = 0; j < i; j++)3407		free_pages(page_list[j], base->lcla_pool.pages);3408 3409	if (i < MAX_LCLA_ALLOC_ATTEMPTS) {3410		base->lcla_pool.base = (void *)page_list[i];3411	} else {3412		/*3413		 * After many attempts and no success with finding the correct3414		 * alignment, try with allocating a big buffer.3415		 */3416		dev_warn(base->dev,3417			 "[%s] Failed to get %d pages @ 18 bit align.\n",3418			 __func__, base->lcla_pool.pages);3419		base->lcla_pool.base_unaligned = kmalloc(SZ_1K *3420							 base->num_phy_chans +3421							 LCLA_ALIGNMENT,3422							 GFP_KERNEL);3423		if (!base->lcla_pool.base_unaligned) {3424			ret = -ENOMEM;3425			goto free_page_list;3426		}3427 3428		base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,3429						 LCLA_ALIGNMENT);3430	}3431 3432	pool->dma_addr = dma_map_single(base->dev, pool->base,3433					SZ_1K * base->num_phy_chans,3434					DMA_TO_DEVICE);3435	if (dma_mapping_error(base->dev, pool->dma_addr)) {3436		pool->dma_addr = 0;3437		ret = -ENOMEM;3438		goto free_page_list;3439	}3440 3441	writel(virt_to_phys(base->lcla_pool.base),3442	       base->virtbase + D40_DREG_LCLA);3443	ret = 0;3444 free_page_list:3445	kfree(page_list);3446	return ret;3447}3448 3449static int __init d40_of_probe(struct device *dev,3450			       struct device_node *np)3451{3452	struct stedma40_platform_data *pdata;3453	int num_phy = 0, num_memcpy = 0, num_disabled = 0;3454	const __be32 *list;3455 3456	pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);3457	if (!pdata)3458		return -ENOMEM;3459 3460	/* If absent this value will be obtained from h/w. */3461	of_property_read_u32(np, "dma-channels", &num_phy);3462	if (num_phy > 0)3463		pdata->num_of_phy_chans = num_phy;3464 3465	list = of_get_property(np, "memcpy-channels", &num_memcpy);3466	num_memcpy /= sizeof(*list);3467 3468	if (num_memcpy > D40_MEMCPY_MAX_CHANS || num_memcpy <= 0) {3469		d40_err(dev,3470			"Invalid number of memcpy channels specified (%d)\n",3471			num_memcpy);3472		return -EINVAL;3473	}3474	pdata->num_of_memcpy_chans = num_memcpy;3475 3476	of_property_read_u32_array(np, "memcpy-channels",3477				   dma40_memcpy_channels,3478				   num_memcpy);3479 3480	list = of_get_property(np, "disabled-channels", &num_disabled);3481	num_disabled /= sizeof(*list);3482 3483	if (num_disabled >= STEDMA40_MAX_PHYS || num_disabled < 0) {3484		d40_err(dev,3485			"Invalid number of disabled channels specified (%d)\n",3486			num_disabled);3487		return -EINVAL;3488	}3489 3490	of_property_read_u32_array(np, "disabled-channels",3491				   pdata->disabled_channels,3492				   num_disabled);3493	pdata->disabled_channels[num_disabled] = -1;3494 3495	dev->platform_data = pdata;3496 3497	return 0;3498}3499 3500static int __init d40_probe(struct platform_device *pdev)3501{3502	struct device *dev = &pdev->dev;3503	struct device_node *np = pdev->dev.of_node;3504	struct device_node *np_lcpa;3505	struct d40_base *base;3506	struct resource *res;3507	struct resource res_lcpa;3508	int num_reserved_chans;3509	u32 val;3510	int ret;3511 3512	if (d40_of_probe(dev, np)) {3513		ret = -ENOMEM;3514		goto report_failure;3515	}3516 3517	ret = d40_hw_detect_init(pdev, &base);3518	if (ret)3519		goto report_failure;3520 3521	num_reserved_chans = d40_phy_res_init(base);3522 3523	platform_set_drvdata(pdev, base);3524 3525	spin_lock_init(&base->interrupt_lock);3526	spin_lock_init(&base->execmd_lock);3527 3528	/* Get IO for logical channel parameter address (LCPA) */3529	np_lcpa = of_parse_phandle(np, "sram", 0);3530	if (!np_lcpa) {3531		dev_err(dev, "no LCPA SRAM node\n");3532		ret = -EINVAL;3533		goto report_failure;3534	}3535	/* This is no device so read the address directly from the node */3536	ret = of_address_to_resource(np_lcpa, 0, &res_lcpa);3537	if (ret) {3538		dev_err(dev, "no LCPA SRAM resource\n");3539		goto report_failure;3540	}3541	base->lcpa_size = resource_size(&res_lcpa);3542	base->phy_lcpa = res_lcpa.start;3543	dev_info(dev, "found LCPA SRAM at %pad, size %pa\n",3544		 &base->phy_lcpa, &base->lcpa_size);3545 3546	/* We make use of ESRAM memory for this. */3547	val = readl(base->virtbase + D40_DREG_LCPA);3548	if (base->phy_lcpa != val && val != 0) {3549		dev_warn(dev,3550			 "[%s] Mismatch LCPA dma 0x%x, def %08x\n",3551			 __func__, val, (u32)base->phy_lcpa);3552	} else3553		writel(base->phy_lcpa, base->virtbase + D40_DREG_LCPA);3554 3555	base->lcpa_base = devm_ioremap(dev, base->phy_lcpa, base->lcpa_size);3556	if (!base->lcpa_base) {3557		ret = -ENOMEM;3558		d40_err(dev, "Failed to ioremap LCPA region\n");3559		goto report_failure;3560	}3561	/* If lcla has to be located in ESRAM we don't need to allocate */3562	if (base->plat_data->use_esram_lcla) {3563		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,3564							"lcla_esram");3565		if (!res) {3566			ret = -ENOENT;3567			d40_err(dev,3568				"No \"lcla_esram\" memory resource\n");3569			goto report_failure;3570		}3571		base->lcla_pool.base = devm_ioremap(dev, res->start,3572						    resource_size(res));3573		if (!base->lcla_pool.base) {3574			ret = -ENOMEM;3575			d40_err(dev, "Failed to ioremap LCLA region\n");3576			goto report_failure;3577		}3578		writel(res->start, base->virtbase + D40_DREG_LCLA);3579 3580	} else {3581		ret = d40_lcla_allocate(base);3582		if (ret) {3583			d40_err(dev, "Failed to allocate LCLA area\n");3584			goto destroy_cache;3585		}3586	}3587 3588	spin_lock_init(&base->lcla_pool.lock);3589 3590	base->irq = platform_get_irq(pdev, 0);3591	if (base->irq < 0) {3592		ret = base->irq;3593		goto destroy_cache;3594	}3595 3596	ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);3597	if (ret) {3598		d40_err(dev, "No IRQ defined\n");3599		goto destroy_cache;3600	}3601 3602	if (base->plat_data->use_esram_lcla) {3603 3604		base->lcpa_regulator = regulator_get(base->dev, "lcla_esram");3605		if (IS_ERR(base->lcpa_regulator)) {3606			d40_err(dev, "Failed to get lcpa_regulator\n");3607			ret = PTR_ERR(base->lcpa_regulator);3608			base->lcpa_regulator = NULL;3609			goto destroy_cache;3610		}3611 3612		ret = regulator_enable(base->lcpa_regulator);3613		if (ret) {3614			d40_err(dev,3615				"Failed to enable lcpa_regulator\n");3616			regulator_put(base->lcpa_regulator);3617			base->lcpa_regulator = NULL;3618			goto destroy_cache;3619		}3620	}3621 3622	writel_relaxed(D40_DREG_GCC_ENABLE_ALL, base->virtbase + D40_DREG_GCC);3623 3624	pm_runtime_irq_safe(base->dev);3625	pm_runtime_set_autosuspend_delay(base->dev, DMA40_AUTOSUSPEND_DELAY);3626	pm_runtime_use_autosuspend(base->dev);3627	pm_runtime_mark_last_busy(base->dev);3628	pm_runtime_set_active(base->dev);3629	pm_runtime_enable(base->dev);3630 3631	ret = d40_dmaengine_init(base, num_reserved_chans);3632	if (ret)3633		goto destroy_cache;3634 3635	dma_set_max_seg_size(base->dev, STEDMA40_MAX_SEG_SIZE);3636 3637	d40_hw_init(base);3638 3639	ret = of_dma_controller_register(np, d40_xlate, NULL);3640	if (ret) {3641		dev_err(dev,3642			"could not register of_dma_controller\n");3643		goto destroy_cache;3644	}3645 3646	dev_info(base->dev, "initialized\n");3647	return 0;3648 3649 destroy_cache:3650	if (base->lcla_pool.dma_addr)3651		dma_unmap_single(base->dev, base->lcla_pool.dma_addr,3652				 SZ_1K * base->num_phy_chans,3653				 DMA_TO_DEVICE);3654 3655	if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)3656		free_pages((unsigned long)base->lcla_pool.base,3657			   base->lcla_pool.pages);3658 3659	kfree(base->lcla_pool.base_unaligned);3660 3661	if (base->lcpa_regulator) {3662		regulator_disable(base->lcpa_regulator);3663		regulator_put(base->lcpa_regulator);3664	}3665	pm_runtime_disable(base->dev);3666 3667 report_failure:3668	d40_err(dev, "probe failed\n");3669	return ret;3670}3671 3672static const struct of_device_id d40_match[] = {3673        { .compatible = "stericsson,dma40", },3674        {}3675};3676 3677static struct platform_driver d40_driver = {3678	.driver = {3679		.name  = D40_NAME,3680		.pm = &dma40_pm_ops,3681		.of_match_table = d40_match,3682	},3683};3684 3685static int __init stedma40_init(void)3686{3687	return platform_driver_probe(&d40_driver, d40_probe);3688}3689subsys_initcall(stedma40_init);3690