brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Synopsys DesignWare Multimedia Card Interface driver4 *  (Based on NXP driver for lpc 31xx)5 *6 * Copyright (C) 2009 NXP Semiconductors7 * Copyright (C) 2009, 2010 Imagination Technologies Ltd.8 */9 10#include <linux/blkdev.h>11#include <linux/clk.h>12#include <linux/debugfs.h>13#include <linux/device.h>14#include <linux/dma-mapping.h>15#include <linux/err.h>16#include <linux/init.h>17#include <linux/interrupt.h>18#include <linux/iopoll.h>19#include <linux/ioport.h>20#include <linux/ktime.h>21#include <linux/module.h>22#include <linux/platform_device.h>23#include <linux/pm_runtime.h>24#include <linux/prandom.h>25#include <linux/seq_file.h>26#include <linux/slab.h>27#include <linux/stat.h>28#include <linux/delay.h>29#include <linux/irq.h>30#include <linux/mmc/card.h>31#include <linux/mmc/host.h>32#include <linux/mmc/mmc.h>33#include <linux/mmc/sd.h>34#include <linux/mmc/sdio.h>35#include <linux/bitops.h>36#include <linux/regulator/consumer.h>37#include <linux/of.h>38#include <linux/mmc/slot-gpio.h>39 40#include "dw_mmc.h"41 42/* Common flag combinations */43#define DW_MCI_DATA_ERROR_FLAGS	(SDMMC_INT_DRTO | SDMMC_INT_DCRC | \44				 SDMMC_INT_HTO | SDMMC_INT_SBE  | \45				 SDMMC_INT_EBE | SDMMC_INT_HLE)46#define DW_MCI_CMD_ERROR_FLAGS	(SDMMC_INT_RTO | SDMMC_INT_RCRC | \47				 SDMMC_INT_RESP_ERR | SDMMC_INT_HLE)48#define DW_MCI_ERROR_FLAGS	(DW_MCI_DATA_ERROR_FLAGS | \49				 DW_MCI_CMD_ERROR_FLAGS)50#define DW_MCI_SEND_STATUS	151#define DW_MCI_RECV_STATUS	252#define DW_MCI_DMA_THRESHOLD	1653 54#define DW_MCI_FREQ_MAX	200000000	/* unit: HZ */55#define DW_MCI_FREQ_MIN	100000		/* unit: HZ */56 57#define IDMAC_INT_CLR		(SDMMC_IDMAC_INT_AI | SDMMC_IDMAC_INT_NI | \58				 SDMMC_IDMAC_INT_CES | SDMMC_IDMAC_INT_DU | \59				 SDMMC_IDMAC_INT_FBE | SDMMC_IDMAC_INT_RI | \60				 SDMMC_IDMAC_INT_TI)61 62#define DESC_RING_BUF_SZ	PAGE_SIZE63 64struct idmac_desc_64addr {65	u32		des0;	/* Control Descriptor */66#define IDMAC_OWN_CLR64(x) \67	!((x) & cpu_to_le32(IDMAC_DES0_OWN))68 69	u32		des1;	/* Reserved */70 71	u32		des2;	/*Buffer sizes */72#define IDMAC_64ADDR_SET_BUFFER1_SIZE(d, s) \73	((d)->des2 = ((d)->des2 & cpu_to_le32(0x03ffe000)) | \74	 ((cpu_to_le32(s)) & cpu_to_le32(0x1fff)))75 76	u32		des3;	/* Reserved */77 78	u32		des4;	/* Lower 32-bits of Buffer Address Pointer 1*/79	u32		des5;	/* Upper 32-bits of Buffer Address Pointer 1*/80 81	u32		des6;	/* Lower 32-bits of Next Descriptor Address */82	u32		des7;	/* Upper 32-bits of Next Descriptor Address */83};84 85struct idmac_desc {86	__le32		des0;	/* Control Descriptor */87#define IDMAC_DES0_DIC	BIT(1)88#define IDMAC_DES0_LD	BIT(2)89#define IDMAC_DES0_FD	BIT(3)90#define IDMAC_DES0_CH	BIT(4)91#define IDMAC_DES0_ER	BIT(5)92#define IDMAC_DES0_CES	BIT(30)93#define IDMAC_DES0_OWN	BIT(31)94 95	__le32		des1;	/* Buffer sizes */96#define IDMAC_SET_BUFFER1_SIZE(d, s) \97	((d)->des1 = ((d)->des1 & cpu_to_le32(0x03ffe000)) | (cpu_to_le32((s) & 0x1fff)))98 99	__le32		des2;	/* buffer 1 physical address */100 101	__le32		des3;	/* buffer 2 physical address */102};103 104/* Each descriptor can transfer up to 4KB of data in chained mode */105#define DW_MCI_DESC_DATA_LENGTH	0x1000106 107#if defined(CONFIG_DEBUG_FS)108static int dw_mci_req_show(struct seq_file *s, void *v)109{110	struct dw_mci_slot *slot = s->private;111	struct mmc_request *mrq;112	struct mmc_command *cmd;113	struct mmc_command *stop;114	struct mmc_data	*data;115 116	/* Make sure we get a consistent snapshot */117	spin_lock_bh(&slot->host->lock);118	mrq = slot->mrq;119 120	if (mrq) {121		cmd = mrq->cmd;122		data = mrq->data;123		stop = mrq->stop;124 125		if (cmd)126			seq_printf(s,127				   "CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n",128				   cmd->opcode, cmd->arg, cmd->flags,129				   cmd->resp[0], cmd->resp[1], cmd->resp[2],130				   cmd->resp[2], cmd->error);131		if (data)132			seq_printf(s, "DATA %u / %u * %u flg %x err %d\n",133				   data->bytes_xfered, data->blocks,134				   data->blksz, data->flags, data->error);135		if (stop)136			seq_printf(s,137				   "CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n",138				   stop->opcode, stop->arg, stop->flags,139				   stop->resp[0], stop->resp[1], stop->resp[2],140				   stop->resp[2], stop->error);141	}142 143	spin_unlock_bh(&slot->host->lock);144 145	return 0;146}147DEFINE_SHOW_ATTRIBUTE(dw_mci_req);148 149static int dw_mci_regs_show(struct seq_file *s, void *v)150{151	struct dw_mci *host = s->private;152 153	pm_runtime_get_sync(host->dev);154 155	seq_printf(s, "STATUS:\t0x%08x\n", mci_readl(host, STATUS));156	seq_printf(s, "RINTSTS:\t0x%08x\n", mci_readl(host, RINTSTS));157	seq_printf(s, "CMD:\t0x%08x\n", mci_readl(host, CMD));158	seq_printf(s, "CTRL:\t0x%08x\n", mci_readl(host, CTRL));159	seq_printf(s, "INTMASK:\t0x%08x\n", mci_readl(host, INTMASK));160	seq_printf(s, "CLKENA:\t0x%08x\n", mci_readl(host, CLKENA));161 162	pm_runtime_put_autosuspend(host->dev);163 164	return 0;165}166DEFINE_SHOW_ATTRIBUTE(dw_mci_regs);167 168static void dw_mci_init_debugfs(struct dw_mci_slot *slot)169{170	struct mmc_host	*mmc = slot->mmc;171	struct dw_mci *host = slot->host;172	struct dentry *root;173 174	root = mmc->debugfs_root;175	if (!root)176		return;177 178	debugfs_create_file("regs", S_IRUSR, root, host, &dw_mci_regs_fops);179	debugfs_create_file("req", S_IRUSR, root, slot, &dw_mci_req_fops);180	debugfs_create_u32("state", S_IRUSR, root, &host->state);181	debugfs_create_xul("pending_events", S_IRUSR, root,182			   &host->pending_events);183	debugfs_create_xul("completed_events", S_IRUSR, root,184			   &host->completed_events);185#ifdef CONFIG_FAULT_INJECTION186	fault_create_debugfs_attr("fail_data_crc", root, &host->fail_data_crc);187#endif188}189#endif /* defined(CONFIG_DEBUG_FS) */190 191static bool dw_mci_ctrl_reset(struct dw_mci *host, u32 reset)192{193	u32 ctrl;194 195	ctrl = mci_readl(host, CTRL);196	ctrl |= reset;197	mci_writel(host, CTRL, ctrl);198 199	/* wait till resets clear */200	if (readl_poll_timeout_atomic(host->regs + SDMMC_CTRL, ctrl,201				      !(ctrl & reset),202				      1, 500 * USEC_PER_MSEC)) {203		dev_err(host->dev,204			"Timeout resetting block (ctrl reset %#x)\n",205			ctrl & reset);206		return false;207	}208 209	return true;210}211 212static void dw_mci_wait_while_busy(struct dw_mci *host, u32 cmd_flags)213{214	u32 status;215 216	/*217	 * Databook says that before issuing a new data transfer command218	 * we need to check to see if the card is busy.  Data transfer commands219	 * all have SDMMC_CMD_PRV_DAT_WAIT set, so we'll key off that.220	 *221	 * ...also allow sending for SDMMC_CMD_VOLT_SWITCH where busy is222	 * expected.223	 */224	if ((cmd_flags & SDMMC_CMD_PRV_DAT_WAIT) &&225	    !(cmd_flags & SDMMC_CMD_VOLT_SWITCH)) {226		if (readl_poll_timeout_atomic(host->regs + SDMMC_STATUS,227					      status,228					      !(status & SDMMC_STATUS_BUSY),229					      10, 500 * USEC_PER_MSEC))230			dev_err(host->dev, "Busy; trying anyway\n");231	}232}233 234static void mci_send_cmd(struct dw_mci_slot *slot, u32 cmd, u32 arg)235{236	struct dw_mci *host = slot->host;237	unsigned int cmd_status = 0;238 239	mci_writel(host, CMDARG, arg);240	wmb(); /* drain writebuffer */241	dw_mci_wait_while_busy(host, cmd);242	mci_writel(host, CMD, SDMMC_CMD_START | cmd);243 244	if (readl_poll_timeout_atomic(host->regs + SDMMC_CMD, cmd_status,245				      !(cmd_status & SDMMC_CMD_START),246				      1, 500 * USEC_PER_MSEC))247		dev_err(&slot->mmc->class_dev,248			"Timeout sending command (cmd %#x arg %#x status %#x)\n",249			cmd, arg, cmd_status);250}251 252static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd)253{254	struct dw_mci_slot *slot = mmc_priv(mmc);255	struct dw_mci *host = slot->host;256	u32 cmdr;257 258	cmd->error = -EINPROGRESS;259	cmdr = cmd->opcode;260 261	if (cmd->opcode == MMC_STOP_TRANSMISSION ||262	    cmd->opcode == MMC_GO_IDLE_STATE ||263	    cmd->opcode == MMC_GO_INACTIVE_STATE ||264	    (cmd->opcode == SD_IO_RW_DIRECT &&265	     ((cmd->arg >> 9) & 0x1FFFF) == SDIO_CCCR_ABORT))266		cmdr |= SDMMC_CMD_STOP;267	else if (cmd->opcode != MMC_SEND_STATUS && cmd->data)268		cmdr |= SDMMC_CMD_PRV_DAT_WAIT;269 270	if (cmd->opcode == SD_SWITCH_VOLTAGE) {271		u32 clk_en_a;272 273		/* Special bit makes CMD11 not die */274		cmdr |= SDMMC_CMD_VOLT_SWITCH;275 276		/* Change state to continue to handle CMD11 weirdness */277		WARN_ON(slot->host->state != STATE_SENDING_CMD);278		slot->host->state = STATE_SENDING_CMD11;279 280		/*281		 * We need to disable low power mode (automatic clock stop)282		 * while doing voltage switch so we don't confuse the card,283		 * since stopping the clock is a specific part of the UHS284		 * voltage change dance.285		 *286		 * Note that low power mode (SDMMC_CLKEN_LOW_PWR) will be287		 * unconditionally turned back on in dw_mci_setup_bus() if it's288		 * ever called with a non-zero clock.  That shouldn't happen289		 * until the voltage change is all done.290		 */291		clk_en_a = mci_readl(host, CLKENA);292		clk_en_a &= ~(SDMMC_CLKEN_LOW_PWR << slot->id);293		mci_writel(host, CLKENA, clk_en_a);294		mci_send_cmd(slot, SDMMC_CMD_UPD_CLK |295			     SDMMC_CMD_PRV_DAT_WAIT, 0);296	}297 298	if (cmd->flags & MMC_RSP_PRESENT) {299		/* We expect a response, so set this bit */300		cmdr |= SDMMC_CMD_RESP_EXP;301		if (cmd->flags & MMC_RSP_136)302			cmdr |= SDMMC_CMD_RESP_LONG;303	}304 305	if (cmd->flags & MMC_RSP_CRC)306		cmdr |= SDMMC_CMD_RESP_CRC;307 308	if (cmd->data) {309		cmdr |= SDMMC_CMD_DAT_EXP;310		if (cmd->data->flags & MMC_DATA_WRITE)311			cmdr |= SDMMC_CMD_DAT_WR;312	}313 314	if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &slot->flags))315		cmdr |= SDMMC_CMD_USE_HOLD_REG;316 317	return cmdr;318}319 320static u32 dw_mci_prep_stop_abort(struct dw_mci *host, struct mmc_command *cmd)321{322	struct mmc_command *stop;323	u32 cmdr;324 325	if (!cmd->data)326		return 0;327 328	stop = &host->stop_abort;329	cmdr = cmd->opcode;330	memset(stop, 0, sizeof(struct mmc_command));331 332	if (cmdr == MMC_READ_SINGLE_BLOCK ||333	    cmdr == MMC_READ_MULTIPLE_BLOCK ||334	    cmdr == MMC_WRITE_BLOCK ||335	    cmdr == MMC_WRITE_MULTIPLE_BLOCK ||336	    mmc_op_tuning(cmdr) ||337	    cmdr == MMC_GEN_CMD) {338		stop->opcode = MMC_STOP_TRANSMISSION;339		stop->arg = 0;340		stop->flags = MMC_RSP_R1B | MMC_CMD_AC;341	} else if (cmdr == SD_IO_RW_EXTENDED) {342		stop->opcode = SD_IO_RW_DIRECT;343		stop->arg |= (1 << 31) | (0 << 28) | (SDIO_CCCR_ABORT << 9) |344			     ((cmd->arg >> 28) & 0x7);345		stop->flags = MMC_RSP_SPI_R5 | MMC_RSP_R5 | MMC_CMD_AC;346	} else {347		return 0;348	}349 350	cmdr = stop->opcode | SDMMC_CMD_STOP |351		SDMMC_CMD_RESP_CRC | SDMMC_CMD_RESP_EXP;352 353	if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &host->slot->flags))354		cmdr |= SDMMC_CMD_USE_HOLD_REG;355 356	return cmdr;357}358 359static inline void dw_mci_set_cto(struct dw_mci *host)360{361	unsigned int cto_clks;362	unsigned int cto_div;363	unsigned int cto_ms;364	unsigned long irqflags;365 366	cto_clks = mci_readl(host, TMOUT) & 0xff;367	cto_div = (mci_readl(host, CLKDIV) & 0xff) * 2;368	if (cto_div == 0)369		cto_div = 1;370 371	cto_ms = DIV_ROUND_UP_ULL((u64)MSEC_PER_SEC * cto_clks * cto_div,372				  host->bus_hz);373 374	/* add a bit spare time */375	cto_ms += 10;376 377	/*378	 * The durations we're working with are fairly short so we have to be379	 * extra careful about synchronization here.  Specifically in hardware a380	 * command timeout is _at most_ 5.1 ms, so that means we expect an381	 * interrupt (either command done or timeout) to come rather quickly382	 * after the mci_writel.  ...but just in case we have a long interrupt383	 * latency let's add a bit of paranoia.384	 *385	 * In general we'll assume that at least an interrupt will be asserted386	 * in hardware by the time the cto_timer runs.  ...and if it hasn't387	 * been asserted in hardware by that time then we'll assume it'll never388	 * come.389	 */390	spin_lock_irqsave(&host->irq_lock, irqflags);391	if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events))392		mod_timer(&host->cto_timer,393			jiffies + msecs_to_jiffies(cto_ms) + 1);394	spin_unlock_irqrestore(&host->irq_lock, irqflags);395}396 397static void dw_mci_start_command(struct dw_mci *host,398				 struct mmc_command *cmd, u32 cmd_flags)399{400	host->cmd = cmd;401	dev_vdbg(host->dev,402		 "start command: ARGR=0x%08x CMDR=0x%08x\n",403		 cmd->arg, cmd_flags);404 405	mci_writel(host, CMDARG, cmd->arg);406	wmb(); /* drain writebuffer */407	dw_mci_wait_while_busy(host, cmd_flags);408 409	mci_writel(host, CMD, cmd_flags | SDMMC_CMD_START);410 411	/* response expected command only */412	if (cmd_flags & SDMMC_CMD_RESP_EXP)413		dw_mci_set_cto(host);414}415 416static inline void send_stop_abort(struct dw_mci *host, struct mmc_data *data)417{418	struct mmc_command *stop = &host->stop_abort;419 420	dw_mci_start_command(host, stop, host->stop_cmdr);421}422 423/* DMA interface functions */424static void dw_mci_stop_dma(struct dw_mci *host)425{426	if (host->using_dma) {427		host->dma_ops->stop(host);428		host->dma_ops->cleanup(host);429	}430 431	/* Data transfer was stopped by the interrupt handler */432	set_bit(EVENT_XFER_COMPLETE, &host->pending_events);433}434 435static void dw_mci_dma_cleanup(struct dw_mci *host)436{437	struct mmc_data *data = host->data;438 439	if (data && data->host_cookie == COOKIE_MAPPED) {440		dma_unmap_sg(host->dev,441			     data->sg,442			     data->sg_len,443			     mmc_get_dma_dir(data));444		data->host_cookie = COOKIE_UNMAPPED;445	}446}447 448static void dw_mci_idmac_reset(struct dw_mci *host)449{450	u32 bmod = mci_readl(host, BMOD);451	/* Software reset of DMA */452	bmod |= SDMMC_IDMAC_SWRESET;453	mci_writel(host, BMOD, bmod);454}455 456static void dw_mci_idmac_stop_dma(struct dw_mci *host)457{458	u32 temp;459 460	/* Disable and reset the IDMAC interface */461	temp = mci_readl(host, CTRL);462	temp &= ~SDMMC_CTRL_USE_IDMAC;463	temp |= SDMMC_CTRL_DMA_RESET;464	mci_writel(host, CTRL, temp);465 466	/* Stop the IDMAC running */467	temp = mci_readl(host, BMOD);468	temp &= ~(SDMMC_IDMAC_ENABLE | SDMMC_IDMAC_FB);469	temp |= SDMMC_IDMAC_SWRESET;470	mci_writel(host, BMOD, temp);471}472 473static void dw_mci_dmac_complete_dma(void *arg)474{475	struct dw_mci *host = arg;476	struct mmc_data *data = host->data;477 478	dev_vdbg(host->dev, "DMA complete\n");479 480	if ((host->use_dma == TRANS_MODE_EDMAC) &&481	    data && (data->flags & MMC_DATA_READ))482		/* Invalidate cache after read */483		dma_sync_sg_for_cpu(mmc_dev(host->slot->mmc),484				    data->sg,485				    data->sg_len,486				    DMA_FROM_DEVICE);487 488	host->dma_ops->cleanup(host);489 490	/*491	 * If the card was removed, data will be NULL. No point in trying to492	 * send the stop command or waiting for NBUSY in this case.493	 */494	if (data) {495		set_bit(EVENT_XFER_COMPLETE, &host->pending_events);496		queue_work(system_bh_wq, &host->bh_work);497	}498}499 500static int dw_mci_idmac_init(struct dw_mci *host)501{502	int i;503 504	if (host->dma_64bit_address == 1) {505		struct idmac_desc_64addr *p;506		/* Number of descriptors in the ring buffer */507		host->ring_size =508			DESC_RING_BUF_SZ / sizeof(struct idmac_desc_64addr);509 510		/* Forward link the descriptor list */511		for (i = 0, p = host->sg_cpu; i < host->ring_size - 1;512								i++, p++) {513			p->des6 = (host->sg_dma +514					(sizeof(struct idmac_desc_64addr) *515							(i + 1))) & 0xffffffff;516 517			p->des7 = (u64)(host->sg_dma +518					(sizeof(struct idmac_desc_64addr) *519							(i + 1))) >> 32;520			/* Initialize reserved and buffer size fields to "0" */521			p->des0 = 0;522			p->des1 = 0;523			p->des2 = 0;524			p->des3 = 0;525		}526 527		/* Set the last descriptor as the end-of-ring descriptor */528		p->des6 = host->sg_dma & 0xffffffff;529		p->des7 = (u64)host->sg_dma >> 32;530		p->des0 = IDMAC_DES0_ER;531 532	} else {533		struct idmac_desc *p;534		/* Number of descriptors in the ring buffer */535		host->ring_size =536			DESC_RING_BUF_SZ / sizeof(struct idmac_desc);537 538		/* Forward link the descriptor list */539		for (i = 0, p = host->sg_cpu;540		     i < host->ring_size - 1;541		     i++, p++) {542			p->des3 = cpu_to_le32(host->sg_dma +543					(sizeof(struct idmac_desc) * (i + 1)));544			p->des0 = 0;545			p->des1 = 0;546		}547 548		/* Set the last descriptor as the end-of-ring descriptor */549		p->des3 = cpu_to_le32(host->sg_dma);550		p->des0 = cpu_to_le32(IDMAC_DES0_ER);551	}552 553	dw_mci_idmac_reset(host);554 555	if (host->dma_64bit_address == 1) {556		/* Mask out interrupts - get Tx & Rx complete only */557		mci_writel(host, IDSTS64, IDMAC_INT_CLR);558		mci_writel(host, IDINTEN64, SDMMC_IDMAC_INT_NI |559				SDMMC_IDMAC_INT_RI | SDMMC_IDMAC_INT_TI);560 561		/* Set the descriptor base address */562		mci_writel(host, DBADDRL, host->sg_dma & 0xffffffff);563		mci_writel(host, DBADDRU, (u64)host->sg_dma >> 32);564 565	} else {566		/* Mask out interrupts - get Tx & Rx complete only */567		mci_writel(host, IDSTS, IDMAC_INT_CLR);568		mci_writel(host, IDINTEN, SDMMC_IDMAC_INT_NI |569				SDMMC_IDMAC_INT_RI | SDMMC_IDMAC_INT_TI);570 571		/* Set the descriptor base address */572		mci_writel(host, DBADDR, host->sg_dma);573	}574 575	return 0;576}577 578static inline int dw_mci_prepare_desc64(struct dw_mci *host,579					 struct mmc_data *data,580					 unsigned int sg_len)581{582	unsigned int desc_len;583	struct idmac_desc_64addr *desc_first, *desc_last, *desc;584	u32 val;585	int i;586 587	desc_first = desc_last = desc = host->sg_cpu;588 589	for (i = 0; i < sg_len; i++) {590		unsigned int length = sg_dma_len(&data->sg[i]);591 592		u64 mem_addr = sg_dma_address(&data->sg[i]);593 594		for ( ; length ; desc++) {595			desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ?596				   length : DW_MCI_DESC_DATA_LENGTH;597 598			length -= desc_len;599 600			/*601			 * Wait for the former clear OWN bit operation602			 * of IDMAC to make sure that this descriptor603			 * isn't still owned by IDMAC as IDMAC's write604			 * ops and CPU's read ops are asynchronous.605			 */606			if (readl_poll_timeout_atomic(&desc->des0, val,607						!(val & IDMAC_DES0_OWN),608						10, 100 * USEC_PER_MSEC))609				goto err_own_bit;610 611			/*612			 * Set the OWN bit and disable interrupts613			 * for this descriptor614			 */615			desc->des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC |616						IDMAC_DES0_CH;617 618			/* Buffer length */619			IDMAC_64ADDR_SET_BUFFER1_SIZE(desc, desc_len);620 621			/* Physical address to DMA to/from */622			desc->des4 = mem_addr & 0xffffffff;623			desc->des5 = mem_addr >> 32;624 625			/* Update physical address for the next desc */626			mem_addr += desc_len;627 628			/* Save pointer to the last descriptor */629			desc_last = desc;630		}631	}632 633	/* Set first descriptor */634	desc_first->des0 |= IDMAC_DES0_FD;635 636	/* Set last descriptor */637	desc_last->des0 &= ~(IDMAC_DES0_CH | IDMAC_DES0_DIC);638	desc_last->des0 |= IDMAC_DES0_LD;639 640	return 0;641err_own_bit:642	/* restore the descriptor chain as it's polluted */643	dev_dbg(host->dev, "descriptor is still owned by IDMAC.\n");644	memset(host->sg_cpu, 0, DESC_RING_BUF_SZ);645	dw_mci_idmac_init(host);646	return -EINVAL;647}648 649 650static inline int dw_mci_prepare_desc32(struct dw_mci *host,651					 struct mmc_data *data,652					 unsigned int sg_len)653{654	unsigned int desc_len;655	struct idmac_desc *desc_first, *desc_last, *desc;656	u32 val;657	int i;658 659	desc_first = desc_last = desc = host->sg_cpu;660 661	for (i = 0; i < sg_len; i++) {662		unsigned int length = sg_dma_len(&data->sg[i]);663 664		u32 mem_addr = sg_dma_address(&data->sg[i]);665 666		for ( ; length ; desc++) {667			desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ?668				   length : DW_MCI_DESC_DATA_LENGTH;669 670			length -= desc_len;671 672			/*673			 * Wait for the former clear OWN bit operation674			 * of IDMAC to make sure that this descriptor675			 * isn't still owned by IDMAC as IDMAC's write676			 * ops and CPU's read ops are asynchronous.677			 */678			if (readl_poll_timeout_atomic(&desc->des0, val,679						      IDMAC_OWN_CLR64(val),680						      10,681						      100 * USEC_PER_MSEC))682				goto err_own_bit;683 684			/*685			 * Set the OWN bit and disable interrupts686			 * for this descriptor687			 */688			desc->des0 = cpu_to_le32(IDMAC_DES0_OWN |689						 IDMAC_DES0_DIC |690						 IDMAC_DES0_CH);691 692			/* Buffer length */693			IDMAC_SET_BUFFER1_SIZE(desc, desc_len);694 695			/* Physical address to DMA to/from */696			desc->des2 = cpu_to_le32(mem_addr);697 698			/* Update physical address for the next desc */699			mem_addr += desc_len;700 701			/* Save pointer to the last descriptor */702			desc_last = desc;703		}704	}705 706	/* Set first descriptor */707	desc_first->des0 |= cpu_to_le32(IDMAC_DES0_FD);708 709	/* Set last descriptor */710	desc_last->des0 &= cpu_to_le32(~(IDMAC_DES0_CH |711				       IDMAC_DES0_DIC));712	desc_last->des0 |= cpu_to_le32(IDMAC_DES0_LD);713 714	return 0;715err_own_bit:716	/* restore the descriptor chain as it's polluted */717	dev_dbg(host->dev, "descriptor is still owned by IDMAC.\n");718	memset(host->sg_cpu, 0, DESC_RING_BUF_SZ);719	dw_mci_idmac_init(host);720	return -EINVAL;721}722 723static int dw_mci_idmac_start_dma(struct dw_mci *host, unsigned int sg_len)724{725	u32 temp;726	int ret;727 728	if (host->dma_64bit_address == 1)729		ret = dw_mci_prepare_desc64(host, host->data, sg_len);730	else731		ret = dw_mci_prepare_desc32(host, host->data, sg_len);732 733	if (ret)734		goto out;735 736	/* drain writebuffer */737	wmb();738 739	/* Make sure to reset DMA in case we did PIO before this */740	dw_mci_ctrl_reset(host, SDMMC_CTRL_DMA_RESET);741	dw_mci_idmac_reset(host);742 743	/* Select IDMAC interface */744	temp = mci_readl(host, CTRL);745	temp |= SDMMC_CTRL_USE_IDMAC;746	mci_writel(host, CTRL, temp);747 748	/* drain writebuffer */749	wmb();750 751	/* Enable the IDMAC */752	temp = mci_readl(host, BMOD);753	temp |= SDMMC_IDMAC_ENABLE | SDMMC_IDMAC_FB;754	mci_writel(host, BMOD, temp);755 756	/* Start it running */757	mci_writel(host, PLDMND, 1);758 759out:760	return ret;761}762 763static const struct dw_mci_dma_ops dw_mci_idmac_ops = {764	.init = dw_mci_idmac_init,765	.start = dw_mci_idmac_start_dma,766	.stop = dw_mci_idmac_stop_dma,767	.complete = dw_mci_dmac_complete_dma,768	.cleanup = dw_mci_dma_cleanup,769};770 771static void dw_mci_edmac_stop_dma(struct dw_mci *host)772{773	dmaengine_terminate_async(host->dms->ch);774}775 776static int dw_mci_edmac_start_dma(struct dw_mci *host,777					    unsigned int sg_len)778{779	struct dma_slave_config cfg;780	struct dma_async_tx_descriptor *desc = NULL;781	struct scatterlist *sgl = host->data->sg;782	static const u32 mszs[] = {1, 4, 8, 16, 32, 64, 128, 256};783	u32 sg_elems = host->data->sg_len;784	u32 fifoth_val;785	u32 fifo_offset = host->fifo_reg - host->regs;786	int ret = 0;787 788	/* Set external dma config: burst size, burst width */789	memset(&cfg, 0, sizeof(cfg));790	cfg.dst_addr = host->phy_regs + fifo_offset;791	cfg.src_addr = cfg.dst_addr;792	cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;793	cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;794 795	/* Match burst msize with external dma config */796	fifoth_val = mci_readl(host, FIFOTH);797	cfg.dst_maxburst = mszs[(fifoth_val >> 28) & 0x7];798	cfg.src_maxburst = cfg.dst_maxburst;799 800	if (host->data->flags & MMC_DATA_WRITE)801		cfg.direction = DMA_MEM_TO_DEV;802	else803		cfg.direction = DMA_DEV_TO_MEM;804 805	ret = dmaengine_slave_config(host->dms->ch, &cfg);806	if (ret) {807		dev_err(host->dev, "Failed to config edmac.\n");808		return -EBUSY;809	}810 811	desc = dmaengine_prep_slave_sg(host->dms->ch, sgl,812				       sg_len, cfg.direction,813				       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);814	if (!desc) {815		dev_err(host->dev, "Can't prepare slave sg.\n");816		return -EBUSY;817	}818 819	/* Set dw_mci_dmac_complete_dma as callback */820	desc->callback = dw_mci_dmac_complete_dma;821	desc->callback_param = (void *)host;822	dmaengine_submit(desc);823 824	/* Flush cache before write */825	if (host->data->flags & MMC_DATA_WRITE)826		dma_sync_sg_for_device(mmc_dev(host->slot->mmc), sgl,827				       sg_elems, DMA_TO_DEVICE);828 829	dma_async_issue_pending(host->dms->ch);830 831	return 0;832}833 834static int dw_mci_edmac_init(struct dw_mci *host)835{836	/* Request external dma channel */837	host->dms = kzalloc(sizeof(struct dw_mci_dma_slave), GFP_KERNEL);838	if (!host->dms)839		return -ENOMEM;840 841	host->dms->ch = dma_request_chan(host->dev, "rx-tx");842	if (IS_ERR(host->dms->ch)) {843		int ret = PTR_ERR(host->dms->ch);844 845		dev_err(host->dev, "Failed to get external DMA channel.\n");846		kfree(host->dms);847		host->dms = NULL;848		return ret;849	}850 851	return 0;852}853 854static void dw_mci_edmac_exit(struct dw_mci *host)855{856	if (host->dms) {857		if (host->dms->ch) {858			dma_release_channel(host->dms->ch);859			host->dms->ch = NULL;860		}861		kfree(host->dms);862		host->dms = NULL;863	}864}865 866static const struct dw_mci_dma_ops dw_mci_edmac_ops = {867	.init = dw_mci_edmac_init,868	.exit = dw_mci_edmac_exit,869	.start = dw_mci_edmac_start_dma,870	.stop = dw_mci_edmac_stop_dma,871	.complete = dw_mci_dmac_complete_dma,872	.cleanup = dw_mci_dma_cleanup,873};874 875static int dw_mci_pre_dma_transfer(struct dw_mci *host,876				   struct mmc_data *data,877				   int cookie)878{879	struct scatterlist *sg;880	unsigned int i, sg_len;881 882	if (data->host_cookie == COOKIE_PRE_MAPPED)883		return data->sg_len;884 885	/*886	 * We don't do DMA on "complex" transfers, i.e. with887	 * non-word-aligned buffers or lengths. Also, we don't bother888	 * with all the DMA setup overhead for short transfers.889	 */890	if (data->blocks * data->blksz < DW_MCI_DMA_THRESHOLD)891		return -EINVAL;892 893	if (data->blksz & 3)894		return -EINVAL;895 896	for_each_sg(data->sg, sg, data->sg_len, i) {897		if (sg->offset & 3 || sg->length & 3)898			return -EINVAL;899	}900 901	sg_len = dma_map_sg(host->dev,902			    data->sg,903			    data->sg_len,904			    mmc_get_dma_dir(data));905	if (sg_len == 0)906		return -EINVAL;907 908	data->host_cookie = cookie;909 910	return sg_len;911}912 913static void dw_mci_pre_req(struct mmc_host *mmc,914			   struct mmc_request *mrq)915{916	struct dw_mci_slot *slot = mmc_priv(mmc);917	struct mmc_data *data = mrq->data;918 919	if (!slot->host->use_dma || !data)920		return;921 922	/* This data might be unmapped at this time */923	data->host_cookie = COOKIE_UNMAPPED;924 925	if (dw_mci_pre_dma_transfer(slot->host, mrq->data,926				COOKIE_PRE_MAPPED) < 0)927		data->host_cookie = COOKIE_UNMAPPED;928}929 930static void dw_mci_post_req(struct mmc_host *mmc,931			    struct mmc_request *mrq,932			    int err)933{934	struct dw_mci_slot *slot = mmc_priv(mmc);935	struct mmc_data *data = mrq->data;936 937	if (!slot->host->use_dma || !data)938		return;939 940	if (data->host_cookie != COOKIE_UNMAPPED)941		dma_unmap_sg(slot->host->dev,942			     data->sg,943			     data->sg_len,944			     mmc_get_dma_dir(data));945	data->host_cookie = COOKIE_UNMAPPED;946}947 948static int dw_mci_get_cd(struct mmc_host *mmc)949{950	int present;951	struct dw_mci_slot *slot = mmc_priv(mmc);952	struct dw_mci *host = slot->host;953	int gpio_cd = mmc_gpio_get_cd(mmc);954 955	/* Use platform get_cd function, else try onboard card detect */956	if (((mmc->caps & MMC_CAP_NEEDS_POLL)957				|| !mmc_card_is_removable(mmc))) {958		present = 1;959 960		if (!test_bit(DW_MMC_CARD_PRESENT, &slot->flags)) {961			if (mmc->caps & MMC_CAP_NEEDS_POLL) {962				dev_info(&mmc->class_dev,963					"card is polling.\n");964			} else {965				dev_info(&mmc->class_dev,966					"card is non-removable.\n");967			}968			set_bit(DW_MMC_CARD_PRESENT, &slot->flags);969		}970 971		return present;972	} else if (gpio_cd >= 0)973		present = gpio_cd;974	else975		present = (mci_readl(slot->host, CDETECT) & (1 << slot->id))976			== 0 ? 1 : 0;977 978	spin_lock_bh(&host->lock);979	if (present && !test_and_set_bit(DW_MMC_CARD_PRESENT, &slot->flags))980		dev_dbg(&mmc->class_dev, "card is present\n");981	else if (!present &&982			!test_and_clear_bit(DW_MMC_CARD_PRESENT, &slot->flags))983		dev_dbg(&mmc->class_dev, "card is not present\n");984	spin_unlock_bh(&host->lock);985 986	return present;987}988 989static void dw_mci_adjust_fifoth(struct dw_mci *host, struct mmc_data *data)990{991	unsigned int blksz = data->blksz;992	static const u32 mszs[] = {1, 4, 8, 16, 32, 64, 128, 256};993	u32 fifo_width = 1 << host->data_shift;994	u32 blksz_depth = blksz / fifo_width, fifoth_val;995	u32 msize = 0, rx_wmark = 1, tx_wmark, tx_wmark_invers;996	int idx = ARRAY_SIZE(mszs) - 1;997 998	/* pio should ship this scenario */999	if (!host->use_dma)1000		return;1001 1002	tx_wmark = (host->fifo_depth) / 2;1003	tx_wmark_invers = host->fifo_depth - tx_wmark;1004 1005	/*1006	 * MSIZE is '1',1007	 * if blksz is not a multiple of the FIFO width1008	 */1009	if (blksz % fifo_width)1010		goto done;1011 1012	do {1013		if (!((blksz_depth % mszs[idx]) ||1014		     (tx_wmark_invers % mszs[idx]))) {1015			msize = idx;1016			rx_wmark = mszs[idx] - 1;1017			break;1018		}1019	} while (--idx > 0);1020	/*1021	 * If idx is '0', it won't be tried1022	 * Thus, initial values are uesed1023	 */1024done:1025	fifoth_val = SDMMC_SET_FIFOTH(msize, rx_wmark, tx_wmark);1026	mci_writel(host, FIFOTH, fifoth_val);1027}1028 1029static void dw_mci_ctrl_thld(struct dw_mci *host, struct mmc_data *data)1030{1031	unsigned int blksz = data->blksz;1032	u32 blksz_depth, fifo_depth;1033	u16 thld_size;1034	u8 enable;1035 1036	/*1037	 * CDTHRCTL doesn't exist prior to 240A (in fact that register offset is1038	 * in the FIFO region, so we really shouldn't access it).1039	 */1040	if (host->verid < DW_MMC_240A ||1041		(host->verid < DW_MMC_280A && data->flags & MMC_DATA_WRITE))1042		return;1043 1044	/*1045	 * Card write Threshold is introduced since 2.80a1046	 * It's used when HS400 mode is enabled.1047	 */1048	if (data->flags & MMC_DATA_WRITE &&1049		host->timing != MMC_TIMING_MMC_HS400)1050		goto disable;1051 1052	if (data->flags & MMC_DATA_WRITE)1053		enable = SDMMC_CARD_WR_THR_EN;1054	else1055		enable = SDMMC_CARD_RD_THR_EN;1056 1057	if (host->timing != MMC_TIMING_MMC_HS200 &&1058	    host->timing != MMC_TIMING_UHS_SDR104 &&1059	    host->timing != MMC_TIMING_MMC_HS400)1060		goto disable;1061 1062	blksz_depth = blksz / (1 << host->data_shift);1063	fifo_depth = host->fifo_depth;1064 1065	if (blksz_depth > fifo_depth)1066		goto disable;1067 1068	/*1069	 * If (blksz_depth) >= (fifo_depth >> 1), should be 'thld_size <= blksz'1070	 * If (blksz_depth) <  (fifo_depth >> 1), should be thld_size = blksz1071	 * Currently just choose blksz.1072	 */1073	thld_size = blksz;1074	mci_writel(host, CDTHRCTL, SDMMC_SET_THLD(thld_size, enable));1075	return;1076 1077disable:1078	mci_writel(host, CDTHRCTL, 0);1079}1080 1081static int dw_mci_submit_data_dma(struct dw_mci *host, struct mmc_data *data)1082{1083	unsigned long irqflags;1084	int sg_len;1085	u32 temp;1086 1087	host->using_dma = 0;1088 1089	/* If we don't have a channel, we can't do DMA */1090	if (!host->use_dma)1091		return -ENODEV;1092 1093	sg_len = dw_mci_pre_dma_transfer(host, data, COOKIE_MAPPED);1094	if (sg_len < 0) {1095		host->dma_ops->stop(host);1096		return sg_len;1097	}1098 1099	host->using_dma = 1;1100 1101	if (host->use_dma == TRANS_MODE_IDMAC)1102		dev_vdbg(host->dev,1103			 "sd sg_cpu: %#lx sg_dma: %#lx sg_len: %d\n",1104			 (unsigned long)host->sg_cpu,1105			 (unsigned long)host->sg_dma,1106			 sg_len);1107 1108	/*1109	 * Decide the MSIZE and RX/TX Watermark.1110	 * If current block size is same with previous size,1111	 * no need to update fifoth.1112	 */1113	if (host->prev_blksz != data->blksz)1114		dw_mci_adjust_fifoth(host, data);1115 1116	/* Enable the DMA interface */1117	temp = mci_readl(host, CTRL);1118	temp |= SDMMC_CTRL_DMA_ENABLE;1119	mci_writel(host, CTRL, temp);1120 1121	/* Disable RX/TX IRQs, let DMA handle it */1122	spin_lock_irqsave(&host->irq_lock, irqflags);1123	temp = mci_readl(host, INTMASK);1124	temp  &= ~(SDMMC_INT_RXDR | SDMMC_INT_TXDR);1125	mci_writel(host, INTMASK, temp);1126	spin_unlock_irqrestore(&host->irq_lock, irqflags);1127 1128	if (host->dma_ops->start(host, sg_len)) {1129		host->dma_ops->stop(host);1130		/* We can't do DMA, try PIO for this one */1131		dev_dbg(host->dev,1132			"%s: fall back to PIO mode for current transfer\n",1133			__func__);1134		return -ENODEV;1135	}1136 1137	return 0;1138}1139 1140static void dw_mci_submit_data(struct dw_mci *host, struct mmc_data *data)1141{1142	unsigned long irqflags;1143	int flags = SG_MITER_ATOMIC;1144	u32 temp;1145 1146	data->error = -EINPROGRESS;1147 1148	WARN_ON(host->data);1149	host->sg = NULL;1150	host->data = data;1151 1152	if (data->flags & MMC_DATA_READ)1153		host->dir_status = DW_MCI_RECV_STATUS;1154	else1155		host->dir_status = DW_MCI_SEND_STATUS;1156 1157	dw_mci_ctrl_thld(host, data);1158 1159	if (dw_mci_submit_data_dma(host, data)) {1160		if (host->data->flags & MMC_DATA_READ)1161			flags |= SG_MITER_TO_SG;1162		else1163			flags |= SG_MITER_FROM_SG;1164 1165		sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);1166		host->sg = data->sg;1167		host->part_buf_start = 0;1168		host->part_buf_count = 0;1169 1170		mci_writel(host, RINTSTS, SDMMC_INT_TXDR | SDMMC_INT_RXDR);1171 1172		spin_lock_irqsave(&host->irq_lock, irqflags);1173		temp = mci_readl(host, INTMASK);1174		temp |= SDMMC_INT_TXDR | SDMMC_INT_RXDR;1175		mci_writel(host, INTMASK, temp);1176		spin_unlock_irqrestore(&host->irq_lock, irqflags);1177 1178		temp = mci_readl(host, CTRL);1179		temp &= ~SDMMC_CTRL_DMA_ENABLE;1180		mci_writel(host, CTRL, temp);1181 1182		/*1183		 * Use the initial fifoth_val for PIO mode. If wm_algined1184		 * is set, we set watermark same as data size.1185		 * If next issued data may be transfered by DMA mode,1186		 * prev_blksz should be invalidated.1187		 */1188		if (host->wm_aligned)1189			dw_mci_adjust_fifoth(host, data);1190		else1191			mci_writel(host, FIFOTH, host->fifoth_val);1192		host->prev_blksz = 0;1193	} else {1194		/*1195		 * Keep the current block size.1196		 * It will be used to decide whether to update1197		 * fifoth register next time.1198		 */1199		host->prev_blksz = data->blksz;1200	}1201}1202 1203static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)1204{1205	struct dw_mci *host = slot->host;1206	unsigned int clock = slot->clock;1207	u32 div;1208	u32 clk_en_a;1209	u32 sdmmc_cmd_bits = SDMMC_CMD_UPD_CLK | SDMMC_CMD_PRV_DAT_WAIT;1210 1211	/* We must continue to set bit 28 in CMD until the change is complete */1212	if (host->state == STATE_WAITING_CMD11_DONE)1213		sdmmc_cmd_bits |= SDMMC_CMD_VOLT_SWITCH;1214 1215	slot->mmc->actual_clock = 0;1216 1217	if (!clock) {1218		mci_writel(host, CLKENA, 0);1219		mci_send_cmd(slot, sdmmc_cmd_bits, 0);1220	} else if (clock != host->current_speed || force_clkinit) {1221		div = host->bus_hz / clock;1222		if (host->bus_hz % clock && host->bus_hz > clock)1223			/*1224			 * move the + 1 after the divide to prevent1225			 * over-clocking the card.1226			 */1227			div += 1;1228 1229		div = (host->bus_hz != clock) ? DIV_ROUND_UP(div, 2) : 0;1230 1231		if ((clock != slot->__clk_old &&1232			!test_bit(DW_MMC_CARD_NEEDS_POLL, &slot->flags)) ||1233			force_clkinit) {1234			/* Silent the verbose log if calling from PM context */1235			if (!force_clkinit)1236				dev_info(&slot->mmc->class_dev,1237					 "Bus speed (slot %d) = %dHz (slot req %dHz, actual %dHZ div = %d)\n",1238					 slot->id, host->bus_hz, clock,1239					 div ? ((host->bus_hz / div) >> 1) :1240					 host->bus_hz, div);1241 1242			/*1243			 * If card is polling, display the message only1244			 * one time at boot time.1245			 */1246			if (slot->mmc->caps & MMC_CAP_NEEDS_POLL &&1247					slot->mmc->f_min == clock)1248				set_bit(DW_MMC_CARD_NEEDS_POLL, &slot->flags);1249		}1250 1251		/* disable clock */1252		mci_writel(host, CLKENA, 0);1253		mci_writel(host, CLKSRC, 0);1254 1255		/* inform CIU */1256		mci_send_cmd(slot, sdmmc_cmd_bits, 0);1257 1258		/* set clock to desired speed */1259		mci_writel(host, CLKDIV, div);1260 1261		/* inform CIU */1262		mci_send_cmd(slot, sdmmc_cmd_bits, 0);1263 1264		/* enable clock; only low power if no SDIO */1265		clk_en_a = SDMMC_CLKEN_ENABLE << slot->id;1266		if (!test_bit(DW_MMC_CARD_NO_LOW_PWR, &slot->flags))1267			clk_en_a |= SDMMC_CLKEN_LOW_PWR << slot->id;1268		mci_writel(host, CLKENA, clk_en_a);1269 1270		/* inform CIU */1271		mci_send_cmd(slot, sdmmc_cmd_bits, 0);1272 1273		/* keep the last clock value that was requested from core */1274		slot->__clk_old = clock;1275		slot->mmc->actual_clock = div ? ((host->bus_hz / div) >> 1) :1276					  host->bus_hz;1277	}1278 1279	host->current_speed = clock;1280 1281	/* Set the current slot bus width */1282	mci_writel(host, CTYPE, (slot->ctype << slot->id));1283}1284 1285static void dw_mci_set_data_timeout(struct dw_mci *host,1286				    unsigned int timeout_ns)1287{1288	const struct dw_mci_drv_data *drv_data = host->drv_data;1289	u32 clk_div, tmout;1290	u64 tmp;1291 1292	if (drv_data && drv_data->set_data_timeout)1293		return drv_data->set_data_timeout(host, timeout_ns);1294 1295	clk_div = (mci_readl(host, CLKDIV) & 0xFF) * 2;1296	if (clk_div == 0)1297		clk_div = 1;1298 1299	tmp = DIV_ROUND_UP_ULL((u64)timeout_ns * host->bus_hz, NSEC_PER_SEC);1300	tmp = DIV_ROUND_UP_ULL(tmp, clk_div);1301 1302	/* TMOUT[7:0] (RESPONSE_TIMEOUT) */1303	tmout = 0xFF; /* Set maximum */1304 1305	/* TMOUT[31:8] (DATA_TIMEOUT) */1306	if (!tmp || tmp > 0xFFFFFF)1307		tmout |= (0xFFFFFF << 8);1308	else1309		tmout |= (tmp & 0xFFFFFF) << 8;1310 1311	mci_writel(host, TMOUT, tmout);1312	dev_dbg(host->dev, "timeout_ns: %u => TMOUT[31:8]: %#08x",1313		timeout_ns, tmout >> 8);1314}1315 1316static void __dw_mci_start_request(struct dw_mci *host,1317				   struct dw_mci_slot *slot,1318				   struct mmc_command *cmd)1319{1320	struct mmc_request *mrq;1321	struct mmc_data	*data;1322	u32 cmdflags;1323 1324	mrq = slot->mrq;1325 1326	host->mrq = mrq;1327 1328	host->pending_events = 0;1329	host->completed_events = 0;1330	host->cmd_status = 0;1331	host->data_status = 0;1332	host->dir_status = 0;1333 1334	data = cmd->data;1335	if (data) {1336		dw_mci_set_data_timeout(host, data->timeout_ns);1337		mci_writel(host, BYTCNT, data->blksz*data->blocks);1338		mci_writel(host, BLKSIZ, data->blksz);1339	}1340 1341	cmdflags = dw_mci_prepare_command(slot->mmc, cmd);1342 1343	/* this is the first command, send the initialization clock */1344	if (test_and_clear_bit(DW_MMC_CARD_NEED_INIT, &slot->flags))1345		cmdflags |= SDMMC_CMD_INIT;1346 1347	if (data) {1348		dw_mci_submit_data(host, data);1349		wmb(); /* drain writebuffer */1350	}1351 1352	dw_mci_start_command(host, cmd, cmdflags);1353 1354	if (cmd->opcode == SD_SWITCH_VOLTAGE) {1355		unsigned long irqflags;1356 1357		/*1358		 * Databook says to fail after 2ms w/ no response, but evidence1359		 * shows that sometimes the cmd11 interrupt takes over 130ms.1360		 * We'll set to 500ms, plus an extra jiffy just in case jiffies1361		 * is just about to roll over.1362		 *1363		 * We do this whole thing under spinlock and only if the1364		 * command hasn't already completed (indicating the irq1365		 * already ran so we don't want the timeout).1366		 */1367		spin_lock_irqsave(&host->irq_lock, irqflags);1368		if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events))1369			mod_timer(&host->cmd11_timer,1370				jiffies + msecs_to_jiffies(500) + 1);1371		spin_unlock_irqrestore(&host->irq_lock, irqflags);1372	}1373 1374	host->stop_cmdr = dw_mci_prep_stop_abort(host, cmd);1375}1376 1377static void dw_mci_start_request(struct dw_mci *host,1378				 struct dw_mci_slot *slot)1379{1380	struct mmc_request *mrq = slot->mrq;1381	struct mmc_command *cmd;1382 1383	cmd = mrq->sbc ? mrq->sbc : mrq->cmd;1384	__dw_mci_start_request(host, slot, cmd);1385}1386 1387/* must be called with host->lock held */1388static void dw_mci_queue_request(struct dw_mci *host, struct dw_mci_slot *slot,1389				 struct mmc_request *mrq)1390{1391	dev_vdbg(&slot->mmc->class_dev, "queue request: state=%d\n",1392		 host->state);1393 1394	slot->mrq = mrq;1395 1396	if (host->state == STATE_WAITING_CMD11_DONE) {1397		dev_warn(&slot->mmc->class_dev,1398			 "Voltage change didn't complete\n");1399		/*1400		 * this case isn't expected to happen, so we can1401		 * either crash here or just try to continue on1402		 * in the closest possible state1403		 */1404		host->state = STATE_IDLE;1405	}1406 1407	if (host->state == STATE_IDLE) {1408		host->state = STATE_SENDING_CMD;1409		dw_mci_start_request(host, slot);1410	} else {1411		list_add_tail(&slot->queue_node, &host->queue);1412	}1413}1414 1415static void dw_mci_request(struct mmc_host *mmc, struct mmc_request *mrq)1416{1417	struct dw_mci_slot *slot = mmc_priv(mmc);1418	struct dw_mci *host = slot->host;1419 1420	WARN_ON(slot->mrq);1421 1422	/*1423	 * The check for card presence and queueing of the request must be1424	 * atomic, otherwise the card could be removed in between and the1425	 * request wouldn't fail until another card was inserted.1426	 */1427 1428	if (!dw_mci_get_cd(mmc)) {1429		mrq->cmd->error = -ENOMEDIUM;1430		mmc_request_done(mmc, mrq);1431		return;1432	}1433 1434	spin_lock_bh(&host->lock);1435 1436	dw_mci_queue_request(host, slot, mrq);1437 1438	spin_unlock_bh(&host->lock);1439}1440 1441static void dw_mci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)1442{1443	struct dw_mci_slot *slot = mmc_priv(mmc);1444	const struct dw_mci_drv_data *drv_data = slot->host->drv_data;1445	u32 regs;1446	int ret;1447 1448	switch (ios->bus_width) {1449	case MMC_BUS_WIDTH_4:1450		slot->ctype = SDMMC_CTYPE_4BIT;1451		break;1452	case MMC_BUS_WIDTH_8:1453		slot->ctype = SDMMC_CTYPE_8BIT;1454		break;1455	default:1456		/* set default 1 bit mode */1457		slot->ctype = SDMMC_CTYPE_1BIT;1458	}1459 1460	regs = mci_readl(slot->host, UHS_REG);1461 1462	/* DDR mode set */1463	if (ios->timing == MMC_TIMING_MMC_DDR52 ||1464	    ios->timing == MMC_TIMING_UHS_DDR50 ||1465	    ios->timing == MMC_TIMING_MMC_HS400)1466		regs |= ((0x1 << slot->id) << 16);1467	else1468		regs &= ~((0x1 << slot->id) << 16);1469 1470	mci_writel(slot->host, UHS_REG, regs);1471	slot->host->timing = ios->timing;1472 1473	/*1474	 * Use mirror of ios->clock to prevent race with mmc1475	 * core ios update when finding the minimum.1476	 */1477	slot->clock = ios->clock;1478 1479	if (drv_data && drv_data->set_ios)1480		drv_data->set_ios(slot->host, ios);1481 1482	switch (ios->power_mode) {1483	case MMC_POWER_UP:1484		if (!IS_ERR(mmc->supply.vmmc)) {1485			ret = mmc_regulator_set_ocr(mmc, mmc->supply.vmmc,1486					ios->vdd);1487			if (ret) {1488				dev_err(slot->host->dev,1489					"failed to enable vmmc regulator\n");1490				/*return, if failed turn on vmmc*/1491				return;1492			}1493		}1494		set_bit(DW_MMC_CARD_NEED_INIT, &slot->flags);1495		regs = mci_readl(slot->host, PWREN);1496		regs |= (1 << slot->id);1497		mci_writel(slot->host, PWREN, regs);1498		break;1499	case MMC_POWER_ON:1500		if (!slot->host->vqmmc_enabled) {1501			if (!IS_ERR(mmc->supply.vqmmc)) {1502				ret = regulator_enable(mmc->supply.vqmmc);1503				if (ret < 0)1504					dev_err(slot->host->dev,1505						"failed to enable vqmmc\n");1506				else1507					slot->host->vqmmc_enabled = true;1508 1509			} else {1510				/* Keep track so we don't reset again */1511				slot->host->vqmmc_enabled = true;1512			}1513 1514			/* Reset our state machine after powering on */1515			dw_mci_ctrl_reset(slot->host,1516					  SDMMC_CTRL_ALL_RESET_FLAGS);1517		}1518 1519		/* Adjust clock / bus width after power is up */1520		dw_mci_setup_bus(slot, false);1521 1522		break;1523	case MMC_POWER_OFF:1524		/* Turn clock off before power goes down */1525		dw_mci_setup_bus(slot, false);1526 1527		if (!IS_ERR(mmc->supply.vmmc))1528			mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);1529 1530		if (!IS_ERR(mmc->supply.vqmmc) && slot->host->vqmmc_enabled)1531			regulator_disable(mmc->supply.vqmmc);1532		slot->host->vqmmc_enabled = false;1533 1534		regs = mci_readl(slot->host, PWREN);1535		regs &= ~(1 << slot->id);1536		mci_writel(slot->host, PWREN, regs);1537		break;1538	default:1539		break;1540	}1541 1542	if (slot->host->state == STATE_WAITING_CMD11_DONE && ios->clock != 0)1543		slot->host->state = STATE_IDLE;1544}1545 1546static int dw_mci_card_busy(struct mmc_host *mmc)1547{1548	struct dw_mci_slot *slot = mmc_priv(mmc);1549	u32 status;1550 1551	/*1552	 * Check the busy bit which is low when DAT[3:0]1553	 * (the data lines) are 00001554	 */1555	status = mci_readl(slot->host, STATUS);1556 1557	return !!(status & SDMMC_STATUS_BUSY);1558}1559 1560static int dw_mci_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)1561{1562	struct dw_mci_slot *slot = mmc_priv(mmc);1563	struct dw_mci *host = slot->host;1564	const struct dw_mci_drv_data *drv_data = host->drv_data;1565	u32 uhs;1566	u32 v18 = SDMMC_UHS_18V << slot->id;1567	int ret;1568 1569	if (drv_data && drv_data->switch_voltage)1570		return drv_data->switch_voltage(mmc, ios);1571 1572	/*1573	 * Program the voltage.  Note that some instances of dw_mmc may use1574	 * the UHS_REG for this.  For other instances (like exynos) the UHS_REG1575	 * does no harm but you need to set the regulator directly.  Try both.1576	 */1577	uhs = mci_readl(host, UHS_REG);1578	if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330)1579		uhs &= ~v18;1580	else1581		uhs |= v18;1582 1583	if (!IS_ERR(mmc->supply.vqmmc)) {1584		ret = mmc_regulator_set_vqmmc(mmc, ios);1585		if (ret < 0) {1586			dev_dbg(&mmc->class_dev,1587					 "Regulator set error %d - %s V\n",1588					 ret, uhs & v18 ? "1.8" : "3.3");1589			return ret;1590		}1591	}1592	mci_writel(host, UHS_REG, uhs);1593 1594	return 0;1595}1596 1597static int dw_mci_get_ro(struct mmc_host *mmc)1598{1599	int read_only;1600	struct dw_mci_slot *slot = mmc_priv(mmc);1601	int gpio_ro = mmc_gpio_get_ro(mmc);1602 1603	/* Use platform get_ro function, else try on board write protect */1604	if (gpio_ro >= 0)1605		read_only = gpio_ro;1606	else1607		read_only =1608			mci_readl(slot->host, WRTPRT) & (1 << slot->id) ? 1 : 0;1609 1610	dev_dbg(&mmc->class_dev, "card is %s\n",1611		read_only ? "read-only" : "read-write");1612 1613	return read_only;1614}1615 1616static void dw_mci_hw_reset(struct mmc_host *mmc)1617{1618	struct dw_mci_slot *slot = mmc_priv(mmc);1619	struct dw_mci *host = slot->host;1620	const struct dw_mci_drv_data *drv_data = host->drv_data;1621	int reset;1622 1623	if (host->use_dma == TRANS_MODE_IDMAC)1624		dw_mci_idmac_reset(host);1625 1626	if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_DMA_RESET |1627				     SDMMC_CTRL_FIFO_RESET))1628		return;1629 1630	if (drv_data && drv_data->hw_reset) {1631		drv_data->hw_reset(host);1632		return;1633	}1634 1635	/*1636	 * According to eMMC spec, card reset procedure:1637	 * tRstW >= 1us:   RST_n pulse width1638	 * tRSCA >= 200us: RST_n to Command time1639	 * tRSTH >= 1us:   RST_n high period1640	 */1641	reset = mci_readl(host, RST_N);1642	reset &= ~(SDMMC_RST_HWACTIVE << slot->id);1643	mci_writel(host, RST_N, reset);1644	usleep_range(1, 2);1645	reset |= SDMMC_RST_HWACTIVE << slot->id;1646	mci_writel(host, RST_N, reset);1647	usleep_range(200, 300);1648}1649 1650static void dw_mci_prepare_sdio_irq(struct dw_mci_slot *slot, bool prepare)1651{1652	struct dw_mci *host = slot->host;1653	const u32 clken_low_pwr = SDMMC_CLKEN_LOW_PWR << slot->id;1654	u32 clk_en_a_old;1655	u32 clk_en_a;1656 1657	/*1658	 * Low power mode will stop the card clock when idle.  According to the1659	 * description of the CLKENA register we should disable low power mode1660	 * for SDIO cards if we need SDIO interrupts to work.1661	 */1662 1663	clk_en_a_old = mci_readl(host, CLKENA);1664	if (prepare) {1665		set_bit(DW_MMC_CARD_NO_LOW_PWR, &slot->flags);1666		clk_en_a = clk_en_a_old & ~clken_low_pwr;1667	} else {1668		clear_bit(DW_MMC_CARD_NO_LOW_PWR, &slot->flags);1669		clk_en_a = clk_en_a_old | clken_low_pwr;1670	}1671 1672	if (clk_en_a != clk_en_a_old) {1673		mci_writel(host, CLKENA, clk_en_a);1674		mci_send_cmd(slot, SDMMC_CMD_UPD_CLK | SDMMC_CMD_PRV_DAT_WAIT,1675			     0);1676	}1677}1678 1679static void __dw_mci_enable_sdio_irq(struct dw_mci_slot *slot, int enb)1680{1681	struct dw_mci *host = slot->host;1682	unsigned long irqflags;1683	u32 int_mask;1684 1685	spin_lock_irqsave(&host->irq_lock, irqflags);1686 1687	/* Enable/disable Slot Specific SDIO interrupt */1688	int_mask = mci_readl(host, INTMASK);1689	if (enb)1690		int_mask |= SDMMC_INT_SDIO(slot->sdio_id);1691	else1692		int_mask &= ~SDMMC_INT_SDIO(slot->sdio_id);1693	mci_writel(host, INTMASK, int_mask);1694 1695	spin_unlock_irqrestore(&host->irq_lock, irqflags);1696}1697 1698static void dw_mci_enable_sdio_irq(struct mmc_host *mmc, int enb)1699{1700	struct dw_mci_slot *slot = mmc_priv(mmc);1701	struct dw_mci *host = slot->host;1702 1703	dw_mci_prepare_sdio_irq(slot, enb);1704	__dw_mci_enable_sdio_irq(slot, enb);1705 1706	/* Avoid runtime suspending the device when SDIO IRQ is enabled */1707	if (enb)1708		pm_runtime_get_noresume(host->dev);1709	else1710		pm_runtime_put_noidle(host->dev);1711}1712 1713static void dw_mci_ack_sdio_irq(struct mmc_host *mmc)1714{1715	struct dw_mci_slot *slot = mmc_priv(mmc);1716 1717	__dw_mci_enable_sdio_irq(slot, 1);1718}1719 1720static int dw_mci_execute_tuning(struct mmc_host *mmc, u32 opcode)1721{1722	struct dw_mci_slot *slot = mmc_priv(mmc);1723	struct dw_mci *host = slot->host;1724	const struct dw_mci_drv_data *drv_data = host->drv_data;1725	int err = -EINVAL;1726 1727	if (drv_data && drv_data->execute_tuning)1728		err = drv_data->execute_tuning(slot, opcode);1729	return err;1730}1731 1732static int dw_mci_prepare_hs400_tuning(struct mmc_host *mmc,1733				       struct mmc_ios *ios)1734{1735	struct dw_mci_slot *slot = mmc_priv(mmc);1736	struct dw_mci *host = slot->host;1737	const struct dw_mci_drv_data *drv_data = host->drv_data;1738 1739	if (drv_data && drv_data->prepare_hs400_tuning)1740		return drv_data->prepare_hs400_tuning(host, ios);1741 1742	return 0;1743}1744 1745static bool dw_mci_reset(struct dw_mci *host)1746{1747	u32 flags = SDMMC_CTRL_RESET | SDMMC_CTRL_FIFO_RESET;1748	bool ret = false;1749	u32 status = 0;1750 1751	/*1752	 * Resetting generates a block interrupt, hence setting1753	 * the scatter-gather pointer to NULL.1754	 */1755	if (host->sg) {1756		sg_miter_stop(&host->sg_miter);1757		host->sg = NULL;1758	}1759 1760	if (host->use_dma)1761		flags |= SDMMC_CTRL_DMA_RESET;1762 1763	if (dw_mci_ctrl_reset(host, flags)) {1764		/*1765		 * In all cases we clear the RAWINTS1766		 * register to clear any interrupts.1767		 */1768		mci_writel(host, RINTSTS, 0xFFFFFFFF);1769 1770		if (!host->use_dma) {1771			ret = true;1772			goto ciu_out;1773		}1774 1775		/* Wait for dma_req to be cleared */1776		if (readl_poll_timeout_atomic(host->regs + SDMMC_STATUS,1777					      status,1778					      !(status & SDMMC_STATUS_DMA_REQ),1779					      1, 500 * USEC_PER_MSEC)) {1780			dev_err(host->dev,1781				"%s: Timeout waiting for dma_req to be cleared\n",1782				__func__);1783			goto ciu_out;1784		}1785 1786		/* when using DMA next we reset the fifo again */1787		if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_FIFO_RESET))1788			goto ciu_out;1789	} else {1790		/* if the controller reset bit did clear, then set clock regs */1791		if (!(mci_readl(host, CTRL) & SDMMC_CTRL_RESET)) {1792			dev_err(host->dev,1793				"%s: fifo/dma reset bits didn't clear but ciu was reset, doing clock update\n",1794				__func__);1795			goto ciu_out;1796		}1797	}1798 1799	if (host->use_dma == TRANS_MODE_IDMAC)1800		/* It is also required that we reinit idmac */1801		dw_mci_idmac_init(host);1802 1803	ret = true;1804 1805ciu_out:1806	/* After a CTRL reset we need to have CIU set clock registers  */1807	mci_send_cmd(host->slot, SDMMC_CMD_UPD_CLK, 0);1808 1809	return ret;1810}1811 1812static const struct mmc_host_ops dw_mci_ops = {1813	.request		= dw_mci_request,1814	.pre_req		= dw_mci_pre_req,1815	.post_req		= dw_mci_post_req,1816	.set_ios		= dw_mci_set_ios,1817	.get_ro			= dw_mci_get_ro,1818	.get_cd			= dw_mci_get_cd,1819	.card_hw_reset          = dw_mci_hw_reset,1820	.enable_sdio_irq	= dw_mci_enable_sdio_irq,1821	.ack_sdio_irq		= dw_mci_ack_sdio_irq,1822	.execute_tuning		= dw_mci_execute_tuning,1823	.card_busy		= dw_mci_card_busy,1824	.start_signal_voltage_switch = dw_mci_switch_voltage,1825	.prepare_hs400_tuning	= dw_mci_prepare_hs400_tuning,1826};1827 1828#ifdef CONFIG_FAULT_INJECTION1829static enum hrtimer_restart dw_mci_fault_timer(struct hrtimer *t)1830{1831	struct dw_mci *host = container_of(t, struct dw_mci, fault_timer);1832	unsigned long flags;1833 1834	spin_lock_irqsave(&host->irq_lock, flags);1835 1836	/*1837	 * Only inject an error if we haven't already got an error or data over1838	 * interrupt.1839	 */1840	if (!host->data_status) {1841		host->data_status = SDMMC_INT_DCRC;1842		set_bit(EVENT_DATA_ERROR, &host->pending_events);1843		queue_work(system_bh_wq, &host->bh_work);1844	}1845 1846	spin_unlock_irqrestore(&host->irq_lock, flags);1847 1848	return HRTIMER_NORESTART;1849}1850 1851static void dw_mci_start_fault_timer(struct dw_mci *host)1852{1853	struct mmc_data *data = host->data;1854 1855	if (!data || data->blocks <= 1)1856		return;1857 1858	if (!should_fail(&host->fail_data_crc, 1))1859		return;1860 1861	/*1862	 * Try to inject the error at random points during the data transfer.1863	 */1864	hrtimer_start(&host->fault_timer,1865		      ms_to_ktime(get_random_u32_below(25)),1866		      HRTIMER_MODE_REL);1867}1868 1869static void dw_mci_stop_fault_timer(struct dw_mci *host)1870{1871	hrtimer_cancel(&host->fault_timer);1872}1873 1874static void dw_mci_init_fault(struct dw_mci *host)1875{1876	host->fail_data_crc = (struct fault_attr) FAULT_ATTR_INITIALIZER;1877 1878	hrtimer_init(&host->fault_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);1879	host->fault_timer.function = dw_mci_fault_timer;1880}1881#else1882static void dw_mci_init_fault(struct dw_mci *host)1883{1884}1885 1886static void dw_mci_start_fault_timer(struct dw_mci *host)1887{1888}1889 1890static void dw_mci_stop_fault_timer(struct dw_mci *host)1891{1892}1893#endif1894 1895static void dw_mci_request_end(struct dw_mci *host, struct mmc_request *mrq)1896	__releases(&host->lock)1897	__acquires(&host->lock)1898{1899	struct dw_mci_slot *slot;1900	struct mmc_host	*prev_mmc = host->slot->mmc;1901 1902	WARN_ON(host->cmd || host->data);1903 1904	host->slot->mrq = NULL;1905	host->mrq = NULL;1906	if (!list_empty(&host->queue)) {1907		slot = list_entry(host->queue.next,1908				  struct dw_mci_slot, queue_node);1909		list_del(&slot->queue_node);1910		dev_vdbg(host->dev, "list not empty: %s is next\n",1911			 mmc_hostname(slot->mmc));1912		host->state = STATE_SENDING_CMD;1913		dw_mci_start_request(host, slot);1914	} else {1915		dev_vdbg(host->dev, "list empty\n");1916 1917		if (host->state == STATE_SENDING_CMD11)1918			host->state = STATE_WAITING_CMD11_DONE;1919		else1920			host->state = STATE_IDLE;1921	}1922 1923	spin_unlock(&host->lock);1924	mmc_request_done(prev_mmc, mrq);1925	spin_lock(&host->lock);1926}1927 1928static int dw_mci_command_complete(struct dw_mci *host, struct mmc_command *cmd)1929{1930	u32 status = host->cmd_status;1931 1932	host->cmd_status = 0;1933 1934	/* Read the response from the card (up to 16 bytes) */1935	if (cmd->flags & MMC_RSP_PRESENT) {1936		if (cmd->flags & MMC_RSP_136) {1937			cmd->resp[3] = mci_readl(host, RESP0);1938			cmd->resp[2] = mci_readl(host, RESP1);1939			cmd->resp[1] = mci_readl(host, RESP2);1940			cmd->resp[0] = mci_readl(host, RESP3);1941		} else {1942			cmd->resp[0] = mci_readl(host, RESP0);1943			cmd->resp[1] = 0;1944			cmd->resp[2] = 0;1945			cmd->resp[3] = 0;1946		}1947	}1948 1949	if (status & SDMMC_INT_RTO)1950		cmd->error = -ETIMEDOUT;1951	else if ((cmd->flags & MMC_RSP_CRC) && (status & SDMMC_INT_RCRC))1952		cmd->error = -EILSEQ;1953	else if (status & SDMMC_INT_RESP_ERR)1954		cmd->error = -EIO;1955	else1956		cmd->error = 0;1957 1958	return cmd->error;1959}1960 1961static int dw_mci_data_complete(struct dw_mci *host, struct mmc_data *data)1962{1963	u32 status = host->data_status;1964 1965	if (status & DW_MCI_DATA_ERROR_FLAGS) {1966		if (status & SDMMC_INT_DRTO) {1967			data->error = -ETIMEDOUT;1968		} else if (status & SDMMC_INT_DCRC) {1969			data->error = -EILSEQ;1970		} else if (status & SDMMC_INT_EBE) {1971			if (host->dir_status ==1972				DW_MCI_SEND_STATUS) {1973				/*1974				 * No data CRC status was returned.1975				 * The number of bytes transferred1976				 * will be exaggerated in PIO mode.1977				 */1978				data->bytes_xfered = 0;1979				data->error = -ETIMEDOUT;1980			} else if (host->dir_status ==1981					DW_MCI_RECV_STATUS) {1982				data->error = -EILSEQ;1983			}1984		} else {1985			/* SDMMC_INT_SBE is included */1986			data->error = -EILSEQ;1987		}1988 1989		dev_dbg(host->dev, "data error, status 0x%08x\n", status);1990 1991		/*1992		 * After an error, there may be data lingering1993		 * in the FIFO1994		 */1995		dw_mci_reset(host);1996	} else {1997		data->bytes_xfered = data->blocks * data->blksz;1998		data->error = 0;1999	}2000 2001	return data->error;2002}2003 2004static void dw_mci_set_drto(struct dw_mci *host)2005{2006	const struct dw_mci_drv_data *drv_data = host->drv_data;2007	unsigned int drto_clks;2008	unsigned int drto_div;2009	unsigned int drto_ms;2010	unsigned long irqflags;2011 2012	if (drv_data && drv_data->get_drto_clks)2013		drto_clks = drv_data->get_drto_clks(host);2014	else2015		drto_clks = mci_readl(host, TMOUT) >> 8;2016	drto_div = (mci_readl(host, CLKDIV) & 0xff) * 2;2017	if (drto_div == 0)2018		drto_div = 1;2019 2020	drto_ms = DIV_ROUND_UP_ULL((u64)MSEC_PER_SEC * drto_clks * drto_div,2021				   host->bus_hz);2022 2023	dev_dbg(host->dev, "drto_ms: %u\n", drto_ms);2024 2025	/* add a bit spare time */2026	drto_ms += 10;2027 2028	spin_lock_irqsave(&host->irq_lock, irqflags);2029	if (!test_bit(EVENT_DATA_COMPLETE, &host->pending_events))2030		mod_timer(&host->dto_timer,2031			  jiffies + msecs_to_jiffies(drto_ms));2032	spin_unlock_irqrestore(&host->irq_lock, irqflags);2033}2034 2035static bool dw_mci_clear_pending_cmd_complete(struct dw_mci *host)2036{2037	if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events))2038		return false;2039 2040	/*2041	 * Really be certain that the timer has stopped.  This is a bit of2042	 * paranoia and could only really happen if we had really bad2043	 * interrupt latency and the interrupt routine and timeout were2044	 * running concurrently so that the del_timer() in the interrupt2045	 * handler couldn't run.2046	 */2047	WARN_ON(del_timer_sync(&host->cto_timer));2048	clear_bit(EVENT_CMD_COMPLETE, &host->pending_events);2049 2050	return true;2051}2052 2053static bool dw_mci_clear_pending_data_complete(struct dw_mci *host)2054{2055	if (!test_bit(EVENT_DATA_COMPLETE, &host->pending_events))2056		return false;2057 2058	/* Extra paranoia just like dw_mci_clear_pending_cmd_complete() */2059	WARN_ON(del_timer_sync(&host->dto_timer));2060	clear_bit(EVENT_DATA_COMPLETE, &host->pending_events);2061 2062	return true;2063}2064 2065static void dw_mci_work_func(struct work_struct *t)2066{2067	struct dw_mci *host = from_work(host, t, bh_work);2068	struct mmc_data	*data;2069	struct mmc_command *cmd;2070	struct mmc_request *mrq;2071	enum dw_mci_state state;2072	enum dw_mci_state prev_state;2073	unsigned int err;2074 2075	spin_lock(&host->lock);2076 2077	state = host->state;2078	data = host->data;2079	mrq = host->mrq;2080 2081	do {2082		prev_state = state;2083 2084		switch (state) {2085		case STATE_IDLE:2086		case STATE_WAITING_CMD11_DONE:2087			break;2088 2089		case STATE_SENDING_CMD11:2090		case STATE_SENDING_CMD:2091			if (!dw_mci_clear_pending_cmd_complete(host))2092				break;2093 2094			cmd = host->cmd;2095			host->cmd = NULL;2096			set_bit(EVENT_CMD_COMPLETE, &host->completed_events);2097			err = dw_mci_command_complete(host, cmd);2098			if (cmd == mrq->sbc && !err) {2099				__dw_mci_start_request(host, host->slot,2100						       mrq->cmd);2101				goto unlock;2102			}2103 2104			if (cmd->data && err) {2105				/*2106				 * During UHS tuning sequence, sending the stop2107				 * command after the response CRC error would2108				 * throw the system into a confused state2109				 * causing all future tuning phases to report2110				 * failure.2111				 *2112				 * In such case controller will move into a data2113				 * transfer state after a response error or2114				 * response CRC error. Let's let that finish2115				 * before trying to send a stop, so we'll go to2116				 * STATE_SENDING_DATA.2117				 *2118				 * Although letting the data transfer take place2119				 * will waste a bit of time (we already know2120				 * the command was bad), it can't cause any2121				 * errors since it's possible it would have2122				 * taken place anyway if this bh work got2123				 * delayed. Allowing the transfer to take place2124				 * avoids races and keeps things simple.2125				 */2126				if (err != -ETIMEDOUT &&2127				    host->dir_status == DW_MCI_RECV_STATUS) {2128					state = STATE_SENDING_DATA;2129					continue;2130				}2131 2132				send_stop_abort(host, data);2133				dw_mci_stop_dma(host);2134				state = STATE_SENDING_STOP;2135				break;2136			}2137 2138			if (!cmd->data || err) {2139				dw_mci_request_end(host, mrq);2140				goto unlock;2141			}2142 2143			prev_state = state = STATE_SENDING_DATA;2144			fallthrough;2145 2146		case STATE_SENDING_DATA:2147			/*2148			 * We could get a data error and never a transfer2149			 * complete so we'd better check for it here.2150			 *2151			 * Note that we don't really care if we also got a2152			 * transfer complete; stopping the DMA and sending an2153			 * abort won't hurt.2154			 */2155			if (test_and_clear_bit(EVENT_DATA_ERROR,2156					       &host->pending_events)) {2157				if (!(host->data_status & (SDMMC_INT_DRTO |2158							   SDMMC_INT_EBE)))2159					send_stop_abort(host, data);2160				dw_mci_stop_dma(host);2161				state = STATE_DATA_ERROR;2162				break;2163			}2164 2165			if (!test_and_clear_bit(EVENT_XFER_COMPLETE,2166						&host->pending_events)) {2167				/*2168				 * If all data-related interrupts don't come2169				 * within the given time in reading data state.2170				 */2171				if (host->dir_status == DW_MCI_RECV_STATUS)2172					dw_mci_set_drto(host);2173				break;2174			}2175 2176			set_bit(EVENT_XFER_COMPLETE, &host->completed_events);2177 2178			/*2179			 * Handle an EVENT_DATA_ERROR that might have shown up2180			 * before the transfer completed.  This might not have2181			 * been caught by the check above because the interrupt2182			 * could have gone off between the previous check and2183			 * the check for transfer complete.2184			 *2185			 * Technically this ought not be needed assuming we2186			 * get a DATA_COMPLETE eventually (we'll notice the2187			 * error and end the request), but it shouldn't hurt.2188			 *2189			 * This has the advantage of sending the stop command.2190			 */2191			if (test_and_clear_bit(EVENT_DATA_ERROR,2192					       &host->pending_events)) {2193				if (!(host->data_status & (SDMMC_INT_DRTO |2194							   SDMMC_INT_EBE)))2195					send_stop_abort(host, data);2196				dw_mci_stop_dma(host);2197				state = STATE_DATA_ERROR;2198				break;2199			}2200			prev_state = state = STATE_DATA_BUSY;2201 2202			fallthrough;2203 2204		case STATE_DATA_BUSY:2205			if (!dw_mci_clear_pending_data_complete(host)) {2206				/*2207				 * If data error interrupt comes but data over2208				 * interrupt doesn't come within the given time.2209				 * in reading data state.2210				 */2211				if (host->dir_status == DW_MCI_RECV_STATUS)2212					dw_mci_set_drto(host);2213				break;2214			}2215 2216			dw_mci_stop_fault_timer(host);2217			host->data = NULL;2218			set_bit(EVENT_DATA_COMPLETE, &host->completed_events);2219			err = dw_mci_data_complete(host, data);2220 2221			if (!err) {2222				if (!data->stop || mrq->sbc) {2223					if (mrq->sbc && data->stop)2224						data->stop->error = 0;2225					dw_mci_request_end(host, mrq);2226					goto unlock;2227				}2228 2229				/* stop command for open-ended transfer*/2230				if (data->stop)2231					send_stop_abort(host, data);2232			} else {2233				/*2234				 * If we don't have a command complete now we'll2235				 * never get one since we just reset everything;2236				 * better end the request.2237				 *2238				 * If we do have a command complete we'll fall2239				 * through to the SENDING_STOP command and2240				 * everything will be peachy keen.2241				 */2242				if (!test_bit(EVENT_CMD_COMPLETE,2243					      &host->pending_events)) {2244					host->cmd = NULL;2245					dw_mci_request_end(host, mrq);2246					goto unlock;2247				}2248			}2249 2250			/*2251			 * If err has non-zero,2252			 * stop-abort command has been already issued.2253			 */2254			prev_state = state = STATE_SENDING_STOP;2255 2256			fallthrough;2257 2258		case STATE_SENDING_STOP:2259			if (!dw_mci_clear_pending_cmd_complete(host))2260				break;2261 2262			/* CMD error in data command */2263			if (mrq->cmd->error && mrq->data)2264				dw_mci_reset(host);2265 2266			dw_mci_stop_fault_timer(host);2267			host->cmd = NULL;2268			host->data = NULL;2269 2270			if (!mrq->sbc && mrq->stop)2271				dw_mci_command_complete(host, mrq->stop);2272			else2273				host->cmd_status = 0;2274 2275			dw_mci_request_end(host, mrq);2276			goto unlock;2277 2278		case STATE_DATA_ERROR:2279			if (!test_and_clear_bit(EVENT_XFER_COMPLETE,2280						&host->pending_events))2281				break;2282 2283			state = STATE_DATA_BUSY;2284			break;2285		}2286	} while (state != prev_state);2287 2288	host->state = state;2289unlock:2290	spin_unlock(&host->lock);2291 2292}2293 2294/* push final bytes to part_buf, only use during push */2295static void dw_mci_set_part_bytes(struct dw_mci *host, void *buf, int cnt)2296{2297	memcpy((void *)&host->part_buf, buf, cnt);2298	host->part_buf_count = cnt;2299}2300 2301/* append bytes to part_buf, only use during push */2302static int dw_mci_push_part_bytes(struct dw_mci *host, void *buf, int cnt)2303{2304	cnt = min(cnt, (1 << host->data_shift) - host->part_buf_count);2305	memcpy((void *)&host->part_buf + host->part_buf_count, buf, cnt);2306	host->part_buf_count += cnt;2307	return cnt;2308}2309 2310/* pull first bytes from part_buf, only use during pull */2311static int dw_mci_pull_part_bytes(struct dw_mci *host, void *buf, int cnt)2312{2313	cnt = min_t(int, cnt, host->part_buf_count);2314	if (cnt) {2315		memcpy(buf, (void *)&host->part_buf + host->part_buf_start,2316		       cnt);2317		host->part_buf_count -= cnt;2318		host->part_buf_start += cnt;2319	}2320	return cnt;2321}2322 2323/* pull final bytes from the part_buf, assuming it's just been filled */2324static void dw_mci_pull_final_bytes(struct dw_mci *host, void *buf, int cnt)2325{2326	memcpy(buf, &host->part_buf, cnt);2327	host->part_buf_start = cnt;2328	host->part_buf_count = (1 << host->data_shift) - cnt;2329}2330 2331static void dw_mci_push_data16(struct dw_mci *host, void *buf, int cnt)2332{2333	struct mmc_data *data = host->data;2334	int init_cnt = cnt;2335 2336	/* try and push anything in the part_buf */2337	if (unlikely(host->part_buf_count)) {2338		int len = dw_mci_push_part_bytes(host, buf, cnt);2339 2340		buf += len;2341		cnt -= len;2342		if (host->part_buf_count == 2) {2343			mci_fifo_writew(host->fifo_reg, host->part_buf16);2344			host->part_buf_count = 0;2345		}2346	}2347#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS2348	if (unlikely((unsigned long)buf & 0x1)) {2349		while (cnt >= 2) {2350			u16 aligned_buf[64];2351			int len = min(cnt & -2, (int)sizeof(aligned_buf));2352			int items = len >> 1;2353			int i;2354			/* memcpy from input buffer into aligned buffer */2355			memcpy(aligned_buf, buf, len);2356			buf += len;2357			cnt -= len;2358			/* push data from aligned buffer into fifo */2359			for (i = 0; i < items; ++i)2360				mci_fifo_writew(host->fifo_reg, aligned_buf[i]);2361		}2362	} else2363#endif2364	{2365		u16 *pdata = buf;2366 2367		for (; cnt >= 2; cnt -= 2)2368			mci_fifo_writew(host->fifo_reg, *pdata++);2369		buf = pdata;2370	}2371	/* put anything remaining in the part_buf */2372	if (cnt) {2373		dw_mci_set_part_bytes(host, buf, cnt);2374		 /* Push data if we have reached the expected data length */2375		if ((data->bytes_xfered + init_cnt) ==2376		    (data->blksz * data->blocks))2377			mci_fifo_writew(host->fifo_reg, host->part_buf16);2378	}2379}2380 2381static void dw_mci_pull_data16(struct dw_mci *host, void *buf, int cnt)2382{2383#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS2384	if (unlikely((unsigned long)buf & 0x1)) {2385		while (cnt >= 2) {2386			/* pull data from fifo into aligned buffer */2387			u16 aligned_buf[64];2388			int len = min(cnt & -2, (int)sizeof(aligned_buf));2389			int items = len >> 1;2390			int i;2391 2392			for (i = 0; i < items; ++i)2393				aligned_buf[i] = mci_fifo_readw(host->fifo_reg);2394			/* memcpy from aligned buffer into output buffer */2395			memcpy(buf, aligned_buf, len);2396			buf += len;2397			cnt -= len;2398		}2399	} else2400#endif2401	{2402		u16 *pdata = buf;2403 2404		for (; cnt >= 2; cnt -= 2)2405			*pdata++ = mci_fifo_readw(host->fifo_reg);2406		buf = pdata;2407	}2408	if (cnt) {2409		host->part_buf16 = mci_fifo_readw(host->fifo_reg);2410		dw_mci_pull_final_bytes(host, buf, cnt);2411	}2412}2413 2414static void dw_mci_push_data32(struct dw_mci *host, void *buf, int cnt)2415{2416	struct mmc_data *data = host->data;2417	int init_cnt = cnt;2418 2419	/* try and push anything in the part_buf */2420	if (unlikely(host->part_buf_count)) {2421		int len = dw_mci_push_part_bytes(host, buf, cnt);2422 2423		buf += len;2424		cnt -= len;2425		if (host->part_buf_count == 4) {2426			mci_fifo_writel(host->fifo_reg,	host->part_buf32);2427			host->part_buf_count = 0;2428		}2429	}2430#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS2431	if (unlikely((unsigned long)buf & 0x3)) {2432		while (cnt >= 4) {2433			u32 aligned_buf[32];2434			int len = min(cnt & -4, (int)sizeof(aligned_buf));2435			int items = len >> 2;2436			int i;2437			/* memcpy from input buffer into aligned buffer */2438			memcpy(aligned_buf, buf, len);2439			buf += len;2440			cnt -= len;2441			/* push data from aligned buffer into fifo */2442			for (i = 0; i < items; ++i)2443				mci_fifo_writel(host->fifo_reg,	aligned_buf[i]);2444		}2445	} else2446#endif2447	{2448		u32 *pdata = buf;2449 2450		for (; cnt >= 4; cnt -= 4)2451			mci_fifo_writel(host->fifo_reg, *pdata++);2452		buf = pdata;2453	}2454	/* put anything remaining in the part_buf */2455	if (cnt) {2456		dw_mci_set_part_bytes(host, buf, cnt);2457		 /* Push data if we have reached the expected data length */2458		if ((data->bytes_xfered + init_cnt) ==2459		    (data->blksz * data->blocks))2460			mci_fifo_writel(host->fifo_reg, host->part_buf32);2461	}2462}2463 2464static void dw_mci_pull_data32(struct dw_mci *host, void *buf, int cnt)2465{2466#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS2467	if (unlikely((unsigned long)buf & 0x3)) {2468		while (cnt >= 4) {2469			/* pull data from fifo into aligned buffer */2470			u32 aligned_buf[32];2471			int len = min(cnt & -4, (int)sizeof(aligned_buf));2472			int items = len >> 2;2473			int i;2474 2475			for (i = 0; i < items; ++i)2476				aligned_buf[i] = mci_fifo_readl(host->fifo_reg);2477			/* memcpy from aligned buffer into output buffer */2478			memcpy(buf, aligned_buf, len);2479			buf += len;2480			cnt -= len;2481		}2482	} else2483#endif2484	{2485		u32 *pdata = buf;2486 2487		for (; cnt >= 4; cnt -= 4)2488			*pdata++ = mci_fifo_readl(host->fifo_reg);2489		buf = pdata;2490	}2491	if (cnt) {2492		host->part_buf32 = mci_fifo_readl(host->fifo_reg);2493		dw_mci_pull_final_bytes(host, buf, cnt);2494	}2495}2496 2497static void dw_mci_push_data64(struct dw_mci *host, void *buf, int cnt)2498{2499	struct mmc_data *data = host->data;2500	int init_cnt = cnt;2501 2502	/* try and push anything in the part_buf */2503	if (unlikely(host->part_buf_count)) {2504		int len = dw_mci_push_part_bytes(host, buf, cnt);2505 2506		buf += len;2507		cnt -= len;2508 2509		if (host->part_buf_count == 8) {2510			mci_fifo_writeq(host->fifo_reg,	host->part_buf);2511			host->part_buf_count = 0;2512		}2513	}2514#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS2515	if (unlikely((unsigned long)buf & 0x7)) {2516		while (cnt >= 8) {2517			u64 aligned_buf[16];2518			int len = min(cnt & -8, (int)sizeof(aligned_buf));2519			int items = len >> 3;2520			int i;2521			/* memcpy from input buffer into aligned buffer */2522			memcpy(aligned_buf, buf, len);2523			buf += len;2524			cnt -= len;2525			/* push data from aligned buffer into fifo */2526			for (i = 0; i < items; ++i)2527				mci_fifo_writeq(host->fifo_reg,	aligned_buf[i]);2528		}2529	} else2530#endif2531	{2532		u64 *pdata = buf;2533 2534		for (; cnt >= 8; cnt -= 8)2535			mci_fifo_writeq(host->fifo_reg, *pdata++);2536		buf = pdata;2537	}2538	/* put anything remaining in the part_buf */2539	if (cnt) {2540		dw_mci_set_part_bytes(host, buf, cnt);2541		/* Push data if we have reached the expected data length */2542		if ((data->bytes_xfered + init_cnt) ==2543		    (data->blksz * data->blocks))2544			mci_fifo_writeq(host->fifo_reg, host->part_buf);2545	}2546}2547 2548static void dw_mci_pull_data64(struct dw_mci *host, void *buf, int cnt)2549{2550#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS2551	if (unlikely((unsigned long)buf & 0x7)) {2552		while (cnt >= 8) {2553			/* pull data from fifo into aligned buffer */2554			u64 aligned_buf[16];2555			int len = min(cnt & -8, (int)sizeof(aligned_buf));2556			int items = len >> 3;2557			int i;2558 2559			for (i = 0; i < items; ++i)2560				aligned_buf[i] = mci_fifo_readq(host->fifo_reg);2561 2562			/* memcpy from aligned buffer into output buffer */2563			memcpy(buf, aligned_buf, len);2564			buf += len;2565			cnt -= len;2566		}2567	} else2568#endif2569	{2570		u64 *pdata = buf;2571 2572		for (; cnt >= 8; cnt -= 8)2573			*pdata++ = mci_fifo_readq(host->fifo_reg);2574		buf = pdata;2575	}2576	if (cnt) {2577		host->part_buf = mci_fifo_readq(host->fifo_reg);2578		dw_mci_pull_final_bytes(host, buf, cnt);2579	}2580}2581 2582static void dw_mci_pull_data(struct dw_mci *host, void *buf, int cnt)2583{2584	int len;2585 2586	/* get remaining partial bytes */2587	len = dw_mci_pull_part_bytes(host, buf, cnt);2588	if (unlikely(len == cnt))2589		return;2590	buf += len;2591	cnt -= len;2592 2593	/* get the rest of the data */2594	host->pull_data(host, buf, cnt);2595}2596 2597static void dw_mci_read_data_pio(struct dw_mci *host, bool dto)2598{2599	struct sg_mapping_iter *sg_miter = &host->sg_miter;2600	void *buf;2601	unsigned int offset;2602	struct mmc_data	*data = host->data;2603	int shift = host->data_shift;2604	u32 status;2605	unsigned int len;2606	unsigned int remain, fcnt;2607 2608	do {2609		if (!sg_miter_next(sg_miter))2610			goto done;2611 2612		host->sg = sg_miter->piter.sg;2613		buf = sg_miter->addr;2614		remain = sg_miter->length;2615		offset = 0;2616 2617		do {2618			fcnt = (SDMMC_GET_FCNT(mci_readl(host, STATUS))2619					<< shift) + host->part_buf_count;2620			len = min(remain, fcnt);2621			if (!len)2622				break;2623			dw_mci_pull_data(host, (void *)(buf + offset), len);2624			data->bytes_xfered += len;2625			offset += len;2626			remain -= len;2627		} while (remain);2628 2629		sg_miter->consumed = offset;2630		status = mci_readl(host, MINTSTS);2631		mci_writel(host, RINTSTS, SDMMC_INT_RXDR);2632	/* if the RXDR is ready read again */2633	} while ((status & SDMMC_INT_RXDR) ||2634		 (dto && SDMMC_GET_FCNT(mci_readl(host, STATUS))));2635 2636	if (!remain) {2637		if (!sg_miter_next(sg_miter))2638			goto done;2639		sg_miter->consumed = 0;2640	}2641	sg_miter_stop(sg_miter);2642	return;2643 2644done:2645	sg_miter_stop(sg_miter);2646	host->sg = NULL;2647	smp_wmb(); /* drain writebuffer */2648	set_bit(EVENT_XFER_COMPLETE, &host->pending_events);2649}2650 2651static void dw_mci_write_data_pio(struct dw_mci *host)2652{2653	struct sg_mapping_iter *sg_miter = &host->sg_miter;2654	void *buf;2655	unsigned int offset;2656	struct mmc_data	*data = host->data;2657	int shift = host->data_shift;2658	u32 status;2659	unsigned int len;2660	unsigned int fifo_depth = host->fifo_depth;2661	unsigned int remain, fcnt;2662 2663	do {2664		if (!sg_miter_next(sg_miter))2665			goto done;2666 2667		host->sg = sg_miter->piter.sg;2668		buf = sg_miter->addr;2669		remain = sg_miter->length;2670		offset = 0;2671 2672		do {2673			fcnt = ((fifo_depth -2674				 SDMMC_GET_FCNT(mci_readl(host, STATUS)))2675					<< shift) - host->part_buf_count;2676			len = min(remain, fcnt);2677			if (!len)2678				break;2679			host->push_data(host, (void *)(buf + offset), len);2680			data->bytes_xfered += len;2681			offset += len;2682			remain -= len;2683		} while (remain);2684 2685		sg_miter->consumed = offset;2686		status = mci_readl(host, MINTSTS);2687		mci_writel(host, RINTSTS, SDMMC_INT_TXDR);2688	} while (status & SDMMC_INT_TXDR); /* if TXDR write again */2689 2690	if (!remain) {2691		if (!sg_miter_next(sg_miter))2692			goto done;2693		sg_miter->consumed = 0;2694	}2695	sg_miter_stop(sg_miter);2696	return;2697 2698done:2699	sg_miter_stop(sg_miter);2700	host->sg = NULL;2701	smp_wmb(); /* drain writebuffer */2702	set_bit(EVENT_XFER_COMPLETE, &host->pending_events);2703}2704 2705static void dw_mci_cmd_interrupt(struct dw_mci *host, u32 status)2706{2707	del_timer(&host->cto_timer);2708 2709	if (!host->cmd_status)2710		host->cmd_status = status;2711 2712	smp_wmb(); /* drain writebuffer */2713 2714	set_bit(EVENT_CMD_COMPLETE, &host->pending_events);2715	queue_work(system_bh_wq, &host->bh_work);2716 2717	dw_mci_start_fault_timer(host);2718}2719 2720static void dw_mci_handle_cd(struct dw_mci *host)2721{2722	struct dw_mci_slot *slot = host->slot;2723 2724	mmc_detect_change(slot->mmc,2725		msecs_to_jiffies(host->pdata->detect_delay_ms));2726}2727 2728static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)2729{2730	struct dw_mci *host = dev_id;2731	u32 pending;2732	struct dw_mci_slot *slot = host->slot;2733 2734	pending = mci_readl(host, MINTSTS); /* read-only mask reg */2735 2736	if (pending) {2737		/* Check volt switch first, since it can look like an error */2738		if ((host->state == STATE_SENDING_CMD11) &&2739		    (pending & SDMMC_INT_VOLT_SWITCH)) {2740			mci_writel(host, RINTSTS, SDMMC_INT_VOLT_SWITCH);2741			pending &= ~SDMMC_INT_VOLT_SWITCH;2742 2743			/*2744			 * Hold the lock; we know cmd11_timer can't be kicked2745			 * off after the lock is released, so safe to delete.2746			 */2747			spin_lock(&host->irq_lock);2748			dw_mci_cmd_interrupt(host, pending);2749			spin_unlock(&host->irq_lock);2750 2751			del_timer(&host->cmd11_timer);2752		}2753 2754		if (pending & DW_MCI_CMD_ERROR_FLAGS) {2755			spin_lock(&host->irq_lock);2756 2757			del_timer(&host->cto_timer);2758			mci_writel(host, RINTSTS, DW_MCI_CMD_ERROR_FLAGS);2759			host->cmd_status = pending;2760			smp_wmb(); /* drain writebuffer */2761			set_bit(EVENT_CMD_COMPLETE, &host->pending_events);2762 2763			spin_unlock(&host->irq_lock);2764		}2765 2766		if (pending & DW_MCI_DATA_ERROR_FLAGS) {2767			spin_lock(&host->irq_lock);2768 2769			if (host->quirks & DW_MMC_QUIRK_EXTENDED_TMOUT)2770				del_timer(&host->dto_timer);2771 2772			/* if there is an error report DATA_ERROR */2773			mci_writel(host, RINTSTS, DW_MCI_DATA_ERROR_FLAGS);2774			host->data_status = pending;2775			smp_wmb(); /* drain writebuffer */2776			set_bit(EVENT_DATA_ERROR, &host->pending_events);2777 2778			if (host->quirks & DW_MMC_QUIRK_EXTENDED_TMOUT)2779				/* In case of error, we cannot expect a DTO */2780				set_bit(EVENT_DATA_COMPLETE,2781					&host->pending_events);2782 2783			queue_work(system_bh_wq, &host->bh_work);2784 2785			spin_unlock(&host->irq_lock);2786		}2787 2788		if (pending & SDMMC_INT_DATA_OVER) {2789			spin_lock(&host->irq_lock);2790 2791			del_timer(&host->dto_timer);2792 2793			mci_writel(host, RINTSTS, SDMMC_INT_DATA_OVER);2794			if (!host->data_status)2795				host->data_status = pending;2796			smp_wmb(); /* drain writebuffer */2797			if (host->dir_status == DW_MCI_RECV_STATUS) {2798				if (host->sg != NULL)2799					dw_mci_read_data_pio(host, true);2800			}2801			set_bit(EVENT_DATA_COMPLETE, &host->pending_events);2802			queue_work(system_bh_wq, &host->bh_work);2803 2804			spin_unlock(&host->irq_lock);2805		}2806 2807		if (pending & SDMMC_INT_RXDR) {2808			mci_writel(host, RINTSTS, SDMMC_INT_RXDR);2809			if (host->dir_status == DW_MCI_RECV_STATUS && host->sg)2810				dw_mci_read_data_pio(host, false);2811		}2812 2813		if (pending & SDMMC_INT_TXDR) {2814			mci_writel(host, RINTSTS, SDMMC_INT_TXDR);2815			if (host->dir_status == DW_MCI_SEND_STATUS && host->sg)2816				dw_mci_write_data_pio(host);2817		}2818 2819		if (pending & SDMMC_INT_CMD_DONE) {2820			spin_lock(&host->irq_lock);2821 2822			mci_writel(host, RINTSTS, SDMMC_INT_CMD_DONE);2823			dw_mci_cmd_interrupt(host, pending);2824 2825			spin_unlock(&host->irq_lock);2826		}2827 2828		if (pending & SDMMC_INT_CD) {2829			mci_writel(host, RINTSTS, SDMMC_INT_CD);2830			dw_mci_handle_cd(host);2831		}2832 2833		if (pending & SDMMC_INT_SDIO(slot->sdio_id)) {2834			mci_writel(host, RINTSTS,2835				   SDMMC_INT_SDIO(slot->sdio_id));2836			__dw_mci_enable_sdio_irq(slot, 0);2837			sdio_signal_irq(slot->mmc);2838		}2839 2840	}2841 2842	if (host->use_dma != TRANS_MODE_IDMAC)2843		return IRQ_HANDLED;2844 2845	/* Handle IDMA interrupts */2846	if (host->dma_64bit_address == 1) {2847		pending = mci_readl(host, IDSTS64);2848		if (pending & (SDMMC_IDMAC_INT_TI | SDMMC_IDMAC_INT_RI)) {2849			mci_writel(host, IDSTS64, SDMMC_IDMAC_INT_TI |2850							SDMMC_IDMAC_INT_RI);2851			mci_writel(host, IDSTS64, SDMMC_IDMAC_INT_NI);2852			if (!test_bit(EVENT_DATA_ERROR, &host->pending_events))2853				host->dma_ops->complete((void *)host);2854		}2855	} else {2856		pending = mci_readl(host, IDSTS);2857		if (pending & (SDMMC_IDMAC_INT_TI | SDMMC_IDMAC_INT_RI)) {2858			mci_writel(host, IDSTS, SDMMC_IDMAC_INT_TI |2859							SDMMC_IDMAC_INT_RI);2860			mci_writel(host, IDSTS, SDMMC_IDMAC_INT_NI);2861			if (!test_bit(EVENT_DATA_ERROR, &host->pending_events))2862				host->dma_ops->complete((void *)host);2863		}2864	}2865 2866	return IRQ_HANDLED;2867}2868 2869static int dw_mci_init_slot_caps(struct dw_mci_slot *slot)2870{2871	struct dw_mci *host = slot->host;2872	const struct dw_mci_drv_data *drv_data = host->drv_data;2873	struct mmc_host *mmc = slot->mmc;2874	int ctrl_id;2875 2876	if (host->pdata->caps)2877		mmc->caps = host->pdata->caps;2878 2879	if (host->pdata->pm_caps)2880		mmc->pm_caps = host->pdata->pm_caps;2881 2882	if (drv_data)2883		mmc->caps |= drv_data->common_caps;2884 2885	if (host->dev->of_node) {2886		ctrl_id = of_alias_get_id(host->dev->of_node, "mshc");2887		if (ctrl_id < 0)2888			ctrl_id = 0;2889	} else {2890		ctrl_id = to_platform_device(host->dev)->id;2891	}2892 2893	if (drv_data && drv_data->caps) {2894		if (ctrl_id >= drv_data->num_caps) {2895			dev_err(host->dev, "invalid controller id %d\n",2896				ctrl_id);2897			return -EINVAL;2898		}2899		mmc->caps |= drv_data->caps[ctrl_id];2900	}2901 2902	if (host->pdata->caps2)2903		mmc->caps2 = host->pdata->caps2;2904 2905	/* if host has set a minimum_freq, we should respect it */2906	if (host->minimum_speed)2907		mmc->f_min = host->minimum_speed;2908	else2909		mmc->f_min = DW_MCI_FREQ_MIN;2910 2911	if (!mmc->f_max)2912		mmc->f_max = DW_MCI_FREQ_MAX;2913 2914	/* Process SDIO IRQs through the sdio_irq_work. */2915	if (mmc->caps & MMC_CAP_SDIO_IRQ)2916		mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;2917 2918	return 0;2919}2920 2921static int dw_mci_init_slot(struct dw_mci *host)2922{2923	struct mmc_host *mmc;2924	struct dw_mci_slot *slot;2925	int ret;2926 2927	mmc = mmc_alloc_host(sizeof(struct dw_mci_slot), host->dev);2928	if (!mmc)2929		return -ENOMEM;2930 2931	slot = mmc_priv(mmc);2932	slot->id = 0;2933	slot->sdio_id = host->sdio_id0 + slot->id;2934	slot->mmc = mmc;2935	slot->host = host;2936	host->slot = slot;2937 2938	mmc->ops = &dw_mci_ops;2939 2940	/*if there are external regulators, get them*/2941	ret = mmc_regulator_get_supply(mmc);2942	if (ret)2943		goto err_host_allocated;2944 2945	if (!mmc->ocr_avail)2946		mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34;2947 2948	ret = mmc_of_parse(mmc);2949	if (ret)2950		goto err_host_allocated;2951 2952	ret = dw_mci_init_slot_caps(slot);2953	if (ret)2954		goto err_host_allocated;2955 2956	/* Useful defaults if platform data is unset. */2957	if (host->use_dma == TRANS_MODE_IDMAC) {2958		mmc->max_segs = host->ring_size;2959		mmc->max_blk_size = 65535;2960		mmc->max_seg_size = 0x1000;2961		mmc->max_req_size = mmc->max_seg_size * host->ring_size;2962		mmc->max_blk_count = mmc->max_req_size / 512;2963	} else if (host->use_dma == TRANS_MODE_EDMAC) {2964		mmc->max_segs = 64;2965		mmc->max_blk_size = 65535;2966		mmc->max_blk_count = 65535;2967		mmc->max_req_size =2968				mmc->max_blk_size * mmc->max_blk_count;2969		mmc->max_seg_size = mmc->max_req_size;2970	} else {2971		/* TRANS_MODE_PIO */2972		mmc->max_segs = 64;2973		mmc->max_blk_size = 65535; /* BLKSIZ is 16 bits */2974		mmc->max_blk_count = 512;2975		mmc->max_req_size = mmc->max_blk_size *2976				    mmc->max_blk_count;2977		mmc->max_seg_size = mmc->max_req_size;2978	}2979 2980	dw_mci_get_cd(mmc);2981 2982	ret = mmc_add_host(mmc);2983	if (ret)2984		goto err_host_allocated;2985 2986#if defined(CONFIG_DEBUG_FS)2987	dw_mci_init_debugfs(slot);2988#endif2989 2990	return 0;2991 2992err_host_allocated:2993	mmc_free_host(mmc);2994	return ret;2995}2996 2997static void dw_mci_cleanup_slot(struct dw_mci_slot *slot)2998{2999	/* Debugfs stuff is cleaned up by mmc core */3000	mmc_remove_host(slot->mmc);3001	slot->host->slot = NULL;3002	mmc_free_host(slot->mmc);3003}3004 3005static void dw_mci_init_dma(struct dw_mci *host)3006{3007	int addr_config;3008	struct device *dev = host->dev;3009 3010	/*3011	* Check tansfer mode from HCON[17:16]3012	* Clear the ambiguous description of dw_mmc databook:3013	* 2b'00: No DMA Interface -> Actually means using Internal DMA block3014	* 2b'01: DesignWare DMA Interface -> Synopsys DW-DMA block3015	* 2b'10: Generic DMA Interface -> non-Synopsys generic DMA block3016	* 2b'11: Non DW DMA Interface -> pio only3017	* Compared to DesignWare DMA Interface, Generic DMA Interface has a3018	* simpler request/acknowledge handshake mechanism and both of them3019	* are regarded as external dma master for dw_mmc.3020	*/3021	host->use_dma = SDMMC_GET_TRANS_MODE(mci_readl(host, HCON));3022	if (host->use_dma == DMA_INTERFACE_IDMA) {3023		host->use_dma = TRANS_MODE_IDMAC;3024	} else if (host->use_dma == DMA_INTERFACE_DWDMA ||3025		   host->use_dma == DMA_INTERFACE_GDMA) {3026		host->use_dma = TRANS_MODE_EDMAC;3027	} else {3028		goto no_dma;3029	}3030 3031	/* Determine which DMA interface to use */3032	if (host->use_dma == TRANS_MODE_IDMAC) {3033		/*3034		* Check ADDR_CONFIG bit in HCON to find3035		* IDMAC address bus width3036		*/3037		addr_config = SDMMC_GET_ADDR_CONFIG(mci_readl(host, HCON));3038 3039		if (addr_config == 1) {3040			/* host supports IDMAC in 64-bit address mode */3041			host->dma_64bit_address = 1;3042			dev_info(host->dev,3043				 "IDMAC supports 64-bit address mode.\n");3044			if (!dma_set_mask(host->dev, DMA_BIT_MASK(64)))3045				dma_set_coherent_mask(host->dev,3046						      DMA_BIT_MASK(64));3047		} else {3048			/* host supports IDMAC in 32-bit address mode */3049			host->dma_64bit_address = 0;3050			dev_info(host->dev,3051				 "IDMAC supports 32-bit address mode.\n");3052		}3053 3054		/* Alloc memory for sg translation */3055		host->sg_cpu = dmam_alloc_coherent(host->dev,3056						   DESC_RING_BUF_SZ,3057						   &host->sg_dma, GFP_KERNEL);3058		if (!host->sg_cpu) {3059			dev_err(host->dev,3060				"%s: could not alloc DMA memory\n",3061				__func__);3062			goto no_dma;3063		}3064 3065		host->dma_ops = &dw_mci_idmac_ops;3066		dev_info(host->dev, "Using internal DMA controller.\n");3067	} else {3068		/* TRANS_MODE_EDMAC: check dma bindings again */3069		if ((device_property_string_array_count(dev, "dma-names") < 0) ||3070		    !device_property_present(dev, "dmas")) {3071			goto no_dma;3072		}3073		host->dma_ops = &dw_mci_edmac_ops;3074		dev_info(host->dev, "Using external DMA controller.\n");3075	}3076 3077	if (host->dma_ops->init && host->dma_ops->start &&3078	    host->dma_ops->stop && host->dma_ops->cleanup) {3079		if (host->dma_ops->init(host)) {3080			dev_err(host->dev, "%s: Unable to initialize DMA Controller.\n",3081				__func__);3082			goto no_dma;3083		}3084	} else {3085		dev_err(host->dev, "DMA initialization not found.\n");3086		goto no_dma;3087	}3088 3089	return;3090 3091no_dma:3092	dev_info(host->dev, "Using PIO mode.\n");3093	host->use_dma = TRANS_MODE_PIO;3094}3095 3096static void dw_mci_cmd11_timer(struct timer_list *t)3097{3098	struct dw_mci *host = from_timer(host, t, cmd11_timer);3099 3100	if (host->state != STATE_SENDING_CMD11) {3101		dev_warn(host->dev, "Unexpected CMD11 timeout\n");3102		return;3103	}3104 3105	host->cmd_status = SDMMC_INT_RTO;3106	set_bit(EVENT_CMD_COMPLETE, &host->pending_events);3107	queue_work(system_bh_wq, &host->bh_work);3108}3109 3110static void dw_mci_cto_timer(struct timer_list *t)3111{3112	struct dw_mci *host = from_timer(host, t, cto_timer);3113	unsigned long irqflags;3114	u32 pending;3115 3116	spin_lock_irqsave(&host->irq_lock, irqflags);3117 3118	/*3119	 * If somehow we have very bad interrupt latency it's remotely possible3120	 * that the timer could fire while the interrupt is still pending or3121	 * while the interrupt is midway through running.  Let's be paranoid3122	 * and detect those two cases.  Note that this is paranoia is somewhat3123	 * justified because in this function we don't actually cancel the3124	 * pending command in the controller--we just assume it will never come.3125	 */3126	pending = mci_readl(host, MINTSTS); /* read-only mask reg */3127	if (pending & (DW_MCI_CMD_ERROR_FLAGS | SDMMC_INT_CMD_DONE)) {3128		/* The interrupt should fire; no need to act but we can warn */3129		dev_warn(host->dev, "Unexpected interrupt latency\n");3130		goto exit;3131	}3132	if (test_bit(EVENT_CMD_COMPLETE, &host->pending_events)) {3133		/* Presumably interrupt handler couldn't delete the timer */3134		dev_warn(host->dev, "CTO timeout when already completed\n");3135		goto exit;3136	}3137 3138	/*3139	 * Continued paranoia to make sure we're in the state we expect.3140	 * This paranoia isn't really justified but it seems good to be safe.3141	 */3142	switch (host->state) {3143	case STATE_SENDING_CMD11:3144	case STATE_SENDING_CMD:3145	case STATE_SENDING_STOP:3146		/*3147		 * If CMD_DONE interrupt does NOT come in sending command3148		 * state, we should notify the driver to terminate current3149		 * transfer and report a command timeout to the core.3150		 */3151		host->cmd_status = SDMMC_INT_RTO;3152		set_bit(EVENT_CMD_COMPLETE, &host->pending_events);3153		queue_work(system_bh_wq, &host->bh_work);3154		break;3155	default:3156		dev_warn(host->dev, "Unexpected command timeout, state %d\n",3157			 host->state);3158		break;3159	}3160 3161exit:3162	spin_unlock_irqrestore(&host->irq_lock, irqflags);3163}3164 3165static void dw_mci_dto_timer(struct timer_list *t)3166{3167	struct dw_mci *host = from_timer(host, t, dto_timer);3168	unsigned long irqflags;3169	u32 pending;3170 3171	spin_lock_irqsave(&host->irq_lock, irqflags);3172 3173	/*3174	 * The DTO timer is much longer than the CTO timer, so it's even less3175	 * likely that we'll these cases, but it pays to be paranoid.3176	 */3177	pending = mci_readl(host, MINTSTS); /* read-only mask reg */3178	if (pending & SDMMC_INT_DATA_OVER) {3179		/* The interrupt should fire; no need to act but we can warn */3180		dev_warn(host->dev, "Unexpected data interrupt latency\n");3181		goto exit;3182	}3183	if (test_bit(EVENT_DATA_COMPLETE, &host->pending_events)) {3184		/* Presumably interrupt handler couldn't delete the timer */3185		dev_warn(host->dev, "DTO timeout when already completed\n");3186		goto exit;3187	}3188 3189	/*3190	 * Continued paranoia to make sure we're in the state we expect.3191	 * This paranoia isn't really justified but it seems good to be safe.3192	 */3193	switch (host->state) {3194	case STATE_SENDING_DATA:3195	case STATE_DATA_BUSY:3196		/*3197		 * If DTO interrupt does NOT come in sending data state,3198		 * we should notify the driver to terminate current transfer3199		 * and report a data timeout to the core.3200		 */3201		host->data_status = SDMMC_INT_DRTO;3202		set_bit(EVENT_DATA_ERROR, &host->pending_events);3203		set_bit(EVENT_DATA_COMPLETE, &host->pending_events);3204		queue_work(system_bh_wq, &host->bh_work);3205		break;3206	default:3207		dev_warn(host->dev, "Unexpected data timeout, state %d\n",3208			 host->state);3209		break;3210	}3211 3212exit:3213	spin_unlock_irqrestore(&host->irq_lock, irqflags);3214}3215 3216#ifdef CONFIG_OF3217static struct dw_mci_board *dw_mci_parse_dt(struct dw_mci *host)3218{3219	struct dw_mci_board *pdata;3220	struct device *dev = host->dev;3221	const struct dw_mci_drv_data *drv_data = host->drv_data;3222	int ret;3223	u32 clock_frequency;3224 3225	pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);3226	if (!pdata)3227		return ERR_PTR(-ENOMEM);3228 3229	/* find reset controller when exist */3230	pdata->rstc = devm_reset_control_get_optional_exclusive(dev, "reset");3231	if (IS_ERR(pdata->rstc))3232		return ERR_CAST(pdata->rstc);3233 3234	if (device_property_read_u32(dev, "fifo-depth", &pdata->fifo_depth))3235		dev_info(dev,3236			 "fifo-depth property not found, using value of FIFOTH register as default\n");3237 3238	device_property_read_u32(dev, "card-detect-delay",3239				 &pdata->detect_delay_ms);3240 3241	device_property_read_u32(dev, "data-addr", &host->data_addr_override);3242 3243	if (device_property_present(dev, "fifo-watermark-aligned"))3244		host->wm_aligned = true;3245 3246	if (!device_property_read_u32(dev, "clock-frequency", &clock_frequency))3247		pdata->bus_hz = clock_frequency;3248 3249	if (drv_data && drv_data->parse_dt) {3250		ret = drv_data->parse_dt(host);3251		if (ret)3252			return ERR_PTR(ret);3253	}3254 3255	return pdata;3256}3257 3258#else /* CONFIG_OF */3259static struct dw_mci_board *dw_mci_parse_dt(struct dw_mci *host)3260{3261	return ERR_PTR(-EINVAL);3262}3263#endif /* CONFIG_OF */3264 3265static void dw_mci_enable_cd(struct dw_mci *host)3266{3267	unsigned long irqflags;3268	u32 temp;3269 3270	/*3271	 * No need for CD if all slots have a non-error GPIO3272	 * as well as broken card detection is found.3273	 */3274	if (host->slot->mmc->caps & MMC_CAP_NEEDS_POLL)3275		return;3276 3277	if (mmc_gpio_get_cd(host->slot->mmc) < 0) {3278		spin_lock_irqsave(&host->irq_lock, irqflags);3279		temp = mci_readl(host, INTMASK);3280		temp  |= SDMMC_INT_CD;3281		mci_writel(host, INTMASK, temp);3282		spin_unlock_irqrestore(&host->irq_lock, irqflags);3283	}3284}3285 3286int dw_mci_probe(struct dw_mci *host)3287{3288	const struct dw_mci_drv_data *drv_data = host->drv_data;3289	int width, i, ret = 0;3290	u32 fifo_size;3291 3292	if (!host->pdata) {3293		host->pdata = dw_mci_parse_dt(host);3294		if (IS_ERR(host->pdata))3295			return dev_err_probe(host->dev, PTR_ERR(host->pdata),3296					     "platform data not available\n");3297	}3298 3299	host->biu_clk = devm_clk_get(host->dev, "biu");3300	if (IS_ERR(host->biu_clk)) {3301		dev_dbg(host->dev, "biu clock not available\n");3302		ret = PTR_ERR(host->biu_clk);3303		if (ret == -EPROBE_DEFER)3304			return ret;3305 3306	} else {3307		ret = clk_prepare_enable(host->biu_clk);3308		if (ret) {3309			dev_err(host->dev, "failed to enable biu clock\n");3310			return ret;3311		}3312	}3313 3314	host->ciu_clk = devm_clk_get(host->dev, "ciu");3315	if (IS_ERR(host->ciu_clk)) {3316		dev_dbg(host->dev, "ciu clock not available\n");3317		ret = PTR_ERR(host->ciu_clk);3318		if (ret == -EPROBE_DEFER)3319			goto err_clk_biu;3320 3321		host->bus_hz = host->pdata->bus_hz;3322	} else {3323		ret = clk_prepare_enable(host->ciu_clk);3324		if (ret) {3325			dev_err(host->dev, "failed to enable ciu clock\n");3326			goto err_clk_biu;3327		}3328 3329		if (host->pdata->bus_hz) {3330			ret = clk_set_rate(host->ciu_clk, host->pdata->bus_hz);3331			if (ret)3332				dev_warn(host->dev,3333					 "Unable to set bus rate to %uHz\n",3334					 host->pdata->bus_hz);3335		}3336		host->bus_hz = clk_get_rate(host->ciu_clk);3337	}3338 3339	if (!host->bus_hz) {3340		dev_err(host->dev,3341			"Platform data must supply bus speed\n");3342		ret = -ENODEV;3343		goto err_clk_ciu;3344	}3345 3346	if (host->pdata->rstc) {3347		reset_control_assert(host->pdata->rstc);3348		usleep_range(10, 50);3349		reset_control_deassert(host->pdata->rstc);3350	}3351 3352	if (drv_data && drv_data->init) {3353		ret = drv_data->init(host);3354		if (ret) {3355			dev_err(host->dev,3356				"implementation specific init failed\n");3357			goto err_clk_ciu;3358		}3359	}3360 3361	timer_setup(&host->cmd11_timer, dw_mci_cmd11_timer, 0);3362	timer_setup(&host->cto_timer, dw_mci_cto_timer, 0);3363	timer_setup(&host->dto_timer, dw_mci_dto_timer, 0);3364 3365	spin_lock_init(&host->lock);3366	spin_lock_init(&host->irq_lock);3367	INIT_LIST_HEAD(&host->queue);3368 3369	dw_mci_init_fault(host);3370 3371	/*3372	 * Get the host data width - this assumes that HCON has been set with3373	 * the correct values.3374	 */3375	i = SDMMC_GET_HDATA_WIDTH(mci_readl(host, HCON));3376	if (!i) {3377		host->push_data = dw_mci_push_data16;3378		host->pull_data = dw_mci_pull_data16;3379		width = 16;3380		host->data_shift = 1;3381	} else if (i == 2) {3382		host->push_data = dw_mci_push_data64;3383		host->pull_data = dw_mci_pull_data64;3384		width = 64;3385		host->data_shift = 3;3386	} else {3387		/* Check for a reserved value, and warn if it is */3388		WARN((i != 1),3389		     "HCON reports a reserved host data width!\n"3390		     "Defaulting to 32-bit access.\n");3391		host->push_data = dw_mci_push_data32;3392		host->pull_data = dw_mci_pull_data32;3393		width = 32;3394		host->data_shift = 2;3395	}3396 3397	/* Reset all blocks */3398	if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS)) {3399		ret = -ENODEV;3400		goto err_clk_ciu;3401	}3402 3403	host->dma_ops = host->pdata->dma_ops;3404	dw_mci_init_dma(host);3405 3406	/* Clear the interrupts for the host controller */3407	mci_writel(host, RINTSTS, 0xFFFFFFFF);3408	mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */3409 3410	/* Put in max timeout */3411	mci_writel(host, TMOUT, 0xFFFFFFFF);3412 3413	/*3414	 * FIFO threshold settings  RxMark  = fifo_size / 2 - 1,3415	 *                          Tx Mark = fifo_size / 2 DMA Size = 83416	 */3417	if (!host->pdata->fifo_depth) {3418		/*3419		 * Power-on value of RX_WMark is FIFO_DEPTH-1, but this may3420		 * have been overwritten by the bootloader, just like we're3421		 * about to do, so if you know the value for your hardware, you3422		 * should put it in the platform data.3423		 */3424		fifo_size = mci_readl(host, FIFOTH);3425		fifo_size = 1 + ((fifo_size >> 16) & 0xfff);3426	} else {3427		fifo_size = host->pdata->fifo_depth;3428	}3429	host->fifo_depth = fifo_size;3430	host->fifoth_val =3431		SDMMC_SET_FIFOTH(0x2, fifo_size / 2 - 1, fifo_size / 2);3432	mci_writel(host, FIFOTH, host->fifoth_val);3433 3434	/* disable clock to CIU */3435	mci_writel(host, CLKENA, 0);3436	mci_writel(host, CLKSRC, 0);3437 3438	/*3439	 * In 2.40a spec, Data offset is changed.3440	 * Need to check the version-id and set data-offset for DATA register.3441	 */3442	host->verid = SDMMC_GET_VERID(mci_readl(host, VERID));3443	dev_info(host->dev, "Version ID is %04x\n", host->verid);3444 3445	if (host->data_addr_override)3446		host->fifo_reg = host->regs + host->data_addr_override;3447	else if (host->verid < DW_MMC_240A)3448		host->fifo_reg = host->regs + DATA_OFFSET;3449	else3450		host->fifo_reg = host->regs + DATA_240A_OFFSET;3451 3452	INIT_WORK(&host->bh_work, dw_mci_work_func);3453	ret = devm_request_irq(host->dev, host->irq, dw_mci_interrupt,3454			       host->irq_flags, "dw-mci", host);3455	if (ret)3456		goto err_dmaunmap;3457 3458	/*3459	 * Enable interrupts for command done, data over, data empty,3460	 * receive ready and error such as transmit, receive timeout, crc error3461	 */3462	mci_writel(host, INTMASK, SDMMC_INT_CMD_DONE | SDMMC_INT_DATA_OVER |3463		   SDMMC_INT_TXDR | SDMMC_INT_RXDR |3464		   DW_MCI_ERROR_FLAGS);3465	/* Enable mci interrupt */3466	mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE);3467 3468	dev_info(host->dev,3469		 "DW MMC controller at irq %d,%d bit host data width,%u deep fifo\n",3470		 host->irq, width, fifo_size);3471 3472	/* We need at least one slot to succeed */3473	ret = dw_mci_init_slot(host);3474	if (ret) {3475		dev_dbg(host->dev, "slot %d init failed\n", i);3476		goto err_dmaunmap;3477	}3478 3479	/* Now that slots are all setup, we can enable card detect */3480	dw_mci_enable_cd(host);3481 3482	return 0;3483 3484err_dmaunmap:3485	if (host->use_dma && host->dma_ops->exit)3486		host->dma_ops->exit(host);3487 3488	reset_control_assert(host->pdata->rstc);3489 3490err_clk_ciu:3491	clk_disable_unprepare(host->ciu_clk);3492 3493err_clk_biu:3494	clk_disable_unprepare(host->biu_clk);3495 3496	return ret;3497}3498EXPORT_SYMBOL(dw_mci_probe);3499 3500void dw_mci_remove(struct dw_mci *host)3501{3502	dev_dbg(host->dev, "remove slot\n");3503	if (host->slot)3504		dw_mci_cleanup_slot(host->slot);3505 3506	mci_writel(host, RINTSTS, 0xFFFFFFFF);3507	mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */3508 3509	/* disable clock to CIU */3510	mci_writel(host, CLKENA, 0);3511	mci_writel(host, CLKSRC, 0);3512 3513	if (host->use_dma && host->dma_ops->exit)3514		host->dma_ops->exit(host);3515 3516	reset_control_assert(host->pdata->rstc);3517 3518	clk_disable_unprepare(host->ciu_clk);3519	clk_disable_unprepare(host->biu_clk);3520}3521EXPORT_SYMBOL(dw_mci_remove);3522 3523 3524 3525#ifdef CONFIG_PM3526int dw_mci_runtime_suspend(struct device *dev)3527{3528	struct dw_mci *host = dev_get_drvdata(dev);3529 3530	if (host->use_dma && host->dma_ops->exit)3531		host->dma_ops->exit(host);3532 3533	clk_disable_unprepare(host->ciu_clk);3534 3535	if (host->slot &&3536	    (mmc_can_gpio_cd(host->slot->mmc) ||3537	     !mmc_card_is_removable(host->slot->mmc)))3538		clk_disable_unprepare(host->biu_clk);3539 3540	return 0;3541}3542EXPORT_SYMBOL(dw_mci_runtime_suspend);3543 3544int dw_mci_runtime_resume(struct device *dev)3545{3546	int ret = 0;3547	struct dw_mci *host = dev_get_drvdata(dev);3548 3549	if (host->slot &&3550	    (mmc_can_gpio_cd(host->slot->mmc) ||3551	     !mmc_card_is_removable(host->slot->mmc))) {3552		ret = clk_prepare_enable(host->biu_clk);3553		if (ret)3554			return ret;3555	}3556 3557	ret = clk_prepare_enable(host->ciu_clk);3558	if (ret)3559		goto err;3560 3561	if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS)) {3562		clk_disable_unprepare(host->ciu_clk);3563		ret = -ENODEV;3564		goto err;3565	}3566 3567	if (host->use_dma && host->dma_ops->init)3568		host->dma_ops->init(host);3569 3570	/*3571	 * Restore the initial value at FIFOTH register3572	 * And Invalidate the prev_blksz with zero3573	 */3574	mci_writel(host, FIFOTH, host->fifoth_val);3575	host->prev_blksz = 0;3576 3577	/* Put in max timeout */3578	mci_writel(host, TMOUT, 0xFFFFFFFF);3579 3580	mci_writel(host, RINTSTS, 0xFFFFFFFF);3581	mci_writel(host, INTMASK, SDMMC_INT_CMD_DONE | SDMMC_INT_DATA_OVER |3582		   SDMMC_INT_TXDR | SDMMC_INT_RXDR |3583		   DW_MCI_ERROR_FLAGS);3584	mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE);3585 3586 3587	if (host->slot && host->slot->mmc->pm_flags & MMC_PM_KEEP_POWER)3588		dw_mci_set_ios(host->slot->mmc, &host->slot->mmc->ios);3589 3590	/* Force setup bus to guarantee available clock output */3591	dw_mci_setup_bus(host->slot, true);3592 3593	/* Re-enable SDIO interrupts. */3594	if (sdio_irq_claimed(host->slot->mmc))3595		__dw_mci_enable_sdio_irq(host->slot, 1);3596 3597	/* Now that slots are all setup, we can enable card detect */3598	dw_mci_enable_cd(host);3599 3600	return 0;3601 3602err:3603	if (host->slot &&3604	    (mmc_can_gpio_cd(host->slot->mmc) ||3605	     !mmc_card_is_removable(host->slot->mmc)))3606		clk_disable_unprepare(host->biu_clk);3607 3608	return ret;3609}3610EXPORT_SYMBOL(dw_mci_runtime_resume);3611#endif /* CONFIG_PM */3612 3613static int __init dw_mci_init(void)3614{3615	pr_info("Synopsys Designware Multimedia Card Interface Driver\n");3616	return 0;3617}3618 3619static void __exit dw_mci_exit(void)3620{3621}3622 3623module_init(dw_mci_init);3624module_exit(dw_mci_exit);3625 3626MODULE_DESCRIPTION("DW Multimedia Card Interface driver");3627MODULE_AUTHOR("NXP Semiconductor VietNam");3628MODULE_AUTHOR("Imagination Technologies Ltd");3629MODULE_LICENSE("GPL v2");3630