brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0+2/*3 * Driver for Motorola/Freescale IMX serial ports4 *5 * Based on drivers/char/serial.c, by Linus Torvalds, Theodore Ts'o.6 *7 * Author: Sascha Hauer <sascha@saschahauer.de>8 * Copyright (C) 2004 Pengutronix9 */10 11#include <linux/circ_buf.h>12#include <linux/module.h>13#include <linux/ioport.h>14#include <linux/init.h>15#include <linux/console.h>16#include <linux/sysrq.h>17#include <linux/platform_device.h>18#include <linux/tty.h>19#include <linux/tty_flip.h>20#include <linux/serial_core.h>21#include <linux/serial.h>22#include <linux/clk.h>23#include <linux/delay.h>24#include <linux/ktime.h>25#include <linux/pinctrl/consumer.h>26#include <linux/rational.h>27#include <linux/slab.h>28#include <linux/of.h>29#include <linux/io.h>30#include <linux/iopoll.h>31#include <linux/dma-mapping.h>32 33#include <asm/irq.h>34#include <linux/dma/imx-dma.h>35 36#include "serial_mctrl_gpio.h"37 38/* Register definitions */39#define URXD0 0x0  /* Receiver Register */40#define URTX0 0x40 /* Transmitter Register */41#define UCR1  0x80 /* Control Register 1 */42#define UCR2  0x84 /* Control Register 2 */43#define UCR3  0x88 /* Control Register 3 */44#define UCR4  0x8c /* Control Register 4 */45#define UFCR  0x90 /* FIFO Control Register */46#define USR1  0x94 /* Status Register 1 */47#define USR2  0x98 /* Status Register 2 */48#define UESC  0x9c /* Escape Character Register */49#define UTIM  0xa0 /* Escape Timer Register */50#define UBIR  0xa4 /* BRM Incremental Register */51#define UBMR  0xa8 /* BRM Modulator Register */52#define UBRC  0xac /* Baud Rate Count Register */53#define IMX21_ONEMS 0xb0 /* One Millisecond register */54#define IMX1_UTS 0xd0 /* UART Test Register on i.mx1 */55#define IMX21_UTS 0xb4 /* UART Test Register on all other i.mx*/56 57/* UART Control Register Bit Fields.*/58#define URXD_DUMMY_READ (1<<16)59#define URXD_CHARRDY	(1<<15)60#define URXD_ERR	(1<<14)61#define URXD_OVRRUN	(1<<13)62#define URXD_FRMERR	(1<<12)63#define URXD_BRK	(1<<11)64#define URXD_PRERR	(1<<10)65#define URXD_RX_DATA	(0xFF<<0)66#define UCR1_ADEN	(1<<15) /* Auto detect interrupt */67#define UCR1_ADBR	(1<<14) /* Auto detect baud rate */68#define UCR1_TRDYEN	(1<<13) /* Transmitter ready interrupt enable */69#define UCR1_IDEN	(1<<12) /* Idle condition interrupt */70#define UCR1_ICD_REG(x) (((x) & 3) << 10) /* idle condition detect */71#define UCR1_RRDYEN	(1<<9)	/* Recv ready interrupt enable */72#define UCR1_RXDMAEN	(1<<8)	/* Recv ready DMA enable */73#define UCR1_IREN	(1<<7)	/* Infrared interface enable */74#define UCR1_TXMPTYEN	(1<<6)	/* Transimitter empty interrupt enable */75#define UCR1_RTSDEN	(1<<5)	/* RTS delta interrupt enable */76#define UCR1_SNDBRK	(1<<4)	/* Send break */77#define UCR1_TXDMAEN	(1<<3)	/* Transmitter ready DMA enable */78#define IMX1_UCR1_UARTCLKEN (1<<2) /* UART clock enabled, i.mx1 only */79#define UCR1_ATDMAEN    (1<<2)  /* Aging DMA Timer Enable */80#define UCR1_DOZE	(1<<1)	/* Doze */81#define UCR1_UARTEN	(1<<0)	/* UART enabled */82#define UCR2_ESCI	(1<<15)	/* Escape seq interrupt enable */83#define UCR2_IRTS	(1<<14)	/* Ignore RTS pin */84#define UCR2_CTSC	(1<<13)	/* CTS pin control */85#define UCR2_CTS	(1<<12)	/* Clear to send */86#define UCR2_ESCEN	(1<<11)	/* Escape enable */87#define UCR2_PREN	(1<<8)	/* Parity enable */88#define UCR2_PROE	(1<<7)	/* Parity odd/even */89#define UCR2_STPB	(1<<6)	/* Stop */90#define UCR2_WS		(1<<5)	/* Word size */91#define UCR2_RTSEN	(1<<4)	/* Request to send interrupt enable */92#define UCR2_ATEN	(1<<3)	/* Aging Timer Enable */93#define UCR2_TXEN	(1<<2)	/* Transmitter enabled */94#define UCR2_RXEN	(1<<1)	/* Receiver enabled */95#define UCR2_SRST	(1<<0)	/* SW reset */96#define UCR3_DTREN	(1<<13) /* DTR interrupt enable */97#define UCR3_PARERREN	(1<<12) /* Parity enable */98#define UCR3_FRAERREN	(1<<11) /* Frame error interrupt enable */99#define UCR3_DSR	(1<<10) /* Data set ready */100#define UCR3_DCD	(1<<9)	/* Data carrier detect */101#define UCR3_RI		(1<<8)	/* Ring indicator */102#define UCR3_ADNIMP	(1<<7)	/* Autobaud Detection Not Improved */103#define UCR3_RXDSEN	(1<<6)	/* Receive status interrupt enable */104#define UCR3_AIRINTEN	(1<<5)	/* Async IR wake interrupt enable */105#define UCR3_AWAKEN	(1<<4)	/* Async wake interrupt enable */106#define UCR3_DTRDEN	(1<<3)	/* Data Terminal Ready Delta Enable. */107#define IMX21_UCR3_RXDMUXSEL	(1<<2)	/* RXD Muxed Input Select */108#define UCR3_INVT	(1<<1)	/* Inverted Infrared transmission */109#define UCR3_BPEN	(1<<0)	/* Preset registers enable */110#define UCR4_CTSTL_SHF	10	/* CTS trigger level shift */111#define UCR4_CTSTL_MASK	0x3F	/* CTS trigger is 6 bits wide */112#define UCR4_INVR	(1<<9)	/* Inverted infrared reception */113#define UCR4_ENIRI	(1<<8)	/* Serial infrared interrupt enable */114#define UCR4_WKEN	(1<<7)	/* Wake interrupt enable */115#define UCR4_REF16	(1<<6)	/* Ref freq 16 MHz */116#define UCR4_IDDMAEN    (1<<6)  /* DMA IDLE Condition Detected */117#define UCR4_IRSC	(1<<5)	/* IR special case */118#define UCR4_TCEN	(1<<3)	/* Transmit complete interrupt enable */119#define UCR4_BKEN	(1<<2)	/* Break condition interrupt enable */120#define UCR4_OREN	(1<<1)	/* Receiver overrun interrupt enable */121#define UCR4_DREN	(1<<0)	/* Recv data ready interrupt enable */122#define UFCR_RXTL_SHF	0	/* Receiver trigger level shift */123#define UFCR_RXTL_MASK	0x3F	/* Receiver trigger 6 bits wide */124#define UFCR_DCEDTE	(1<<6)	/* DCE/DTE mode select */125#define UFCR_RFDIV	(7<<7)	/* Reference freq divider mask */126#define UFCR_RFDIV_REG(x)	(((x) < 7 ? 6 - (x) : 6) << 7)127#define UFCR_TXTL_SHF	10	/* Transmitter trigger level shift */128#define USR1_PARITYERR	(1<<15) /* Parity error interrupt flag */129#define USR1_RTSS	(1<<14) /* RTS pin status */130#define USR1_TRDY	(1<<13) /* Transmitter ready interrupt/dma flag */131#define USR1_RTSD	(1<<12) /* RTS delta */132#define USR1_ESCF	(1<<11) /* Escape seq interrupt flag */133#define USR1_FRAMERR	(1<<10) /* Frame error interrupt flag */134#define USR1_RRDY	(1<<9)	 /* Receiver ready interrupt/dma flag */135#define USR1_AGTIM	(1<<8)	 /* Ageing timer interrupt flag */136#define USR1_DTRD	(1<<7)	 /* DTR Delta */137#define USR1_RXDS	 (1<<6)	 /* Receiver idle interrupt flag */138#define USR1_AIRINT	 (1<<5)	 /* Async IR wake interrupt flag */139#define USR1_AWAKE	 (1<<4)	 /* Aysnc wake interrupt flag */140#define USR2_ADET	 (1<<15) /* Auto baud rate detect complete */141#define USR2_TXFE	 (1<<14) /* Transmit buffer FIFO empty */142#define USR2_DTRF	 (1<<13) /* DTR edge interrupt flag */143#define USR2_IDLE	 (1<<12) /* Idle condition */144#define USR2_RIDELT	 (1<<10) /* Ring Interrupt Delta */145#define USR2_RIIN	 (1<<9)	 /* Ring Indicator Input */146#define USR2_IRINT	 (1<<8)	 /* Serial infrared interrupt flag */147#define USR2_WAKE	 (1<<7)	 /* Wake */148#define USR2_DCDIN	 (1<<5)	 /* Data Carrier Detect Input */149#define USR2_RTSF	 (1<<4)	 /* RTS edge interrupt flag */150#define USR2_TXDC	 (1<<3)	 /* Transmitter complete */151#define USR2_BRCD	 (1<<2)	 /* Break condition */152#define USR2_ORE	(1<<1)	 /* Overrun error */153#define USR2_RDR	(1<<0)	 /* Recv data ready */154#define UTS_FRCPERR	(1<<13) /* Force parity error */155#define UTS_LOOP	(1<<12)	 /* Loop tx and rx */156#define UTS_TXEMPTY	 (1<<6)	 /* TxFIFO empty */157#define UTS_RXEMPTY	 (1<<5)	 /* RxFIFO empty */158#define UTS_TXFULL	 (1<<4)	 /* TxFIFO full */159#define UTS_RXFULL	 (1<<3)	 /* RxFIFO full */160#define UTS_SOFTRST	 (1<<0)	 /* Software reset */161 162/* We've been assigned a range on the "Low-density serial ports" major */163#define SERIAL_IMX_MAJOR	207164#define MINOR_START		16165#define DEV_NAME		"ttymxc"166 167/*168 * This determines how often we check the modem status signals169 * for any change.  They generally aren't connected to an IRQ170 * so we have to poll them.  We also check immediately before171 * filling the TX fifo incase CTS has been dropped.172 */173#define MCTRL_TIMEOUT	(250*HZ/1000)174 175#define DRIVER_NAME "IMX-uart"176 177#define UART_NR 8178 179/* i.MX21 type uart runs on all i.mx except i.MX1 and i.MX6q */180enum imx_uart_type {181	IMX1_UART,182	IMX21_UART,183};184 185/* device type dependent stuff */186struct imx_uart_data {187	unsigned uts_reg;188	enum imx_uart_type devtype;189};190 191enum imx_tx_state {192	OFF,193	WAIT_AFTER_RTS,194	SEND,195	WAIT_AFTER_SEND,196};197 198struct imx_port {199	struct uart_port	port;200	struct timer_list	timer;201	unsigned int		old_status;202	unsigned int		have_rtscts:1;203	unsigned int		have_rtsgpio:1;204	unsigned int		dte_mode:1;205	unsigned int		inverted_tx:1;206	unsigned int		inverted_rx:1;207	struct clk		*clk_ipg;208	struct clk		*clk_per;209	const struct imx_uart_data *devdata;210 211	struct mctrl_gpios *gpios;212 213	/* counter to stop 0xff flood */214	int idle_counter;215 216	/* DMA fields */217	unsigned int		dma_is_enabled:1;218	unsigned int		dma_is_rxing:1;219	unsigned int		dma_is_txing:1;220	struct dma_chan		*dma_chan_rx, *dma_chan_tx;221	struct scatterlist	rx_sgl, tx_sgl[2];222	void			*rx_buf;223	struct circ_buf		rx_ring;224	unsigned int		rx_buf_size;225	unsigned int		rx_period_length;226	unsigned int		rx_periods;227	dma_cookie_t		rx_cookie;228	unsigned int		tx_bytes;229	unsigned int		dma_tx_nents;230	unsigned int            saved_reg[10];231	bool			context_saved;232 233	enum imx_tx_state	tx_state;234	struct hrtimer		trigger_start_tx;235	struct hrtimer		trigger_stop_tx;236};237 238struct imx_port_ucrs {239	unsigned int	ucr1;240	unsigned int	ucr2;241	unsigned int	ucr3;242};243 244static const struct imx_uart_data imx_uart_imx1_devdata = {245	.uts_reg = IMX1_UTS,246	.devtype = IMX1_UART,247};248 249static const struct imx_uart_data imx_uart_imx21_devdata = {250	.uts_reg = IMX21_UTS,251	.devtype = IMX21_UART,252};253 254static const struct of_device_id imx_uart_dt_ids[] = {255	/*256	 * For reasons unknown to me, some UART devices (e.g. imx6ul's) are257	 * compatible to fsl,imx6q-uart, but not fsl,imx21-uart, while the258	 * original imx6q's UART is compatible to fsl,imx21-uart. This driver259	 * doesn't make any distinction between these two variants.260	 */261	{ .compatible = "fsl,imx6q-uart", .data = &imx_uart_imx21_devdata, },262	{ .compatible = "fsl,imx1-uart", .data = &imx_uart_imx1_devdata, },263	{ .compatible = "fsl,imx21-uart", .data = &imx_uart_imx21_devdata, },264	{ /* sentinel */ }265};266MODULE_DEVICE_TABLE(of, imx_uart_dt_ids);267 268static inline struct imx_port *to_imx_port(struct uart_port *port)269{270        return container_of(port, struct imx_port, port);271}272 273static inline void imx_uart_writel(struct imx_port *sport, u32 val, u32 offset)274{275	writel(val, sport->port.membase + offset);276}277 278static inline u32 imx_uart_readl(struct imx_port *sport, u32 offset)279{280	return readl(sport->port.membase + offset);281}282 283static inline unsigned imx_uart_uts_reg(struct imx_port *sport)284{285	return sport->devdata->uts_reg;286}287 288static inline int imx_uart_is_imx1(struct imx_port *sport)289{290	return sport->devdata->devtype == IMX1_UART;291}292 293/*294 * Save and restore functions for UCR1, UCR2 and UCR3 registers295 */296#if IS_ENABLED(CONFIG_SERIAL_IMX_CONSOLE)297static void imx_uart_ucrs_save(struct imx_port *sport,298			       struct imx_port_ucrs *ucr)299{300	/* save control registers */301	ucr->ucr1 = imx_uart_readl(sport, UCR1);302	ucr->ucr2 = imx_uart_readl(sport, UCR2);303	ucr->ucr3 = imx_uart_readl(sport, UCR3);304}305 306static void imx_uart_ucrs_restore(struct imx_port *sport,307				  struct imx_port_ucrs *ucr)308{309	/* restore control registers */310	imx_uart_writel(sport, ucr->ucr1, UCR1);311	imx_uart_writel(sport, ucr->ucr2, UCR2);312	imx_uart_writel(sport, ucr->ucr3, UCR3);313}314#endif315 316/* called with port.lock taken and irqs caller dependent */317static void imx_uart_rts_active(struct imx_port *sport, u32 *ucr2)318{319	*ucr2 &= ~(UCR2_CTSC | UCR2_CTS);320 321	mctrl_gpio_set(sport->gpios, sport->port.mctrl | TIOCM_RTS);322}323 324/* called with port.lock taken and irqs caller dependent */325static void imx_uart_rts_inactive(struct imx_port *sport, u32 *ucr2)326{327	*ucr2 &= ~UCR2_CTSC;328	*ucr2 |= UCR2_CTS;329 330	mctrl_gpio_set(sport->gpios, sport->port.mctrl & ~TIOCM_RTS);331}332 333static void start_hrtimer_ms(struct hrtimer *hrt, unsigned long msec)334{335       hrtimer_start(hrt, ms_to_ktime(msec), HRTIMER_MODE_REL);336}337 338/* called with port.lock taken and irqs off */339static void imx_uart_soft_reset(struct imx_port *sport)340{341	int i = 10;342	u32 ucr2, ubir, ubmr, uts;343 344	/*345	 * According to the Reference Manual description of the UART SRST bit:346	 *347	 * "Reset the transmit and receive state machines,348	 * all FIFOs and register USR1, USR2, UBIR, UBMR, UBRC, URXD, UTXD349	 * and UTS[6-3]".350	 *351	 * We don't need to restore the old values from USR1, USR2, URXD and352	 * UTXD. UBRC is read only, so only save/restore the other three353	 * registers.354	 */355	ubir = imx_uart_readl(sport, UBIR);356	ubmr = imx_uart_readl(sport, UBMR);357	uts = imx_uart_readl(sport, IMX21_UTS);358 359	ucr2 = imx_uart_readl(sport, UCR2);360	imx_uart_writel(sport, ucr2 & ~UCR2_SRST, UCR2);361 362	while (!(imx_uart_readl(sport, UCR2) & UCR2_SRST) && (--i > 0))363		udelay(1);364 365	/* Restore the registers */366	imx_uart_writel(sport, ubir, UBIR);367	imx_uart_writel(sport, ubmr, UBMR);368	imx_uart_writel(sport, uts, IMX21_UTS);369 370	sport->idle_counter = 0;371}372 373static void imx_uart_disable_loopback_rs485(struct imx_port *sport)374{375	unsigned int uts;376 377	/* See SER_RS485_ENABLED/UTS_LOOP comment in imx_uart_probe() */378	uts = imx_uart_readl(sport, imx_uart_uts_reg(sport));379	uts &= ~UTS_LOOP;380	imx_uart_writel(sport, uts, imx_uart_uts_reg(sport));381}382 383/* called with port.lock taken and irqs off */384static void imx_uart_start_rx(struct uart_port *port)385{386	struct imx_port *sport = to_imx_port(port);387	unsigned int ucr1, ucr2;388 389	ucr1 = imx_uart_readl(sport, UCR1);390	ucr2 = imx_uart_readl(sport, UCR2);391 392	ucr2 |= UCR2_RXEN;393 394	if (sport->dma_is_enabled) {395		ucr1 |= UCR1_RXDMAEN | UCR1_ATDMAEN;396	} else {397		ucr1 |= UCR1_RRDYEN;398		ucr2 |= UCR2_ATEN;399	}400 401	/* Write UCR2 first as it includes RXEN */402	imx_uart_writel(sport, ucr2, UCR2);403	imx_uart_writel(sport, ucr1, UCR1);404	imx_uart_disable_loopback_rs485(sport);405}406 407/* called with port.lock taken and irqs off */408static void imx_uart_stop_tx(struct uart_port *port)409{410	struct imx_port *sport = to_imx_port(port);411	u32 ucr1, ucr4, usr2;412 413	if (sport->tx_state == OFF)414		return;415 416	/*417	 * We are maybe in the SMP context, so if the DMA TX thread is running418	 * on other cpu, we have to wait for it to finish.419	 */420	if (sport->dma_is_txing)421		return;422 423	ucr1 = imx_uart_readl(sport, UCR1);424	imx_uart_writel(sport, ucr1 & ~UCR1_TRDYEN, UCR1);425 426	ucr4 = imx_uart_readl(sport, UCR4);427	usr2 = imx_uart_readl(sport, USR2);428	if ((!(usr2 & USR2_TXDC)) && (ucr4 & UCR4_TCEN)) {429		/* The shifter is still busy, so retry once TC triggers */430		return;431	}432 433	ucr4 &= ~UCR4_TCEN;434	imx_uart_writel(sport, ucr4, UCR4);435 436	/* in rs485 mode disable transmitter */437	if (port->rs485.flags & SER_RS485_ENABLED) {438		if (sport->tx_state == SEND) {439			sport->tx_state = WAIT_AFTER_SEND;440 441			if (port->rs485.delay_rts_after_send > 0) {442				start_hrtimer_ms(&sport->trigger_stop_tx,443					 port->rs485.delay_rts_after_send);444				return;445			}446 447			/* continue without any delay */448		}449 450		if (sport->tx_state == WAIT_AFTER_RTS ||451		    sport->tx_state == WAIT_AFTER_SEND) {452			u32 ucr2;453 454			hrtimer_try_to_cancel(&sport->trigger_start_tx);455 456			ucr2 = imx_uart_readl(sport, UCR2);457			if (port->rs485.flags & SER_RS485_RTS_AFTER_SEND)458				imx_uart_rts_active(sport, &ucr2);459			else460				imx_uart_rts_inactive(sport, &ucr2);461			imx_uart_writel(sport, ucr2, UCR2);462 463			if (!port->rs485_rx_during_tx_gpio)464				imx_uart_start_rx(port);465 466			sport->tx_state = OFF;467		}468	} else {469		sport->tx_state = OFF;470	}471}472 473static void imx_uart_stop_rx_with_loopback_ctrl(struct uart_port *port, bool loopback)474{475	struct imx_port *sport = to_imx_port(port);476	u32 ucr1, ucr2, ucr4, uts;477 478	ucr1 = imx_uart_readl(sport, UCR1);479	ucr2 = imx_uart_readl(sport, UCR2);480	ucr4 = imx_uart_readl(sport, UCR4);481 482	if (sport->dma_is_enabled) {483		ucr1 &= ~(UCR1_RXDMAEN | UCR1_ATDMAEN);484	} else {485		ucr1 &= ~UCR1_RRDYEN;486		ucr2 &= ~UCR2_ATEN;487		ucr4 &= ~UCR4_OREN;488	}489	imx_uart_writel(sport, ucr1, UCR1);490	imx_uart_writel(sport, ucr4, UCR4);491 492	/* See SER_RS485_ENABLED/UTS_LOOP comment in imx_uart_probe() */493	if (port->rs485.flags & SER_RS485_ENABLED &&494	    port->rs485.flags & SER_RS485_RTS_ON_SEND &&495	    sport->have_rtscts && !sport->have_rtsgpio && loopback) {496		uts = imx_uart_readl(sport, imx_uart_uts_reg(sport));497		uts |= UTS_LOOP;498		imx_uart_writel(sport, uts, imx_uart_uts_reg(sport));499		ucr2 |= UCR2_RXEN;500	} else {501		ucr2 &= ~UCR2_RXEN;502	}503 504	imx_uart_writel(sport, ucr2, UCR2);505}506 507/* called with port.lock taken and irqs off */508static void imx_uart_stop_rx(struct uart_port *port)509{510	/*511	 * Stop RX and enable loopback in order to make sure RS485 bus512	 * is not blocked. Se comment in imx_uart_probe().513	 */514	imx_uart_stop_rx_with_loopback_ctrl(port, true);515}516 517/* called with port.lock taken and irqs off */518static void imx_uart_enable_ms(struct uart_port *port)519{520	struct imx_port *sport = to_imx_port(port);521 522	mod_timer(&sport->timer, jiffies);523 524	mctrl_gpio_enable_ms(sport->gpios);525}526 527static void imx_uart_dma_tx(struct imx_port *sport);528 529/* called with port.lock taken and irqs off */530static inline void imx_uart_transmit_buffer(struct imx_port *sport)531{532	struct tty_port *tport = &sport->port.state->port;533	unsigned char c;534 535	if (sport->port.x_char) {536		/* Send next char */537		imx_uart_writel(sport, sport->port.x_char, URTX0);538		sport->port.icount.tx++;539		sport->port.x_char = 0;540		return;541	}542 543	if (kfifo_is_empty(&tport->xmit_fifo) ||544			uart_tx_stopped(&sport->port)) {545		imx_uart_stop_tx(&sport->port);546		return;547	}548 549	if (sport->dma_is_enabled) {550		u32 ucr1;551		/*552		 * We've just sent a X-char Ensure the TX DMA is enabled553		 * and the TX IRQ is disabled.554		 **/555		ucr1 = imx_uart_readl(sport, UCR1);556		ucr1 &= ~UCR1_TRDYEN;557		if (sport->dma_is_txing) {558			ucr1 |= UCR1_TXDMAEN;559			imx_uart_writel(sport, ucr1, UCR1);560		} else {561			imx_uart_writel(sport, ucr1, UCR1);562			imx_uart_dma_tx(sport);563		}564 565		return;566	}567 568	while (!(imx_uart_readl(sport, imx_uart_uts_reg(sport)) & UTS_TXFULL) &&569			uart_fifo_get(&sport->port, &c))570		imx_uart_writel(sport, c, URTX0);571 572	if (kfifo_len(&tport->xmit_fifo) < WAKEUP_CHARS)573		uart_write_wakeup(&sport->port);574 575	if (kfifo_is_empty(&tport->xmit_fifo))576		imx_uart_stop_tx(&sport->port);577}578 579static void imx_uart_dma_tx_callback(void *data)580{581	struct imx_port *sport = data;582	struct tty_port *tport = &sport->port.state->port;583	struct scatterlist *sgl = &sport->tx_sgl[0];584	unsigned long flags;585	u32 ucr1;586 587	uart_port_lock_irqsave(&sport->port, &flags);588 589	dma_unmap_sg(sport->port.dev, sgl, sport->dma_tx_nents, DMA_TO_DEVICE);590 591	ucr1 = imx_uart_readl(sport, UCR1);592	ucr1 &= ~UCR1_TXDMAEN;593	imx_uart_writel(sport, ucr1, UCR1);594 595	uart_xmit_advance(&sport->port, sport->tx_bytes);596 597	dev_dbg(sport->port.dev, "we finish the TX DMA.\n");598 599	sport->dma_is_txing = 0;600 601	if (kfifo_len(&tport->xmit_fifo) < WAKEUP_CHARS)602		uart_write_wakeup(&sport->port);603 604	if (!kfifo_is_empty(&tport->xmit_fifo) &&605			!uart_tx_stopped(&sport->port))606		imx_uart_dma_tx(sport);607	else if (sport->port.rs485.flags & SER_RS485_ENABLED) {608		u32 ucr4 = imx_uart_readl(sport, UCR4);609		ucr4 |= UCR4_TCEN;610		imx_uart_writel(sport, ucr4, UCR4);611	}612 613	uart_port_unlock_irqrestore(&sport->port, flags);614}615 616/* called with port.lock taken and irqs off */617static void imx_uart_dma_tx(struct imx_port *sport)618{619	struct tty_port *tport = &sport->port.state->port;620	struct scatterlist *sgl = sport->tx_sgl;621	struct dma_async_tx_descriptor *desc;622	struct dma_chan	*chan = sport->dma_chan_tx;623	struct device *dev = sport->port.dev;624	u32 ucr1, ucr4;625	int ret;626 627	if (sport->dma_is_txing)628		return;629 630	ucr4 = imx_uart_readl(sport, UCR4);631	ucr4 &= ~UCR4_TCEN;632	imx_uart_writel(sport, ucr4, UCR4);633 634	sg_init_table(sgl, ARRAY_SIZE(sport->tx_sgl));635	sport->tx_bytes = kfifo_len(&tport->xmit_fifo);636	sport->dma_tx_nents = kfifo_dma_out_prepare(&tport->xmit_fifo, sgl,637			ARRAY_SIZE(sport->tx_sgl), sport->tx_bytes);638 639	ret = dma_map_sg(dev, sgl, sport->dma_tx_nents, DMA_TO_DEVICE);640	if (ret == 0) {641		dev_err(dev, "DMA mapping error for TX.\n");642		return;643	}644	desc = dmaengine_prep_slave_sg(chan, sgl, ret,645					DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT);646	if (!desc) {647		dma_unmap_sg(dev, sgl, sport->dma_tx_nents,648			     DMA_TO_DEVICE);649		dev_err(dev, "We cannot prepare for the TX slave dma!\n");650		return;651	}652	desc->callback = imx_uart_dma_tx_callback;653	desc->callback_param = sport;654 655	dev_dbg(dev, "TX: prepare to send %u bytes by DMA.\n", sport->tx_bytes);656 657	ucr1 = imx_uart_readl(sport, UCR1);658	ucr1 |= UCR1_TXDMAEN;659	imx_uart_writel(sport, ucr1, UCR1);660 661	/* fire it */662	sport->dma_is_txing = 1;663	dmaengine_submit(desc);664	dma_async_issue_pending(chan);665	return;666}667 668/* called with port.lock taken and irqs off */669static void imx_uart_start_tx(struct uart_port *port)670{671	struct imx_port *sport = to_imx_port(port);672	struct tty_port *tport = &sport->port.state->port;673	u32 ucr1;674 675	if (!sport->port.x_char && kfifo_is_empty(&tport->xmit_fifo))676		return;677 678	/*679	 * We cannot simply do nothing here if sport->tx_state == SEND already680	 * because UCR1_TXMPTYEN might already have been cleared in681	 * imx_uart_stop_tx(), but tx_state is still SEND.682	 */683 684	if (port->rs485.flags & SER_RS485_ENABLED) {685		if (sport->tx_state == OFF) {686			u32 ucr2 = imx_uart_readl(sport, UCR2);687			if (port->rs485.flags & SER_RS485_RTS_ON_SEND)688				imx_uart_rts_active(sport, &ucr2);689			else690				imx_uart_rts_inactive(sport, &ucr2);691			imx_uart_writel(sport, ucr2, UCR2);692 693			/*694			 * Since we are about to transmit we can not stop RX695			 * with loopback enabled because that will make our696			 * transmitted data being just looped to RX.697			 */698			if (!(port->rs485.flags & SER_RS485_RX_DURING_TX) &&699			    !port->rs485_rx_during_tx_gpio)700				imx_uart_stop_rx_with_loopback_ctrl(port, false);701 702			sport->tx_state = WAIT_AFTER_RTS;703 704			if (port->rs485.delay_rts_before_send > 0) {705				start_hrtimer_ms(&sport->trigger_start_tx,706					 port->rs485.delay_rts_before_send);707				return;708			}709 710			/* continue without any delay */711		}712 713		if (sport->tx_state == WAIT_AFTER_SEND714		    || sport->tx_state == WAIT_AFTER_RTS) {715 716			hrtimer_try_to_cancel(&sport->trigger_stop_tx);717 718			/*719			 * Enable transmitter and shifter empty irq only if DMA720			 * is off.  In the DMA case this is done in the721			 * tx-callback.722			 */723			if (!sport->dma_is_enabled) {724				u32 ucr4 = imx_uart_readl(sport, UCR4);725				ucr4 |= UCR4_TCEN;726				imx_uart_writel(sport, ucr4, UCR4);727			}728 729			sport->tx_state = SEND;730		}731	} else {732		sport->tx_state = SEND;733	}734 735	if (!sport->dma_is_enabled) {736		ucr1 = imx_uart_readl(sport, UCR1);737		imx_uart_writel(sport, ucr1 | UCR1_TRDYEN, UCR1);738	}739 740	if (sport->dma_is_enabled) {741		if (sport->port.x_char) {742			/* We have X-char to send, so enable TX IRQ and743			 * disable TX DMA to let TX interrupt to send X-char */744			ucr1 = imx_uart_readl(sport, UCR1);745			ucr1 &= ~UCR1_TXDMAEN;746			ucr1 |= UCR1_TRDYEN;747			imx_uart_writel(sport, ucr1, UCR1);748			return;749		}750 751		if (!kfifo_is_empty(&tport->xmit_fifo) &&752		    !uart_tx_stopped(port))753			imx_uart_dma_tx(sport);754		return;755	}756}757 758static irqreturn_t __imx_uart_rtsint(int irq, void *dev_id)759{760	struct imx_port *sport = dev_id;761	u32 usr1;762 763	imx_uart_writel(sport, USR1_RTSD, USR1);764	usr1 = imx_uart_readl(sport, USR1) & USR1_RTSS;765	/*766	 * Update sport->old_status here, so any follow-up calls to767	 * imx_uart_mctrl_check() will be able to recognize that RTS768	 * state changed since last imx_uart_mctrl_check() call.769	 *770	 * In case RTS has been detected as asserted here and later on771	 * deasserted by the time imx_uart_mctrl_check() was called,772	 * imx_uart_mctrl_check() can detect the RTS state change and773	 * trigger uart_handle_cts_change() to unblock the port for774	 * further TX transfers.775	 */776	if (usr1 & USR1_RTSS)777		sport->old_status |= TIOCM_CTS;778	else779		sport->old_status &= ~TIOCM_CTS;780	uart_handle_cts_change(&sport->port, usr1);781	wake_up_interruptible(&sport->port.state->port.delta_msr_wait);782 783	return IRQ_HANDLED;784}785 786static irqreturn_t imx_uart_rtsint(int irq, void *dev_id)787{788	struct imx_port *sport = dev_id;789	irqreturn_t ret;790 791	uart_port_lock(&sport->port);792 793	ret = __imx_uart_rtsint(irq, dev_id);794 795	uart_port_unlock(&sport->port);796 797	return ret;798}799 800static irqreturn_t imx_uart_txint(int irq, void *dev_id)801{802	struct imx_port *sport = dev_id;803 804	uart_port_lock(&sport->port);805	imx_uart_transmit_buffer(sport);806	uart_port_unlock(&sport->port);807	return IRQ_HANDLED;808}809 810/* Check if hardware Rx flood is in progress, and issue soft reset to stop it.811 * This is to be called from Rx ISRs only when some bytes were actually812 * received.813 *814 * A way to reproduce the flood (checked on iMX6SX) is: open iMX UART at 9600815 * 8N1, and from external source send 0xf0 char at 115200 8N1. In about 90% of816 * cases this starts a flood of "receiving" of 0xff characters by the iMX6 UART817 * that is terminated by any activity on RxD line, or could be stopped by818 * issuing soft reset to the UART (just stop/start of RX does not help). Note819 * that what we do here is sending isolated start bit about 2.4 times shorter820 * than it is to be on UART configured baud rate.821 */822static void imx_uart_check_flood(struct imx_port *sport, u32 usr2)823{824	/* To detect hardware 0xff flood we monitor RxD line between RX825	 * interrupts to isolate "receiving" of char(s) with no activity826	 * on RxD line, that'd never happen on actual data transfers.827	 *828	 * We use USR2_WAKE bit to check for activity on RxD line, but we have a829	 * race here if we clear USR2_WAKE when receiving of a char is in830	 * progress, so we might get RX interrupt later with USR2_WAKE bit831	 * cleared. Note though that as we don't try to clear USR2_WAKE when we832	 * detected no activity, this race may hide actual activity only once.833	 *834	 * Yet another case where receive interrupt may occur without RxD835	 * activity is expiration of aging timer, so we consider this as well.836	 *837	 * We use 'idle_counter' to ensure that we got at least so many RX838	 * interrupts without any detected activity on RxD line. 2 cases839	 * described plus 1 to be on the safe side gives us a margin of 3,840	 * below. In practice I was not able to produce a false positive to841	 * induce soft reset at regular data transfers even using 1 as the842	 * margin, so 3 is actually very strong.843	 *844	 * We count interrupts, not chars in 'idle-counter' for simplicity.845	 */846 847	if (usr2 & USR2_WAKE) {848		imx_uart_writel(sport, USR2_WAKE, USR2);849		sport->idle_counter = 0;850	} else if (++sport->idle_counter > 3) {851		dev_warn(sport->port.dev, "RX flood detected: soft reset.");852		imx_uart_soft_reset(sport); /* also clears 'sport->idle_counter' */853	}854}855 856static irqreturn_t __imx_uart_rxint(int irq, void *dev_id)857{858	struct imx_port *sport = dev_id;859	struct tty_port *port = &sport->port.state->port;860	u32 usr2, rx;861 862	/* If we received something, check for 0xff flood */863	usr2 = imx_uart_readl(sport, USR2);864	if (usr2 & USR2_RDR)865		imx_uart_check_flood(sport, usr2);866 867	while ((rx = imx_uart_readl(sport, URXD0)) & URXD_CHARRDY) {868		unsigned int flg = TTY_NORMAL;869		sport->port.icount.rx++;870 871		if (unlikely(rx & URXD_ERR)) {872			if (rx & URXD_BRK) {873				sport->port.icount.brk++;874				if (uart_handle_break(&sport->port))875					continue;876			}877			else if (rx & URXD_PRERR)878				sport->port.icount.parity++;879			else if (rx & URXD_FRMERR)880				sport->port.icount.frame++;881			if (rx & URXD_OVRRUN)882				sport->port.icount.overrun++;883 884			if (rx & sport->port.ignore_status_mask)885				continue;886 887			rx &= (sport->port.read_status_mask | 0xFF);888 889			if (rx & URXD_BRK)890				flg = TTY_BREAK;891			else if (rx & URXD_PRERR)892				flg = TTY_PARITY;893			else if (rx & URXD_FRMERR)894				flg = TTY_FRAME;895			if (rx & URXD_OVRRUN)896				flg = TTY_OVERRUN;897 898			sport->port.sysrq = 0;899		} else if (uart_handle_sysrq_char(&sport->port, (unsigned char)rx)) {900			continue;901		}902 903		if (sport->port.ignore_status_mask & URXD_DUMMY_READ)904			continue;905 906		if (tty_insert_flip_char(port, rx, flg) == 0)907			sport->port.icount.buf_overrun++;908	}909 910	tty_flip_buffer_push(port);911 912	return IRQ_HANDLED;913}914 915static irqreturn_t imx_uart_rxint(int irq, void *dev_id)916{917	struct imx_port *sport = dev_id;918	irqreturn_t ret;919 920	uart_port_lock(&sport->port);921 922	ret = __imx_uart_rxint(irq, dev_id);923 924	uart_port_unlock(&sport->port);925 926	return ret;927}928 929static void imx_uart_clear_rx_errors(struct imx_port *sport);930 931/*932 * We have a modem side uart, so the meanings of RTS and CTS are inverted.933 */934static unsigned int imx_uart_get_hwmctrl(struct imx_port *sport)935{936	unsigned int tmp = TIOCM_DSR;937	unsigned usr1 = imx_uart_readl(sport, USR1);938	unsigned usr2 = imx_uart_readl(sport, USR2);939 940	if (usr1 & USR1_RTSS)941		tmp |= TIOCM_CTS;942 943	/* in DCE mode DCDIN is always 0 */944	if (!(usr2 & USR2_DCDIN))945		tmp |= TIOCM_CAR;946 947	if (sport->dte_mode)948		if (!(imx_uart_readl(sport, USR2) & USR2_RIIN))949			tmp |= TIOCM_RI;950 951	return tmp;952}953 954/*955 * Handle any change of modem status signal since we were last called.956 */957static void imx_uart_mctrl_check(struct imx_port *sport)958{959	unsigned int status, changed;960 961	status = imx_uart_get_hwmctrl(sport);962	changed = status ^ sport->old_status;963 964	if (changed == 0)965		return;966 967	sport->old_status = status;968 969	if (changed & TIOCM_RI && status & TIOCM_RI)970		sport->port.icount.rng++;971	if (changed & TIOCM_DSR)972		sport->port.icount.dsr++;973	if (changed & TIOCM_CAR)974		uart_handle_dcd_change(&sport->port, status & TIOCM_CAR);975	if (changed & TIOCM_CTS)976		uart_handle_cts_change(&sport->port, status & TIOCM_CTS);977 978	wake_up_interruptible(&sport->port.state->port.delta_msr_wait);979}980 981static irqreturn_t imx_uart_int(int irq, void *dev_id)982{983	struct imx_port *sport = dev_id;984	unsigned int usr1, usr2, ucr1, ucr2, ucr3, ucr4;985	irqreturn_t ret = IRQ_NONE;986 987	uart_port_lock(&sport->port);988 989	usr1 = imx_uart_readl(sport, USR1);990	usr2 = imx_uart_readl(sport, USR2);991	ucr1 = imx_uart_readl(sport, UCR1);992	ucr2 = imx_uart_readl(sport, UCR2);993	ucr3 = imx_uart_readl(sport, UCR3);994	ucr4 = imx_uart_readl(sport, UCR4);995 996	/*997	 * Even if a condition is true that can trigger an irq only handle it if998	 * the respective irq source is enabled. This prevents some undesired999	 * actions, for example if a character that sits in the RX FIFO and that1000	 * should be fetched via DMA is tried to be fetched using PIO. Or the1001	 * receiver is currently off and so reading from URXD0 results in an1002	 * exception. So just mask the (raw) status bits for disabled irqs.1003	 */1004	if ((ucr1 & UCR1_RRDYEN) == 0)1005		usr1 &= ~USR1_RRDY;1006	if ((ucr2 & UCR2_ATEN) == 0)1007		usr1 &= ~USR1_AGTIM;1008	if ((ucr1 & UCR1_TRDYEN) == 0)1009		usr1 &= ~USR1_TRDY;1010	if ((ucr4 & UCR4_TCEN) == 0)1011		usr2 &= ~USR2_TXDC;1012	if ((ucr3 & UCR3_DTRDEN) == 0)1013		usr1 &= ~USR1_DTRD;1014	if ((ucr1 & UCR1_RTSDEN) == 0)1015		usr1 &= ~USR1_RTSD;1016	if ((ucr3 & UCR3_AWAKEN) == 0)1017		usr1 &= ~USR1_AWAKE;1018	if ((ucr4 & UCR4_OREN) == 0)1019		usr2 &= ~USR2_ORE;1020 1021	if (usr1 & (USR1_RRDY | USR1_AGTIM)) {1022		imx_uart_writel(sport, USR1_AGTIM, USR1);1023 1024		__imx_uart_rxint(irq, dev_id);1025		ret = IRQ_HANDLED;1026	}1027 1028	if ((usr1 & USR1_TRDY) || (usr2 & USR2_TXDC)) {1029		imx_uart_transmit_buffer(sport);1030		ret = IRQ_HANDLED;1031	}1032 1033	if (usr1 & USR1_DTRD) {1034		imx_uart_writel(sport, USR1_DTRD, USR1);1035 1036		imx_uart_mctrl_check(sport);1037 1038		ret = IRQ_HANDLED;1039	}1040 1041	if (usr1 & USR1_RTSD) {1042		__imx_uart_rtsint(irq, dev_id);1043		ret = IRQ_HANDLED;1044	}1045 1046	if (usr1 & USR1_AWAKE) {1047		imx_uart_writel(sport, USR1_AWAKE, USR1);1048		ret = IRQ_HANDLED;1049	}1050 1051	if (usr2 & USR2_ORE) {1052		sport->port.icount.overrun++;1053		imx_uart_writel(sport, USR2_ORE, USR2);1054		ret = IRQ_HANDLED;1055	}1056 1057	uart_port_unlock(&sport->port);1058 1059	return ret;1060}1061 1062/*1063 * Return TIOCSER_TEMT when transmitter is not busy.1064 */1065static unsigned int imx_uart_tx_empty(struct uart_port *port)1066{1067	struct imx_port *sport = to_imx_port(port);1068	unsigned int ret;1069 1070	ret = (imx_uart_readl(sport, USR2) & USR2_TXDC) ?  TIOCSER_TEMT : 0;1071 1072	/* If the TX DMA is working, return 0. */1073	if (sport->dma_is_txing)1074		ret = 0;1075 1076	return ret;1077}1078 1079/* called with port.lock taken and irqs off */1080static unsigned int imx_uart_get_mctrl(struct uart_port *port)1081{1082	struct imx_port *sport = to_imx_port(port);1083	unsigned int ret = imx_uart_get_hwmctrl(sport);1084 1085	mctrl_gpio_get(sport->gpios, &ret);1086 1087	return ret;1088}1089 1090/* called with port.lock taken and irqs off */1091static void imx_uart_set_mctrl(struct uart_port *port, unsigned int mctrl)1092{1093	struct imx_port *sport = to_imx_port(port);1094	u32 ucr3, uts;1095 1096	if (!(port->rs485.flags & SER_RS485_ENABLED)) {1097		u32 ucr2;1098 1099		/*1100		 * Turn off autoRTS if RTS is lowered and restore autoRTS1101		 * setting if RTS is raised.1102		 */1103		ucr2 = imx_uart_readl(sport, UCR2);1104		ucr2 &= ~(UCR2_CTS | UCR2_CTSC);1105		if (mctrl & TIOCM_RTS) {1106			ucr2 |= UCR2_CTS;1107			/*1108			 * UCR2_IRTS is unset if and only if the port is1109			 * configured for CRTSCTS, so we use inverted UCR2_IRTS1110			 * to get the state to restore to.1111			 */1112			if (!(ucr2 & UCR2_IRTS))1113				ucr2 |= UCR2_CTSC;1114		}1115		imx_uart_writel(sport, ucr2, UCR2);1116	}1117 1118	ucr3 = imx_uart_readl(sport, UCR3) & ~UCR3_DSR;1119	if (!(mctrl & TIOCM_DTR))1120		ucr3 |= UCR3_DSR;1121	imx_uart_writel(sport, ucr3, UCR3);1122 1123	uts = imx_uart_readl(sport, imx_uart_uts_reg(sport)) & ~UTS_LOOP;1124	if (mctrl & TIOCM_LOOP)1125		uts |= UTS_LOOP;1126	imx_uart_writel(sport, uts, imx_uart_uts_reg(sport));1127 1128	mctrl_gpio_set(sport->gpios, mctrl);1129}1130 1131/*1132 * Interrupts always disabled.1133 */1134static void imx_uart_break_ctl(struct uart_port *port, int break_state)1135{1136	struct imx_port *sport = to_imx_port(port);1137	unsigned long flags;1138	u32 ucr1;1139 1140	uart_port_lock_irqsave(&sport->port, &flags);1141 1142	ucr1 = imx_uart_readl(sport, UCR1) & ~UCR1_SNDBRK;1143 1144	if (break_state != 0)1145		ucr1 |= UCR1_SNDBRK;1146 1147	imx_uart_writel(sport, ucr1, UCR1);1148 1149	uart_port_unlock_irqrestore(&sport->port, flags);1150}1151 1152/*1153 * This is our per-port timeout handler, for checking the1154 * modem status signals.1155 */1156static void imx_uart_timeout(struct timer_list *t)1157{1158	struct imx_port *sport = from_timer(sport, t, timer);1159	unsigned long flags;1160 1161	if (sport->port.state) {1162		uart_port_lock_irqsave(&sport->port, &flags);1163		imx_uart_mctrl_check(sport);1164		uart_port_unlock_irqrestore(&sport->port, flags);1165 1166		mod_timer(&sport->timer, jiffies + MCTRL_TIMEOUT);1167	}1168}1169 1170/*1171 * There are two kinds of RX DMA interrupts(such as in the MX6Q):1172 *   [1] the RX DMA buffer is full.1173 *   [2] the aging timer expires1174 *1175 * Condition [2] is triggered when a character has been sitting in the FIFO1176 * for at least 8 byte durations.1177 */1178static void imx_uart_dma_rx_callback(void *data)1179{1180	struct imx_port *sport = data;1181	struct dma_chan	*chan = sport->dma_chan_rx;1182	struct scatterlist *sgl = &sport->rx_sgl;1183	struct tty_port *port = &sport->port.state->port;1184	struct dma_tx_state state;1185	struct circ_buf *rx_ring = &sport->rx_ring;1186	enum dma_status status;1187	unsigned int w_bytes = 0;1188	unsigned int r_bytes;1189	unsigned int bd_size;1190 1191	status = dmaengine_tx_status(chan, sport->rx_cookie, &state);1192 1193	if (status == DMA_ERROR) {1194		uart_port_lock(&sport->port);1195		imx_uart_clear_rx_errors(sport);1196		uart_port_unlock(&sport->port);1197		return;1198	}1199 1200	/*1201	 * The state-residue variable represents the empty space1202	 * relative to the entire buffer. Taking this in consideration1203	 * the head is always calculated base on the buffer total1204	 * length - DMA transaction residue. The UART script from the1205	 * SDMA firmware will jump to the next buffer descriptor,1206	 * once a DMA transaction if finalized (IMX53 RM - A.4.1.2.4).1207	 * Taking this in consideration the tail is always at the1208	 * beginning of the buffer descriptor that contains the head.1209	 */1210 1211	/* Calculate the head */1212	rx_ring->head = sg_dma_len(sgl) - state.residue;1213 1214	/* Calculate the tail. */1215	bd_size = sg_dma_len(sgl) / sport->rx_periods;1216	rx_ring->tail = ((rx_ring->head-1) / bd_size) * bd_size;1217 1218	if (rx_ring->head <= sg_dma_len(sgl) &&1219	    rx_ring->head > rx_ring->tail) {1220 1221		/* Move data from tail to head */1222		r_bytes = rx_ring->head - rx_ring->tail;1223 1224		/* If we received something, check for 0xff flood */1225		uart_port_lock(&sport->port);1226		imx_uart_check_flood(sport, imx_uart_readl(sport, USR2));1227		uart_port_unlock(&sport->port);1228 1229		if (!(sport->port.ignore_status_mask & URXD_DUMMY_READ)) {1230 1231			/* CPU claims ownership of RX DMA buffer */1232			dma_sync_sg_for_cpu(sport->port.dev, sgl, 1,1233					    DMA_FROM_DEVICE);1234 1235			w_bytes = tty_insert_flip_string(port,1236							 sport->rx_buf + rx_ring->tail, r_bytes);1237 1238			/* UART retrieves ownership of RX DMA buffer */1239			dma_sync_sg_for_device(sport->port.dev, sgl, 1,1240					       DMA_FROM_DEVICE);1241 1242			if (w_bytes != r_bytes)1243				sport->port.icount.buf_overrun++;1244 1245			sport->port.icount.rx += w_bytes;1246		}1247	} else	{1248		WARN_ON(rx_ring->head > sg_dma_len(sgl));1249		WARN_ON(rx_ring->head <= rx_ring->tail);1250	}1251 1252	if (w_bytes) {1253		tty_flip_buffer_push(port);1254		dev_dbg(sport->port.dev, "We get %d bytes.\n", w_bytes);1255	}1256}1257 1258static int imx_uart_start_rx_dma(struct imx_port *sport)1259{1260	struct scatterlist *sgl = &sport->rx_sgl;1261	struct dma_chan	*chan = sport->dma_chan_rx;1262	struct device *dev = sport->port.dev;1263	struct dma_async_tx_descriptor *desc;1264	int ret;1265 1266	sport->rx_ring.head = 0;1267	sport->rx_ring.tail = 0;1268 1269	sg_init_one(sgl, sport->rx_buf, sport->rx_buf_size);1270	ret = dma_map_sg(dev, sgl, 1, DMA_FROM_DEVICE);1271	if (ret == 0) {1272		dev_err(dev, "DMA mapping error for RX.\n");1273		return -EINVAL;1274	}1275 1276	desc = dmaengine_prep_dma_cyclic(chan, sg_dma_address(sgl),1277		sg_dma_len(sgl), sg_dma_len(sgl) / sport->rx_periods,1278		DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT);1279 1280	if (!desc) {1281		dma_unmap_sg(dev, sgl, 1, DMA_FROM_DEVICE);1282		dev_err(dev, "We cannot prepare for the RX slave dma!\n");1283		return -EINVAL;1284	}1285	desc->callback = imx_uart_dma_rx_callback;1286	desc->callback_param = sport;1287 1288	dev_dbg(dev, "RX: prepare for the DMA.\n");1289	sport->dma_is_rxing = 1;1290	sport->rx_cookie = dmaengine_submit(desc);1291	dma_async_issue_pending(chan);1292	return 0;1293}1294 1295static void imx_uart_clear_rx_errors(struct imx_port *sport)1296{1297	struct tty_port *port = &sport->port.state->port;1298	u32 usr1, usr2;1299 1300	usr1 = imx_uart_readl(sport, USR1);1301	usr2 = imx_uart_readl(sport, USR2);1302 1303	if (usr2 & USR2_BRCD) {1304		sport->port.icount.brk++;1305		imx_uart_writel(sport, USR2_BRCD, USR2);1306		uart_handle_break(&sport->port);1307		if (tty_insert_flip_char(port, 0, TTY_BREAK) == 0)1308			sport->port.icount.buf_overrun++;1309		tty_flip_buffer_push(port);1310	} else {1311		if (usr1 & USR1_FRAMERR) {1312			sport->port.icount.frame++;1313			imx_uart_writel(sport, USR1_FRAMERR, USR1);1314		} else if (usr1 & USR1_PARITYERR) {1315			sport->port.icount.parity++;1316			imx_uart_writel(sport, USR1_PARITYERR, USR1);1317		}1318	}1319 1320	if (usr2 & USR2_ORE) {1321		sport->port.icount.overrun++;1322		imx_uart_writel(sport, USR2_ORE, USR2);1323	}1324 1325	sport->idle_counter = 0;1326 1327}1328 1329#define TXTL_DEFAULT 81330#define RXTL_DEFAULT 8 /* 8 characters or aging timer */1331#define TXTL_DMA 8 /* DMA burst setting */1332#define RXTL_DMA 9 /* DMA burst setting */1333 1334static void imx_uart_setup_ufcr(struct imx_port *sport,1335				unsigned char txwl, unsigned char rxwl)1336{1337	unsigned int val;1338 1339	/* set receiver / transmitter trigger level */1340	val = imx_uart_readl(sport, UFCR) & (UFCR_RFDIV | UFCR_DCEDTE);1341	val |= txwl << UFCR_TXTL_SHF | rxwl;1342	imx_uart_writel(sport, val, UFCR);1343}1344 1345static void imx_uart_dma_exit(struct imx_port *sport)1346{1347	if (sport->dma_chan_rx) {1348		dmaengine_terminate_sync(sport->dma_chan_rx);1349		dma_release_channel(sport->dma_chan_rx);1350		sport->dma_chan_rx = NULL;1351		sport->rx_cookie = -EINVAL;1352		kfree(sport->rx_buf);1353		sport->rx_buf = NULL;1354	}1355 1356	if (sport->dma_chan_tx) {1357		dmaengine_terminate_sync(sport->dma_chan_tx);1358		dma_release_channel(sport->dma_chan_tx);1359		sport->dma_chan_tx = NULL;1360	}1361}1362 1363static int imx_uart_dma_init(struct imx_port *sport)1364{1365	struct dma_slave_config slave_config = {};1366	struct device *dev = sport->port.dev;1367	struct dma_chan *chan;1368	int ret;1369 1370	/* Prepare for RX : */1371	chan = dma_request_chan(dev, "rx");1372	if (IS_ERR(chan)) {1373		dev_dbg(dev, "cannot get the DMA channel.\n");1374		sport->dma_chan_rx = NULL;1375		ret = PTR_ERR(chan);1376		goto err;1377	}1378	sport->dma_chan_rx = chan;1379 1380	slave_config.direction = DMA_DEV_TO_MEM;1381	slave_config.src_addr = sport->port.mapbase + URXD0;1382	slave_config.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;1383	/* one byte less than the watermark level to enable the aging timer */1384	slave_config.src_maxburst = RXTL_DMA - 1;1385	ret = dmaengine_slave_config(sport->dma_chan_rx, &slave_config);1386	if (ret) {1387		dev_err(dev, "error in RX dma configuration.\n");1388		goto err;1389	}1390 1391	sport->rx_buf_size = sport->rx_period_length * sport->rx_periods;1392	sport->rx_buf = kzalloc(sport->rx_buf_size, GFP_KERNEL);1393	if (!sport->rx_buf) {1394		ret = -ENOMEM;1395		goto err;1396	}1397	sport->rx_ring.buf = sport->rx_buf;1398 1399	/* Prepare for TX : */1400	chan = dma_request_chan(dev, "tx");1401	if (IS_ERR(chan)) {1402		dev_err(dev, "cannot get the TX DMA channel!\n");1403		sport->dma_chan_tx = NULL;1404		ret = PTR_ERR(chan);1405		goto err;1406	}1407	sport->dma_chan_tx = chan;1408 1409	slave_config.direction = DMA_MEM_TO_DEV;1410	slave_config.dst_addr = sport->port.mapbase + URTX0;1411	slave_config.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;1412	slave_config.dst_maxburst = TXTL_DMA;1413	ret = dmaengine_slave_config(sport->dma_chan_tx, &slave_config);1414	if (ret) {1415		dev_err(dev, "error in TX dma configuration.");1416		goto err;1417	}1418 1419	return 0;1420err:1421	imx_uart_dma_exit(sport);1422	return ret;1423}1424 1425static void imx_uart_enable_dma(struct imx_port *sport)1426{1427	u32 ucr1;1428 1429	imx_uart_setup_ufcr(sport, TXTL_DMA, RXTL_DMA);1430 1431	/* set UCR1 */1432	ucr1 = imx_uart_readl(sport, UCR1);1433	ucr1 |= UCR1_RXDMAEN | UCR1_TXDMAEN | UCR1_ATDMAEN;1434	imx_uart_writel(sport, ucr1, UCR1);1435 1436	sport->dma_is_enabled = 1;1437}1438 1439static void imx_uart_disable_dma(struct imx_port *sport)1440{1441	u32 ucr1;1442 1443	/* clear UCR1 */1444	ucr1 = imx_uart_readl(sport, UCR1);1445	ucr1 &= ~(UCR1_RXDMAEN | UCR1_TXDMAEN | UCR1_ATDMAEN);1446	imx_uart_writel(sport, ucr1, UCR1);1447 1448	imx_uart_setup_ufcr(sport, TXTL_DEFAULT, RXTL_DEFAULT);1449 1450	sport->dma_is_enabled = 0;1451}1452 1453/* half the RX buffer size */1454#define CTSTL 161455 1456static int imx_uart_startup(struct uart_port *port)1457{1458	struct imx_port *sport = to_imx_port(port);1459	int retval;1460	unsigned long flags;1461	int dma_is_inited = 0;1462	u32 ucr1, ucr2, ucr3, ucr4;1463 1464	retval = clk_prepare_enable(sport->clk_per);1465	if (retval)1466		return retval;1467	retval = clk_prepare_enable(sport->clk_ipg);1468	if (retval) {1469		clk_disable_unprepare(sport->clk_per);1470		return retval;1471	}1472 1473	imx_uart_setup_ufcr(sport, TXTL_DEFAULT, RXTL_DEFAULT);1474 1475	/* disable the DREN bit (Data Ready interrupt enable) before1476	 * requesting IRQs1477	 */1478	ucr4 = imx_uart_readl(sport, UCR4);1479 1480	/* set the trigger level for CTS */1481	ucr4 &= ~(UCR4_CTSTL_MASK << UCR4_CTSTL_SHF);1482	ucr4 |= CTSTL << UCR4_CTSTL_SHF;1483 1484	imx_uart_writel(sport, ucr4 & ~UCR4_DREN, UCR4);1485 1486	/* Can we enable the DMA support? */1487	if (!uart_console(port) && imx_uart_dma_init(sport) == 0) {1488		lockdep_set_subclass(&port->lock, 1);1489		dma_is_inited = 1;1490	}1491 1492	uart_port_lock_irqsave(&sport->port, &flags);1493 1494	/* Reset fifo's and state machines */1495	imx_uart_soft_reset(sport);1496 1497	/*1498	 * Finally, clear and enable interrupts1499	 */1500	imx_uart_writel(sport, USR1_RTSD | USR1_DTRD, USR1);1501	imx_uart_writel(sport, USR2_ORE, USR2);1502 1503	ucr1 = imx_uart_readl(sport, UCR1) & ~UCR1_RRDYEN;1504	ucr1 |= UCR1_UARTEN;1505	if (sport->have_rtscts)1506		ucr1 |= UCR1_RTSDEN;1507 1508	imx_uart_writel(sport, ucr1, UCR1);1509 1510	ucr4 = imx_uart_readl(sport, UCR4) & ~(UCR4_OREN | UCR4_INVR);1511	if (!dma_is_inited)1512		ucr4 |= UCR4_OREN;1513	if (sport->inverted_rx)1514		ucr4 |= UCR4_INVR;1515	imx_uart_writel(sport, ucr4, UCR4);1516 1517	ucr3 = imx_uart_readl(sport, UCR3) & ~UCR3_INVT;1518	/*1519	 * configure tx polarity before enabling tx1520	 */1521	if (sport->inverted_tx)1522		ucr3 |= UCR3_INVT;1523 1524	if (!imx_uart_is_imx1(sport)) {1525		ucr3 |= UCR3_DTRDEN | UCR3_RI | UCR3_DCD;1526 1527		if (sport->dte_mode)1528			/* disable broken interrupts */1529			ucr3 &= ~(UCR3_RI | UCR3_DCD);1530	}1531	imx_uart_writel(sport, ucr3, UCR3);1532 1533	ucr2 = imx_uart_readl(sport, UCR2) & ~UCR2_ATEN;1534	ucr2 |= (UCR2_RXEN | UCR2_TXEN);1535	if (!sport->have_rtscts)1536		ucr2 |= UCR2_IRTS;1537	/*1538	 * make sure the edge sensitive RTS-irq is disabled,1539	 * we're using RTSD instead.1540	 */1541	if (!imx_uart_is_imx1(sport))1542		ucr2 &= ~UCR2_RTSEN;1543	imx_uart_writel(sport, ucr2, UCR2);1544 1545	/*1546	 * Enable modem status interrupts1547	 */1548	imx_uart_enable_ms(&sport->port);1549 1550	if (dma_is_inited) {1551		imx_uart_enable_dma(sport);1552		imx_uart_start_rx_dma(sport);1553	} else {1554		ucr1 = imx_uart_readl(sport, UCR1);1555		ucr1 |= UCR1_RRDYEN;1556		imx_uart_writel(sport, ucr1, UCR1);1557 1558		ucr2 = imx_uart_readl(sport, UCR2);1559		ucr2 |= UCR2_ATEN;1560		imx_uart_writel(sport, ucr2, UCR2);1561	}1562 1563	imx_uart_disable_loopback_rs485(sport);1564 1565	uart_port_unlock_irqrestore(&sport->port, flags);1566 1567	return 0;1568}1569 1570static void imx_uart_shutdown(struct uart_port *port)1571{1572	struct imx_port *sport = to_imx_port(port);1573	unsigned long flags;1574	u32 ucr1, ucr2, ucr4, uts;1575	int loops;1576 1577	if (sport->dma_is_enabled) {1578		dmaengine_terminate_sync(sport->dma_chan_tx);1579		if (sport->dma_is_txing) {1580			dma_unmap_sg(sport->port.dev, &sport->tx_sgl[0],1581				     sport->dma_tx_nents, DMA_TO_DEVICE);1582			sport->dma_is_txing = 0;1583		}1584		dmaengine_terminate_sync(sport->dma_chan_rx);1585		if (sport->dma_is_rxing) {1586			dma_unmap_sg(sport->port.dev, &sport->rx_sgl,1587				     1, DMA_FROM_DEVICE);1588			sport->dma_is_rxing = 0;1589		}1590 1591		uart_port_lock_irqsave(&sport->port, &flags);1592		imx_uart_stop_tx(port);1593		imx_uart_stop_rx(port);1594		imx_uart_disable_dma(sport);1595		uart_port_unlock_irqrestore(&sport->port, flags);1596		imx_uart_dma_exit(sport);1597	}1598 1599	mctrl_gpio_disable_ms(sport->gpios);1600 1601	uart_port_lock_irqsave(&sport->port, &flags);1602	ucr2 = imx_uart_readl(sport, UCR2);1603	ucr2 &= ~(UCR2_TXEN | UCR2_ATEN);1604	imx_uart_writel(sport, ucr2, UCR2);1605	uart_port_unlock_irqrestore(&sport->port, flags);1606 1607	/*1608	 * Stop our timer.1609	 */1610	del_timer_sync(&sport->timer);1611 1612	/*1613	 * Disable all interrupts, port and break condition.1614	 */1615 1616	uart_port_lock_irqsave(&sport->port, &flags);1617 1618	ucr1 = imx_uart_readl(sport, UCR1);1619	ucr1 &= ~(UCR1_TRDYEN | UCR1_RRDYEN | UCR1_RTSDEN | UCR1_RXDMAEN |1620		  UCR1_ATDMAEN | UCR1_SNDBRK);1621	/* See SER_RS485_ENABLED/UTS_LOOP comment in imx_uart_probe() */1622	if (port->rs485.flags & SER_RS485_ENABLED &&1623	    port->rs485.flags & SER_RS485_RTS_ON_SEND &&1624	    sport->have_rtscts && !sport->have_rtsgpio) {1625		uts = imx_uart_readl(sport, imx_uart_uts_reg(sport));1626		uts |= UTS_LOOP;1627		imx_uart_writel(sport, uts, imx_uart_uts_reg(sport));1628		ucr1 |= UCR1_UARTEN;1629	} else {1630		ucr1 &= ~UCR1_UARTEN;1631	}1632	imx_uart_writel(sport, ucr1, UCR1);1633 1634	ucr4 = imx_uart_readl(sport, UCR4);1635	ucr4 &= ~UCR4_TCEN;1636	imx_uart_writel(sport, ucr4, UCR4);1637 1638	/*1639	 * We have to ensure the tx state machine ends up in OFF. This1640	 * is especially important for rs485 where we must not leave1641	 * the RTS signal high, blocking the bus indefinitely.1642	 *1643	 * All interrupts are now disabled, so imx_uart_stop_tx() will1644	 * no longer be called from imx_uart_transmit_buffer(). It may1645	 * still be called via the hrtimers, and if those are in play,1646	 * we have to honour the delays.1647	 */1648	if (sport->tx_state == WAIT_AFTER_RTS || sport->tx_state == SEND)1649		imx_uart_stop_tx(port);1650 1651	/*1652	 * In many cases (rs232 mode, or if tx_state was1653	 * WAIT_AFTER_RTS, or if tx_state was SEND and there is no1654	 * delay_rts_after_send), this will have moved directly to1655	 * OFF. In rs485 mode, tx_state might already have been1656	 * WAIT_AFTER_SEND and the hrtimer thus already started, or1657	 * the above imx_uart_stop_tx() call could have started it. In1658	 * those cases, we have to wait for the hrtimer to fire and1659	 * complete the transition to OFF.1660	 */1661	loops = port->rs485.flags & SER_RS485_ENABLED ?1662		port->rs485.delay_rts_after_send : 0;1663	while (sport->tx_state != OFF && loops--) {1664		uart_port_unlock_irqrestore(&sport->port, flags);1665		msleep(1);1666		uart_port_lock_irqsave(&sport->port, &flags);1667	}1668 1669	if (sport->tx_state != OFF) {1670		dev_warn(sport->port.dev, "unexpected tx_state %d\n",1671			 sport->tx_state);1672		/*1673		 * This machine may be busted, but ensure the RTS1674		 * signal is inactive in order not to block other1675		 * devices.1676		 */1677		if (port->rs485.flags & SER_RS485_ENABLED) {1678			ucr2 = imx_uart_readl(sport, UCR2);1679			if (port->rs485.flags & SER_RS485_RTS_AFTER_SEND)1680				imx_uart_rts_active(sport, &ucr2);1681			else1682				imx_uart_rts_inactive(sport, &ucr2);1683			imx_uart_writel(sport, ucr2, UCR2);1684		}1685		sport->tx_state = OFF;1686	}1687 1688	uart_port_unlock_irqrestore(&sport->port, flags);1689 1690	clk_disable_unprepare(sport->clk_per);1691	clk_disable_unprepare(sport->clk_ipg);1692}1693 1694/* called with port.lock taken and irqs off */1695static void imx_uart_flush_buffer(struct uart_port *port)1696{1697	struct imx_port *sport = to_imx_port(port);1698	struct scatterlist *sgl = &sport->tx_sgl[0];1699 1700	if (!sport->dma_chan_tx)1701		return;1702 1703	sport->tx_bytes = 0;1704	dmaengine_terminate_all(sport->dma_chan_tx);1705	if (sport->dma_is_txing) {1706		u32 ucr1;1707 1708		dma_unmap_sg(sport->port.dev, sgl, sport->dma_tx_nents,1709			     DMA_TO_DEVICE);1710		ucr1 = imx_uart_readl(sport, UCR1);1711		ucr1 &= ~UCR1_TXDMAEN;1712		imx_uart_writel(sport, ucr1, UCR1);1713		sport->dma_is_txing = 0;1714	}1715 1716	imx_uart_soft_reset(sport);1717 1718}1719 1720static void1721imx_uart_set_termios(struct uart_port *port, struct ktermios *termios,1722		     const struct ktermios *old)1723{1724	struct imx_port *sport = to_imx_port(port);1725	unsigned long flags;1726	u32 ucr2, old_ucr2, ufcr;1727	unsigned int baud, quot;1728	unsigned int old_csize = old ? old->c_cflag & CSIZE : CS8;1729	unsigned long div;1730	unsigned long num, denom, old_ubir, old_ubmr;1731	uint64_t tdiv64;1732 1733	/*1734	 * We only support CS7 and CS8.1735	 */1736	while ((termios->c_cflag & CSIZE) != CS7 &&1737	       (termios->c_cflag & CSIZE) != CS8) {1738		termios->c_cflag &= ~CSIZE;1739		termios->c_cflag |= old_csize;1740		old_csize = CS8;1741	}1742 1743	del_timer_sync(&sport->timer);1744 1745	/*1746	 * Ask the core to calculate the divisor for us.1747	 */1748	baud = uart_get_baud_rate(port, termios, old, 50, port->uartclk / 16);1749	quot = uart_get_divisor(port, baud);1750 1751	uart_port_lock_irqsave(&sport->port, &flags);1752 1753	/*1754	 * Read current UCR2 and save it for future use, then clear all the bits1755	 * except those we will or may need to preserve.1756	 */1757	old_ucr2 = imx_uart_readl(sport, UCR2);1758	ucr2 = old_ucr2 & (UCR2_TXEN | UCR2_RXEN | UCR2_ATEN | UCR2_CTS);1759 1760	ucr2 |= UCR2_SRST | UCR2_IRTS;1761	if ((termios->c_cflag & CSIZE) == CS8)1762		ucr2 |= UCR2_WS;1763 1764	if (!sport->have_rtscts)1765		termios->c_cflag &= ~CRTSCTS;1766 1767	if (port->rs485.flags & SER_RS485_ENABLED) {1768		/*1769		 * RTS is mandatory for rs485 operation, so keep1770		 * it under manual control and keep transmitter1771		 * disabled.1772		 */1773		if (port->rs485.flags & SER_RS485_RTS_AFTER_SEND)1774			imx_uart_rts_active(sport, &ucr2);1775		else1776			imx_uart_rts_inactive(sport, &ucr2);1777 1778	} else if (termios->c_cflag & CRTSCTS) {1779		/*1780		 * Only let receiver control RTS output if we were not requested1781		 * to have RTS inactive (which then should take precedence).1782		 */1783		if (ucr2 & UCR2_CTS)1784			ucr2 |= UCR2_CTSC;1785	}1786 1787	if (termios->c_cflag & CRTSCTS)1788		ucr2 &= ~UCR2_IRTS;1789	if (termios->c_cflag & CSTOPB)1790		ucr2 |= UCR2_STPB;1791	if (termios->c_cflag & PARENB) {1792		ucr2 |= UCR2_PREN;1793		if (termios->c_cflag & PARODD)1794			ucr2 |= UCR2_PROE;1795	}1796 1797	sport->port.read_status_mask = 0;1798	if (termios->c_iflag & INPCK)1799		sport->port.read_status_mask |= (URXD_FRMERR | URXD_PRERR);1800	if (termios->c_iflag & (BRKINT | PARMRK))1801		sport->port.read_status_mask |= URXD_BRK;1802 1803	/*1804	 * Characters to ignore1805	 */1806	sport->port.ignore_status_mask = 0;1807	if (termios->c_iflag & IGNPAR)1808		sport->port.ignore_status_mask |= URXD_PRERR | URXD_FRMERR;1809	if (termios->c_iflag & IGNBRK) {1810		sport->port.ignore_status_mask |= URXD_BRK;1811		/*1812		 * If we're ignoring parity and break indicators,1813		 * ignore overruns too (for real raw support).1814		 */1815		if (termios->c_iflag & IGNPAR)1816			sport->port.ignore_status_mask |= URXD_OVRRUN;1817	}1818 1819	if ((termios->c_cflag & CREAD) == 0)1820		sport->port.ignore_status_mask |= URXD_DUMMY_READ;1821 1822	/*1823	 * Update the per-port timeout.1824	 */1825	uart_update_timeout(port, termios->c_cflag, baud);1826 1827	/* custom-baudrate handling */1828	div = sport->port.uartclk / (baud * 16);1829	if (baud == 38400 && quot != div)1830		baud = sport->port.uartclk / (quot * 16);1831 1832	div = sport->port.uartclk / (baud * 16);1833	if (div > 7)1834		div = 7;1835	if (!div)1836		div = 1;1837 1838	rational_best_approximation(16 * div * baud, sport->port.uartclk,1839		1 << 16, 1 << 16, &num, &denom);1840 1841	tdiv64 = sport->port.uartclk;1842	tdiv64 *= num;1843	do_div(tdiv64, denom * 16 * div);1844	tty_termios_encode_baud_rate(termios,1845				(speed_t)tdiv64, (speed_t)tdiv64);1846 1847	num -= 1;1848	denom -= 1;1849 1850	ufcr = imx_uart_readl(sport, UFCR);1851	ufcr = (ufcr & (~UFCR_RFDIV)) | UFCR_RFDIV_REG(div);1852	imx_uart_writel(sport, ufcr, UFCR);1853 1854	/*1855	 *  Two registers below should always be written both and in this1856	 *  particular order. One consequence is that we need to check if any of1857	 *  them changes and then update both. We do need the check for change1858	 *  as even writing the same values seem to "restart"1859	 *  transmission/receiving logic in the hardware, that leads to data1860	 *  breakage even when rate doesn't in fact change. E.g., user switches1861	 *  RTS/CTS handshake and suddenly gets broken bytes.1862	 */1863	old_ubir = imx_uart_readl(sport, UBIR);1864	old_ubmr = imx_uart_readl(sport, UBMR);1865	if (old_ubir != num || old_ubmr != denom) {1866		imx_uart_writel(sport, num, UBIR);1867		imx_uart_writel(sport, denom, UBMR);1868	}1869 1870	if (!imx_uart_is_imx1(sport))1871		imx_uart_writel(sport, sport->port.uartclk / div / 1000,1872				IMX21_ONEMS);1873 1874	imx_uart_writel(sport, ucr2, UCR2);1875 1876	if (UART_ENABLE_MS(&sport->port, termios->c_cflag))1877		imx_uart_enable_ms(&sport->port);1878 1879	uart_port_unlock_irqrestore(&sport->port, flags);1880}1881 1882static const char *imx_uart_type(struct uart_port *port)1883{1884	return port->type == PORT_IMX ? "IMX" : NULL;1885}1886 1887/*1888 * Configure/autoconfigure the port.1889 */1890static void imx_uart_config_port(struct uart_port *port, int flags)1891{1892	if (flags & UART_CONFIG_TYPE)1893		port->type = PORT_IMX;1894}1895 1896/*1897 * Verify the new serial_struct (for TIOCSSERIAL).1898 * The only change we allow are to the flags and type, and1899 * even then only between PORT_IMX and PORT_UNKNOWN1900 */1901static int1902imx_uart_verify_port(struct uart_port *port, struct serial_struct *ser)1903{1904	int ret = 0;1905 1906	if (ser->type != PORT_UNKNOWN && ser->type != PORT_IMX)1907		ret = -EINVAL;1908	if (port->irq != ser->irq)1909		ret = -EINVAL;1910	if (ser->io_type != UPIO_MEM)1911		ret = -EINVAL;1912	if (port->uartclk / 16 != ser->baud_base)1913		ret = -EINVAL;1914	if (port->mapbase != (unsigned long)ser->iomem_base)1915		ret = -EINVAL;1916	if (port->iobase != ser->port)1917		ret = -EINVAL;1918	if (ser->hub6 != 0)1919		ret = -EINVAL;1920	return ret;1921}1922 1923#if defined(CONFIG_CONSOLE_POLL)1924 1925static int imx_uart_poll_init(struct uart_port *port)1926{1927	struct imx_port *sport = to_imx_port(port);1928	unsigned long flags;1929	u32 ucr1, ucr2;1930	int retval;1931 1932	retval = clk_prepare_enable(sport->clk_ipg);1933	if (retval)1934		return retval;1935	retval = clk_prepare_enable(sport->clk_per);1936	if (retval)1937		clk_disable_unprepare(sport->clk_ipg);1938 1939	imx_uart_setup_ufcr(sport, TXTL_DEFAULT, RXTL_DEFAULT);1940 1941	uart_port_lock_irqsave(&sport->port, &flags);1942 1943	/*1944	 * Be careful about the order of enabling bits here. First enable the1945	 * receiver (UARTEN + RXEN) and only then the corresponding irqs.1946	 * This prevents that a character that already sits in the RX fifo is1947	 * triggering an irq but the try to fetch it from there results in an1948	 * exception because UARTEN or RXEN is still off.1949	 */1950	ucr1 = imx_uart_readl(sport, UCR1);1951	ucr2 = imx_uart_readl(sport, UCR2);1952 1953	if (imx_uart_is_imx1(sport))1954		ucr1 |= IMX1_UCR1_UARTCLKEN;1955 1956	ucr1 |= UCR1_UARTEN;1957	ucr1 &= ~(UCR1_TRDYEN | UCR1_RTSDEN | UCR1_RRDYEN);1958 1959	ucr2 |= UCR2_RXEN | UCR2_TXEN;1960	ucr2 &= ~UCR2_ATEN;1961 1962	imx_uart_writel(sport, ucr1, UCR1);1963	imx_uart_writel(sport, ucr2, UCR2);1964 1965	/* now enable irqs */1966	imx_uart_writel(sport, ucr1 | UCR1_RRDYEN, UCR1);1967	imx_uart_writel(sport, ucr2 | UCR2_ATEN, UCR2);1968 1969	uart_port_unlock_irqrestore(&sport->port, flags);1970 1971	return 0;1972}1973 1974static int imx_uart_poll_get_char(struct uart_port *port)1975{1976	struct imx_port *sport = to_imx_port(port);1977	if (!(imx_uart_readl(sport, USR2) & USR2_RDR))1978		return NO_POLL_CHAR;1979 1980	return imx_uart_readl(sport, URXD0) & URXD_RX_DATA;1981}1982 1983static void imx_uart_poll_put_char(struct uart_port *port, unsigned char c)1984{1985	struct imx_port *sport = to_imx_port(port);1986	unsigned int status;1987 1988	/* drain */1989	do {1990		status = imx_uart_readl(sport, USR1);1991	} while (~status & USR1_TRDY);1992 1993	/* write */1994	imx_uart_writel(sport, c, URTX0);1995 1996	/* flush */1997	do {1998		status = imx_uart_readl(sport, USR2);1999	} while (~status & USR2_TXDC);2000}2001#endif2002 2003/* called with port.lock taken and irqs off or from .probe without locking */2004static int imx_uart_rs485_config(struct uart_port *port, struct ktermios *termios,2005				 struct serial_rs485 *rs485conf)2006{2007	struct imx_port *sport = to_imx_port(port);2008	u32 ucr2, ufcr;2009 2010	if (rs485conf->flags & SER_RS485_ENABLED) {2011		/* Enable receiver if low-active RTS signal is requested */2012		if (sport->have_rtscts &&  !sport->have_rtsgpio &&2013		    !(rs485conf->flags & SER_RS485_RTS_ON_SEND))2014			rs485conf->flags |= SER_RS485_RX_DURING_TX;2015 2016		/* disable transmitter */2017		ucr2 = imx_uart_readl(sport, UCR2);2018		if (rs485conf->flags & SER_RS485_RTS_AFTER_SEND)2019			imx_uart_rts_active(sport, &ucr2);2020		else2021			imx_uart_rts_inactive(sport, &ucr2);2022		imx_uart_writel(sport, ucr2, UCR2);2023	}2024 2025	/* Make sure Rx is enabled in case Tx is active with Rx disabled */2026	if (!(rs485conf->flags & SER_RS485_ENABLED) ||2027	    rs485conf->flags & SER_RS485_RX_DURING_TX) {2028		/* If the receiver trigger is 0, set it to a default value */2029		ufcr = imx_uart_readl(sport, UFCR);2030		if ((ufcr & UFCR_RXTL_MASK) == 0)2031			imx_uart_setup_ufcr(sport, TXTL_DEFAULT, RXTL_DEFAULT);2032		imx_uart_start_rx(port);2033	}2034 2035	return 0;2036}2037 2038static const struct uart_ops imx_uart_pops = {2039	.tx_empty	= imx_uart_tx_empty,2040	.set_mctrl	= imx_uart_set_mctrl,2041	.get_mctrl	= imx_uart_get_mctrl,2042	.stop_tx	= imx_uart_stop_tx,2043	.start_tx	= imx_uart_start_tx,2044	.stop_rx	= imx_uart_stop_rx,2045	.enable_ms	= imx_uart_enable_ms,2046	.break_ctl	= imx_uart_break_ctl,2047	.startup	= imx_uart_startup,2048	.shutdown	= imx_uart_shutdown,2049	.flush_buffer	= imx_uart_flush_buffer,2050	.set_termios	= imx_uart_set_termios,2051	.type		= imx_uart_type,2052	.config_port	= imx_uart_config_port,2053	.verify_port	= imx_uart_verify_port,2054#if defined(CONFIG_CONSOLE_POLL)2055	.poll_init      = imx_uart_poll_init,2056	.poll_get_char  = imx_uart_poll_get_char,2057	.poll_put_char  = imx_uart_poll_put_char,2058#endif2059};2060 2061static struct imx_port *imx_uart_ports[UART_NR];2062 2063#if IS_ENABLED(CONFIG_SERIAL_IMX_CONSOLE)2064static void imx_uart_console_putchar(struct uart_port *port, unsigned char ch)2065{2066	struct imx_port *sport = to_imx_port(port);2067 2068	while (imx_uart_readl(sport, imx_uart_uts_reg(sport)) & UTS_TXFULL)2069		barrier();2070 2071	imx_uart_writel(sport, ch, URTX0);2072}2073 2074/*2075 * Interrupts are disabled on entering2076 */2077static void2078imx_uart_console_write(struct console *co, const char *s, unsigned int count)2079{2080	struct imx_port *sport = imx_uart_ports[co->index];2081	struct imx_port_ucrs old_ucr;2082	unsigned long flags;2083	unsigned int ucr1, usr2;2084	int locked = 1;2085 2086	if (sport->port.sysrq)2087		locked = 0;2088	else if (oops_in_progress)2089		locked = uart_port_trylock_irqsave(&sport->port, &flags);2090	else2091		uart_port_lock_irqsave(&sport->port, &flags);2092 2093	/*2094	 *	First, save UCR1/2/3 and then disable interrupts2095	 */2096	imx_uart_ucrs_save(sport, &old_ucr);2097	ucr1 = old_ucr.ucr1;2098 2099	if (imx_uart_is_imx1(sport))2100		ucr1 |= IMX1_UCR1_UARTCLKEN;2101	ucr1 |= UCR1_UARTEN;2102	ucr1 &= ~(UCR1_TRDYEN | UCR1_RRDYEN | UCR1_RTSDEN);2103 2104	imx_uart_writel(sport, ucr1, UCR1);2105 2106	imx_uart_writel(sport, old_ucr.ucr2 | UCR2_TXEN, UCR2);2107 2108	uart_console_write(&sport->port, s, count, imx_uart_console_putchar);2109 2110	/*2111	 *	Finally, wait for transmitter to become empty2112	 *	and restore UCR1/2/32113	 */2114	read_poll_timeout_atomic(imx_uart_readl, usr2, usr2 & USR2_TXDC,2115				 0, USEC_PER_SEC, false, sport, USR2);2116	imx_uart_ucrs_restore(sport, &old_ucr);2117 2118	if (locked)2119		uart_port_unlock_irqrestore(&sport->port, flags);2120}2121 2122/*2123 * If the port was already initialised (eg, by a boot loader),2124 * try to determine the current setup.2125 */2126static void2127imx_uart_console_get_options(struct imx_port *sport, int *baud,2128			     int *parity, int *bits)2129{2130 2131	if (imx_uart_readl(sport, UCR1) & UCR1_UARTEN) {2132		/* ok, the port was enabled */2133		unsigned int ucr2, ubir, ubmr, uartclk;2134		unsigned int baud_raw;2135		unsigned int ucfr_rfdiv;2136 2137		ucr2 = imx_uart_readl(sport, UCR2);2138 2139		*parity = 'n';2140		if (ucr2 & UCR2_PREN) {2141			if (ucr2 & UCR2_PROE)2142				*parity = 'o';2143			else2144				*parity = 'e';2145		}2146 2147		if (ucr2 & UCR2_WS)2148			*bits = 8;2149		else2150			*bits = 7;2151 2152		ubir = imx_uart_readl(sport, UBIR) & 0xffff;2153		ubmr = imx_uart_readl(sport, UBMR) & 0xffff;2154 2155		ucfr_rfdiv = (imx_uart_readl(sport, UFCR) & UFCR_RFDIV) >> 7;2156		if (ucfr_rfdiv == 6)2157			ucfr_rfdiv = 7;2158		else2159			ucfr_rfdiv = 6 - ucfr_rfdiv;2160 2161		uartclk = clk_get_rate(sport->clk_per);2162		uartclk /= ucfr_rfdiv;2163 2164		{	/*2165			 * The next code provides exact computation of2166			 *   baud_raw = round(((uartclk/16) * (ubir + 1)) / (ubmr + 1))2167			 * without need of float support or long long division,2168			 * which would be required to prevent 32bit arithmetic overflow2169			 */2170			unsigned int mul = ubir + 1;2171			unsigned int div = 16 * (ubmr + 1);2172			unsigned int rem = uartclk % div;2173 2174			baud_raw = (uartclk / div) * mul;2175			baud_raw += (rem * mul + div / 2) / div;2176			*baud = (baud_raw + 50) / 100 * 100;2177		}2178 2179		if (*baud != baud_raw)2180			dev_info(sport->port.dev, "Console IMX rounded baud rate from %d to %d\n",2181				baud_raw, *baud);2182	}2183}2184 2185static int2186imx_uart_console_setup(struct console *co, char *options)2187{2188	struct imx_port *sport;2189	int baud = 9600;2190	int bits = 8;2191	int parity = 'n';2192	int flow = 'n';2193	int retval;2194 2195	/*2196	 * Check whether an invalid uart number has been specified, and2197	 * if so, search for the first available port that does have2198	 * console support.2199	 */2200	if (co->index == -1 || co->index >= ARRAY_SIZE(imx_uart_ports))2201		co->index = 0;2202	sport = imx_uart_ports[co->index];2203	if (sport == NULL)2204		return -ENODEV;2205 2206	/* For setting the registers, we only need to enable the ipg clock. */2207	retval = clk_prepare_enable(sport->clk_ipg);2208	if (retval)2209		goto error_console;2210 2211	if (options)2212		uart_parse_options(options, &baud, &parity, &bits, &flow);2213	else2214		imx_uart_console_get_options(sport, &baud, &parity, &bits);2215 2216	imx_uart_setup_ufcr(sport, TXTL_DEFAULT, RXTL_DEFAULT);2217 2218	retval = uart_set_options(&sport->port, co, baud, parity, bits, flow);2219 2220	if (retval) {2221		clk_disable_unprepare(sport->clk_ipg);2222		goto error_console;2223	}2224 2225	retval = clk_prepare_enable(sport->clk_per);2226	if (retval)2227		clk_disable_unprepare(sport->clk_ipg);2228 2229error_console:2230	return retval;2231}2232 2233static int2234imx_uart_console_exit(struct console *co)2235{2236	struct imx_port *sport = imx_uart_ports[co->index];2237 2238	clk_disable_unprepare(sport->clk_per);2239	clk_disable_unprepare(sport->clk_ipg);2240 2241	return 0;2242}2243 2244static struct uart_driver imx_uart_uart_driver;2245static struct console imx_uart_console = {2246	.name		= DEV_NAME,2247	.write		= imx_uart_console_write,2248	.device		= uart_console_device,2249	.setup		= imx_uart_console_setup,2250	.exit		= imx_uart_console_exit,2251	.flags		= CON_PRINTBUFFER,2252	.index		= -1,2253	.data		= &imx_uart_uart_driver,2254};2255 2256#define IMX_CONSOLE	&imx_uart_console2257 2258#else2259#define IMX_CONSOLE	NULL2260#endif2261 2262static struct uart_driver imx_uart_uart_driver = {2263	.owner          = THIS_MODULE,2264	.driver_name    = DRIVER_NAME,2265	.dev_name       = DEV_NAME,2266	.major          = SERIAL_IMX_MAJOR,2267	.minor          = MINOR_START,2268	.nr             = ARRAY_SIZE(imx_uart_ports),2269	.cons           = IMX_CONSOLE,2270};2271 2272static enum hrtimer_restart imx_trigger_start_tx(struct hrtimer *t)2273{2274	struct imx_port *sport = container_of(t, struct imx_port, trigger_start_tx);2275	unsigned long flags;2276 2277	uart_port_lock_irqsave(&sport->port, &flags);2278	if (sport->tx_state == WAIT_AFTER_RTS)2279		imx_uart_start_tx(&sport->port);2280	uart_port_unlock_irqrestore(&sport->port, flags);2281 2282	return HRTIMER_NORESTART;2283}2284 2285static enum hrtimer_restart imx_trigger_stop_tx(struct hrtimer *t)2286{2287	struct imx_port *sport = container_of(t, struct imx_port, trigger_stop_tx);2288	unsigned long flags;2289 2290	uart_port_lock_irqsave(&sport->port, &flags);2291	if (sport->tx_state == WAIT_AFTER_SEND)2292		imx_uart_stop_tx(&sport->port);2293	uart_port_unlock_irqrestore(&sport->port, flags);2294 2295	return HRTIMER_NORESTART;2296}2297 2298static const struct serial_rs485 imx_rs485_supported = {2299	.flags = SER_RS485_ENABLED | SER_RS485_RTS_ON_SEND | SER_RS485_RTS_AFTER_SEND |2300		 SER_RS485_RX_DURING_TX,2301	.delay_rts_before_send = 1,2302	.delay_rts_after_send = 1,2303};2304 2305/* Default RX DMA buffer configuration */2306#define RX_DMA_PERIODS		162307#define RX_DMA_PERIOD_LEN	(PAGE_SIZE / 4)2308 2309static int imx_uart_probe(struct platform_device *pdev)2310{2311	struct device_node *np = pdev->dev.of_node;2312	struct imx_port *sport;2313	void __iomem *base;2314	u32 dma_buf_conf[2];2315	int ret = 0;2316	u32 ucr1, ucr2, uts;2317	struct resource *res;2318	int txirq, rxirq, rtsirq;2319 2320	sport = devm_kzalloc(&pdev->dev, sizeof(*sport), GFP_KERNEL);2321	if (!sport)2322		return -ENOMEM;2323 2324	sport->devdata = of_device_get_match_data(&pdev->dev);2325 2326	ret = of_alias_get_id(np, "serial");2327	if (ret < 0) {2328		dev_err(&pdev->dev, "failed to get alias id, errno %d\n", ret);2329		return ret;2330	}2331	sport->port.line = ret;2332 2333	sport->have_rtscts = of_property_read_bool(np, "uart-has-rtscts") ||2334		of_property_read_bool(np, "fsl,uart-has-rtscts"); /* deprecated */2335 2336	sport->dte_mode = of_property_read_bool(np, "fsl,dte-mode");2337 2338	sport->have_rtsgpio = of_property_present(np, "rts-gpios");2339 2340	sport->inverted_tx = of_property_read_bool(np, "fsl,inverted-tx");2341 2342	sport->inverted_rx = of_property_read_bool(np, "fsl,inverted-rx");2343 2344	if (!of_property_read_u32_array(np, "fsl,dma-info", dma_buf_conf, 2)) {2345		sport->rx_period_length = dma_buf_conf[0];2346		sport->rx_periods = dma_buf_conf[1];2347	} else {2348		sport->rx_period_length = RX_DMA_PERIOD_LEN;2349		sport->rx_periods = RX_DMA_PERIODS;2350	}2351 2352	if (sport->port.line >= ARRAY_SIZE(imx_uart_ports)) {2353		dev_err(&pdev->dev, "serial%d out of range\n",2354			sport->port.line);2355		return -EINVAL;2356	}2357 2358	base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);2359	if (IS_ERR(base))2360		return PTR_ERR(base);2361 2362	rxirq = platform_get_irq(pdev, 0);2363	if (rxirq < 0)2364		return rxirq;2365	txirq = platform_get_irq_optional(pdev, 1);2366	rtsirq = platform_get_irq_optional(pdev, 2);2367 2368	sport->port.dev = &pdev->dev;2369	sport->port.mapbase = res->start;2370	sport->port.membase = base;2371	sport->port.type = PORT_IMX;2372	sport->port.iotype = UPIO_MEM;2373	sport->port.irq = rxirq;2374	sport->port.fifosize = 32;2375	sport->port.has_sysrq = IS_ENABLED(CONFIG_SERIAL_IMX_CONSOLE);2376	sport->port.ops = &imx_uart_pops;2377	sport->port.rs485_config = imx_uart_rs485_config;2378	/* RTS is required to control the RS485 transmitter */2379	if (sport->have_rtscts || sport->have_rtsgpio)2380		sport->port.rs485_supported = imx_rs485_supported;2381	sport->port.flags = UPF_BOOT_AUTOCONF;2382	timer_setup(&sport->timer, imx_uart_timeout, 0);2383 2384	sport->gpios = mctrl_gpio_init(&sport->port, 0);2385	if (IS_ERR(sport->gpios))2386		return PTR_ERR(sport->gpios);2387 2388	sport->clk_ipg = devm_clk_get(&pdev->dev, "ipg");2389	if (IS_ERR(sport->clk_ipg)) {2390		ret = PTR_ERR(sport->clk_ipg);2391		dev_err(&pdev->dev, "failed to get ipg clk: %d\n", ret);2392		return ret;2393	}2394 2395	sport->clk_per = devm_clk_get(&pdev->dev, "per");2396	if (IS_ERR(sport->clk_per)) {2397		ret = PTR_ERR(sport->clk_per);2398		dev_err(&pdev->dev, "failed to get per clk: %d\n", ret);2399		return ret;2400	}2401 2402	sport->port.uartclk = clk_get_rate(sport->clk_per);2403 2404	/* For register access, we only need to enable the ipg clock. */2405	ret = clk_prepare_enable(sport->clk_ipg);2406	if (ret) {2407		dev_err(&pdev->dev, "failed to enable ipg clk: %d\n", ret);2408		return ret;2409	}2410 2411	ret = uart_get_rs485_mode(&sport->port);2412	if (ret)2413		goto err_clk;2414 2415	/*2416	 * If using the i.MX UART RTS/CTS control then the RTS (CTS_B)2417	 * signal cannot be set low during transmission in case the2418	 * receiver is off (limitation of the i.MX UART IP).2419	 */2420	if (sport->port.rs485.flags & SER_RS485_ENABLED &&2421	    sport->have_rtscts && !sport->have_rtsgpio &&2422	    (!(sport->port.rs485.flags & SER_RS485_RTS_ON_SEND) &&2423	     !(sport->port.rs485.flags & SER_RS485_RX_DURING_TX)))2424		dev_err(&pdev->dev,2425			"low-active RTS not possible when receiver is off, enabling receiver\n");2426 2427	/* Disable interrupts before requesting them */2428	ucr1 = imx_uart_readl(sport, UCR1);2429	ucr1 &= ~(UCR1_ADEN | UCR1_TRDYEN | UCR1_IDEN | UCR1_RRDYEN | UCR1_RTSDEN);2430	imx_uart_writel(sport, ucr1, UCR1);2431 2432	/* Disable Ageing Timer interrupt */2433	ucr2 = imx_uart_readl(sport, UCR2);2434	ucr2 &= ~UCR2_ATEN;2435	imx_uart_writel(sport, ucr2, UCR2);2436 2437	/*2438	 * In case RS485 is enabled without GPIO RTS control, the UART IP2439	 * is used to control CTS signal. Keep both the UART and Receiver2440	 * enabled, otherwise the UART IP pulls CTS signal always HIGH no2441	 * matter how the UCR2 CTSC and CTS bits are set. To prevent any2442	 * data from being fed into the RX FIFO, enable loopback mode in2443	 * UTS register, which disconnects the RX path from external RXD2444	 * pin and connects it to the Transceiver, which is disabled, so2445	 * no data can be fed to the RX FIFO that way.2446	 */2447	if (sport->port.rs485.flags & SER_RS485_ENABLED &&2448	    sport->have_rtscts && !sport->have_rtsgpio) {2449		uts = imx_uart_readl(sport, imx_uart_uts_reg(sport));2450		uts |= UTS_LOOP;2451		imx_uart_writel(sport, uts, imx_uart_uts_reg(sport));2452 2453		ucr1 = imx_uart_readl(sport, UCR1);2454		ucr1 |= UCR1_UARTEN;2455		imx_uart_writel(sport, ucr1, UCR1);2456 2457		ucr2 = imx_uart_readl(sport, UCR2);2458		ucr2 |= UCR2_RXEN;2459		imx_uart_writel(sport, ucr2, UCR2);2460	}2461 2462	if (!imx_uart_is_imx1(sport) && sport->dte_mode) {2463		/*2464		 * The DCEDTE bit changes the direction of DSR, DCD, DTR and RI2465		 * and influences if UCR3_RI and UCR3_DCD changes the level of RI2466		 * and DCD (when they are outputs) or enables the respective2467		 * irqs. So set this bit early, i.e. before requesting irqs.2468		 */2469		u32 ufcr = imx_uart_readl(sport, UFCR);2470		if (!(ufcr & UFCR_DCEDTE))2471			imx_uart_writel(sport, ufcr | UFCR_DCEDTE, UFCR);2472 2473		/*2474		 * Disable UCR3_RI and UCR3_DCD irqs. They are also not2475		 * enabled later because they cannot be cleared2476		 * (confirmed on i.MX25) which makes them unusable.2477		 */2478		imx_uart_writel(sport,2479				IMX21_UCR3_RXDMUXSEL | UCR3_ADNIMP | UCR3_DSR,2480				UCR3);2481 2482	} else {2483		u32 ucr3 = UCR3_DSR;2484		u32 ufcr = imx_uart_readl(sport, UFCR);2485		if (ufcr & UFCR_DCEDTE)2486			imx_uart_writel(sport, ufcr & ~UFCR_DCEDTE, UFCR);2487 2488		if (!imx_uart_is_imx1(sport))2489			ucr3 |= IMX21_UCR3_RXDMUXSEL | UCR3_ADNIMP;2490		imx_uart_writel(sport, ucr3, UCR3);2491	}2492 2493	hrtimer_init(&sport->trigger_start_tx, CLOCK_MONOTONIC, HRTIMER_MODE_REL);2494	hrtimer_init(&sport->trigger_stop_tx, CLOCK_MONOTONIC, HRTIMER_MODE_REL);2495	sport->trigger_start_tx.function = imx_trigger_start_tx;2496	sport->trigger_stop_tx.function = imx_trigger_stop_tx;2497 2498	/*2499	 * Allocate the IRQ(s) i.MX1 has three interrupts whereas later2500	 * chips only have one interrupt.2501	 */2502	if (txirq > 0) {2503		ret = devm_request_irq(&pdev->dev, rxirq, imx_uart_rxint, 0,2504				       dev_name(&pdev->dev), sport);2505		if (ret) {2506			dev_err(&pdev->dev, "failed to request rx irq: %d\n",2507				ret);2508			goto err_clk;2509		}2510 2511		ret = devm_request_irq(&pdev->dev, txirq, imx_uart_txint, 0,2512				       dev_name(&pdev->dev), sport);2513		if (ret) {2514			dev_err(&pdev->dev, "failed to request tx irq: %d\n",2515				ret);2516			goto err_clk;2517		}2518 2519		ret = devm_request_irq(&pdev->dev, rtsirq, imx_uart_rtsint, 0,2520				       dev_name(&pdev->dev), sport);2521		if (ret) {2522			dev_err(&pdev->dev, "failed to request rts irq: %d\n",2523				ret);2524			goto err_clk;2525		}2526	} else {2527		ret = devm_request_irq(&pdev->dev, rxirq, imx_uart_int, 0,2528				       dev_name(&pdev->dev), sport);2529		if (ret) {2530			dev_err(&pdev->dev, "failed to request irq: %d\n", ret);2531			goto err_clk;2532		}2533	}2534 2535	imx_uart_ports[sport->port.line] = sport;2536 2537	platform_set_drvdata(pdev, sport);2538 2539	ret = uart_add_one_port(&imx_uart_uart_driver, &sport->port);2540 2541err_clk:2542	clk_disable_unprepare(sport->clk_ipg);2543 2544	return ret;2545}2546 2547static void imx_uart_remove(struct platform_device *pdev)2548{2549	struct imx_port *sport = platform_get_drvdata(pdev);2550 2551	uart_remove_one_port(&imx_uart_uart_driver, &sport->port);2552}2553 2554static void imx_uart_restore_context(struct imx_port *sport)2555{2556	unsigned long flags;2557 2558	uart_port_lock_irqsave(&sport->port, &flags);2559	if (!sport->context_saved) {2560		uart_port_unlock_irqrestore(&sport->port, flags);2561		return;2562	}2563 2564	imx_uart_writel(sport, sport->saved_reg[4], UFCR);2565	imx_uart_writel(sport, sport->saved_reg[5], UESC);2566	imx_uart_writel(sport, sport->saved_reg[6], UTIM);2567	imx_uart_writel(sport, sport->saved_reg[7], UBIR);2568	imx_uart_writel(sport, sport->saved_reg[8], UBMR);2569	imx_uart_writel(sport, sport->saved_reg[9], IMX21_UTS);2570	imx_uart_writel(sport, sport->saved_reg[0], UCR1);2571	imx_uart_writel(sport, sport->saved_reg[1] | UCR2_SRST, UCR2);2572	imx_uart_writel(sport, sport->saved_reg[2], UCR3);2573	imx_uart_writel(sport, sport->saved_reg[3], UCR4);2574	sport->context_saved = false;2575	uart_port_unlock_irqrestore(&sport->port, flags);2576}2577 2578static void imx_uart_save_context(struct imx_port *sport)2579{2580	unsigned long flags;2581 2582	/* Save necessary regs */2583	uart_port_lock_irqsave(&sport->port, &flags);2584	sport->saved_reg[0] = imx_uart_readl(sport, UCR1);2585	sport->saved_reg[1] = imx_uart_readl(sport, UCR2);2586	sport->saved_reg[2] = imx_uart_readl(sport, UCR3);2587	sport->saved_reg[3] = imx_uart_readl(sport, UCR4);2588	sport->saved_reg[4] = imx_uart_readl(sport, UFCR);2589	sport->saved_reg[5] = imx_uart_readl(sport, UESC);2590	sport->saved_reg[6] = imx_uart_readl(sport, UTIM);2591	sport->saved_reg[7] = imx_uart_readl(sport, UBIR);2592	sport->saved_reg[8] = imx_uart_readl(sport, UBMR);2593	sport->saved_reg[9] = imx_uart_readl(sport, IMX21_UTS);2594	sport->context_saved = true;2595	uart_port_unlock_irqrestore(&sport->port, flags);2596}2597 2598static void imx_uart_enable_wakeup(struct imx_port *sport, bool on)2599{2600	u32 ucr3;2601 2602	ucr3 = imx_uart_readl(sport, UCR3);2603	if (on) {2604		imx_uart_writel(sport, USR1_AWAKE, USR1);2605		ucr3 |= UCR3_AWAKEN;2606	} else {2607		ucr3 &= ~UCR3_AWAKEN;2608	}2609	imx_uart_writel(sport, ucr3, UCR3);2610 2611	if (sport->have_rtscts) {2612		u32 ucr1 = imx_uart_readl(sport, UCR1);2613		if (on) {2614			imx_uart_writel(sport, USR1_RTSD, USR1);2615			ucr1 |= UCR1_RTSDEN;2616		} else {2617			ucr1 &= ~UCR1_RTSDEN;2618		}2619		imx_uart_writel(sport, ucr1, UCR1);2620	}2621}2622 2623static int imx_uart_suspend_noirq(struct device *dev)2624{2625	struct imx_port *sport = dev_get_drvdata(dev);2626 2627	imx_uart_save_context(sport);2628 2629	clk_disable(sport->clk_ipg);2630 2631	pinctrl_pm_select_sleep_state(dev);2632 2633	return 0;2634}2635 2636static int imx_uart_resume_noirq(struct device *dev)2637{2638	struct imx_port *sport = dev_get_drvdata(dev);2639	int ret;2640 2641	pinctrl_pm_select_default_state(dev);2642 2643	ret = clk_enable(sport->clk_ipg);2644	if (ret)2645		return ret;2646 2647	imx_uart_restore_context(sport);2648 2649	return 0;2650}2651 2652static int imx_uart_suspend(struct device *dev)2653{2654	struct imx_port *sport = dev_get_drvdata(dev);2655	int ret;2656 2657	uart_suspend_port(&imx_uart_uart_driver, &sport->port);2658	disable_irq(sport->port.irq);2659 2660	ret = clk_prepare_enable(sport->clk_ipg);2661	if (ret)2662		return ret;2663 2664	/* enable wakeup from i.MX UART */2665	imx_uart_enable_wakeup(sport, true);2666 2667	return 0;2668}2669 2670static int imx_uart_resume(struct device *dev)2671{2672	struct imx_port *sport = dev_get_drvdata(dev);2673 2674	/* disable wakeup from i.MX UART */2675	imx_uart_enable_wakeup(sport, false);2676 2677	uart_resume_port(&imx_uart_uart_driver, &sport->port);2678	enable_irq(sport->port.irq);2679 2680	clk_disable_unprepare(sport->clk_ipg);2681 2682	return 0;2683}2684 2685static int imx_uart_freeze(struct device *dev)2686{2687	struct imx_port *sport = dev_get_drvdata(dev);2688 2689	uart_suspend_port(&imx_uart_uart_driver, &sport->port);2690 2691	return clk_prepare_enable(sport->clk_ipg);2692}2693 2694static int imx_uart_thaw(struct device *dev)2695{2696	struct imx_port *sport = dev_get_drvdata(dev);2697 2698	uart_resume_port(&imx_uart_uart_driver, &sport->port);2699 2700	clk_disable_unprepare(sport->clk_ipg);2701 2702	return 0;2703}2704 2705static const struct dev_pm_ops imx_uart_pm_ops = {2706	.suspend_noirq = imx_uart_suspend_noirq,2707	.resume_noirq = imx_uart_resume_noirq,2708	.freeze_noirq = imx_uart_suspend_noirq,2709	.thaw_noirq = imx_uart_resume_noirq,2710	.restore_noirq = imx_uart_resume_noirq,2711	.suspend = imx_uart_suspend,2712	.resume = imx_uart_resume,2713	.freeze = imx_uart_freeze,2714	.thaw = imx_uart_thaw,2715	.restore = imx_uart_thaw,2716};2717 2718static struct platform_driver imx_uart_platform_driver = {2719	.probe = imx_uart_probe,2720	.remove_new = imx_uart_remove,2721 2722	.driver = {2723		.name = "imx-uart",2724		.of_match_table = imx_uart_dt_ids,2725		.pm = &imx_uart_pm_ops,2726	},2727};2728 2729static int __init imx_uart_init(void)2730{2731	int ret = uart_register_driver(&imx_uart_uart_driver);2732 2733	if (ret)2734		return ret;2735 2736	ret = platform_driver_register(&imx_uart_platform_driver);2737	if (ret != 0)2738		uart_unregister_driver(&imx_uart_uart_driver);2739 2740	return ret;2741}2742 2743static void __exit imx_uart_exit(void)2744{2745	platform_driver_unregister(&imx_uart_platform_driver);2746	uart_unregister_driver(&imx_uart_uart_driver);2747}2748 2749module_init(imx_uart_init);2750module_exit(imx_uart_exit);2751 2752MODULE_AUTHOR("Sascha Hauer");2753MODULE_DESCRIPTION("IMX generic serial port driver");2754MODULE_LICENSE("GPL");2755MODULE_ALIAS("platform:imx-uart");2756