1877 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * 8250-core based driver for the OMAP internal UART4 *5 * based on omap-serial.c, Copyright (C) 2010 Texas Instruments.6 *7 * Copyright (C) 2014 Sebastian Andrzej Siewior8 *9 */10 11#include <linux/atomic.h>12#include <linux/clk.h>13#include <linux/device.h>14#include <linux/io.h>15#include <linux/module.h>16#include <linux/serial_8250.h>17#include <linux/serial_reg.h>18#include <linux/tty_flip.h>19#include <linux/platform_device.h>20#include <linux/slab.h>21#include <linux/of.h>22#include <linux/of_irq.h>23#include <linux/delay.h>24#include <linux/pm_runtime.h>25#include <linux/console.h>26#include <linux/pm_qos.h>27#include <linux/pm_wakeirq.h>28#include <linux/dma-mapping.h>29#include <linux/sys_soc.h>30 31#include "8250.h"32 33#define DEFAULT_CLK_SPEED 4800000034#define OMAP_UART_REGSHIFT 235 36#define UART_ERRATA_i202_MDR1_ACCESS (1 << 0)37#define OMAP_UART_WER_HAS_TX_WAKEUP (1 << 1)38#define OMAP_DMA_TX_KICK (1 << 2)39/*40 * See Advisory 21 in AM437x errata SPRZ408B, updated April 2015.41 * The same errata is applicable to AM335x and DRA7x processors too.42 */43#define UART_ERRATA_CLOCK_DISABLE (1 << 3)44#define UART_HAS_EFR2 BIT(4)45#define UART_HAS_RHR_IT_DIS BIT(5)46#define UART_RX_TIMEOUT_QUIRK BIT(6)47#define UART_HAS_NATIVE_RS485 BIT(7)48 49#define OMAP_UART_FCR_RX_TRIG 650#define OMAP_UART_FCR_TX_TRIG 451 52/* SCR register bitmasks */53#define OMAP_UART_SCR_RX_TRIG_GRANU1_MASK (1 << 7)54#define OMAP_UART_SCR_TX_TRIG_GRANU1_MASK (1 << 6)55#define OMAP_UART_SCR_TX_EMPTY (1 << 3)56#define OMAP_UART_SCR_DMAMODE_MASK (3 << 1)57#define OMAP_UART_SCR_DMAMODE_1 (1 << 1)58#define OMAP_UART_SCR_DMAMODE_CTL (1 << 0)59 60/* MVR register bitmasks */61#define OMAP_UART_MVR_SCHEME_SHIFT 3062#define OMAP_UART_LEGACY_MVR_MAJ_MASK 0xf063#define OMAP_UART_LEGACY_MVR_MAJ_SHIFT 464#define OMAP_UART_LEGACY_MVR_MIN_MASK 0x0f65#define OMAP_UART_MVR_MAJ_MASK 0x70066#define OMAP_UART_MVR_MAJ_SHIFT 867#define OMAP_UART_MVR_MIN_MASK 0x3f68 69/* SYSC register bitmasks */70#define OMAP_UART_SYSC_SOFTRESET (1 << 1)71 72/* SYSS register bitmasks */73#define OMAP_UART_SYSS_RESETDONE (1 << 0)74 75#define UART_TI752_TLR_TX 076#define UART_TI752_TLR_RX 477 78#define TRIGGER_TLR_MASK(x) ((x & 0x3c) >> 2)79#define TRIGGER_FCR_MASK(x) (x & 3)80 81/* Enable XON/XOFF flow control on output */82#define OMAP_UART_SW_TX 0x0883/* Enable XON/XOFF flow control on input */84#define OMAP_UART_SW_RX 0x0285 86#define OMAP_UART_WER_MOD_WKUP 0x7f87#define OMAP_UART_TX_WAKEUP_EN (1 << 7)88 89#define TX_TRIGGER 190#define RX_TRIGGER 4891 92#define OMAP_UART_TCR_RESTORE(x) ((x / 4) << 4)93#define OMAP_UART_TCR_HALT(x) ((x / 4) << 0)94 95#define UART_BUILD_REVISION(x, y) (((x) << 8) | (y))96 97#define OMAP_UART_REV_46 0x040698#define OMAP_UART_REV_52 0x050299#define OMAP_UART_REV_63 0x0603100 101/* Interrupt Enable Register 2 */102#define UART_OMAP_IER2 0x1B103#define UART_OMAP_IER2_RHR_IT_DIS BIT(2)104 105/* Mode Definition Register 3 */106#define UART_OMAP_MDR3 0x20107#define UART_OMAP_MDR3_DIR_POL BIT(3)108#define UART_OMAP_MDR3_DIR_EN BIT(4)109 110/* Enhanced features register 2 */111#define UART_OMAP_EFR2 0x23112#define UART_OMAP_EFR2_TIMEOUT_BEHAVE BIT(6)113 114/* RX FIFO occupancy indicator */115#define UART_OMAP_RX_LVL 0x19116 117/* Timeout low and High */118#define UART_OMAP_TO_L 0x26119#define UART_OMAP_TO_H 0x27120 121struct omap8250_priv {122 void __iomem *membase;123 int line;124 u8 habit;125 u8 mdr1;126 u8 mdr3;127 u8 efr;128 u8 scr;129 u8 wer;130 u8 xon;131 u8 xoff;132 u8 delayed_restore;133 u16 quot;134 135 u8 tx_trigger;136 u8 rx_trigger;137 atomic_t active;138 bool is_suspending;139 int wakeirq;140 u32 latency;141 u32 calc_latency;142 struct pm_qos_request pm_qos_request;143 struct work_struct qos_work;144 struct uart_8250_dma omap8250_dma;145 spinlock_t rx_dma_lock;146 bool rx_dma_broken;147 bool throttled;148};149 150struct omap8250_dma_params {151 u32 rx_size;152 u8 rx_trigger;153 u8 tx_trigger;154};155 156struct omap8250_platdata {157 struct omap8250_dma_params *dma_params;158 u8 habit;159};160 161#ifdef CONFIG_SERIAL_8250_DMA162static void omap_8250_rx_dma_flush(struct uart_8250_port *p);163#else164static inline void omap_8250_rx_dma_flush(struct uart_8250_port *p) { }165#endif166 167static u32 uart_read(struct omap8250_priv *priv, u32 reg)168{169 return readl(priv->membase + (reg << OMAP_UART_REGSHIFT));170}171 172/*173 * Called on runtime PM resume path from omap8250_restore_regs(), and174 * omap8250_set_mctrl().175 */176static void __omap8250_set_mctrl(struct uart_port *port, unsigned int mctrl)177{178 struct uart_8250_port *up = up_to_u8250p(port);179 struct omap8250_priv *priv = up->port.private_data;180 u8 lcr;181 182 serial8250_do_set_mctrl(port, mctrl);183 184 if (!mctrl_gpio_to_gpiod(up->gpios, UART_GPIO_RTS)) {185 /*186 * Turn off autoRTS if RTS is lowered and restore autoRTS187 * setting if RTS is raised188 */189 lcr = serial_in(up, UART_LCR);190 serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);191 if ((mctrl & TIOCM_RTS) && (port->status & UPSTAT_AUTORTS))192 priv->efr |= UART_EFR_RTS;193 else194 priv->efr &= ~UART_EFR_RTS;195 serial_out(up, UART_EFR, priv->efr);196 serial_out(up, UART_LCR, lcr);197 }198}199 200static void omap8250_set_mctrl(struct uart_port *port, unsigned int mctrl)201{202 int err;203 204 err = pm_runtime_resume_and_get(port->dev);205 if (err)206 return;207 208 __omap8250_set_mctrl(port, mctrl);209 210 pm_runtime_mark_last_busy(port->dev);211 pm_runtime_put_autosuspend(port->dev);212}213 214/*215 * Work Around for Errata i202 (2430, 3430, 3630, 4430 and 4460)216 * The access to uart register after MDR1 Access217 * causes UART to corrupt data.218 *219 * Need a delay =220 * 5 L4 clock cycles + 5 UART functional clock cycle (@48MHz = ~0.2uS)221 * give 10 times as much222 */223static void omap_8250_mdr1_errataset(struct uart_8250_port *up,224 struct omap8250_priv *priv)225{226 serial_out(up, UART_OMAP_MDR1, priv->mdr1);227 udelay(2);228 serial_out(up, UART_FCR, up->fcr | UART_FCR_CLEAR_XMIT |229 UART_FCR_CLEAR_RCVR);230}231 232static void omap_8250_get_divisor(struct uart_port *port, unsigned int baud,233 struct omap8250_priv *priv)234{235 unsigned int uartclk = port->uartclk;236 unsigned int div_13, div_16;237 unsigned int abs_d13, abs_d16;238 239 /*240 * Old custom speed handling.241 */242 if (baud == 38400 && (port->flags & UPF_SPD_MASK) == UPF_SPD_CUST) {243 priv->quot = port->custom_divisor & UART_DIV_MAX;244 /*245 * I assume that nobody is using this. But hey, if somebody246 * would like to specify the divisor _and_ the mode then the247 * driver is ready and waiting for it.248 */249 if (port->custom_divisor & (1 << 16))250 priv->mdr1 = UART_OMAP_MDR1_13X_MODE;251 else252 priv->mdr1 = UART_OMAP_MDR1_16X_MODE;253 return;254 }255 div_13 = DIV_ROUND_CLOSEST(uartclk, 13 * baud);256 div_16 = DIV_ROUND_CLOSEST(uartclk, 16 * baud);257 258 if (!div_13)259 div_13 = 1;260 if (!div_16)261 div_16 = 1;262 263 abs_d13 = abs(baud - uartclk / 13 / div_13);264 abs_d16 = abs(baud - uartclk / 16 / div_16);265 266 if (abs_d13 >= abs_d16) {267 priv->mdr1 = UART_OMAP_MDR1_16X_MODE;268 priv->quot = div_16;269 } else {270 priv->mdr1 = UART_OMAP_MDR1_13X_MODE;271 priv->quot = div_13;272 }273}274 275static void omap8250_update_scr(struct uart_8250_port *up,276 struct omap8250_priv *priv)277{278 u8 old_scr;279 280 old_scr = serial_in(up, UART_OMAP_SCR);281 if (old_scr == priv->scr)282 return;283 284 /*285 * The manual recommends not to enable the DMA mode selector in the SCR286 * (instead of the FCR) register _and_ selecting the DMA mode as one287 * register write because this may lead to malfunction.288 */289 if (priv->scr & OMAP_UART_SCR_DMAMODE_MASK)290 serial_out(up, UART_OMAP_SCR,291 priv->scr & ~OMAP_UART_SCR_DMAMODE_MASK);292 serial_out(up, UART_OMAP_SCR, priv->scr);293}294 295static void omap8250_update_mdr1(struct uart_8250_port *up,296 struct omap8250_priv *priv)297{298 if (priv->habit & UART_ERRATA_i202_MDR1_ACCESS)299 omap_8250_mdr1_errataset(up, priv);300 else301 serial_out(up, UART_OMAP_MDR1, priv->mdr1);302}303 304static void omap8250_restore_regs(struct uart_8250_port *up)305{306 struct omap8250_priv *priv = up->port.private_data;307 struct uart_8250_dma *dma = up->dma;308 u8 mcr = serial8250_in_MCR(up);309 310 /* Port locked to synchronize UART_IER access against the console. */311 lockdep_assert_held_once(&up->port.lock);312 313 if (dma && dma->tx_running) {314 /*315 * TCSANOW requests the change to occur immediately however if316 * we have a TX-DMA operation in progress then it has been317 * observed that it might stall and never complete. Therefore we318 * delay DMA completes to prevent this hang from happen.319 */320 priv->delayed_restore = 1;321 return;322 }323 324 serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);325 serial_out(up, UART_EFR, UART_EFR_ECB);326 327 serial_out(up, UART_LCR, UART_LCR_CONF_MODE_A);328 serial8250_out_MCR(up, mcr | UART_MCR_TCRTLR);329 serial_out(up, UART_FCR, up->fcr);330 331 omap8250_update_scr(up, priv);332 333 serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);334 335 serial_out(up, UART_TI752_TCR, OMAP_UART_TCR_RESTORE(16) |336 OMAP_UART_TCR_HALT(52));337 serial_out(up, UART_TI752_TLR,338 TRIGGER_TLR_MASK(priv->tx_trigger) << UART_TI752_TLR_TX |339 TRIGGER_TLR_MASK(priv->rx_trigger) << UART_TI752_TLR_RX);340 341 serial_out(up, UART_LCR, 0);342 343 /* drop TCR + TLR access, we setup XON/XOFF later */344 serial8250_out_MCR(up, mcr);345 346 serial_out(up, UART_IER, up->ier);347 348 serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);349 serial_dl_write(up, priv->quot);350 351 serial_out(up, UART_EFR, priv->efr);352 353 /* Configure flow control */354 serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);355 serial_out(up, UART_XON1, priv->xon);356 serial_out(up, UART_XOFF1, priv->xoff);357 358 serial_out(up, UART_LCR, up->lcr);359 360 omap8250_update_mdr1(up, priv);361 362 __omap8250_set_mctrl(&up->port, up->port.mctrl);363 364 serial_out(up, UART_OMAP_MDR3, priv->mdr3);365 366 if (up->port.rs485.flags & SER_RS485_ENABLED &&367 up->port.rs485_config == serial8250_em485_config)368 serial8250_em485_stop_tx(up);369}370 371/*372 * OMAP can use "CLK / (16 or 13) / div" for baud rate. And then we have have373 * some differences in how we want to handle flow control.374 */375static void omap_8250_set_termios(struct uart_port *port,376 struct ktermios *termios,377 const struct ktermios *old)378{379 struct uart_8250_port *up = up_to_u8250p(port);380 struct omap8250_priv *priv = up->port.private_data;381 unsigned char cval = 0;382 unsigned int baud;383 384 cval = UART_LCR_WLEN(tty_get_char_size(termios->c_cflag));385 386 if (termios->c_cflag & CSTOPB)387 cval |= UART_LCR_STOP;388 if (termios->c_cflag & PARENB)389 cval |= UART_LCR_PARITY;390 if (!(termios->c_cflag & PARODD))391 cval |= UART_LCR_EPAR;392 if (termios->c_cflag & CMSPAR)393 cval |= UART_LCR_SPAR;394 395 /*396 * Ask the core to calculate the divisor for us.397 */398 baud = uart_get_baud_rate(port, termios, old,399 port->uartclk / 16 / UART_DIV_MAX,400 port->uartclk / 13);401 omap_8250_get_divisor(port, baud, priv);402 403 /*404 * Ok, we're now changing the port state. Do it with405 * interrupts disabled.406 */407 pm_runtime_get_sync(port->dev);408 uart_port_lock_irq(port);409 410 /*411 * Update the per-port timeout.412 */413 uart_update_timeout(port, termios->c_cflag, baud);414 415 up->port.read_status_mask = UART_LSR_OE | UART_LSR_THRE | UART_LSR_DR;416 if (termios->c_iflag & INPCK)417 up->port.read_status_mask |= UART_LSR_FE | UART_LSR_PE;418 if (termios->c_iflag & (IGNBRK | PARMRK))419 up->port.read_status_mask |= UART_LSR_BI;420 421 /*422 * Characters to ignore423 */424 up->port.ignore_status_mask = 0;425 if (termios->c_iflag & IGNPAR)426 up->port.ignore_status_mask |= UART_LSR_PE | UART_LSR_FE;427 if (termios->c_iflag & IGNBRK) {428 up->port.ignore_status_mask |= UART_LSR_BI;429 /*430 * If we're ignoring parity and break indicators,431 * ignore overruns too (for real raw support).432 */433 if (termios->c_iflag & IGNPAR)434 up->port.ignore_status_mask |= UART_LSR_OE;435 }436 437 /*438 * ignore all characters if CREAD is not set439 */440 if ((termios->c_cflag & CREAD) == 0)441 up->port.ignore_status_mask |= UART_LSR_DR;442 443 /*444 * Modem status interrupts445 */446 up->ier &= ~UART_IER_MSI;447 if (UART_ENABLE_MS(&up->port, termios->c_cflag))448 up->ier |= UART_IER_MSI;449 450 up->lcr = cval;451 /* Up to here it was mostly serial8250_do_set_termios() */452 453 /*454 * We enable TRIG_GRANU for RX and TX and additionally we set455 * SCR_TX_EMPTY bit. The result is the following:456 * - RX_TRIGGER amount of bytes in the FIFO will cause an interrupt.457 * - less than RX_TRIGGER number of bytes will also cause an interrupt458 * once the UART decides that there no new bytes arriving.459 * - Once THRE is enabled, the interrupt will be fired once the FIFO is460 * empty - the trigger level is ignored here.461 *462 * Once DMA is enabled:463 * - UART will assert the TX DMA line once there is room for TX_TRIGGER464 * bytes in the TX FIFO. On each assert the DMA engine will move465 * TX_TRIGGER bytes into the FIFO.466 * - UART will assert the RX DMA line once there are RX_TRIGGER bytes in467 * the FIFO and move RX_TRIGGER bytes.468 * This is because threshold and trigger values are the same.469 */470 up->fcr = UART_FCR_ENABLE_FIFO;471 up->fcr |= TRIGGER_FCR_MASK(priv->tx_trigger) << OMAP_UART_FCR_TX_TRIG;472 up->fcr |= TRIGGER_FCR_MASK(priv->rx_trigger) << OMAP_UART_FCR_RX_TRIG;473 474 priv->scr = OMAP_UART_SCR_RX_TRIG_GRANU1_MASK | OMAP_UART_SCR_TX_EMPTY |475 OMAP_UART_SCR_TX_TRIG_GRANU1_MASK;476 477 if (up->dma)478 priv->scr |= OMAP_UART_SCR_DMAMODE_1 |479 OMAP_UART_SCR_DMAMODE_CTL;480 481 priv->xon = termios->c_cc[VSTART];482 priv->xoff = termios->c_cc[VSTOP];483 484 priv->efr = 0;485 up->port.status &= ~(UPSTAT_AUTOCTS | UPSTAT_AUTORTS | UPSTAT_AUTOXOFF);486 487 if (termios->c_cflag & CRTSCTS && up->port.flags & UPF_HARD_FLOW &&488 !mctrl_gpio_to_gpiod(up->gpios, UART_GPIO_RTS) &&489 !mctrl_gpio_to_gpiod(up->gpios, UART_GPIO_CTS)) {490 /* Enable AUTOCTS (autoRTS is enabled when RTS is raised) */491 up->port.status |= UPSTAT_AUTOCTS | UPSTAT_AUTORTS;492 priv->efr |= UART_EFR_CTS;493 } else if (up->port.flags & UPF_SOFT_FLOW) {494 /*495 * OMAP rx s/w flow control is borked; the transmitter remains496 * stuck off even if rx flow control is subsequently disabled497 */498 499 /*500 * IXOFF Flag:501 * Enable XON/XOFF flow control on output.502 * Transmit XON1, XOFF1503 */504 if (termios->c_iflag & IXOFF) {505 up->port.status |= UPSTAT_AUTOXOFF;506 priv->efr |= OMAP_UART_SW_TX;507 }508 }509 omap8250_restore_regs(up);510 511 uart_port_unlock_irq(&up->port);512 pm_runtime_mark_last_busy(port->dev);513 pm_runtime_put_autosuspend(port->dev);514 515 /* calculate wakeup latency constraint */516 priv->calc_latency = USEC_PER_SEC * 64 * 8 / baud;517 priv->latency = priv->calc_latency;518 519 schedule_work(&priv->qos_work);520 521 /* Don't rewrite B0 */522 if (tty_termios_baud_rate(termios))523 tty_termios_encode_baud_rate(termios, baud, baud);524}525 526/* same as 8250 except that we may have extra flow bits set in EFR */527static void omap_8250_pm(struct uart_port *port, unsigned int state,528 unsigned int oldstate)529{530 struct uart_8250_port *up = up_to_u8250p(port);531 u8 efr;532 533 pm_runtime_get_sync(port->dev);534 535 /* Synchronize UART_IER access against the console. */536 uart_port_lock_irq(port);537 538 serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);539 efr = serial_in(up, UART_EFR);540 serial_out(up, UART_EFR, efr | UART_EFR_ECB);541 serial_out(up, UART_LCR, 0);542 543 serial_out(up, UART_IER, (state != 0) ? UART_IERX_SLEEP : 0);544 serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);545 serial_out(up, UART_EFR, efr);546 serial_out(up, UART_LCR, 0);547 548 uart_port_unlock_irq(port);549 550 pm_runtime_mark_last_busy(port->dev);551 pm_runtime_put_autosuspend(port->dev);552}553 554static void omap_serial_fill_features_erratas(struct uart_8250_port *up,555 struct omap8250_priv *priv)556{557 static const struct soc_device_attribute k3_soc_devices[] = {558 { .family = "AM65X", },559 { .family = "J721E", .revision = "SR1.0" },560 { /* sentinel */ }561 };562 u32 mvr, scheme;563 u16 revision, major, minor;564 565 mvr = uart_read(priv, UART_OMAP_MVER);566 567 /* Check revision register scheme */568 scheme = mvr >> OMAP_UART_MVR_SCHEME_SHIFT;569 570 switch (scheme) {571 case 0: /* Legacy Scheme: OMAP2/3 */572 /* MINOR_REV[0:4], MAJOR_REV[4:7] */573 major = (mvr & OMAP_UART_LEGACY_MVR_MAJ_MASK) >>574 OMAP_UART_LEGACY_MVR_MAJ_SHIFT;575 minor = (mvr & OMAP_UART_LEGACY_MVR_MIN_MASK);576 break;577 case 1:578 /* New Scheme: OMAP4+ */579 /* MINOR_REV[0:5], MAJOR_REV[8:10] */580 major = (mvr & OMAP_UART_MVR_MAJ_MASK) >>581 OMAP_UART_MVR_MAJ_SHIFT;582 minor = (mvr & OMAP_UART_MVR_MIN_MASK);583 break;584 default:585 dev_warn(up->port.dev,586 "Unknown revision, defaulting to highest\n");587 /* highest possible revision */588 major = 0xff;589 minor = 0xff;590 }591 /* normalize revision for the driver */592 revision = UART_BUILD_REVISION(major, minor);593 594 switch (revision) {595 case OMAP_UART_REV_46:596 priv->habit |= UART_ERRATA_i202_MDR1_ACCESS;597 break;598 case OMAP_UART_REV_52:599 priv->habit |= UART_ERRATA_i202_MDR1_ACCESS |600 OMAP_UART_WER_HAS_TX_WAKEUP;601 break;602 case OMAP_UART_REV_63:603 priv->habit |= UART_ERRATA_i202_MDR1_ACCESS |604 OMAP_UART_WER_HAS_TX_WAKEUP;605 break;606 default:607 break;608 }609 610 /*611 * AM65x SR1.0, AM65x SR2.0 and J721e SR1.0 don't612 * don't have RHR_IT_DIS bit in IER2 register. So drop to flag613 * to enable errata workaround.614 */615 if (soc_device_match(k3_soc_devices))616 priv->habit &= ~UART_HAS_RHR_IT_DIS;617}618 619static void omap8250_uart_qos_work(struct work_struct *work)620{621 struct omap8250_priv *priv;622 623 priv = container_of(work, struct omap8250_priv, qos_work);624 cpu_latency_qos_update_request(&priv->pm_qos_request, priv->latency);625}626 627#ifdef CONFIG_SERIAL_8250_DMA628static int omap_8250_dma_handle_irq(struct uart_port *port);629#endif630 631static irqreturn_t omap8250_irq(int irq, void *dev_id)632{633 struct omap8250_priv *priv = dev_id;634 struct uart_8250_port *up = serial8250_get_port(priv->line);635 struct uart_port *port = &up->port;636 unsigned int iir, lsr;637 int ret;638 639 pm_runtime_get_noresume(port->dev);640 641 /* Shallow idle state wake-up to an IO interrupt? */642 if (atomic_add_unless(&priv->active, 1, 1)) {643 priv->latency = priv->calc_latency;644 schedule_work(&priv->qos_work);645 }646 647#ifdef CONFIG_SERIAL_8250_DMA648 if (up->dma) {649 ret = omap_8250_dma_handle_irq(port);650 pm_runtime_mark_last_busy(port->dev);651 pm_runtime_put(port->dev);652 return IRQ_RETVAL(ret);653 }654#endif655 656 lsr = serial_port_in(port, UART_LSR);657 iir = serial_port_in(port, UART_IIR);658 ret = serial8250_handle_irq(port, iir);659 660 /*661 * On K3 SoCs, it is observed that RX TIMEOUT is signalled after662 * FIFO has been drained or erroneously.663 * So apply solution of Errata i2310 as mentioned in664 * https://www.ti.com/lit/pdf/sprz536665 */666 if (priv->habit & UART_RX_TIMEOUT_QUIRK &&667 (iir & UART_IIR_RX_TIMEOUT) == UART_IIR_RX_TIMEOUT &&668 serial_port_in(port, UART_OMAP_RX_LVL) == 0) {669 unsigned char efr2, timeout_h, timeout_l;670 671 efr2 = serial_in(up, UART_OMAP_EFR2);672 timeout_h = serial_in(up, UART_OMAP_TO_H);673 timeout_l = serial_in(up, UART_OMAP_TO_L);674 serial_out(up, UART_OMAP_TO_H, 0xFF);675 serial_out(up, UART_OMAP_TO_L, 0xFF);676 serial_out(up, UART_OMAP_EFR2, UART_OMAP_EFR2_TIMEOUT_BEHAVE);677 serial_in(up, UART_IIR);678 serial_out(up, UART_OMAP_EFR2, efr2);679 serial_out(up, UART_OMAP_TO_H, timeout_h);680 serial_out(up, UART_OMAP_TO_L, timeout_l);681 }682 683 /* Stop processing interrupts on input overrun */684 if ((lsr & UART_LSR_OE) && up->overrun_backoff_time_ms > 0) {685 unsigned long delay;686 687 /* Synchronize UART_IER access against the console. */688 uart_port_lock(port);689 up->ier = port->serial_in(port, UART_IER);690 if (up->ier & (UART_IER_RLSI | UART_IER_RDI)) {691 port->ops->stop_rx(port);692 } else {693 /* Keep restarting the timer until694 * the input overrun subsides.695 */696 cancel_delayed_work(&up->overrun_backoff);697 }698 uart_port_unlock(port);699 700 delay = msecs_to_jiffies(up->overrun_backoff_time_ms);701 schedule_delayed_work(&up->overrun_backoff, delay);702 }703 704 pm_runtime_mark_last_busy(port->dev);705 pm_runtime_put(port->dev);706 707 return IRQ_RETVAL(ret);708}709 710static int omap_8250_startup(struct uart_port *port)711{712 struct uart_8250_port *up = up_to_u8250p(port);713 struct omap8250_priv *priv = port->private_data;714 struct uart_8250_dma *dma = &priv->omap8250_dma;715 int ret;716 717 if (priv->wakeirq) {718 ret = dev_pm_set_dedicated_wake_irq(port->dev, priv->wakeirq);719 if (ret)720 return ret;721 }722 723 pm_runtime_get_sync(port->dev);724 725 serial_out(up, UART_FCR, UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT);726 727 serial_out(up, UART_LCR, UART_LCR_WLEN8);728 729 up->lsr_saved_flags = 0;730 up->msr_saved_flags = 0;731 732 /* Disable DMA for console UART */733 if (dma->fn && !uart_console(port)) {734 up->dma = &priv->omap8250_dma;735 ret = serial8250_request_dma(up);736 if (ret) {737 dev_warn_ratelimited(port->dev,738 "failed to request DMA\n");739 up->dma = NULL;740 }741 } else {742 up->dma = NULL;743 }744 745 /* Synchronize UART_IER access against the console. */746 uart_port_lock_irq(port);747 up->ier = UART_IER_RLSI | UART_IER_RDI;748 serial_out(up, UART_IER, up->ier);749 uart_port_unlock_irq(port);750 751#ifdef CONFIG_PM752 up->capabilities |= UART_CAP_RPM;753#endif754 755 /* Enable module level wake up */756 priv->wer = OMAP_UART_WER_MOD_WKUP;757 if (priv->habit & OMAP_UART_WER_HAS_TX_WAKEUP)758 priv->wer |= OMAP_UART_TX_WAKEUP_EN;759 serial_out(up, UART_OMAP_WER, priv->wer);760 761 if (up->dma && !(priv->habit & UART_HAS_EFR2)) {762 uart_port_lock_irq(port);763 up->dma->rx_dma(up);764 uart_port_unlock_irq(port);765 }766 767 enable_irq(up->port.irq);768 769 pm_runtime_mark_last_busy(port->dev);770 pm_runtime_put_autosuspend(port->dev);771 return 0;772}773 774static void omap_8250_shutdown(struct uart_port *port)775{776 struct uart_8250_port *up = up_to_u8250p(port);777 struct omap8250_priv *priv = port->private_data;778 779 flush_work(&priv->qos_work);780 if (up->dma)781 omap_8250_rx_dma_flush(up);782 783 pm_runtime_get_sync(port->dev);784 785 serial_out(up, UART_OMAP_WER, 0);786 if (priv->habit & UART_HAS_EFR2)787 serial_out(up, UART_OMAP_EFR2, 0x0);788 789 /* Synchronize UART_IER access against the console. */790 uart_port_lock_irq(port);791 up->ier = 0;792 serial_out(up, UART_IER, 0);793 uart_port_unlock_irq(port);794 disable_irq_nosync(up->port.irq);795 dev_pm_clear_wake_irq(port->dev);796 797 serial8250_release_dma(up);798 up->dma = NULL;799 800 /*801 * Disable break condition and FIFOs802 */803 if (up->lcr & UART_LCR_SBC)804 serial_out(up, UART_LCR, up->lcr & ~UART_LCR_SBC);805 serial_out(up, UART_FCR, UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT);806 807 pm_runtime_mark_last_busy(port->dev);808 pm_runtime_put_autosuspend(port->dev);809}810 811static void omap_8250_throttle(struct uart_port *port)812{813 struct omap8250_priv *priv = port->private_data;814 unsigned long flags;815 816 pm_runtime_get_sync(port->dev);817 818 uart_port_lock_irqsave(port, &flags);819 port->ops->stop_rx(port);820 priv->throttled = true;821 uart_port_unlock_irqrestore(port, flags);822 823 pm_runtime_mark_last_busy(port->dev);824 pm_runtime_put_autosuspend(port->dev);825}826 827static void omap_8250_unthrottle(struct uart_port *port)828{829 struct omap8250_priv *priv = port->private_data;830 struct uart_8250_port *up = up_to_u8250p(port);831 unsigned long flags;832 833 pm_runtime_get_sync(port->dev);834 835 /* Synchronize UART_IER access against the console. */836 uart_port_lock_irqsave(port, &flags);837 priv->throttled = false;838 if (up->dma)839 up->dma->rx_dma(up);840 up->ier |= UART_IER_RLSI | UART_IER_RDI;841 port->read_status_mask |= UART_LSR_DR;842 serial_out(up, UART_IER, up->ier);843 uart_port_unlock_irqrestore(port, flags);844 845 pm_runtime_mark_last_busy(port->dev);846 pm_runtime_put_autosuspend(port->dev);847}848 849static int omap8250_rs485_config(struct uart_port *port,850 struct ktermios *termios,851 struct serial_rs485 *rs485)852{853 struct omap8250_priv *priv = port->private_data;854 struct uart_8250_port *up = up_to_u8250p(port);855 u32 fixed_delay_rts_before_send = 0;856 u32 fixed_delay_rts_after_send = 0;857 unsigned int baud;858 859 /*860 * There is a fixed delay of 3 bit clock cycles after the TX shift861 * register is going empty to allow time for the stop bit to transition862 * through the transceiver before direction is changed to receive.863 *864 * Additionally there appears to be a 1 bit clock delay between writing865 * to the THR register and transmission of the start bit, per page 8783866 * of the AM65 TRM: https://www.ti.com/lit/ug/spruid7e/spruid7e.pdf867 */868 if (priv->quot) {869 if (priv->mdr1 == UART_OMAP_MDR1_16X_MODE)870 baud = port->uartclk / (16 * priv->quot);871 else872 baud = port->uartclk / (13 * priv->quot);873 874 fixed_delay_rts_after_send = 3 * MSEC_PER_SEC / baud;875 fixed_delay_rts_before_send = 1 * MSEC_PER_SEC / baud;876 }877 878 /*879 * Fall back to RS485 software emulation if the UART is missing880 * hardware support, if the device tree specifies an mctrl_gpio881 * (indicates that RTS is unavailable due to a pinmux conflict)882 * or if the requested delays exceed the fixed hardware delays.883 */884 if (!(priv->habit & UART_HAS_NATIVE_RS485) ||885 mctrl_gpio_to_gpiod(up->gpios, UART_GPIO_RTS) ||886 rs485->delay_rts_after_send > fixed_delay_rts_after_send ||887 rs485->delay_rts_before_send > fixed_delay_rts_before_send) {888 priv->mdr3 &= ~UART_OMAP_MDR3_DIR_EN;889 serial_out(up, UART_OMAP_MDR3, priv->mdr3);890 891 port->rs485_config = serial8250_em485_config;892 return serial8250_em485_config(port, termios, rs485);893 }894 895 rs485->delay_rts_after_send = fixed_delay_rts_after_send;896 rs485->delay_rts_before_send = fixed_delay_rts_before_send;897 898 if (rs485->flags & SER_RS485_ENABLED)899 priv->mdr3 |= UART_OMAP_MDR3_DIR_EN;900 else901 priv->mdr3 &= ~UART_OMAP_MDR3_DIR_EN;902 903 /*904 * Retain same polarity semantics as RS485 software emulation,905 * i.e. SER_RS485_RTS_ON_SEND means driving RTS low on send.906 */907 if (rs485->flags & SER_RS485_RTS_ON_SEND)908 priv->mdr3 &= ~UART_OMAP_MDR3_DIR_POL;909 else910 priv->mdr3 |= UART_OMAP_MDR3_DIR_POL;911 912 serial_out(up, UART_OMAP_MDR3, priv->mdr3);913 914 return 0;915}916 917#ifdef CONFIG_SERIAL_8250_DMA918static int omap_8250_rx_dma(struct uart_8250_port *p);919 920/* Must be called while priv->rx_dma_lock is held */921static void __dma_rx_do_complete(struct uart_8250_port *p)922{923 struct uart_8250_dma *dma = p->dma;924 struct tty_port *tty_port = &p->port.state->port;925 struct omap8250_priv *priv = p->port.private_data;926 struct dma_chan *rxchan = dma->rxchan;927 dma_cookie_t cookie;928 struct dma_tx_state state;929 int count;930 int ret;931 u32 reg;932 933 if (!dma->rx_running)934 goto out;935 936 cookie = dma->rx_cookie;937 dma->rx_running = 0;938 939 /* Re-enable RX FIFO interrupt now that transfer is complete */940 if (priv->habit & UART_HAS_RHR_IT_DIS) {941 reg = serial_in(p, UART_OMAP_IER2);942 reg &= ~UART_OMAP_IER2_RHR_IT_DIS;943 serial_out(p, UART_OMAP_IER2, reg);944 }945 946 dmaengine_tx_status(rxchan, cookie, &state);947 948 count = dma->rx_size - state.residue + state.in_flight_bytes;949 if (count < dma->rx_size) {950 dmaengine_terminate_async(rxchan);951 952 /*953 * Poll for teardown to complete which guarantees in954 * flight data is drained.955 */956 if (state.in_flight_bytes) {957 int poll_count = 25;958 959 while (dmaengine_tx_status(rxchan, cookie, NULL) &&960 poll_count--)961 cpu_relax();962 963 if (poll_count == -1)964 dev_err(p->port.dev, "teardown incomplete\n");965 }966 }967 if (!count)968 goto out;969 ret = tty_insert_flip_string(tty_port, dma->rx_buf, count);970 971 p->port.icount.rx += ret;972 p->port.icount.buf_overrun += count - ret;973out:974 975 tty_flip_buffer_push(tty_port);976}977 978static void __dma_rx_complete(void *param)979{980 struct uart_8250_port *p = param;981 struct omap8250_priv *priv = p->port.private_data;982 struct uart_8250_dma *dma = p->dma;983 struct dma_tx_state state;984 unsigned long flags;985 986 /* Synchronize UART_IER access against the console. */987 uart_port_lock_irqsave(&p->port, &flags);988 989 /*990 * If the tx status is not DMA_COMPLETE, then this is a delayed991 * completion callback. A previous RX timeout flush would have992 * already pushed the data, so exit.993 */994 if (dmaengine_tx_status(dma->rxchan, dma->rx_cookie, &state) !=995 DMA_COMPLETE) {996 uart_port_unlock_irqrestore(&p->port, flags);997 return;998 }999 __dma_rx_do_complete(p);1000 if (!priv->throttled) {1001 p->ier |= UART_IER_RLSI | UART_IER_RDI;1002 serial_out(p, UART_IER, p->ier);1003 if (!(priv->habit & UART_HAS_EFR2))1004 omap_8250_rx_dma(p);1005 }1006 1007 uart_port_unlock_irqrestore(&p->port, flags);1008}1009 1010static void omap_8250_rx_dma_flush(struct uart_8250_port *p)1011{1012 struct omap8250_priv *priv = p->port.private_data;1013 struct uart_8250_dma *dma = p->dma;1014 struct dma_tx_state state;1015 unsigned long flags;1016 int ret;1017 1018 spin_lock_irqsave(&priv->rx_dma_lock, flags);1019 1020 if (!dma->rx_running) {1021 spin_unlock_irqrestore(&priv->rx_dma_lock, flags);1022 return;1023 }1024 1025 ret = dmaengine_tx_status(dma->rxchan, dma->rx_cookie, &state);1026 if (ret == DMA_IN_PROGRESS) {1027 ret = dmaengine_pause(dma->rxchan);1028 if (WARN_ON_ONCE(ret))1029 priv->rx_dma_broken = true;1030 }1031 __dma_rx_do_complete(p);1032 spin_unlock_irqrestore(&priv->rx_dma_lock, flags);1033}1034 1035static int omap_8250_rx_dma(struct uart_8250_port *p)1036{1037 struct omap8250_priv *priv = p->port.private_data;1038 struct uart_8250_dma *dma = p->dma;1039 int err = 0;1040 struct dma_async_tx_descriptor *desc;1041 unsigned long flags;1042 u32 reg;1043 1044 /* Port locked to synchronize UART_IER access against the console. */1045 lockdep_assert_held_once(&p->port.lock);1046 1047 if (priv->rx_dma_broken)1048 return -EINVAL;1049 1050 spin_lock_irqsave(&priv->rx_dma_lock, flags);1051 1052 if (dma->rx_running) {1053 enum dma_status state;1054 1055 state = dmaengine_tx_status(dma->rxchan, dma->rx_cookie, NULL);1056 if (state == DMA_COMPLETE) {1057 /*1058 * Disable RX interrupts to allow RX DMA completion1059 * callback to run.1060 */1061 p->ier &= ~(UART_IER_RLSI | UART_IER_RDI);1062 serial_out(p, UART_IER, p->ier);1063 }1064 goto out;1065 }1066 1067 desc = dmaengine_prep_slave_single(dma->rxchan, dma->rx_addr,1068 dma->rx_size, DMA_DEV_TO_MEM,1069 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);1070 if (!desc) {1071 err = -EBUSY;1072 goto out;1073 }1074 1075 dma->rx_running = 1;1076 desc->callback = __dma_rx_complete;1077 desc->callback_param = p;1078 1079 dma->rx_cookie = dmaengine_submit(desc);1080 1081 /*1082 * Disable RX FIFO interrupt while RX DMA is enabled, else1083 * spurious interrupt may be raised when data is in the RX FIFO1084 * but is yet to be drained by DMA.1085 */1086 if (priv->habit & UART_HAS_RHR_IT_DIS) {1087 reg = serial_in(p, UART_OMAP_IER2);1088 reg |= UART_OMAP_IER2_RHR_IT_DIS;1089 serial_out(p, UART_OMAP_IER2, reg);1090 }1091 1092 dma_async_issue_pending(dma->rxchan);1093out:1094 spin_unlock_irqrestore(&priv->rx_dma_lock, flags);1095 return err;1096}1097 1098static int omap_8250_tx_dma(struct uart_8250_port *p);1099 1100static void omap_8250_dma_tx_complete(void *param)1101{1102 struct uart_8250_port *p = param;1103 struct uart_8250_dma *dma = p->dma;1104 struct tty_port *tport = &p->port.state->port;1105 unsigned long flags;1106 bool en_thri = false;1107 struct omap8250_priv *priv = p->port.private_data;1108 1109 dma_sync_single_for_cpu(dma->txchan->device->dev, dma->tx_addr,1110 UART_XMIT_SIZE, DMA_TO_DEVICE);1111 1112 uart_port_lock_irqsave(&p->port, &flags);1113 1114 dma->tx_running = 0;1115 1116 uart_xmit_advance(&p->port, dma->tx_size);1117 1118 if (priv->delayed_restore) {1119 priv->delayed_restore = 0;1120 omap8250_restore_regs(p);1121 }1122 1123 if (kfifo_len(&tport->xmit_fifo) < WAKEUP_CHARS)1124 uart_write_wakeup(&p->port);1125 1126 if (!kfifo_is_empty(&tport->xmit_fifo) && !uart_tx_stopped(&p->port)) {1127 int ret;1128 1129 ret = omap_8250_tx_dma(p);1130 if (ret)1131 en_thri = true;1132 } else if (p->capabilities & UART_CAP_RPM) {1133 en_thri = true;1134 }1135 1136 if (en_thri) {1137 dma->tx_err = 1;1138 serial8250_set_THRI(p);1139 }1140 1141 uart_port_unlock_irqrestore(&p->port, flags);1142}1143 1144static int omap_8250_tx_dma(struct uart_8250_port *p)1145{1146 struct uart_8250_dma *dma = p->dma;1147 struct omap8250_priv *priv = p->port.private_data;1148 struct tty_port *tport = &p->port.state->port;1149 struct dma_async_tx_descriptor *desc;1150 struct scatterlist sg;1151 int skip_byte = -1;1152 int ret;1153 1154 if (dma->tx_running)1155 return 0;1156 if (uart_tx_stopped(&p->port) || kfifo_is_empty(&tport->xmit_fifo)) {1157 1158 /*1159 * Even if no data, we need to return an error for the two cases1160 * below so serial8250_tx_chars() is invoked and properly clears1161 * THRI and/or runtime suspend.1162 */1163 if (dma->tx_err || p->capabilities & UART_CAP_RPM) {1164 ret = -EBUSY;1165 goto err;1166 }1167 serial8250_clear_THRI(p);1168 return 0;1169 }1170 1171 sg_init_table(&sg, 1);1172 ret = kfifo_dma_out_prepare_mapped(&tport->xmit_fifo, &sg, 1,1173 UART_XMIT_SIZE, dma->tx_addr);1174 if (ret != 1) {1175 serial8250_clear_THRI(p);1176 return 0;1177 }1178 1179 dma->tx_size = sg_dma_len(&sg);1180 1181 if (priv->habit & OMAP_DMA_TX_KICK) {1182 unsigned char c;1183 u8 tx_lvl;1184 1185 /*1186 * We need to put the first byte into the FIFO in order to start1187 * the DMA transfer. For transfers smaller than four bytes we1188 * don't bother doing DMA at all. It seem not matter if there1189 * are still bytes in the FIFO from the last transfer (in case1190 * we got here directly from omap_8250_dma_tx_complete()). Bytes1191 * leaving the FIFO seem not to trigger the DMA transfer. It is1192 * really the byte that we put into the FIFO.1193 * If the FIFO is already full then we most likely got here from1194 * omap_8250_dma_tx_complete(). And this means the DMA engine1195 * just completed its work. We don't have to wait the complete1196 * 86us at 115200,8n1 but around 60us (not to mention lower1197 * baudrates). So in that case we take the interrupt and try1198 * again with an empty FIFO.1199 */1200 tx_lvl = serial_in(p, UART_OMAP_TX_LVL);1201 if (tx_lvl == p->tx_loadsz) {1202 ret = -EBUSY;1203 goto err;1204 }1205 if (dma->tx_size < 4) {1206 ret = -EINVAL;1207 goto err;1208 }1209 if (!kfifo_get(&tport->xmit_fifo, &c)) {1210 ret = -EINVAL;1211 goto err;1212 }1213 skip_byte = c;1214 /* now we need to recompute due to kfifo_get */1215 kfifo_dma_out_prepare_mapped(&tport->xmit_fifo, &sg, 1,1216 UART_XMIT_SIZE, dma->tx_addr);1217 }1218 1219 desc = dmaengine_prep_slave_sg(dma->txchan, &sg, 1, DMA_MEM_TO_DEV,1220 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);1221 if (!desc) {1222 ret = -EBUSY;1223 goto err;1224 }1225 1226 dma->tx_running = 1;1227 1228 desc->callback = omap_8250_dma_tx_complete;1229 desc->callback_param = p;1230 1231 dma->tx_cookie = dmaengine_submit(desc);1232 1233 dma_sync_single_for_device(dma->txchan->device->dev, dma->tx_addr,1234 UART_XMIT_SIZE, DMA_TO_DEVICE);1235 1236 dma_async_issue_pending(dma->txchan);1237 if (dma->tx_err)1238 dma->tx_err = 0;1239 1240 serial8250_clear_THRI(p);1241 ret = 0;1242 goto out_skip;1243err:1244 dma->tx_err = 1;1245out_skip:1246 if (skip_byte >= 0)1247 serial_out(p, UART_TX, skip_byte);1248 return ret;1249}1250 1251static bool handle_rx_dma(struct uart_8250_port *up, unsigned int iir)1252{1253 switch (iir & 0x3f) {1254 case UART_IIR_RLSI:1255 case UART_IIR_RX_TIMEOUT:1256 case UART_IIR_RDI:1257 omap_8250_rx_dma_flush(up);1258 return true;1259 }1260 return omap_8250_rx_dma(up);1261}1262 1263static u16 omap_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, u16 status)1264{1265 if ((status & (UART_LSR_DR | UART_LSR_BI)) &&1266 (iir & UART_IIR_RDI)) {1267 if (handle_rx_dma(up, iir)) {1268 status = serial8250_rx_chars(up, status);1269 omap_8250_rx_dma(up);1270 }1271 }1272 1273 return status;1274}1275 1276static void am654_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir,1277 u16 status)1278{1279 /* Port locked to synchronize UART_IER access against the console. */1280 lockdep_assert_held_once(&up->port.lock);1281 1282 /*1283 * Queue a new transfer if FIFO has data.1284 */1285 if ((status & (UART_LSR_DR | UART_LSR_BI)) &&1286 (up->ier & UART_IER_RDI)) {1287 omap_8250_rx_dma(up);1288 serial_out(up, UART_OMAP_EFR2, UART_OMAP_EFR2_TIMEOUT_BEHAVE);1289 } else if ((iir & 0x3f) == UART_IIR_RX_TIMEOUT) {1290 /*1291 * Disable RX timeout, read IIR to clear1292 * current timeout condition, clear EFR2 to1293 * periodic timeouts, re-enable interrupts.1294 */1295 up->ier &= ~(UART_IER_RLSI | UART_IER_RDI);1296 serial_out(up, UART_IER, up->ier);1297 omap_8250_rx_dma_flush(up);1298 serial_in(up, UART_IIR);1299 serial_out(up, UART_OMAP_EFR2, 0x0);1300 up->ier |= UART_IER_RLSI | UART_IER_RDI;1301 serial_out(up, UART_IER, up->ier);1302 }1303}1304 1305/*1306 * This is mostly serial8250_handle_irq(). We have a slightly different DMA1307 * hoook for RX/TX and need different logic for them in the ISR. Therefore we1308 * use the default routine in the non-DMA case and this one for with DMA.1309 */1310static int omap_8250_dma_handle_irq(struct uart_port *port)1311{1312 struct uart_8250_port *up = up_to_u8250p(port);1313 struct omap8250_priv *priv = up->port.private_data;1314 u16 status;1315 u8 iir;1316 1317 iir = serial_port_in(port, UART_IIR);1318 if (iir & UART_IIR_NO_INT) {1319 return IRQ_HANDLED;1320 }1321 1322 uart_port_lock(port);1323 1324 status = serial_port_in(port, UART_LSR);1325 1326 if ((iir & 0x3f) != UART_IIR_THRI) {1327 if (priv->habit & UART_HAS_EFR2)1328 am654_8250_handle_rx_dma(up, iir, status);1329 else1330 status = omap_8250_handle_rx_dma(up, iir, status);1331 }1332 1333 serial8250_modem_status(up);1334 if (status & UART_LSR_THRE && up->dma->tx_err) {1335 if (uart_tx_stopped(&up->port) ||1336 kfifo_is_empty(&up->port.state->port.xmit_fifo)) {1337 up->dma->tx_err = 0;1338 serial8250_tx_chars(up);1339 } else {1340 /*1341 * try again due to an earlier failer which1342 * might have been resolved by now.1343 */1344 if (omap_8250_tx_dma(up))1345 serial8250_tx_chars(up);1346 }1347 }1348 1349 uart_unlock_and_check_sysrq(port);1350 1351 return 1;1352}1353 1354static bool the_no_dma_filter_fn(struct dma_chan *chan, void *param)1355{1356 return false;1357}1358 1359#else1360 1361static inline int omap_8250_rx_dma(struct uart_8250_port *p)1362{1363 return -EINVAL;1364}1365#endif1366 1367static int omap8250_no_handle_irq(struct uart_port *port)1368{1369 /* IRQ has not been requested but handling irq? */1370 WARN_ONCE(1, "Unexpected irq handling before port startup\n");1371 return 0;1372}1373 1374static struct omap8250_dma_params am654_dma = {1375 .rx_size = SZ_2K,1376 .rx_trigger = 1,1377 .tx_trigger = TX_TRIGGER,1378};1379 1380static struct omap8250_dma_params am33xx_dma = {1381 .rx_size = RX_TRIGGER,1382 .rx_trigger = RX_TRIGGER,1383 .tx_trigger = TX_TRIGGER,1384};1385 1386static struct omap8250_platdata am654_platdata = {1387 .dma_params = &am654_dma,1388 .habit = UART_HAS_EFR2 | UART_HAS_RHR_IT_DIS |1389 UART_RX_TIMEOUT_QUIRK | UART_HAS_NATIVE_RS485,1390};1391 1392static struct omap8250_platdata am33xx_platdata = {1393 .dma_params = &am33xx_dma,1394 .habit = OMAP_DMA_TX_KICK | UART_ERRATA_CLOCK_DISABLE,1395};1396 1397static struct omap8250_platdata omap4_platdata = {1398 .dma_params = &am33xx_dma,1399 .habit = UART_ERRATA_CLOCK_DISABLE,1400};1401 1402static const struct of_device_id omap8250_dt_ids[] = {1403 { .compatible = "ti,am654-uart", .data = &am654_platdata, },1404 { .compatible = "ti,omap2-uart" },1405 { .compatible = "ti,omap3-uart" },1406 { .compatible = "ti,omap4-uart", .data = &omap4_platdata, },1407 { .compatible = "ti,am3352-uart", .data = &am33xx_platdata, },1408 { .compatible = "ti,am4372-uart", .data = &am33xx_platdata, },1409 { .compatible = "ti,dra742-uart", .data = &omap4_platdata, },1410 {},1411};1412MODULE_DEVICE_TABLE(of, omap8250_dt_ids);1413 1414static int omap8250_probe(struct platform_device *pdev)1415{1416 struct device_node *np = pdev->dev.of_node;1417 struct omap8250_priv *priv;1418 const struct omap8250_platdata *pdata;1419 struct uart_8250_port up;1420 struct resource *regs;1421 void __iomem *membase;1422 int ret;1423 1424 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);1425 if (!regs) {1426 dev_err(&pdev->dev, "missing registers\n");1427 return -EINVAL;1428 }1429 1430 priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);1431 if (!priv)1432 return -ENOMEM;1433 1434 membase = devm_ioremap(&pdev->dev, regs->start,1435 resource_size(regs));1436 if (!membase)1437 return -ENODEV;1438 1439 memset(&up, 0, sizeof(up));1440 up.port.dev = &pdev->dev;1441 up.port.mapbase = regs->start;1442 up.port.membase = membase;1443 /*1444 * It claims to be 16C750 compatible however it is a little different.1445 * It has EFR and has no FCR7_64byte bit. The AFE (which it claims to1446 * have) is enabled via EFR instead of MCR. The type is set here 82501447 * just to get things going. UNKNOWN does not work for a few reasons and1448 * we don't need our own type since we don't use 8250's set_termios()1449 * or pm callback.1450 */1451 up.port.type = PORT_8250;1452 up.port.flags = UPF_FIXED_PORT | UPF_FIXED_TYPE | UPF_SOFT_FLOW | UPF_HARD_FLOW;1453 up.port.private_data = priv;1454 1455 up.tx_loadsz = 64;1456 up.capabilities = UART_CAP_FIFO;1457#ifdef CONFIG_PM1458 /*1459 * Runtime PM is mostly transparent. However to do it right we need to a1460 * TX empty interrupt before we can put the device to auto idle. So if1461 * PM is not enabled we don't add that flag and can spare that one extra1462 * interrupt in the TX path.1463 */1464 up.capabilities |= UART_CAP_RPM;1465#endif1466 up.port.set_termios = omap_8250_set_termios;1467 up.port.set_mctrl = omap8250_set_mctrl;1468 up.port.pm = omap_8250_pm;1469 up.port.startup = omap_8250_startup;1470 up.port.shutdown = omap_8250_shutdown;1471 up.port.throttle = omap_8250_throttle;1472 up.port.unthrottle = omap_8250_unthrottle;1473 up.port.rs485_config = omap8250_rs485_config;1474 /* same rs485_supported for software emulation and native RS485 */1475 up.port.rs485_supported = serial8250_em485_supported;1476 up.rs485_start_tx = serial8250_em485_start_tx;1477 up.rs485_stop_tx = serial8250_em485_stop_tx;1478 up.port.has_sysrq = IS_ENABLED(CONFIG_SERIAL_8250_CONSOLE);1479 1480 ret = uart_read_port_properties(&up.port);1481 if (ret)1482 return ret;1483 1484 up.port.regshift = OMAP_UART_REGSHIFT;1485 up.port.fifosize = 64;1486 1487 if (!up.port.uartclk) {1488 struct clk *clk;1489 1490 clk = devm_clk_get(&pdev->dev, NULL);1491 if (IS_ERR(clk)) {1492 if (PTR_ERR(clk) == -EPROBE_DEFER)1493 return -EPROBE_DEFER;1494 } else {1495 up.port.uartclk = clk_get_rate(clk);1496 }1497 }1498 1499 if (of_property_read_u32(np, "overrun-throttle-ms",1500 &up.overrun_backoff_time_ms) != 0)1501 up.overrun_backoff_time_ms = 0;1502 1503 pdata = of_device_get_match_data(&pdev->dev);1504 if (pdata)1505 priv->habit |= pdata->habit;1506 1507 if (!up.port.uartclk) {1508 up.port.uartclk = DEFAULT_CLK_SPEED;1509 dev_warn(&pdev->dev,1510 "No clock speed specified: using default: %d\n",1511 DEFAULT_CLK_SPEED);1512 }1513 1514 priv->membase = membase;1515 priv->line = -ENODEV;1516 priv->latency = PM_QOS_CPU_LATENCY_DEFAULT_VALUE;1517 priv->calc_latency = PM_QOS_CPU_LATENCY_DEFAULT_VALUE;1518 cpu_latency_qos_add_request(&priv->pm_qos_request, priv->latency);1519 INIT_WORK(&priv->qos_work, omap8250_uart_qos_work);1520 1521 spin_lock_init(&priv->rx_dma_lock);1522 1523 platform_set_drvdata(pdev, priv);1524 1525 device_set_wakeup_capable(&pdev->dev, true);1526 if (of_property_read_bool(np, "wakeup-source"))1527 device_set_wakeup_enable(&pdev->dev, true);1528 1529 pm_runtime_enable(&pdev->dev);1530 pm_runtime_use_autosuspend(&pdev->dev);1531 1532 /*1533 * Disable runtime PM until autosuspend delay unless specifically1534 * enabled by the user via sysfs. This is the historic way to1535 * prevent an unsafe default policy with lossy characters on wake-up.1536 * For serdev devices this is not needed, the policy can be managed by1537 * the serdev driver.1538 */1539 if (!of_get_available_child_count(pdev->dev.of_node))1540 pm_runtime_set_autosuspend_delay(&pdev->dev, -1);1541 1542 pm_runtime_get_sync(&pdev->dev);1543 1544 omap_serial_fill_features_erratas(&up, priv);1545 up.port.handle_irq = omap8250_no_handle_irq;1546 priv->rx_trigger = RX_TRIGGER;1547 priv->tx_trigger = TX_TRIGGER;1548#ifdef CONFIG_SERIAL_8250_DMA1549 /*1550 * Oh DMA support. If there are no DMA properties in the DT then1551 * we will fall back to a generic DMA channel which does not1552 * really work here. To ensure that we do not get a generic DMA1553 * channel assigned, we have the the_no_dma_filter_fn() here.1554 * To avoid "failed to request DMA" messages we check for DMA1555 * properties in DT.1556 */1557 ret = of_property_count_strings(np, "dma-names");1558 if (ret == 2) {1559 struct omap8250_dma_params *dma_params = NULL;1560 struct uart_8250_dma *dma = &priv->omap8250_dma;1561 1562 dma->fn = the_no_dma_filter_fn;1563 dma->tx_dma = omap_8250_tx_dma;1564 dma->rx_dma = omap_8250_rx_dma;1565 if (pdata)1566 dma_params = pdata->dma_params;1567 1568 if (dma_params) {1569 dma->rx_size = dma_params->rx_size;1570 dma->rxconf.src_maxburst = dma_params->rx_trigger;1571 dma->txconf.dst_maxburst = dma_params->tx_trigger;1572 priv->rx_trigger = dma_params->rx_trigger;1573 priv->tx_trigger = dma_params->tx_trigger;1574 } else {1575 dma->rx_size = RX_TRIGGER;1576 dma->rxconf.src_maxburst = RX_TRIGGER;1577 dma->txconf.dst_maxburst = TX_TRIGGER;1578 }1579 }1580#endif1581 1582 irq_set_status_flags(up.port.irq, IRQ_NOAUTOEN);1583 ret = devm_request_irq(&pdev->dev, up.port.irq, omap8250_irq, 0,1584 dev_name(&pdev->dev), priv);1585 if (ret < 0)1586 goto err;1587 1588 priv->wakeirq = irq_of_parse_and_map(np, 1);1589 1590 ret = serial8250_register_8250_port(&up);1591 if (ret < 0) {1592 dev_err(&pdev->dev, "unable to register 8250 port\n");1593 goto err;1594 }1595 priv->line = ret;1596 pm_runtime_mark_last_busy(&pdev->dev);1597 pm_runtime_put_autosuspend(&pdev->dev);1598 return 0;1599err:1600 pm_runtime_dont_use_autosuspend(&pdev->dev);1601 pm_runtime_put_sync(&pdev->dev);1602 flush_work(&priv->qos_work);1603 pm_runtime_disable(&pdev->dev);1604 cpu_latency_qos_remove_request(&priv->pm_qos_request);1605 return ret;1606}1607 1608static void omap8250_remove(struct platform_device *pdev)1609{1610 struct omap8250_priv *priv = platform_get_drvdata(pdev);1611 struct uart_8250_port *up;1612 int err;1613 1614 err = pm_runtime_resume_and_get(&pdev->dev);1615 if (err)1616 dev_err(&pdev->dev, "Failed to resume hardware\n");1617 1618 up = serial8250_get_port(priv->line);1619 omap_8250_shutdown(&up->port);1620 serial8250_unregister_port(priv->line);1621 priv->line = -ENODEV;1622 pm_runtime_dont_use_autosuspend(&pdev->dev);1623 pm_runtime_put_sync(&pdev->dev);1624 flush_work(&priv->qos_work);1625 pm_runtime_disable(&pdev->dev);1626 cpu_latency_qos_remove_request(&priv->pm_qos_request);1627 device_set_wakeup_capable(&pdev->dev, false);1628}1629 1630static int omap8250_prepare(struct device *dev)1631{1632 struct omap8250_priv *priv = dev_get_drvdata(dev);1633 1634 if (!priv)1635 return 0;1636 priv->is_suspending = true;1637 return 0;1638}1639 1640static void omap8250_complete(struct device *dev)1641{1642 struct omap8250_priv *priv = dev_get_drvdata(dev);1643 1644 if (!priv)1645 return;1646 priv->is_suspending = false;1647}1648 1649static int omap8250_suspend(struct device *dev)1650{1651 struct omap8250_priv *priv = dev_get_drvdata(dev);1652 struct uart_8250_port *up = serial8250_get_port(priv->line);1653 int err = 0;1654 1655 serial8250_suspend_port(priv->line);1656 1657 err = pm_runtime_resume_and_get(dev);1658 if (err)1659 return err;1660 if (!device_may_wakeup(dev))1661 priv->wer = 0;1662 serial_out(up, UART_OMAP_WER, priv->wer);1663 if (uart_console(&up->port) && console_suspend_enabled)1664 err = pm_runtime_force_suspend(dev);1665 flush_work(&priv->qos_work);1666 1667 return err;1668}1669 1670static int omap8250_resume(struct device *dev)1671{1672 struct omap8250_priv *priv = dev_get_drvdata(dev);1673 struct uart_8250_port *up = serial8250_get_port(priv->line);1674 int err;1675 1676 if (uart_console(&up->port) && console_suspend_enabled) {1677 err = pm_runtime_force_resume(dev);1678 if (err)1679 return err;1680 }1681 1682 serial8250_resume_port(priv->line);1683 /* Paired with pm_runtime_resume_and_get() in omap8250_suspend() */1684 pm_runtime_mark_last_busy(dev);1685 pm_runtime_put_autosuspend(dev);1686 1687 return 0;1688}1689 1690static int omap8250_lost_context(struct uart_8250_port *up)1691{1692 u32 val;1693 1694 val = serial_in(up, UART_OMAP_SCR);1695 /*1696 * If we lose context, then SCR is set to its reset value of zero.1697 * After set_termios() we set bit 3 of SCR (TX_EMPTY_CTL_IT) to 1,1698 * among other bits, to never set the register back to zero again.1699 */1700 if (!val)1701 return 1;1702 return 0;1703}1704 1705static void uart_write(struct omap8250_priv *priv, u32 reg, u32 val)1706{1707 writel(val, priv->membase + (reg << OMAP_UART_REGSHIFT));1708}1709 1710/* TODO: in future, this should happen via API in drivers/reset/ */1711static int omap8250_soft_reset(struct device *dev)1712{1713 struct omap8250_priv *priv = dev_get_drvdata(dev);1714 int timeout = 100;1715 int sysc;1716 int syss;1717 1718 /*1719 * At least on omap4, unused uarts may not idle after reset without1720 * a basic scr dma configuration even with no dma in use. The1721 * module clkctrl status bits will be 1 instead of 3 blocking idle1722 * for the whole clockdomain. The softreset below will clear scr,1723 * and we restore it on resume so this is safe to do on all SoCs1724 * needing omap8250_soft_reset() quirk. Do it in two writes as1725 * recommended in the comment for omap8250_update_scr().1726 */1727 uart_write(priv, UART_OMAP_SCR, OMAP_UART_SCR_DMAMODE_1);1728 uart_write(priv, UART_OMAP_SCR,1729 OMAP_UART_SCR_DMAMODE_1 | OMAP_UART_SCR_DMAMODE_CTL);1730 1731 sysc = uart_read(priv, UART_OMAP_SYSC);1732 1733 /* softreset the UART */1734 sysc |= OMAP_UART_SYSC_SOFTRESET;1735 uart_write(priv, UART_OMAP_SYSC, sysc);1736 1737 /* By experiments, 1us enough for reset complete on AM335x */1738 do {1739 udelay(1);1740 syss = uart_read(priv, UART_OMAP_SYSS);1741 } while (--timeout && !(syss & OMAP_UART_SYSS_RESETDONE));1742 1743 if (!timeout) {1744 dev_err(dev, "timed out waiting for reset done\n");1745 return -ETIMEDOUT;1746 }1747 1748 return 0;1749}1750 1751static int omap8250_runtime_suspend(struct device *dev)1752{1753 struct omap8250_priv *priv = dev_get_drvdata(dev);1754 struct uart_8250_port *up = NULL;1755 1756 if (priv->line >= 0)1757 up = serial8250_get_port(priv->line);1758 1759 if (priv->habit & UART_ERRATA_CLOCK_DISABLE) {1760 int ret;1761 1762 ret = omap8250_soft_reset(dev);1763 if (ret)1764 return ret;1765 1766 if (up) {1767 /* Restore to UART mode after reset (for wakeup) */1768 omap8250_update_mdr1(up, priv);1769 /* Restore wakeup enable register */1770 serial_out(up, UART_OMAP_WER, priv->wer);1771 }1772 }1773 1774 if (up && up->dma && up->dma->rxchan)1775 omap_8250_rx_dma_flush(up);1776 1777 priv->latency = PM_QOS_CPU_LATENCY_DEFAULT_VALUE;1778 schedule_work(&priv->qos_work);1779 atomic_set(&priv->active, 0);1780 1781 return 0;1782}1783 1784static int omap8250_runtime_resume(struct device *dev)1785{1786 struct omap8250_priv *priv = dev_get_drvdata(dev);1787 struct uart_8250_port *up = NULL;1788 1789 /* Did the hardware wake to a device IO interrupt before a wakeirq? */1790 if (atomic_read(&priv->active))1791 return 0;1792 1793 if (priv->line >= 0)1794 up = serial8250_get_port(priv->line);1795 1796 if (up && omap8250_lost_context(up)) {1797 uart_port_lock_irq(&up->port);1798 omap8250_restore_regs(up);1799 uart_port_unlock_irq(&up->port);1800 }1801 1802 if (up && up->dma && up->dma->rxchan && !(priv->habit & UART_HAS_EFR2)) {1803 uart_port_lock_irq(&up->port);1804 omap_8250_rx_dma(up);1805 uart_port_unlock_irq(&up->port);1806 }1807 1808 atomic_set(&priv->active, 1);1809 priv->latency = priv->calc_latency;1810 schedule_work(&priv->qos_work);1811 1812 return 0;1813}1814 1815#ifdef CONFIG_SERIAL_8250_OMAP_TTYO_FIXUP1816static int __init omap8250_console_fixup(void)1817{1818 char *omap_str;1819 char *options;1820 u8 idx;1821 1822 if (strstr(boot_command_line, "console=ttyS"))1823 /* user set a ttyS based name for the console */1824 return 0;1825 1826 omap_str = strstr(boot_command_line, "console=ttyO");1827 if (!omap_str)1828 /* user did not set ttyO based console, so we don't care */1829 return 0;1830 1831 omap_str += 12;1832 if ('0' <= *omap_str && *omap_str <= '9')1833 idx = *omap_str - '0';1834 else1835 return 0;1836 1837 omap_str++;1838 if (omap_str[0] == ',') {1839 omap_str++;1840 options = omap_str;1841 } else {1842 options = NULL;1843 }1844 1845 add_preferred_console("ttyS", idx, options);1846 pr_err("WARNING: Your 'console=ttyO%d' has been replaced by 'ttyS%d'\n",1847 idx, idx);1848 pr_err("This ensures that you still see kernel messages. Please\n");1849 pr_err("update your kernel commandline.\n");1850 return 0;1851}1852console_initcall(omap8250_console_fixup);1853#endif1854 1855static const struct dev_pm_ops omap8250_dev_pm_ops = {1856 SYSTEM_SLEEP_PM_OPS(omap8250_suspend, omap8250_resume)1857 RUNTIME_PM_OPS(omap8250_runtime_suspend,1858 omap8250_runtime_resume, NULL)1859 .prepare = pm_sleep_ptr(omap8250_prepare),1860 .complete = pm_sleep_ptr(omap8250_complete),1861};1862 1863static struct platform_driver omap8250_platform_driver = {1864 .driver = {1865 .name = "omap8250",1866 .pm = pm_ptr(&omap8250_dev_pm_ops),1867 .of_match_table = omap8250_dt_ids,1868 },1869 .probe = omap8250_probe,1870 .remove_new = omap8250_remove,1871};1872module_platform_driver(omap8250_platform_driver);1873 1874MODULE_AUTHOR("Sebastian Andrzej Siewior");1875MODULE_DESCRIPTION("OMAP 8250 Driver");1876MODULE_LICENSE("GPL v2");1877