1236 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright (c) 2020, Broadcom */3/*4 * 8250-core based driver for Broadcom ns16550a UARTs5 *6 * This driver uses the standard 8250 driver core but adds additional7 * optional features including the ability to use a baud rate clock8 * mux for more accurate high speed baud rate selection and also9 * an optional DMA engine.10 *11 */12 13#include <linux/module.h>14#include <linux/types.h>15#include <linux/tty.h>16#include <linux/errno.h>17#include <linux/device.h>18#include <linux/io.h>19#include <linux/of.h>20#include <linux/dma-mapping.h>21#include <linux/tty_flip.h>22#include <linux/delay.h>23#include <linux/clk.h>24#include <linux/debugfs.h>25#include <linux/units.h>26 27#include "8250.h"28 29/* Register definitions for UART DMA block. Version 1.1 or later. */30#define UDMA_ARB_RX 0x0031#define UDMA_ARB_TX 0x0432#define UDMA_ARB_REQ 0x0000000133#define UDMA_ARB_GRANT 0x0000000234 35#define UDMA_RX_REVISION 0x0036#define UDMA_RX_REVISION_REQUIRED 0x0000010137#define UDMA_RX_CTRL 0x0438#define UDMA_RX_CTRL_BUF_CLOSE_MODE 0x0001000039#define UDMA_RX_CTRL_MASK_WR_DONE 0x0000800040#define UDMA_RX_CTRL_ENDIAN_OVERRIDE 0x0000400041#define UDMA_RX_CTRL_ENDIAN 0x0000200042#define UDMA_RX_CTRL_OE_IS_ERR 0x0000100043#define UDMA_RX_CTRL_PE_IS_ERR 0x0000080044#define UDMA_RX_CTRL_FE_IS_ERR 0x0000040045#define UDMA_RX_CTRL_NUM_BUF_USED_MASK 0x000003c046#define UDMA_RX_CTRL_NUM_BUF_USED_SHIFT 647#define UDMA_RX_CTRL_BUF_CLOSE_CLK_SEL_SYS 0x0000002048#define UDMA_RX_CTRL_BUF_CLOSE_ENA 0x0000001049#define UDMA_RX_CTRL_TIMEOUT_CLK_SEL_SYS 0x0000000850#define UDMA_RX_CTRL_TIMEOUT_ENA 0x0000000451#define UDMA_RX_CTRL_ABORT 0x0000000252#define UDMA_RX_CTRL_ENA 0x0000000153#define UDMA_RX_STATUS 0x0854#define UDMA_RX_STATUS_ACTIVE_BUF_MASK 0x0000000f55#define UDMA_RX_TRANSFER_LEN 0x0c56#define UDMA_RX_TRANSFER_TOTAL 0x1057#define UDMA_RX_BUFFER_SIZE 0x1458#define UDMA_RX_SRC_ADDR 0x1859#define UDMA_RX_TIMEOUT 0x1c60#define UDMA_RX_BUFFER_CLOSE 0x2061#define UDMA_RX_BLOCKOUT_COUNTER 0x2462#define UDMA_RX_BUF0_PTR_LO 0x2863#define UDMA_RX_BUF0_PTR_HI 0x2c64#define UDMA_RX_BUF0_STATUS 0x3065#define UDMA_RX_BUFX_STATUS_OVERRUN_ERR 0x0000001066#define UDMA_RX_BUFX_STATUS_FRAME_ERR 0x0000000867#define UDMA_RX_BUFX_STATUS_PARITY_ERR 0x0000000468#define UDMA_RX_BUFX_STATUS_CLOSE_EXPIRED 0x0000000269#define UDMA_RX_BUFX_STATUS_DATA_RDY 0x0000000170#define UDMA_RX_BUF0_DATA_LEN 0x3471#define UDMA_RX_BUF1_PTR_LO 0x3872#define UDMA_RX_BUF1_PTR_HI 0x3c73#define UDMA_RX_BUF1_STATUS 0x4074#define UDMA_RX_BUF1_DATA_LEN 0x4475 76#define UDMA_TX_REVISION 0x0077#define UDMA_TX_REVISION_REQUIRED 0x0000010178#define UDMA_TX_CTRL 0x0479#define UDMA_TX_CTRL_ENDIAN_OVERRIDE 0x0000008080#define UDMA_TX_CTRL_ENDIAN 0x0000004081#define UDMA_TX_CTRL_NUM_BUF_USED_MASK 0x0000003082#define UDMA_TX_CTRL_NUM_BUF_USED_1 0x0000001083#define UDMA_TX_CTRL_ABORT 0x0000000284#define UDMA_TX_CTRL_ENA 0x0000000185#define UDMA_TX_DST_ADDR 0x0886#define UDMA_TX_BLOCKOUT_COUNTER 0x1087#define UDMA_TX_TRANSFER_LEN 0x1488#define UDMA_TX_TRANSFER_TOTAL 0x1889#define UDMA_TX_STATUS 0x2090#define UDMA_TX_BUF0_PTR_LO 0x2491#define UDMA_TX_BUF0_PTR_HI 0x2892#define UDMA_TX_BUF0_STATUS 0x2c93#define UDMA_TX_BUFX_LAST 0x0000000294#define UDMA_TX_BUFX_EMPTY 0x0000000195#define UDMA_TX_BUF0_DATA_LEN 0x3096#define UDMA_TX_BUF0_DATA_SENT 0x3497#define UDMA_TX_BUF1_PTR_LO 0x3898 99#define UDMA_INTR_STATUS 0x00100#define UDMA_INTR_ARB_TX_GRANT 0x00040000101#define UDMA_INTR_ARB_RX_GRANT 0x00020000102#define UDMA_INTR_TX_ALL_EMPTY 0x00010000103#define UDMA_INTR_TX_EMPTY_BUF1 0x00008000104#define UDMA_INTR_TX_EMPTY_BUF0 0x00004000105#define UDMA_INTR_TX_ABORT 0x00002000106#define UDMA_INTR_TX_DONE 0x00001000107#define UDMA_INTR_RX_ERROR 0x00000800108#define UDMA_INTR_RX_TIMEOUT 0x00000400109#define UDMA_INTR_RX_READY_BUF7 0x00000200110#define UDMA_INTR_RX_READY_BUF6 0x00000100111#define UDMA_INTR_RX_READY_BUF5 0x00000080112#define UDMA_INTR_RX_READY_BUF4 0x00000040113#define UDMA_INTR_RX_READY_BUF3 0x00000020114#define UDMA_INTR_RX_READY_BUF2 0x00000010115#define UDMA_INTR_RX_READY_BUF1 0x00000008116#define UDMA_INTR_RX_READY_BUF0 0x00000004117#define UDMA_INTR_RX_READY_MASK 0x000003fc118#define UDMA_INTR_RX_READY_SHIFT 2119#define UDMA_INTR_RX_ABORT 0x00000002120#define UDMA_INTR_RX_DONE 0x00000001121#define UDMA_INTR_SET 0x04122#define UDMA_INTR_CLEAR 0x08123#define UDMA_INTR_MASK_STATUS 0x0c124#define UDMA_INTR_MASK_SET 0x10125#define UDMA_INTR_MASK_CLEAR 0x14126 127 128#define UDMA_RX_INTERRUPTS ( \129 UDMA_INTR_RX_ERROR | \130 UDMA_INTR_RX_TIMEOUT | \131 UDMA_INTR_RX_READY_BUF0 | \132 UDMA_INTR_RX_READY_BUF1 | \133 UDMA_INTR_RX_READY_BUF2 | \134 UDMA_INTR_RX_READY_BUF3 | \135 UDMA_INTR_RX_READY_BUF4 | \136 UDMA_INTR_RX_READY_BUF5 | \137 UDMA_INTR_RX_READY_BUF6 | \138 UDMA_INTR_RX_READY_BUF7 | \139 UDMA_INTR_RX_ABORT | \140 UDMA_INTR_RX_DONE)141 142#define UDMA_RX_ERR_INTERRUPTS ( \143 UDMA_INTR_RX_ERROR | \144 UDMA_INTR_RX_TIMEOUT | \145 UDMA_INTR_RX_ABORT | \146 UDMA_INTR_RX_DONE)147 148#define UDMA_TX_INTERRUPTS ( \149 UDMA_INTR_TX_ABORT | \150 UDMA_INTR_TX_DONE)151 152#define UDMA_IS_RX_INTERRUPT(status) ((status) & UDMA_RX_INTERRUPTS)153#define UDMA_IS_TX_INTERRUPT(status) ((status) & UDMA_TX_INTERRUPTS)154 155 156/* Current devices have 8 sets of RX buffer registers */157#define UDMA_RX_BUFS_COUNT 8158#define UDMA_RX_BUFS_REG_OFFSET (UDMA_RX_BUF1_PTR_LO - UDMA_RX_BUF0_PTR_LO)159#define UDMA_RX_BUFx_PTR_LO(x) (UDMA_RX_BUF0_PTR_LO + \160 ((x) * UDMA_RX_BUFS_REG_OFFSET))161#define UDMA_RX_BUFx_PTR_HI(x) (UDMA_RX_BUF0_PTR_HI + \162 ((x) * UDMA_RX_BUFS_REG_OFFSET))163#define UDMA_RX_BUFx_STATUS(x) (UDMA_RX_BUF0_STATUS + \164 ((x) * UDMA_RX_BUFS_REG_OFFSET))165#define UDMA_RX_BUFx_DATA_LEN(x) (UDMA_RX_BUF0_DATA_LEN + \166 ((x) * UDMA_RX_BUFS_REG_OFFSET))167 168/* Current devices have 2 sets of TX buffer registers */169#define UDMA_TX_BUFS_COUNT 2170#define UDMA_TX_BUFS_REG_OFFSET (UDMA_TX_BUF1_PTR_LO - UDMA_TX_BUF0_PTR_LO)171#define UDMA_TX_BUFx_PTR_LO(x) (UDMA_TX_BUF0_PTR_LO + \172 ((x) * UDMA_TX_BUFS_REG_OFFSET))173#define UDMA_TX_BUFx_PTR_HI(x) (UDMA_TX_BUF0_PTR_HI + \174 ((x) * UDMA_TX_BUFS_REG_OFFSET))175#define UDMA_TX_BUFx_STATUS(x) (UDMA_TX_BUF0_STATUS + \176 ((x) * UDMA_TX_BUFS_REG_OFFSET))177#define UDMA_TX_BUFx_DATA_LEN(x) (UDMA_TX_BUF0_DATA_LEN + \178 ((x) * UDMA_TX_BUFS_REG_OFFSET))179#define UDMA_TX_BUFx_DATA_SENT(x) (UDMA_TX_BUF0_DATA_SENT + \180 ((x) * UDMA_TX_BUFS_REG_OFFSET))181#define REGS_8250 0182#define REGS_DMA_RX 1183#define REGS_DMA_TX 2184#define REGS_DMA_ISR 3185#define REGS_DMA_ARB 4186#define REGS_MAX 5187 188#define TX_BUF_SIZE 4096189#define RX_BUF_SIZE 4096190#define RX_BUFS_COUNT 2191 192static const u32 brcmstb_rate_table[] = {193 81 * HZ_PER_MHZ,194 108 * HZ_PER_MHZ,195 64 * HZ_PER_MHZ, /* Actually 64285715 for some chips */196 48 * HZ_PER_MHZ,197};198 199static const u32 brcmstb_rate_table_7278[] = {200 81 * HZ_PER_MHZ,201 108 * HZ_PER_MHZ,202 0,203 48 * HZ_PER_MHZ,204};205 206struct brcmuart_priv {207 int line;208 struct clk *baud_mux_clk;209 unsigned long default_mux_rate;210 u32 real_rates[ARRAY_SIZE(brcmstb_rate_table)];211 const u32 *rate_table;212 ktime_t char_wait;213 struct uart_port *up;214 struct hrtimer hrt;215 bool shutdown;216 bool dma_enabled;217 struct uart_8250_dma dma;218 void __iomem *regs[REGS_MAX];219 dma_addr_t rx_addr;220 void *rx_bufs;221 size_t rx_size;222 int rx_next_buf;223 dma_addr_t tx_addr;224 void *tx_buf;225 size_t tx_size;226 bool tx_running;227 bool rx_running;228 struct dentry *debugfs_dir;229 230 /* stats exposed through debugfs */231 u64 dma_rx_partial_buf;232 u64 dma_rx_full_buf;233 u32 rx_bad_timeout_late_char;234 u32 rx_bad_timeout_no_char;235 u32 rx_missing_close_timeout;236 u32 rx_err;237 u32 rx_timeout;238 u32 rx_abort;239 u32 saved_mctrl;240};241 242static struct dentry *brcmuart_debugfs_root;243 244/*245 * Register access routines246 */247static u32 udma_readl(struct brcmuart_priv *priv,248 int reg_type, int offset)249{250 return readl(priv->regs[reg_type] + offset);251}252 253static void udma_writel(struct brcmuart_priv *priv,254 int reg_type, int offset, u32 value)255{256 writel(value, priv->regs[reg_type] + offset);257}258 259static void udma_set(struct brcmuart_priv *priv,260 int reg_type, int offset, u32 bits)261{262 void __iomem *reg = priv->regs[reg_type] + offset;263 u32 value;264 265 value = readl(reg);266 value |= bits;267 writel(value, reg);268}269 270static void udma_unset(struct brcmuart_priv *priv,271 int reg_type, int offset, u32 bits)272{273 void __iomem *reg = priv->regs[reg_type] + offset;274 u32 value;275 276 value = readl(reg);277 value &= ~bits;278 writel(value, reg);279}280 281/*282 * The UART DMA engine hardware can be used by multiple UARTS, but283 * only one at a time. Sharing is not currently supported so284 * the first UART to request the DMA engine will get it and any285 * subsequent requests by other UARTS will fail.286 */287static int brcmuart_arbitration(struct brcmuart_priv *priv, bool acquire)288{289 u32 rx_grant;290 u32 tx_grant;291 int waits;292 int ret = 0;293 294 if (acquire) {295 udma_set(priv, REGS_DMA_ARB, UDMA_ARB_RX, UDMA_ARB_REQ);296 udma_set(priv, REGS_DMA_ARB, UDMA_ARB_TX, UDMA_ARB_REQ);297 298 waits = 1;299 while (1) {300 rx_grant = udma_readl(priv, REGS_DMA_ARB, UDMA_ARB_RX);301 tx_grant = udma_readl(priv, REGS_DMA_ARB, UDMA_ARB_TX);302 if (rx_grant & tx_grant & UDMA_ARB_GRANT)303 return 0;304 if (waits-- == 0)305 break;306 msleep(1);307 }308 ret = 1;309 }310 311 udma_unset(priv, REGS_DMA_ARB, UDMA_ARB_RX, UDMA_ARB_REQ);312 udma_unset(priv, REGS_DMA_ARB, UDMA_ARB_TX, UDMA_ARB_REQ);313 return ret;314}315 316static void brcmuart_init_dma_hardware(struct brcmuart_priv *priv)317{318 u32 daddr;319 u32 value;320 int x;321 322 /* Start with all interrupts disabled */323 udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_SET, 0xffffffff);324 325 udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFFER_SIZE, RX_BUF_SIZE);326 327 /*328 * Setup buffer close to happen when 32 character times have329 * elapsed since the last character was received.330 */331 udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFFER_CLOSE, 16*10*32);332 value = (RX_BUFS_COUNT << UDMA_RX_CTRL_NUM_BUF_USED_SHIFT)333 | UDMA_RX_CTRL_BUF_CLOSE_MODE334 | UDMA_RX_CTRL_BUF_CLOSE_ENA;335 udma_writel(priv, REGS_DMA_RX, UDMA_RX_CTRL, value);336 337 udma_writel(priv, REGS_DMA_RX, UDMA_RX_BLOCKOUT_COUNTER, 0);338 daddr = priv->rx_addr;339 for (x = 0; x < RX_BUFS_COUNT; x++) {340 341 /* Set RX transfer length to 0 for unknown */342 udma_writel(priv, REGS_DMA_RX, UDMA_RX_TRANSFER_LEN, 0);343 344 udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFx_PTR_LO(x),345 lower_32_bits(daddr));346 udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFx_PTR_HI(x),347 upper_32_bits(daddr));348 daddr += RX_BUF_SIZE;349 }350 351 daddr = priv->tx_addr;352 udma_writel(priv, REGS_DMA_TX, UDMA_TX_BUFx_PTR_LO(0),353 lower_32_bits(daddr));354 udma_writel(priv, REGS_DMA_TX, UDMA_TX_BUFx_PTR_HI(0),355 upper_32_bits(daddr));356 udma_writel(priv, REGS_DMA_TX, UDMA_TX_CTRL,357 UDMA_TX_CTRL_NUM_BUF_USED_1);358 359 /* clear all interrupts then enable them */360 udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_CLEAR, 0xffffffff);361 udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_CLEAR,362 UDMA_RX_INTERRUPTS | UDMA_TX_INTERRUPTS);363 364}365 366static void start_rx_dma(struct uart_8250_port *p)367{368 struct brcmuart_priv *priv = p->port.private_data;369 int x;370 371 udma_unset(priv, REGS_DMA_RX, UDMA_RX_CTRL, UDMA_RX_CTRL_ENA);372 373 /* Clear the RX ready bit for all buffers */374 for (x = 0; x < RX_BUFS_COUNT; x++)375 udma_unset(priv, REGS_DMA_RX, UDMA_RX_BUFx_STATUS(x),376 UDMA_RX_BUFX_STATUS_DATA_RDY);377 378 /* always start with buffer 0 */379 udma_unset(priv, REGS_DMA_RX, UDMA_RX_STATUS,380 UDMA_RX_STATUS_ACTIVE_BUF_MASK);381 priv->rx_next_buf = 0;382 383 udma_set(priv, REGS_DMA_RX, UDMA_RX_CTRL, UDMA_RX_CTRL_ENA);384 priv->rx_running = true;385}386 387static void stop_rx_dma(struct uart_8250_port *p)388{389 struct brcmuart_priv *priv = p->port.private_data;390 391 /* If RX is running, set the RX ABORT */392 if (priv->rx_running)393 udma_set(priv, REGS_DMA_RX, UDMA_RX_CTRL, UDMA_RX_CTRL_ABORT);394}395 396static int stop_tx_dma(struct uart_8250_port *p)397{398 struct brcmuart_priv *priv = p->port.private_data;399 u32 value;400 401 /* If TX is running, set the TX ABORT */402 value = udma_readl(priv, REGS_DMA_TX, UDMA_TX_CTRL);403 if (value & UDMA_TX_CTRL_ENA)404 udma_set(priv, REGS_DMA_TX, UDMA_TX_CTRL, UDMA_TX_CTRL_ABORT);405 priv->tx_running = false;406 return 0;407}408 409/*410 * NOTE: printk's in this routine will hang the system if this is411 * the console tty412 */413static int brcmuart_tx_dma(struct uart_8250_port *p)414{415 struct brcmuart_priv *priv = p->port.private_data;416 struct tty_port *tport = &p->port.state->port;417 u32 tx_size;418 419 if (uart_tx_stopped(&p->port) || priv->tx_running ||420 kfifo_is_empty(&tport->xmit_fifo)) {421 return 0;422 }423 424 priv->dma.tx_err = 0;425 tx_size = uart_fifo_out(&p->port, priv->tx_buf, UART_XMIT_SIZE);426 427 if (kfifo_len(&tport->xmit_fifo) < WAKEUP_CHARS)428 uart_write_wakeup(&p->port);429 430 udma_writel(priv, REGS_DMA_TX, UDMA_TX_TRANSFER_LEN, tx_size);431 udma_writel(priv, REGS_DMA_TX, UDMA_TX_BUF0_DATA_LEN, tx_size);432 udma_unset(priv, REGS_DMA_TX, UDMA_TX_BUF0_STATUS, UDMA_TX_BUFX_EMPTY);433 udma_set(priv, REGS_DMA_TX, UDMA_TX_CTRL, UDMA_TX_CTRL_ENA);434 priv->tx_running = true;435 436 return 0;437}438 439static void brcmuart_rx_buf_done_isr(struct uart_port *up, int index)440{441 struct brcmuart_priv *priv = up->private_data;442 struct tty_port *tty_port = &up->state->port;443 u32 status;444 u32 length;445 u32 copied;446 447 /* Make sure we're still in sync with the hardware */448 status = udma_readl(priv, REGS_DMA_RX, UDMA_RX_BUFx_STATUS(index));449 length = udma_readl(priv, REGS_DMA_RX, UDMA_RX_BUFx_DATA_LEN(index));450 451 if ((status & UDMA_RX_BUFX_STATUS_DATA_RDY) == 0) {452 dev_err(up->dev, "RX done interrupt but DATA_RDY not found\n");453 return;454 }455 if (status & (UDMA_RX_BUFX_STATUS_OVERRUN_ERR |456 UDMA_RX_BUFX_STATUS_FRAME_ERR |457 UDMA_RX_BUFX_STATUS_PARITY_ERR)) {458 if (status & UDMA_RX_BUFX_STATUS_OVERRUN_ERR) {459 up->icount.overrun++;460 dev_warn(up->dev, "RX OVERRUN Error\n");461 }462 if (status & UDMA_RX_BUFX_STATUS_FRAME_ERR) {463 up->icount.frame++;464 dev_warn(up->dev, "RX FRAMING Error\n");465 }466 if (status & UDMA_RX_BUFX_STATUS_PARITY_ERR) {467 up->icount.parity++;468 dev_warn(up->dev, "RX PARITY Error\n");469 }470 }471 copied = (u32)tty_insert_flip_string(472 tty_port,473 priv->rx_bufs + (index * RX_BUF_SIZE),474 length);475 if (copied != length) {476 dev_warn(up->dev, "Flip buffer overrun of %d bytes\n",477 length - copied);478 up->icount.overrun += length - copied;479 }480 up->icount.rx += length;481 if (status & UDMA_RX_BUFX_STATUS_CLOSE_EXPIRED)482 priv->dma_rx_partial_buf++;483 else if (length != RX_BUF_SIZE)484 /*485 * This is a bug in the controller that doesn't cause486 * any problems but will be fixed in the future.487 */488 priv->rx_missing_close_timeout++;489 else490 priv->dma_rx_full_buf++;491 492 tty_flip_buffer_push(tty_port);493}494 495static void brcmuart_rx_isr(struct uart_port *up, u32 rx_isr)496{497 struct brcmuart_priv *priv = up->private_data;498 struct device *dev = up->dev;499 u32 rx_done_isr;500 u32 check_isr;501 502 rx_done_isr = (rx_isr & UDMA_INTR_RX_READY_MASK);503 while (rx_done_isr) {504 check_isr = UDMA_INTR_RX_READY_BUF0 << priv->rx_next_buf;505 if (check_isr & rx_done_isr) {506 brcmuart_rx_buf_done_isr(up, priv->rx_next_buf);507 } else {508 dev_err(dev,509 "RX buffer ready out of sequence, restarting RX DMA\n");510 start_rx_dma(up_to_u8250p(up));511 break;512 }513 if (rx_isr & UDMA_RX_ERR_INTERRUPTS) {514 if (rx_isr & UDMA_INTR_RX_ERROR)515 priv->rx_err++;516 if (rx_isr & UDMA_INTR_RX_TIMEOUT) {517 priv->rx_timeout++;518 dev_err(dev, "RX TIMEOUT Error\n");519 }520 if (rx_isr & UDMA_INTR_RX_ABORT)521 priv->rx_abort++;522 priv->rx_running = false;523 }524 /* If not ABORT, re-enable RX buffer */525 if (!(rx_isr & UDMA_INTR_RX_ABORT))526 udma_unset(priv, REGS_DMA_RX,527 UDMA_RX_BUFx_STATUS(priv->rx_next_buf),528 UDMA_RX_BUFX_STATUS_DATA_RDY);529 rx_done_isr &= ~check_isr;530 priv->rx_next_buf++;531 if (priv->rx_next_buf == RX_BUFS_COUNT)532 priv->rx_next_buf = 0;533 }534}535 536static void brcmuart_tx_isr(struct uart_port *up, u32 isr)537{538 struct brcmuart_priv *priv = up->private_data;539 struct device *dev = up->dev;540 struct uart_8250_port *port_8250 = up_to_u8250p(up);541 struct tty_port *tport = &port_8250->port.state->port;542 543 if (isr & UDMA_INTR_TX_ABORT) {544 if (priv->tx_running)545 dev_err(dev, "Unexpected TX_ABORT interrupt\n");546 return;547 }548 priv->tx_running = false;549 if (!kfifo_is_empty(&tport->xmit_fifo) && !uart_tx_stopped(up))550 brcmuart_tx_dma(port_8250);551}552 553static irqreturn_t brcmuart_isr(int irq, void *dev_id)554{555 struct uart_port *up = dev_id;556 struct device *dev = up->dev;557 struct brcmuart_priv *priv = up->private_data;558 unsigned long flags;559 u32 interrupts;560 u32 rval;561 u32 tval;562 563 interrupts = udma_readl(priv, REGS_DMA_ISR, UDMA_INTR_STATUS);564 if (interrupts == 0)565 return IRQ_NONE;566 567 uart_port_lock_irqsave(up, &flags);568 569 /* Clear all interrupts */570 udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_CLEAR, interrupts);571 572 rval = UDMA_IS_RX_INTERRUPT(interrupts);573 if (rval)574 brcmuart_rx_isr(up, rval);575 tval = UDMA_IS_TX_INTERRUPT(interrupts);576 if (tval)577 brcmuart_tx_isr(up, tval);578 if ((rval | tval) == 0)579 dev_warn(dev, "Spurious interrupt: 0x%x\n", interrupts);580 581 uart_port_unlock_irqrestore(up, flags);582 return IRQ_HANDLED;583}584 585static int brcmuart_startup(struct uart_port *port)586{587 int res;588 struct uart_8250_port *up = up_to_u8250p(port);589 struct brcmuart_priv *priv = up->port.private_data;590 591 priv->shutdown = false;592 593 /*594 * prevent serial8250_do_startup() from allocating non-existent595 * DMA resources596 */597 up->dma = NULL;598 599 res = serial8250_do_startup(port);600 if (!priv->dma_enabled)601 return res;602 /*603 * Disable the Receive Data Interrupt because the DMA engine604 * will handle this.605 *606 * Synchronize UART_IER access against the console.607 */608 uart_port_lock_irq(port);609 up->ier &= ~UART_IER_RDI;610 serial_port_out(port, UART_IER, up->ier);611 uart_port_unlock_irq(port);612 613 priv->tx_running = false;614 priv->dma.rx_dma = NULL;615 priv->dma.tx_dma = brcmuart_tx_dma;616 up->dma = &priv->dma;617 618 brcmuart_init_dma_hardware(priv);619 start_rx_dma(up);620 return res;621}622 623static void brcmuart_shutdown(struct uart_port *port)624{625 struct uart_8250_port *up = up_to_u8250p(port);626 struct brcmuart_priv *priv = up->port.private_data;627 unsigned long flags;628 629 uart_port_lock_irqsave(port, &flags);630 priv->shutdown = true;631 if (priv->dma_enabled) {632 stop_rx_dma(up);633 stop_tx_dma(up);634 /* disable all interrupts */635 udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_SET,636 UDMA_RX_INTERRUPTS | UDMA_TX_INTERRUPTS);637 }638 639 /*640 * prevent serial8250_do_shutdown() from trying to free641 * DMA resources that we never alloc'd for this driver.642 */643 up->dma = NULL;644 645 uart_port_unlock_irqrestore(port, flags);646 serial8250_do_shutdown(port);647}648 649/*650 * Not all clocks run at the exact specified rate, so set each requested651 * rate and then get the actual rate.652 */653static void init_real_clk_rates(struct device *dev, struct brcmuart_priv *priv)654{655 int x;656 int rc;657 658 priv->default_mux_rate = clk_get_rate(priv->baud_mux_clk);659 for (x = 0; x < ARRAY_SIZE(priv->real_rates); x++) {660 if (priv->rate_table[x] == 0) {661 priv->real_rates[x] = 0;662 continue;663 }664 rc = clk_set_rate(priv->baud_mux_clk, priv->rate_table[x]);665 if (rc) {666 dev_err(dev, "Error selecting BAUD MUX clock for %u\n",667 priv->rate_table[x]);668 priv->real_rates[x] = priv->rate_table[x];669 } else {670 priv->real_rates[x] = clk_get_rate(priv->baud_mux_clk);671 }672 }673 clk_set_rate(priv->baud_mux_clk, priv->default_mux_rate);674}675 676static u32 find_quot(struct device *dev, u32 freq, u32 baud, u32 *percent)677{678 u32 quot;679 u32 rate;680 u64 hires_rate;681 u64 hires_baud;682 u64 hires_err;683 684 rate = freq / 16;685 quot = DIV_ROUND_CLOSEST(rate, baud);686 if (!quot)687 return 0;688 689 /* increase resolution to get xx.xx percent */690 hires_rate = div_u64((u64)rate * 10000, (u64)quot);691 hires_baud = (u64)baud * 10000;692 693 /* get the delta */694 if (hires_rate > hires_baud)695 hires_err = (hires_rate - hires_baud);696 else697 hires_err = (hires_baud - hires_rate);698 699 *percent = (unsigned long)DIV_ROUND_CLOSEST_ULL(hires_err, baud);700 701 dev_dbg(dev, "Baud rate: %u, MUX Clk: %u, Error: %u.%u%%\n",702 baud, freq, *percent / 100, *percent % 100);703 704 return quot;705}706 707static void set_clock_mux(struct uart_port *up, struct brcmuart_priv *priv,708 u32 baud)709{710 u32 percent;711 u32 best_percent = UINT_MAX;712 u32 quot;713 u32 freq;714 u32 best_quot = 1;715 u32 best_freq = 0;716 int rc;717 int i;718 int real_baud;719 720 /* If the Baud Mux Clock was not specified, just return */721 if (priv->baud_mux_clk == NULL)722 return;723 724 /* Try default_mux_rate first */725 quot = find_quot(up->dev, priv->default_mux_rate, baud, &percent);726 if (quot) {727 best_percent = percent;728 best_freq = priv->default_mux_rate;729 best_quot = quot;730 }731 /* If more than 1% error, find the closest match for specified baud */732 if (best_percent > 100) {733 for (i = 0; i < ARRAY_SIZE(priv->real_rates); i++) {734 freq = priv->real_rates[i];735 if (freq == 0 || freq == priv->default_mux_rate)736 continue;737 quot = find_quot(up->dev, freq, baud, &percent);738 if (!quot)739 continue;740 741 if (percent < best_percent) {742 best_percent = percent;743 best_freq = freq;744 best_quot = quot;745 }746 }747 }748 if (!best_freq) {749 dev_err(up->dev, "Error, %d BAUD rate is too fast.\n", baud);750 return;751 }752 rc = clk_set_rate(priv->baud_mux_clk, best_freq);753 if (rc)754 dev_err(up->dev, "Error selecting BAUD MUX clock\n");755 756 /* Error over 3 percent will cause data errors */757 if (best_percent > 300)758 dev_err(up->dev, "Error, baud: %d has %u.%u%% error\n",759 baud, percent / 100, percent % 100);760 761 real_baud = best_freq / 16 / best_quot;762 dev_dbg(up->dev, "Selecting BAUD MUX rate: %u\n", best_freq);763 dev_dbg(up->dev, "Requested baud: %u, Actual baud: %u\n",764 baud, real_baud);765 766 /* calc nanoseconds for 1.5 characters time at the given baud rate */767 i = NSEC_PER_SEC / real_baud / 10;768 i += (i / 2);769 priv->char_wait = ns_to_ktime(i);770 771 up->uartclk = best_freq;772}773 774static void brcmstb_set_termios(struct uart_port *up,775 struct ktermios *termios,776 const struct ktermios *old)777{778 struct uart_8250_port *p8250 = up_to_u8250p(up);779 struct brcmuart_priv *priv = up->private_data;780 781 if (priv->dma_enabled)782 stop_rx_dma(p8250);783 set_clock_mux(up, priv, tty_termios_baud_rate(termios));784 serial8250_do_set_termios(up, termios, old);785 if (p8250->mcr & UART_MCR_AFE)786 p8250->port.status |= UPSTAT_AUTOCTS;787 if (priv->dma_enabled)788 start_rx_dma(p8250);789}790 791static int brcmuart_handle_irq(struct uart_port *p)792{793 unsigned int iir = serial_port_in(p, UART_IIR);794 struct brcmuart_priv *priv = p->private_data;795 struct uart_8250_port *up = up_to_u8250p(p);796 unsigned int status;797 unsigned long flags;798 unsigned int ier;799 unsigned int mcr;800 int handled = 0;801 802 /*803 * There's a bug in some 8250 cores where we get a timeout804 * interrupt but there is no data ready.805 */806 if (((iir & UART_IIR_ID) == UART_IIR_RX_TIMEOUT) && !(priv->shutdown)) {807 uart_port_lock_irqsave(p, &flags);808 status = serial_port_in(p, UART_LSR);809 if ((status & UART_LSR_DR) == 0) {810 811 ier = serial_port_in(p, UART_IER);812 /*813 * if Receive Data Interrupt is enabled and814 * we're uing hardware flow control, deassert815 * RTS and wait for any chars in the pipline to816 * arrive and then check for DR again.817 */818 if ((ier & UART_IER_RDI) && (up->mcr & UART_MCR_AFE)) {819 ier &= ~(UART_IER_RLSI | UART_IER_RDI);820 serial_port_out(p, UART_IER, ier);821 mcr = serial_port_in(p, UART_MCR);822 mcr &= ~UART_MCR_RTS;823 serial_port_out(p, UART_MCR, mcr);824 hrtimer_start(&priv->hrt, priv->char_wait,825 HRTIMER_MODE_REL);826 } else {827 serial_port_in(p, UART_RX);828 }829 830 handled = 1;831 }832 uart_port_unlock_irqrestore(p, flags);833 if (handled)834 return 1;835 }836 return serial8250_handle_irq(p, iir);837}838 839static enum hrtimer_restart brcmuart_hrtimer_func(struct hrtimer *t)840{841 struct brcmuart_priv *priv = container_of(t, struct brcmuart_priv, hrt);842 struct uart_port *p = priv->up;843 struct uart_8250_port *up = up_to_u8250p(p);844 unsigned int status;845 unsigned long flags;846 847 if (priv->shutdown)848 return HRTIMER_NORESTART;849 850 uart_port_lock_irqsave(p, &flags);851 status = serial_port_in(p, UART_LSR);852 853 /*854 * If a character did not arrive after the timeout, clear the false855 * receive timeout.856 */857 if ((status & UART_LSR_DR) == 0) {858 serial_port_in(p, UART_RX);859 priv->rx_bad_timeout_no_char++;860 } else {861 priv->rx_bad_timeout_late_char++;862 }863 864 /* re-enable receive unless upper layer has disabled it */865 if ((up->ier & (UART_IER_RLSI | UART_IER_RDI)) ==866 (UART_IER_RLSI | UART_IER_RDI)) {867 status = serial_port_in(p, UART_IER);868 status |= (UART_IER_RLSI | UART_IER_RDI);869 serial_port_out(p, UART_IER, status);870 status = serial_port_in(p, UART_MCR);871 status |= UART_MCR_RTS;872 serial_port_out(p, UART_MCR, status);873 }874 uart_port_unlock_irqrestore(p, flags);875 return HRTIMER_NORESTART;876}877 878static const struct of_device_id brcmuart_dt_ids[] = {879 {880 .compatible = "brcm,bcm7278-uart",881 .data = brcmstb_rate_table_7278,882 },883 {884 .compatible = "brcm,bcm7271-uart",885 .data = brcmstb_rate_table,886 },887 {},888};889 890MODULE_DEVICE_TABLE(of, brcmuart_dt_ids);891 892static void brcmuart_free_bufs(struct device *dev, struct brcmuart_priv *priv)893{894 if (priv->rx_bufs)895 dma_free_coherent(dev, priv->rx_size, priv->rx_bufs,896 priv->rx_addr);897 if (priv->tx_buf)898 dma_free_coherent(dev, priv->tx_size, priv->tx_buf,899 priv->tx_addr);900}901 902static void brcmuart_throttle(struct uart_port *port)903{904 struct brcmuart_priv *priv = port->private_data;905 906 udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_SET, UDMA_RX_INTERRUPTS);907}908 909static void brcmuart_unthrottle(struct uart_port *port)910{911 struct brcmuart_priv *priv = port->private_data;912 913 udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_CLEAR,914 UDMA_RX_INTERRUPTS);915}916 917static int debugfs_stats_show(struct seq_file *s, void *unused)918{919 struct brcmuart_priv *priv = s->private;920 921 seq_printf(s, "rx_err:\t\t\t\t%u\n",922 priv->rx_err);923 seq_printf(s, "rx_timeout:\t\t\t%u\n",924 priv->rx_timeout);925 seq_printf(s, "rx_abort:\t\t\t%u\n",926 priv->rx_abort);927 seq_printf(s, "rx_bad_timeout_late_char:\t%u\n",928 priv->rx_bad_timeout_late_char);929 seq_printf(s, "rx_bad_timeout_no_char:\t\t%u\n",930 priv->rx_bad_timeout_no_char);931 seq_printf(s, "rx_missing_close_timeout:\t%u\n",932 priv->rx_missing_close_timeout);933 if (priv->dma_enabled) {934 seq_printf(s, "dma_rx_partial_buf:\t\t%llu\n",935 priv->dma_rx_partial_buf);936 seq_printf(s, "dma_rx_full_buf:\t\t%llu\n",937 priv->dma_rx_full_buf);938 }939 return 0;940}941DEFINE_SHOW_ATTRIBUTE(debugfs_stats);942 943static void brcmuart_init_debugfs(struct brcmuart_priv *priv,944 const char *device)945{946 priv->debugfs_dir = debugfs_create_dir(device, brcmuart_debugfs_root);947 debugfs_create_file("stats", 0444, priv->debugfs_dir, priv,948 &debugfs_stats_fops);949}950 951 952static int brcmuart_probe(struct platform_device *pdev)953{954 struct resource *regs;955 const struct of_device_id *of_id = NULL;956 struct uart_8250_port *new_port;957 struct device *dev = &pdev->dev;958 struct brcmuart_priv *priv;959 struct clk *baud_mux_clk;960 struct uart_8250_port up;961 void __iomem *membase = NULL;962 resource_size_t mapbase = 0;963 int ret;964 int x;965 int dma_irq;966 static const char * const reg_names[REGS_MAX] = {967 "uart", "dma_rx", "dma_tx", "dma_intr2", "dma_arb"968 };969 970 priv = devm_kzalloc(dev, sizeof(struct brcmuart_priv),971 GFP_KERNEL);972 if (!priv)973 return -ENOMEM;974 975 of_id = of_match_node(brcmuart_dt_ids, dev->of_node);976 if (!of_id || !of_id->data)977 priv->rate_table = brcmstb_rate_table;978 else979 priv->rate_table = of_id->data;980 981 for (x = 0; x < REGS_MAX; x++) {982 regs = platform_get_resource_byname(pdev, IORESOURCE_MEM,983 reg_names[x]);984 if (!regs)985 break;986 priv->regs[x] = devm_ioremap(dev, regs->start,987 resource_size(regs));988 if (!priv->regs[x])989 return -ENOMEM;990 if (x == REGS_8250) {991 mapbase = regs->start;992 membase = priv->regs[x];993 }994 }995 996 /* We should have just the uart base registers or all the registers */997 if (x != 1 && x != REGS_MAX)998 return dev_err_probe(dev, -EINVAL, "%s registers not specified\n",999 reg_names[x]);1000 1001 /* if the DMA registers were specified, try to enable DMA */1002 if (x > REGS_DMA_RX) {1003 if (brcmuart_arbitration(priv, 1) == 0) {1004 u32 txrev = 0;1005 u32 rxrev = 0;1006 1007 txrev = udma_readl(priv, REGS_DMA_RX, UDMA_RX_REVISION);1008 rxrev = udma_readl(priv, REGS_DMA_TX, UDMA_TX_REVISION);1009 if ((txrev >= UDMA_TX_REVISION_REQUIRED) &&1010 (rxrev >= UDMA_RX_REVISION_REQUIRED)) {1011 1012 /* Enable the use of the DMA hardware */1013 priv->dma_enabled = true;1014 } else {1015 brcmuart_arbitration(priv, 0);1016 dev_err(dev,1017 "Unsupported DMA Hardware Revision\n");1018 }1019 } else {1020 dev_err(dev,1021 "Timeout arbitrating for UART DMA hardware\n");1022 }1023 }1024 1025 dev_dbg(dev, "DMA is %senabled\n", priv->dma_enabled ? "" : "not ");1026 1027 memset(&up, 0, sizeof(up));1028 up.port.type = PORT_BCM7271;1029 up.port.dev = dev;1030 up.port.mapbase = mapbase;1031 up.port.membase = membase;1032 up.port.handle_irq = brcmuart_handle_irq;1033 up.port.flags = UPF_BOOT_AUTOCONF | UPF_FIXED_PORT | UPF_FIXED_TYPE;1034 up.port.private_data = priv;1035 1036 ret = uart_read_port_properties(&up.port);1037 if (ret)1038 goto release_dma;1039 1040 up.port.regshift = 2;1041 up.port.iotype = device_is_big_endian(dev) ? UPIO_MEM32BE : UPIO_MEM32;1042 1043 /* See if a Baud clock has been specified */1044 baud_mux_clk = devm_clk_get_optional_enabled(dev, "sw_baud");1045 ret = PTR_ERR_OR_ZERO(baud_mux_clk);1046 if (ret)1047 goto release_dma;1048 if (baud_mux_clk) {1049 dev_dbg(dev, "BAUD MUX clock found\n");1050 1051 priv->baud_mux_clk = baud_mux_clk;1052 init_real_clk_rates(dev, priv);1053 up.port.uartclk = priv->default_mux_rate;1054 } else {1055 dev_dbg(dev, "BAUD MUX clock not specified\n");1056 }1057 1058 /* setup HR timer */1059 hrtimer_init(&priv->hrt, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);1060 priv->hrt.function = brcmuart_hrtimer_func;1061 1062 up.port.shutdown = brcmuart_shutdown;1063 up.port.startup = brcmuart_startup;1064 up.port.throttle = brcmuart_throttle;1065 up.port.unthrottle = brcmuart_unthrottle;1066 up.port.set_termios = brcmstb_set_termios;1067 1068 if (priv->dma_enabled) {1069 priv->rx_size = RX_BUF_SIZE * RX_BUFS_COUNT;1070 priv->rx_bufs = dma_alloc_coherent(dev,1071 priv->rx_size,1072 &priv->rx_addr, GFP_KERNEL);1073 if (!priv->rx_bufs) {1074 ret = -ENOMEM;1075 goto err;1076 }1077 priv->tx_size = UART_XMIT_SIZE;1078 priv->tx_buf = dma_alloc_coherent(dev,1079 priv->tx_size,1080 &priv->tx_addr, GFP_KERNEL);1081 if (!priv->tx_buf) {1082 ret = -ENOMEM;1083 goto err;1084 }1085 }1086 1087 ret = serial8250_register_8250_port(&up);1088 if (ret < 0) {1089 dev_err_probe(dev, ret, "unable to register 8250 port\n");1090 goto err;1091 }1092 priv->line = ret;1093 new_port = serial8250_get_port(ret);1094 priv->up = &new_port->port;1095 if (priv->dma_enabled) {1096 dma_irq = platform_get_irq_byname(pdev, "dma");1097 if (dma_irq < 0) {1098 ret = dev_err_probe(dev, dma_irq, "no IRQ resource info\n");1099 goto err1;1100 }1101 ret = devm_request_irq(dev, dma_irq, brcmuart_isr,1102 IRQF_SHARED, "uart DMA irq", &new_port->port);1103 if (ret) {1104 dev_err_probe(dev, ret, "unable to register IRQ handler\n");1105 goto err1;1106 }1107 }1108 platform_set_drvdata(pdev, priv);1109 brcmuart_init_debugfs(priv, dev_name(&pdev->dev));1110 return 0;1111 1112err1:1113 serial8250_unregister_port(priv->line);1114err:1115 brcmuart_free_bufs(dev, priv);1116release_dma:1117 if (priv->dma_enabled)1118 brcmuart_arbitration(priv, 0);1119 return ret;1120}1121 1122static void brcmuart_remove(struct platform_device *pdev)1123{1124 struct brcmuart_priv *priv = platform_get_drvdata(pdev);1125 1126 debugfs_remove_recursive(priv->debugfs_dir);1127 hrtimer_cancel(&priv->hrt);1128 serial8250_unregister_port(priv->line);1129 brcmuart_free_bufs(&pdev->dev, priv);1130 if (priv->dma_enabled)1131 brcmuart_arbitration(priv, 0);1132}1133 1134static int __maybe_unused brcmuart_suspend(struct device *dev)1135{1136 struct brcmuart_priv *priv = dev_get_drvdata(dev);1137 struct uart_8250_port *up = serial8250_get_port(priv->line);1138 struct uart_port *port = &up->port;1139 unsigned long flags;1140 1141 /*1142 * This will prevent resume from enabling RTS before the1143 * baud rate has been restored.1144 */1145 uart_port_lock_irqsave(port, &flags);1146 priv->saved_mctrl = port->mctrl;1147 port->mctrl &= ~TIOCM_RTS;1148 uart_port_unlock_irqrestore(port, flags);1149 1150 serial8250_suspend_port(priv->line);1151 clk_disable_unprepare(priv->baud_mux_clk);1152 1153 return 0;1154}1155 1156static int __maybe_unused brcmuart_resume(struct device *dev)1157{1158 struct brcmuart_priv *priv = dev_get_drvdata(dev);1159 struct uart_8250_port *up = serial8250_get_port(priv->line);1160 struct uart_port *port = &up->port;1161 unsigned long flags;1162 int ret;1163 1164 ret = clk_prepare_enable(priv->baud_mux_clk);1165 if (ret)1166 dev_err(dev, "Error enabling BAUD MUX clock\n");1167 1168 /*1169 * The hardware goes back to it's default after suspend1170 * so get the "clk" back in sync.1171 */1172 ret = clk_set_rate(priv->baud_mux_clk, priv->default_mux_rate);1173 if (ret)1174 dev_err(dev, "Error restoring default BAUD MUX clock\n");1175 if (priv->dma_enabled) {1176 if (brcmuart_arbitration(priv, 1)) {1177 dev_err(dev, "Timeout arbitrating for DMA hardware on resume\n");1178 return(-EBUSY);1179 }1180 brcmuart_init_dma_hardware(priv);1181 start_rx_dma(serial8250_get_port(priv->line));1182 }1183 serial8250_resume_port(priv->line);1184 1185 if (priv->saved_mctrl & TIOCM_RTS) {1186 /* Restore RTS */1187 uart_port_lock_irqsave(port, &flags);1188 port->mctrl |= TIOCM_RTS;1189 port->ops->set_mctrl(port, port->mctrl);1190 uart_port_unlock_irqrestore(port, flags);1191 }1192 1193 return 0;1194}1195 1196static const struct dev_pm_ops brcmuart_dev_pm_ops = {1197 SET_SYSTEM_SLEEP_PM_OPS(brcmuart_suspend, brcmuart_resume)1198};1199 1200static struct platform_driver brcmuart_platform_driver = {1201 .driver = {1202 .name = "bcm7271-uart",1203 .pm = &brcmuart_dev_pm_ops,1204 .of_match_table = brcmuart_dt_ids,1205 },1206 .probe = brcmuart_probe,1207 .remove_new = brcmuart_remove,1208};1209 1210static int __init brcmuart_init(void)1211{1212 int ret;1213 1214 brcmuart_debugfs_root = debugfs_create_dir(1215 brcmuart_platform_driver.driver.name, NULL);1216 ret = platform_driver_register(&brcmuart_platform_driver);1217 if (ret) {1218 debugfs_remove_recursive(brcmuart_debugfs_root);1219 return ret;1220 }1221 1222 return 0;1223}1224module_init(brcmuart_init);1225 1226static void __exit brcmuart_deinit(void)1227{1228 platform_driver_unregister(&brcmuart_platform_driver);1229 debugfs_remove_recursive(brcmuart_debugfs_root);1230}1231module_exit(brcmuart_deinit);1232 1233MODULE_AUTHOR("Al Cooper");1234MODULE_DESCRIPTION("Broadcom NS16550A compatible serial port driver");1235MODULE_LICENSE("GPL v2");1236