312 lines · c
1// SPDX-License-Identifier: GPL-2.0+2/*3 * 8250_dma.c - DMA Engine API support for 8250.c4 *5 * Copyright (C) 2013 Intel Corporation6 */7#include <linux/tty.h>8#include <linux/tty_flip.h>9#include <linux/serial_reg.h>10#include <linux/dma-mapping.h>11 12#include "8250.h"13 14static void __dma_tx_complete(void *param)15{16 struct uart_8250_port *p = param;17 struct uart_8250_dma *dma = p->dma;18 struct tty_port *tport = &p->port.state->port;19 unsigned long flags;20 int ret;21 22 dma_sync_single_for_cpu(dma->txchan->device->dev, dma->tx_addr,23 UART_XMIT_SIZE, DMA_TO_DEVICE);24 25 uart_port_lock_irqsave(&p->port, &flags);26 27 dma->tx_running = 0;28 29 uart_xmit_advance(&p->port, dma->tx_size);30 31 if (kfifo_len(&tport->xmit_fifo) < WAKEUP_CHARS)32 uart_write_wakeup(&p->port);33 34 ret = serial8250_tx_dma(p);35 if (ret || !dma->tx_running)36 serial8250_set_THRI(p);37 38 uart_port_unlock_irqrestore(&p->port, flags);39}40 41static void __dma_rx_complete(struct uart_8250_port *p)42{43 struct uart_8250_dma *dma = p->dma;44 struct tty_port *tty_port = &p->port.state->port;45 struct dma_tx_state state;46 enum dma_status dma_status;47 int count;48 49 /*50 * New DMA Rx can be started during the completion handler before it51 * could acquire port's lock and it might still be ongoing. Don't to52 * anything in such case.53 */54 dma_status = dmaengine_tx_status(dma->rxchan, dma->rx_cookie, &state);55 if (dma_status == DMA_IN_PROGRESS)56 return;57 58 count = dma->rx_size - state.residue;59 60 tty_insert_flip_string(tty_port, dma->rx_buf, count);61 p->port.icount.rx += count;62 dma->rx_running = 0;63 64 tty_flip_buffer_push(tty_port);65}66 67static void dma_rx_complete(void *param)68{69 struct uart_8250_port *p = param;70 struct uart_8250_dma *dma = p->dma;71 unsigned long flags;72 73 uart_port_lock_irqsave(&p->port, &flags);74 if (dma->rx_running)75 __dma_rx_complete(p);76 77 /*78 * Cannot be combined with the previous check because __dma_rx_complete()79 * changes dma->rx_running.80 */81 if (!dma->rx_running && (serial_lsr_in(p) & UART_LSR_DR))82 p->dma->rx_dma(p);83 uart_port_unlock_irqrestore(&p->port, flags);84}85 86int serial8250_tx_dma(struct uart_8250_port *p)87{88 struct uart_8250_dma *dma = p->dma;89 struct tty_port *tport = &p->port.state->port;90 struct dma_async_tx_descriptor *desc;91 struct uart_port *up = &p->port;92 struct scatterlist *sg;93 struct scatterlist sgl[2];94 int i;95 int ret;96 97 if (dma->tx_running) {98 if (up->x_char) {99 dmaengine_pause(dma->txchan);100 uart_xchar_out(up, UART_TX);101 dmaengine_resume(dma->txchan);102 }103 return 0;104 } else if (up->x_char) {105 uart_xchar_out(up, UART_TX);106 }107 108 if (uart_tx_stopped(&p->port) || kfifo_is_empty(&tport->xmit_fifo)) {109 /* We have been called from __dma_tx_complete() */110 return 0;111 }112 113 serial8250_do_prepare_tx_dma(p);114 115 sg_init_table(sgl, ARRAY_SIZE(sgl));116 117 ret = kfifo_dma_out_prepare_mapped(&tport->xmit_fifo, sgl, ARRAY_SIZE(sgl),118 UART_XMIT_SIZE, dma->tx_addr);119 120 dma->tx_size = 0;121 122 for_each_sg(sgl, sg, ret, i)123 dma->tx_size += sg_dma_len(sg);124 125 desc = dmaengine_prep_slave_sg(dma->txchan, sgl, ret,126 DMA_MEM_TO_DEV,127 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);128 if (!desc) {129 ret = -EBUSY;130 goto err;131 }132 133 dma->tx_running = 1;134 desc->callback = __dma_tx_complete;135 desc->callback_param = p;136 137 dma->tx_cookie = dmaengine_submit(desc);138 139 dma_sync_single_for_device(dma->txchan->device->dev, dma->tx_addr,140 UART_XMIT_SIZE, DMA_TO_DEVICE);141 142 dma_async_issue_pending(dma->txchan);143 serial8250_clear_THRI(p);144 dma->tx_err = 0;145 146 return 0;147err:148 dma->tx_err = 1;149 return ret;150}151 152int serial8250_rx_dma(struct uart_8250_port *p)153{154 struct uart_8250_dma *dma = p->dma;155 struct dma_async_tx_descriptor *desc;156 157 if (dma->rx_running)158 return 0;159 160 serial8250_do_prepare_rx_dma(p);161 162 desc = dmaengine_prep_slave_single(dma->rxchan, dma->rx_addr,163 dma->rx_size, DMA_DEV_TO_MEM,164 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);165 if (!desc)166 return -EBUSY;167 168 dma->rx_running = 1;169 desc->callback = dma_rx_complete;170 desc->callback_param = p;171 172 dma->rx_cookie = dmaengine_submit(desc);173 174 dma_async_issue_pending(dma->rxchan);175 176 return 0;177}178 179void serial8250_rx_dma_flush(struct uart_8250_port *p)180{181 struct uart_8250_dma *dma = p->dma;182 183 if (dma->rx_running) {184 dmaengine_pause(dma->rxchan);185 __dma_rx_complete(p);186 dmaengine_terminate_async(dma->rxchan);187 }188}189EXPORT_SYMBOL_GPL(serial8250_rx_dma_flush);190 191int serial8250_request_dma(struct uart_8250_port *p)192{193 struct uart_8250_dma *dma = p->dma;194 phys_addr_t rx_dma_addr = dma->rx_dma_addr ?195 dma->rx_dma_addr : p->port.mapbase;196 phys_addr_t tx_dma_addr = dma->tx_dma_addr ?197 dma->tx_dma_addr : p->port.mapbase;198 dma_cap_mask_t mask;199 struct dma_slave_caps caps;200 int ret;201 202 /* Default slave configuration parameters */203 dma->rxconf.direction = DMA_DEV_TO_MEM;204 dma->rxconf.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;205 dma->rxconf.src_addr = rx_dma_addr + UART_RX;206 207 dma->txconf.direction = DMA_MEM_TO_DEV;208 dma->txconf.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;209 dma->txconf.dst_addr = tx_dma_addr + UART_TX;210 211 dma_cap_zero(mask);212 dma_cap_set(DMA_SLAVE, mask);213 214 /* Get a channel for RX */215 dma->rxchan = dma_request_slave_channel_compat(mask,216 dma->fn, dma->rx_param,217 p->port.dev, "rx");218 if (!dma->rxchan)219 return -ENODEV;220 221 /* 8250 rx dma requires dmaengine driver to support pause/terminate */222 ret = dma_get_slave_caps(dma->rxchan, &caps);223 if (ret)224 goto release_rx;225 if (!caps.cmd_pause || !caps.cmd_terminate ||226 caps.residue_granularity == DMA_RESIDUE_GRANULARITY_DESCRIPTOR) {227 ret = -EINVAL;228 goto release_rx;229 }230 231 dmaengine_slave_config(dma->rxchan, &dma->rxconf);232 233 /* Get a channel for TX */234 dma->txchan = dma_request_slave_channel_compat(mask,235 dma->fn, dma->tx_param,236 p->port.dev, "tx");237 if (!dma->txchan) {238 ret = -ENODEV;239 goto release_rx;240 }241 242 /* 8250 tx dma requires dmaengine driver to support terminate */243 ret = dma_get_slave_caps(dma->txchan, &caps);244 if (ret)245 goto err;246 if (!caps.cmd_terminate) {247 ret = -EINVAL;248 goto err;249 }250 251 dmaengine_slave_config(dma->txchan, &dma->txconf);252 253 /* RX buffer */254 if (!dma->rx_size)255 dma->rx_size = PAGE_SIZE;256 257 dma->rx_buf = dma_alloc_coherent(dma->rxchan->device->dev, dma->rx_size,258 &dma->rx_addr, GFP_KERNEL);259 if (!dma->rx_buf) {260 ret = -ENOMEM;261 goto err;262 }263 264 /* TX buffer */265 dma->tx_addr = dma_map_single(dma->txchan->device->dev,266 p->port.state->port.xmit_buf,267 UART_XMIT_SIZE,268 DMA_TO_DEVICE);269 if (dma_mapping_error(dma->txchan->device->dev, dma->tx_addr)) {270 dma_free_coherent(dma->rxchan->device->dev, dma->rx_size,271 dma->rx_buf, dma->rx_addr);272 ret = -ENOMEM;273 goto err;274 }275 276 dev_dbg_ratelimited(p->port.dev, "got both dma channels\n");277 278 return 0;279err:280 dma_release_channel(dma->txchan);281release_rx:282 dma_release_channel(dma->rxchan);283 return ret;284}285EXPORT_SYMBOL_GPL(serial8250_request_dma);286 287void serial8250_release_dma(struct uart_8250_port *p)288{289 struct uart_8250_dma *dma = p->dma;290 291 if (!dma)292 return;293 294 /* Release RX resources */295 dmaengine_terminate_sync(dma->rxchan);296 dma_free_coherent(dma->rxchan->device->dev, dma->rx_size, dma->rx_buf,297 dma->rx_addr);298 dma_release_channel(dma->rxchan);299 dma->rxchan = NULL;300 301 /* Release TX resources */302 dmaengine_terminate_sync(dma->txchan);303 dma_unmap_single(dma->txchan->device->dev, dma->tx_addr,304 UART_XMIT_SIZE, DMA_TO_DEVICE);305 dma_release_channel(dma->txchan);306 dma->txchan = NULL;307 dma->tx_running = 0;308 309 dev_dbg_ratelimited(p->port.dev, "dma channels released\n");310}311EXPORT_SYMBOL_GPL(serial8250_release_dma);312