513 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Core driver for the High Speed UART DMA4 *5 * Copyright (C) 2015 Intel Corporation6 * Author: Andy Shevchenko <andriy.shevchenko@linux.intel.com>7 *8 * Partially based on the bits found in drivers/tty/serial/mfd.c.9 */10 11/*12 * DMA channel allocation:13 * 1. Even number chans are used for DMA Read (UART TX), odd chans for DMA14 * Write (UART RX).15 * 2. 0/1 channel are assigned to port 0, 2/3 chan to port 1, 4/5 chan to16 * port 3, and so on.17 */18 19#include <linux/bits.h>20#include <linux/delay.h>21#include <linux/device.h>22#include <linux/dmaengine.h>23#include <linux/dma-mapping.h>24#include <linux/init.h>25#include <linux/interrupt.h>26#include <linux/list.h>27#include <linux/module.h>28#include <linux/percpu-defs.h>29#include <linux/scatterlist.h>30#include <linux/slab.h>31#include <linux/string.h>32#include <linux/spinlock.h>33 34#include "hsu.h"35 36#define HSU_DMA_BUSWIDTHS \37 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \38 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \39 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \40 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \41 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \42 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES) | \43 BIT(DMA_SLAVE_BUSWIDTH_16_BYTES)44 45static inline void hsu_chan_disable(struct hsu_dma_chan *hsuc)46{47 hsu_chan_writel(hsuc, HSU_CH_CR, 0);48}49 50static inline void hsu_chan_enable(struct hsu_dma_chan *hsuc)51{52 u32 cr = HSU_CH_CR_CHA;53 54 if (hsuc->direction == DMA_MEM_TO_DEV)55 cr &= ~HSU_CH_CR_CHD;56 else if (hsuc->direction == DMA_DEV_TO_MEM)57 cr |= HSU_CH_CR_CHD;58 59 hsu_chan_writel(hsuc, HSU_CH_CR, cr);60}61 62static void hsu_dma_chan_start(struct hsu_dma_chan *hsuc)63{64 struct dma_slave_config *config = &hsuc->config;65 struct hsu_dma_desc *desc = hsuc->desc;66 u32 bsr = 0, mtsr = 0; /* to shut the compiler up */67 u32 dcr = HSU_CH_DCR_CHSOE | HSU_CH_DCR_CHEI;68 unsigned int i, count;69 70 if (hsuc->direction == DMA_MEM_TO_DEV) {71 bsr = config->dst_maxburst;72 mtsr = config->dst_addr_width;73 } else if (hsuc->direction == DMA_DEV_TO_MEM) {74 bsr = config->src_maxburst;75 mtsr = config->src_addr_width;76 }77 78 hsu_chan_disable(hsuc);79 80 hsu_chan_writel(hsuc, HSU_CH_DCR, 0);81 hsu_chan_writel(hsuc, HSU_CH_BSR, bsr);82 hsu_chan_writel(hsuc, HSU_CH_MTSR, mtsr);83 84 /* Set descriptors */85 count = desc->nents - desc->active;86 for (i = 0; i < count && i < HSU_DMA_CHAN_NR_DESC; i++) {87 hsu_chan_writel(hsuc, HSU_CH_DxSAR(i), desc->sg[i].addr);88 hsu_chan_writel(hsuc, HSU_CH_DxTSR(i), desc->sg[i].len);89 90 /* Prepare value for DCR */91 dcr |= HSU_CH_DCR_DESCA(i);92 dcr |= HSU_CH_DCR_CHTOI(i); /* timeout bit, see HSU Errata 1 */93 94 desc->active++;95 }96 /* Only for the last descriptor in the chain */97 dcr |= HSU_CH_DCR_CHSOD(count - 1);98 dcr |= HSU_CH_DCR_CHDI(count - 1);99 100 hsu_chan_writel(hsuc, HSU_CH_DCR, dcr);101 102 hsu_chan_enable(hsuc);103}104 105static void hsu_dma_stop_channel(struct hsu_dma_chan *hsuc)106{107 hsu_chan_disable(hsuc);108 hsu_chan_writel(hsuc, HSU_CH_DCR, 0);109}110 111static void hsu_dma_start_channel(struct hsu_dma_chan *hsuc)112{113 hsu_dma_chan_start(hsuc);114}115 116static void hsu_dma_start_transfer(struct hsu_dma_chan *hsuc)117{118 struct virt_dma_desc *vdesc;119 120 /* Get the next descriptor */121 vdesc = vchan_next_desc(&hsuc->vchan);122 if (!vdesc) {123 hsuc->desc = NULL;124 return;125 }126 127 list_del(&vdesc->node);128 hsuc->desc = to_hsu_dma_desc(vdesc);129 130 /* Start the channel with a new descriptor */131 hsu_dma_start_channel(hsuc);132}133 134/*135 * hsu_dma_get_status() - get DMA channel status136 * @chip: HSUART DMA chip137 * @nr: DMA channel number138 * @status: pointer for DMA Channel Status Register value139 *140 * Description:141 * The function reads and clears the DMA Channel Status Register, checks142 * if it was a timeout interrupt and returns a corresponding value.143 *144 * Caller should provide a valid pointer for the DMA Channel Status145 * Register value that will be returned in @status.146 *147 * Return:148 * 1 for DMA timeout status, 0 for other DMA status, or error code for149 * invalid parameters or no interrupt pending.150 */151int hsu_dma_get_status(struct hsu_dma_chip *chip, unsigned short nr,152 u32 *status)153{154 struct hsu_dma_chan *hsuc;155 unsigned long flags;156 u32 sr;157 158 /* Sanity check */159 if (nr >= chip->hsu->nr_channels)160 return -EINVAL;161 162 hsuc = &chip->hsu->chan[nr];163 164 /*165 * No matter what situation, need read clear the IRQ status166 * There is a bug, see Errata 5, HSD 2900918167 */168 spin_lock_irqsave(&hsuc->vchan.lock, flags);169 sr = hsu_chan_readl(hsuc, HSU_CH_SR);170 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);171 172 /* Check if any interrupt is pending */173 sr &= ~(HSU_CH_SR_DESCE_ANY | HSU_CH_SR_CDESC_ANY);174 if (!sr)175 return -EIO;176 177 /* Timeout IRQ, need wait some time, see Errata 2 */178 if (sr & HSU_CH_SR_DESCTO_ANY)179 udelay(2);180 181 /*182 * At this point, at least one of Descriptor Time Out, Channel Error183 * or Descriptor Done bits must be set. Clear the Descriptor Time Out184 * bits and if sr is still non-zero, it must be channel error or185 * descriptor done which are higher priority than timeout and handled186 * in hsu_dma_do_irq(). Else, it must be a timeout.187 */188 sr &= ~HSU_CH_SR_DESCTO_ANY;189 190 *status = sr;191 192 return sr ? 0 : 1;193}194EXPORT_SYMBOL_GPL(hsu_dma_get_status);195 196/*197 * hsu_dma_do_irq() - DMA interrupt handler198 * @chip: HSUART DMA chip199 * @nr: DMA channel number200 * @status: Channel Status Register value201 *202 * Description:203 * This function handles Channel Error and Descriptor Done interrupts.204 * This function should be called after determining that the DMA interrupt205 * is not a normal timeout interrupt, ie. hsu_dma_get_status() returned 0.206 *207 * Return:208 * 0 for invalid channel number, 1 otherwise.209 */210int hsu_dma_do_irq(struct hsu_dma_chip *chip, unsigned short nr, u32 status)211{212 struct dma_chan_percpu *stat;213 struct hsu_dma_chan *hsuc;214 struct hsu_dma_desc *desc;215 unsigned long flags;216 217 /* Sanity check */218 if (nr >= chip->hsu->nr_channels)219 return 0;220 221 hsuc = &chip->hsu->chan[nr];222 stat = this_cpu_ptr(hsuc->vchan.chan.local);223 224 spin_lock_irqsave(&hsuc->vchan.lock, flags);225 desc = hsuc->desc;226 if (desc) {227 if (status & HSU_CH_SR_CHE) {228 desc->status = DMA_ERROR;229 } else if (desc->active < desc->nents) {230 hsu_dma_start_channel(hsuc);231 } else {232 vchan_cookie_complete(&desc->vdesc);233 desc->status = DMA_COMPLETE;234 stat->bytes_transferred += desc->length;235 hsu_dma_start_transfer(hsuc);236 }237 }238 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);239 240 return 1;241}242EXPORT_SYMBOL_GPL(hsu_dma_do_irq);243 244static struct hsu_dma_desc *hsu_dma_alloc_desc(unsigned int nents)245{246 struct hsu_dma_desc *desc;247 248 desc = kzalloc(sizeof(*desc), GFP_NOWAIT);249 if (!desc)250 return NULL;251 252 desc->sg = kcalloc(nents, sizeof(*desc->sg), GFP_NOWAIT);253 if (!desc->sg) {254 kfree(desc);255 return NULL;256 }257 258 return desc;259}260 261static void hsu_dma_desc_free(struct virt_dma_desc *vdesc)262{263 struct hsu_dma_desc *desc = to_hsu_dma_desc(vdesc);264 265 kfree(desc->sg);266 kfree(desc);267}268 269static struct dma_async_tx_descriptor *hsu_dma_prep_slave_sg(270 struct dma_chan *chan, struct scatterlist *sgl,271 unsigned int sg_len, enum dma_transfer_direction direction,272 unsigned long flags, void *context)273{274 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);275 struct hsu_dma_desc *desc;276 struct scatterlist *sg;277 unsigned int i;278 279 desc = hsu_dma_alloc_desc(sg_len);280 if (!desc)281 return NULL;282 283 for_each_sg(sgl, sg, sg_len, i) {284 desc->sg[i].addr = sg_dma_address(sg);285 desc->sg[i].len = sg_dma_len(sg);286 287 desc->length += sg_dma_len(sg);288 }289 290 desc->nents = sg_len;291 desc->direction = direction;292 /* desc->active = 0 by kzalloc */293 desc->status = DMA_IN_PROGRESS;294 295 return vchan_tx_prep(&hsuc->vchan, &desc->vdesc, flags);296}297 298static void hsu_dma_issue_pending(struct dma_chan *chan)299{300 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);301 unsigned long flags;302 303 spin_lock_irqsave(&hsuc->vchan.lock, flags);304 if (vchan_issue_pending(&hsuc->vchan) && !hsuc->desc)305 hsu_dma_start_transfer(hsuc);306 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);307}308 309static size_t hsu_dma_active_desc_size(struct hsu_dma_chan *hsuc)310{311 struct hsu_dma_desc *desc = hsuc->desc;312 size_t bytes = 0;313 int i;314 315 for (i = desc->active; i < desc->nents; i++)316 bytes += desc->sg[i].len;317 318 i = HSU_DMA_CHAN_NR_DESC - 1;319 do {320 bytes += hsu_chan_readl(hsuc, HSU_CH_DxTSR(i));321 } while (--i >= 0);322 323 return bytes;324}325 326static enum dma_status hsu_dma_tx_status(struct dma_chan *chan,327 dma_cookie_t cookie, struct dma_tx_state *state)328{329 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);330 struct virt_dma_desc *vdesc;331 enum dma_status status;332 size_t bytes;333 unsigned long flags;334 335 status = dma_cookie_status(chan, cookie, state);336 if (status == DMA_COMPLETE)337 return status;338 339 spin_lock_irqsave(&hsuc->vchan.lock, flags);340 vdesc = vchan_find_desc(&hsuc->vchan, cookie);341 if (hsuc->desc && cookie == hsuc->desc->vdesc.tx.cookie) {342 bytes = hsu_dma_active_desc_size(hsuc);343 dma_set_residue(state, bytes);344 status = hsuc->desc->status;345 } else if (vdesc) {346 bytes = to_hsu_dma_desc(vdesc)->length;347 dma_set_residue(state, bytes);348 }349 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);350 351 return status;352}353 354static int hsu_dma_slave_config(struct dma_chan *chan,355 struct dma_slave_config *config)356{357 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);358 359 memcpy(&hsuc->config, config, sizeof(hsuc->config));360 361 return 0;362}363 364static int hsu_dma_pause(struct dma_chan *chan)365{366 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);367 unsigned long flags;368 369 spin_lock_irqsave(&hsuc->vchan.lock, flags);370 if (hsuc->desc && hsuc->desc->status == DMA_IN_PROGRESS) {371 hsu_chan_disable(hsuc);372 hsuc->desc->status = DMA_PAUSED;373 }374 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);375 376 return 0;377}378 379static int hsu_dma_resume(struct dma_chan *chan)380{381 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);382 unsigned long flags;383 384 spin_lock_irqsave(&hsuc->vchan.lock, flags);385 if (hsuc->desc && hsuc->desc->status == DMA_PAUSED) {386 hsuc->desc->status = DMA_IN_PROGRESS;387 hsu_chan_enable(hsuc);388 }389 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);390 391 return 0;392}393 394static int hsu_dma_terminate_all(struct dma_chan *chan)395{396 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);397 unsigned long flags;398 LIST_HEAD(head);399 400 spin_lock_irqsave(&hsuc->vchan.lock, flags);401 402 hsu_dma_stop_channel(hsuc);403 if (hsuc->desc) {404 hsu_dma_desc_free(&hsuc->desc->vdesc);405 hsuc->desc = NULL;406 }407 408 vchan_get_all_descriptors(&hsuc->vchan, &head);409 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);410 vchan_dma_desc_free_list(&hsuc->vchan, &head);411 412 return 0;413}414 415static void hsu_dma_free_chan_resources(struct dma_chan *chan)416{417 vchan_free_chan_resources(to_virt_chan(chan));418}419 420static void hsu_dma_synchronize(struct dma_chan *chan)421{422 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);423 424 vchan_synchronize(&hsuc->vchan);425}426 427int hsu_dma_probe(struct hsu_dma_chip *chip)428{429 struct hsu_dma *hsu;430 void __iomem *addr = chip->regs + chip->offset;431 unsigned short i;432 int ret;433 434 hsu = devm_kzalloc(chip->dev, sizeof(*hsu), GFP_KERNEL);435 if (!hsu)436 return -ENOMEM;437 438 chip->hsu = hsu;439 440 /* Calculate nr_channels from the IO space length */441 hsu->nr_channels = (chip->length - chip->offset) / HSU_DMA_CHAN_LENGTH;442 443 hsu->chan = devm_kcalloc(chip->dev, hsu->nr_channels,444 sizeof(*hsu->chan), GFP_KERNEL);445 if (!hsu->chan)446 return -ENOMEM;447 448 INIT_LIST_HEAD(&hsu->dma.channels);449 for (i = 0; i < hsu->nr_channels; i++) {450 struct hsu_dma_chan *hsuc = &hsu->chan[i];451 452 hsuc->vchan.desc_free = hsu_dma_desc_free;453 vchan_init(&hsuc->vchan, &hsu->dma);454 455 hsuc->direction = (i & 0x1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV;456 hsuc->reg = addr + i * HSU_DMA_CHAN_LENGTH;457 }458 459 dma_cap_set(DMA_SLAVE, hsu->dma.cap_mask);460 dma_cap_set(DMA_PRIVATE, hsu->dma.cap_mask);461 462 hsu->dma.device_free_chan_resources = hsu_dma_free_chan_resources;463 464 hsu->dma.device_prep_slave_sg = hsu_dma_prep_slave_sg;465 466 hsu->dma.device_issue_pending = hsu_dma_issue_pending;467 hsu->dma.device_tx_status = hsu_dma_tx_status;468 469 hsu->dma.device_config = hsu_dma_slave_config;470 hsu->dma.device_pause = hsu_dma_pause;471 hsu->dma.device_resume = hsu_dma_resume;472 hsu->dma.device_terminate_all = hsu_dma_terminate_all;473 hsu->dma.device_synchronize = hsu_dma_synchronize;474 475 hsu->dma.src_addr_widths = HSU_DMA_BUSWIDTHS;476 hsu->dma.dst_addr_widths = HSU_DMA_BUSWIDTHS;477 hsu->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);478 hsu->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;479 480 hsu->dma.dev = chip->dev;481 482 dma_set_max_seg_size(hsu->dma.dev, HSU_CH_DxTSR_MASK);483 484 ret = dma_async_device_register(&hsu->dma);485 if (ret)486 return ret;487 488 dev_info(chip->dev, "Found HSU DMA, %d channels\n", hsu->nr_channels);489 return 0;490}491EXPORT_SYMBOL_GPL(hsu_dma_probe);492 493int hsu_dma_remove(struct hsu_dma_chip *chip)494{495 struct hsu_dma *hsu = chip->hsu;496 unsigned short i;497 498 dma_async_device_unregister(&hsu->dma);499 500 for (i = 0; i < hsu->nr_channels; i++) {501 struct hsu_dma_chan *hsuc = &hsu->chan[i];502 503 tasklet_kill(&hsuc->vchan.task);504 }505 506 return 0;507}508EXPORT_SYMBOL_GPL(hsu_dma_remove);509 510MODULE_LICENSE("GPL v2");511MODULE_DESCRIPTION("High Speed UART DMA core driver");512MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");513