617 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * External DMA controller driver for UniPhier SoCs4 * Copyright 2019 Socionext Inc.5 * Author: Kunihiko Hayashi <hayashi.kunihiko@socionext.com>6 */7 8#include <linux/bitops.h>9#include <linux/bitfield.h>10#include <linux/iopoll.h>11#include <linux/module.h>12#include <linux/of.h>13#include <linux/of_dma.h>14#include <linux/platform_device.h>15#include <linux/slab.h>16 17#include "dmaengine.h"18#include "virt-dma.h"19 20#define XDMAC_CH_WIDTH 0x10021 22#define XDMAC_TFA 0x0823#define XDMAC_TFA_MCNT_MASK GENMASK(23, 16)24#define XDMAC_TFA_MASK GENMASK(5, 0)25#define XDMAC_SADM 0x1026#define XDMAC_SADM_STW_MASK GENMASK(25, 24)27#define XDMAC_SADM_SAM BIT(4)28#define XDMAC_SADM_SAM_FIXED XDMAC_SADM_SAM29#define XDMAC_SADM_SAM_INC 030#define XDMAC_DADM 0x1431#define XDMAC_DADM_DTW_MASK XDMAC_SADM_STW_MASK32#define XDMAC_DADM_DAM XDMAC_SADM_SAM33#define XDMAC_DADM_DAM_FIXED XDMAC_SADM_SAM_FIXED34#define XDMAC_DADM_DAM_INC XDMAC_SADM_SAM_INC35#define XDMAC_EXSAD 0x1836#define XDMAC_EXDAD 0x1c37#define XDMAC_SAD 0x2038#define XDMAC_DAD 0x2439#define XDMAC_ITS 0x2840#define XDMAC_ITS_MASK GENMASK(25, 0)41#define XDMAC_TNUM 0x2c42#define XDMAC_TNUM_MASK GENMASK(15, 0)43#define XDMAC_TSS 0x3044#define XDMAC_TSS_REQ BIT(0)45#define XDMAC_IEN 0x3446#define XDMAC_IEN_ERRIEN BIT(1)47#define XDMAC_IEN_ENDIEN BIT(0)48#define XDMAC_STAT 0x4049#define XDMAC_STAT_TENF BIT(0)50#define XDMAC_IR 0x4451#define XDMAC_IR_ERRF BIT(1)52#define XDMAC_IR_ENDF BIT(0)53#define XDMAC_ID 0x4854#define XDMAC_ID_ERRIDF BIT(1)55#define XDMAC_ID_ENDIDF BIT(0)56 57#define XDMAC_MAX_CHANS 1658#define XDMAC_INTERVAL_CLKS 2059#define XDMAC_MAX_WORDS XDMAC_TNUM_MASK60 61/* cut lower bit for maintain alignment of maximum transfer size */62#define XDMAC_MAX_WORD_SIZE (XDMAC_ITS_MASK & ~GENMASK(3, 0))63 64#define UNIPHIER_XDMAC_BUSWIDTHS \65 (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \66 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \67 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \68 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))69 70struct uniphier_xdmac_desc_node {71 dma_addr_t src;72 dma_addr_t dst;73 u32 burst_size;74 u32 nr_burst;75};76 77struct uniphier_xdmac_desc {78 struct virt_dma_desc vd;79 80 unsigned int nr_node;81 unsigned int cur_node;82 enum dma_transfer_direction dir;83 struct uniphier_xdmac_desc_node nodes[] __counted_by(nr_node);84};85 86struct uniphier_xdmac_chan {87 struct virt_dma_chan vc;88 struct uniphier_xdmac_device *xdev;89 struct uniphier_xdmac_desc *xd;90 void __iomem *reg_ch_base;91 struct dma_slave_config sconfig;92 int id;93 unsigned int req_factor;94};95 96struct uniphier_xdmac_device {97 struct dma_device ddev;98 void __iomem *reg_base;99 int nr_chans;100 struct uniphier_xdmac_chan channels[] __counted_by(nr_chans);101};102 103static struct uniphier_xdmac_chan *104to_uniphier_xdmac_chan(struct virt_dma_chan *vc)105{106 return container_of(vc, struct uniphier_xdmac_chan, vc);107}108 109static struct uniphier_xdmac_desc *110to_uniphier_xdmac_desc(struct virt_dma_desc *vd)111{112 return container_of(vd, struct uniphier_xdmac_desc, vd);113}114 115/* xc->vc.lock must be held by caller */116static struct uniphier_xdmac_desc *117uniphier_xdmac_next_desc(struct uniphier_xdmac_chan *xc)118{119 struct virt_dma_desc *vd;120 121 vd = vchan_next_desc(&xc->vc);122 if (!vd)123 return NULL;124 125 list_del(&vd->node);126 127 return to_uniphier_xdmac_desc(vd);128}129 130/* xc->vc.lock must be held by caller */131static void uniphier_xdmac_chan_start(struct uniphier_xdmac_chan *xc,132 struct uniphier_xdmac_desc *xd)133{134 u32 src_mode, src_width;135 u32 dst_mode, dst_width;136 dma_addr_t src_addr, dst_addr;137 u32 val, its, tnum;138 enum dma_slave_buswidth buswidth;139 140 src_addr = xd->nodes[xd->cur_node].src;141 dst_addr = xd->nodes[xd->cur_node].dst;142 its = xd->nodes[xd->cur_node].burst_size;143 tnum = xd->nodes[xd->cur_node].nr_burst;144 145 /*146 * The width of MEM side must be 4 or 8 bytes, that does not147 * affect that of DEV side and transfer size.148 */149 if (xd->dir == DMA_DEV_TO_MEM) {150 src_mode = XDMAC_SADM_SAM_FIXED;151 buswidth = xc->sconfig.src_addr_width;152 } else {153 src_mode = XDMAC_SADM_SAM_INC;154 buswidth = DMA_SLAVE_BUSWIDTH_8_BYTES;155 }156 src_width = FIELD_PREP(XDMAC_SADM_STW_MASK, __ffs(buswidth));157 158 if (xd->dir == DMA_MEM_TO_DEV) {159 dst_mode = XDMAC_DADM_DAM_FIXED;160 buswidth = xc->sconfig.dst_addr_width;161 } else {162 dst_mode = XDMAC_DADM_DAM_INC;163 buswidth = DMA_SLAVE_BUSWIDTH_8_BYTES;164 }165 dst_width = FIELD_PREP(XDMAC_DADM_DTW_MASK, __ffs(buswidth));166 167 /* setup transfer factor */168 val = FIELD_PREP(XDMAC_TFA_MCNT_MASK, XDMAC_INTERVAL_CLKS);169 val |= FIELD_PREP(XDMAC_TFA_MASK, xc->req_factor);170 writel(val, xc->reg_ch_base + XDMAC_TFA);171 172 /* setup the channel */173 writel(lower_32_bits(src_addr), xc->reg_ch_base + XDMAC_SAD);174 writel(upper_32_bits(src_addr), xc->reg_ch_base + XDMAC_EXSAD);175 176 writel(lower_32_bits(dst_addr), xc->reg_ch_base + XDMAC_DAD);177 writel(upper_32_bits(dst_addr), xc->reg_ch_base + XDMAC_EXDAD);178 179 src_mode |= src_width;180 dst_mode |= dst_width;181 writel(src_mode, xc->reg_ch_base + XDMAC_SADM);182 writel(dst_mode, xc->reg_ch_base + XDMAC_DADM);183 184 writel(its, xc->reg_ch_base + XDMAC_ITS);185 writel(tnum, xc->reg_ch_base + XDMAC_TNUM);186 187 /* enable interrupt */188 writel(XDMAC_IEN_ENDIEN | XDMAC_IEN_ERRIEN,189 xc->reg_ch_base + XDMAC_IEN);190 191 /* start XDMAC */192 val = readl(xc->reg_ch_base + XDMAC_TSS);193 val |= XDMAC_TSS_REQ;194 writel(val, xc->reg_ch_base + XDMAC_TSS);195}196 197/* xc->vc.lock must be held by caller */198static int uniphier_xdmac_chan_stop(struct uniphier_xdmac_chan *xc)199{200 u32 val;201 202 /* disable interrupt */203 val = readl(xc->reg_ch_base + XDMAC_IEN);204 val &= ~(XDMAC_IEN_ENDIEN | XDMAC_IEN_ERRIEN);205 writel(val, xc->reg_ch_base + XDMAC_IEN);206 207 /* stop XDMAC */208 val = readl(xc->reg_ch_base + XDMAC_TSS);209 val &= ~XDMAC_TSS_REQ;210 writel(0, xc->reg_ch_base + XDMAC_TSS);211 212 /* wait until transfer is stopped */213 return readl_poll_timeout_atomic(xc->reg_ch_base + XDMAC_STAT, val,214 !(val & XDMAC_STAT_TENF), 100, 1000);215}216 217/* xc->vc.lock must be held by caller */218static void uniphier_xdmac_start(struct uniphier_xdmac_chan *xc)219{220 struct uniphier_xdmac_desc *xd;221 222 xd = uniphier_xdmac_next_desc(xc);223 if (xd)224 uniphier_xdmac_chan_start(xc, xd);225 226 /* set desc to chan regardless of xd is null */227 xc->xd = xd;228}229 230static void uniphier_xdmac_chan_irq(struct uniphier_xdmac_chan *xc)231{232 u32 stat;233 int ret;234 235 spin_lock(&xc->vc.lock);236 237 stat = readl(xc->reg_ch_base + XDMAC_ID);238 239 if (stat & XDMAC_ID_ERRIDF) {240 ret = uniphier_xdmac_chan_stop(xc);241 if (ret)242 dev_err(xc->xdev->ddev.dev,243 "DMA transfer error with aborting issue\n");244 else245 dev_err(xc->xdev->ddev.dev,246 "DMA transfer error\n");247 248 } else if ((stat & XDMAC_ID_ENDIDF) && xc->xd) {249 xc->xd->cur_node++;250 if (xc->xd->cur_node >= xc->xd->nr_node) {251 vchan_cookie_complete(&xc->xd->vd);252 uniphier_xdmac_start(xc);253 } else {254 uniphier_xdmac_chan_start(xc, xc->xd);255 }256 }257 258 /* write bits to clear */259 writel(stat, xc->reg_ch_base + XDMAC_IR);260 261 spin_unlock(&xc->vc.lock);262}263 264static irqreturn_t uniphier_xdmac_irq_handler(int irq, void *dev_id)265{266 struct uniphier_xdmac_device *xdev = dev_id;267 int i;268 269 for (i = 0; i < xdev->nr_chans; i++)270 uniphier_xdmac_chan_irq(&xdev->channels[i]);271 272 return IRQ_HANDLED;273}274 275static void uniphier_xdmac_free_chan_resources(struct dma_chan *chan)276{277 vchan_free_chan_resources(to_virt_chan(chan));278}279 280static struct dma_async_tx_descriptor *281uniphier_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst,282 dma_addr_t src, size_t len, unsigned long flags)283{284 struct virt_dma_chan *vc = to_virt_chan(chan);285 struct uniphier_xdmac_desc *xd;286 unsigned int nr;287 size_t burst_size, tlen;288 int i;289 290 if (len > XDMAC_MAX_WORD_SIZE * XDMAC_MAX_WORDS)291 return NULL;292 293 nr = 1 + len / XDMAC_MAX_WORD_SIZE;294 295 xd = kzalloc(struct_size(xd, nodes, nr), GFP_NOWAIT);296 if (!xd)297 return NULL;298 xd->nr_node = nr;299 300 for (i = 0; i < nr; i++) {301 burst_size = min_t(size_t, len, XDMAC_MAX_WORD_SIZE);302 xd->nodes[i].src = src;303 xd->nodes[i].dst = dst;304 xd->nodes[i].burst_size = burst_size;305 xd->nodes[i].nr_burst = len / burst_size;306 tlen = rounddown(len, burst_size);307 src += tlen;308 dst += tlen;309 len -= tlen;310 }311 312 xd->dir = DMA_MEM_TO_MEM;313 xd->cur_node = 0;314 315 return vchan_tx_prep(vc, &xd->vd, flags);316}317 318static struct dma_async_tx_descriptor *319uniphier_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,320 unsigned int sg_len,321 enum dma_transfer_direction direction,322 unsigned long flags, void *context)323{324 struct virt_dma_chan *vc = to_virt_chan(chan);325 struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc);326 struct uniphier_xdmac_desc *xd;327 struct scatterlist *sg;328 enum dma_slave_buswidth buswidth;329 u32 maxburst;330 int i;331 332 if (!is_slave_direction(direction))333 return NULL;334 335 if (direction == DMA_DEV_TO_MEM) {336 buswidth = xc->sconfig.src_addr_width;337 maxburst = xc->sconfig.src_maxburst;338 } else {339 buswidth = xc->sconfig.dst_addr_width;340 maxburst = xc->sconfig.dst_maxburst;341 }342 343 if (!maxburst)344 maxburst = 1;345 if (maxburst > xc->xdev->ddev.max_burst) {346 dev_err(xc->xdev->ddev.dev,347 "Exceed maximum number of burst words\n");348 return NULL;349 }350 351 xd = kzalloc(struct_size(xd, nodes, sg_len), GFP_NOWAIT);352 if (!xd)353 return NULL;354 xd->nr_node = sg_len;355 356 for_each_sg(sgl, sg, sg_len, i) {357 xd->nodes[i].src = (direction == DMA_DEV_TO_MEM)358 ? xc->sconfig.src_addr : sg_dma_address(sg);359 xd->nodes[i].dst = (direction == DMA_MEM_TO_DEV)360 ? xc->sconfig.dst_addr : sg_dma_address(sg);361 xd->nodes[i].burst_size = maxburst * buswidth;362 xd->nodes[i].nr_burst =363 sg_dma_len(sg) / xd->nodes[i].burst_size;364 365 /*366 * Currently transfer that size doesn't align the unit size367 * (the number of burst words * bus-width) is not allowed,368 * because the driver does not support the way to transfer369 * residue size. As a matter of fact, in order to transfer370 * arbitrary size, 'src_maxburst' or 'dst_maxburst' of371 * dma_slave_config must be 1.372 */373 if (sg_dma_len(sg) % xd->nodes[i].burst_size) {374 dev_err(xc->xdev->ddev.dev,375 "Unaligned transfer size: %d", sg_dma_len(sg));376 kfree(xd);377 return NULL;378 }379 380 if (xd->nodes[i].nr_burst > XDMAC_MAX_WORDS) {381 dev_err(xc->xdev->ddev.dev,382 "Exceed maximum transfer size");383 kfree(xd);384 return NULL;385 }386 }387 388 xd->dir = direction;389 xd->cur_node = 0;390 391 return vchan_tx_prep(vc, &xd->vd, flags);392}393 394static int uniphier_xdmac_slave_config(struct dma_chan *chan,395 struct dma_slave_config *config)396{397 struct virt_dma_chan *vc = to_virt_chan(chan);398 struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc);399 400 memcpy(&xc->sconfig, config, sizeof(*config));401 402 return 0;403}404 405static int uniphier_xdmac_terminate_all(struct dma_chan *chan)406{407 struct virt_dma_chan *vc = to_virt_chan(chan);408 struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc);409 unsigned long flags;410 int ret = 0;411 LIST_HEAD(head);412 413 spin_lock_irqsave(&vc->lock, flags);414 415 if (xc->xd) {416 vchan_terminate_vdesc(&xc->xd->vd);417 xc->xd = NULL;418 ret = uniphier_xdmac_chan_stop(xc);419 }420 421 vchan_get_all_descriptors(vc, &head);422 423 spin_unlock_irqrestore(&vc->lock, flags);424 425 vchan_dma_desc_free_list(vc, &head);426 427 return ret;428}429 430static void uniphier_xdmac_synchronize(struct dma_chan *chan)431{432 vchan_synchronize(to_virt_chan(chan));433}434 435static void uniphier_xdmac_issue_pending(struct dma_chan *chan)436{437 struct virt_dma_chan *vc = to_virt_chan(chan);438 struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc);439 unsigned long flags;440 441 spin_lock_irqsave(&vc->lock, flags);442 443 if (vchan_issue_pending(vc) && !xc->xd)444 uniphier_xdmac_start(xc);445 446 spin_unlock_irqrestore(&vc->lock, flags);447}448 449static void uniphier_xdmac_desc_free(struct virt_dma_desc *vd)450{451 kfree(to_uniphier_xdmac_desc(vd));452}453 454static void uniphier_xdmac_chan_init(struct uniphier_xdmac_device *xdev,455 int ch)456{457 struct uniphier_xdmac_chan *xc = &xdev->channels[ch];458 459 xc->xdev = xdev;460 xc->reg_ch_base = xdev->reg_base + XDMAC_CH_WIDTH * ch;461 xc->vc.desc_free = uniphier_xdmac_desc_free;462 463 vchan_init(&xc->vc, &xdev->ddev);464}465 466static struct dma_chan *of_dma_uniphier_xlate(struct of_phandle_args *dma_spec,467 struct of_dma *ofdma)468{469 struct uniphier_xdmac_device *xdev = ofdma->of_dma_data;470 int chan_id = dma_spec->args[0];471 472 if (chan_id >= xdev->nr_chans)473 return NULL;474 475 xdev->channels[chan_id].id = chan_id;476 xdev->channels[chan_id].req_factor = dma_spec->args[1];477 478 return dma_get_slave_channel(&xdev->channels[chan_id].vc.chan);479}480 481static int uniphier_xdmac_probe(struct platform_device *pdev)482{483 struct uniphier_xdmac_device *xdev;484 struct device *dev = &pdev->dev;485 struct dma_device *ddev;486 int irq;487 int nr_chans;488 int i, ret;489 490 if (of_property_read_u32(dev->of_node, "dma-channels", &nr_chans))491 return -EINVAL;492 if (nr_chans > XDMAC_MAX_CHANS)493 nr_chans = XDMAC_MAX_CHANS;494 495 xdev = devm_kzalloc(dev, struct_size(xdev, channels, nr_chans),496 GFP_KERNEL);497 if (!xdev)498 return -ENOMEM;499 500 xdev->nr_chans = nr_chans;501 xdev->reg_base = devm_platform_ioremap_resource(pdev, 0);502 if (IS_ERR(xdev->reg_base))503 return PTR_ERR(xdev->reg_base);504 505 ddev = &xdev->ddev;506 ddev->dev = dev;507 dma_cap_zero(ddev->cap_mask);508 dma_cap_set(DMA_MEMCPY, ddev->cap_mask);509 dma_cap_set(DMA_SLAVE, ddev->cap_mask);510 ddev->src_addr_widths = UNIPHIER_XDMAC_BUSWIDTHS;511 ddev->dst_addr_widths = UNIPHIER_XDMAC_BUSWIDTHS;512 ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |513 BIT(DMA_MEM_TO_MEM);514 ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;515 ddev->max_burst = XDMAC_MAX_WORDS;516 ddev->device_free_chan_resources = uniphier_xdmac_free_chan_resources;517 ddev->device_prep_dma_memcpy = uniphier_xdmac_prep_dma_memcpy;518 ddev->device_prep_slave_sg = uniphier_xdmac_prep_slave_sg;519 ddev->device_config = uniphier_xdmac_slave_config;520 ddev->device_terminate_all = uniphier_xdmac_terminate_all;521 ddev->device_synchronize = uniphier_xdmac_synchronize;522 ddev->device_tx_status = dma_cookie_status;523 ddev->device_issue_pending = uniphier_xdmac_issue_pending;524 INIT_LIST_HEAD(&ddev->channels);525 526 for (i = 0; i < nr_chans; i++)527 uniphier_xdmac_chan_init(xdev, i);528 529 irq = platform_get_irq(pdev, 0);530 if (irq < 0)531 return irq;532 533 ret = devm_request_irq(dev, irq, uniphier_xdmac_irq_handler,534 IRQF_SHARED, "xdmac", xdev);535 if (ret) {536 dev_err(dev, "Failed to request IRQ\n");537 return ret;538 }539 540 ret = dma_async_device_register(ddev);541 if (ret) {542 dev_err(dev, "Failed to register XDMA device\n");543 return ret;544 }545 546 ret = of_dma_controller_register(dev->of_node,547 of_dma_uniphier_xlate, xdev);548 if (ret) {549 dev_err(dev, "Failed to register XDMA controller\n");550 goto out_unregister_dmac;551 }552 553 platform_set_drvdata(pdev, xdev);554 555 dev_info(&pdev->dev, "UniPhier XDMAC driver (%d channels)\n",556 nr_chans);557 558 return 0;559 560out_unregister_dmac:561 dma_async_device_unregister(ddev);562 563 return ret;564}565 566static void uniphier_xdmac_remove(struct platform_device *pdev)567{568 struct uniphier_xdmac_device *xdev = platform_get_drvdata(pdev);569 struct dma_device *ddev = &xdev->ddev;570 struct dma_chan *chan;571 int ret;572 573 /*574 * Before reaching here, almost all descriptors have been freed by the575 * ->device_free_chan_resources() hook. However, each channel might576 * be still holding one descriptor that was on-flight at that moment.577 * Terminate it to make sure this hardware is no longer running. Then,578 * free the channel resources once again to avoid memory leak.579 */580 list_for_each_entry(chan, &ddev->channels, device_node) {581 ret = dmaengine_terminate_sync(chan);582 if (ret) {583 /*584 * This results in resource leakage and maybe also585 * use-after-free errors as e.g. *xdev is kfreed.586 */587 dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n",588 chan->chan_id, ERR_PTR(ret));589 return;590 }591 uniphier_xdmac_free_chan_resources(chan);592 }593 594 of_dma_controller_free(pdev->dev.of_node);595 dma_async_device_unregister(ddev);596}597 598static const struct of_device_id uniphier_xdmac_match[] = {599 { .compatible = "socionext,uniphier-xdmac" },600 { /* sentinel */ }601};602MODULE_DEVICE_TABLE(of, uniphier_xdmac_match);603 604static struct platform_driver uniphier_xdmac_driver = {605 .probe = uniphier_xdmac_probe,606 .remove_new = uniphier_xdmac_remove,607 .driver = {608 .name = "uniphier-xdmac",609 .of_match_table = uniphier_xdmac_match,610 },611};612module_platform_driver(uniphier_xdmac_driver);613 614MODULE_AUTHOR("Kunihiko Hayashi <hayashi.kunihiko@socionext.com>");615MODULE_DESCRIPTION("UniPhier external DMA controller driver");616MODULE_LICENSE("GPL v2");617