709 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Core driver for the Intel integrated DMA 64-bit4 *5 * Copyright (C) 2015 Intel Corporation6 * Author: Andy Shevchenko <andriy.shevchenko@linux.intel.com>7 */8 9#include <linux/bitops.h>10#include <linux/delay.h>11#include <linux/dmaengine.h>12#include <linux/dma-mapping.h>13#include <linux/dmapool.h>14#include <linux/init.h>15#include <linux/module.h>16#include <linux/platform_device.h>17#include <linux/slab.h>18 19#include <linux/dma/idma64.h>20 21#include "idma64.h"22 23/* For now we support only two channels */24#define IDMA64_NR_CHAN 225 26/* ---------------------------------------------------------------------- */27 28static struct device *chan2dev(struct dma_chan *chan)29{30 return &chan->dev->device;31}32 33/* ---------------------------------------------------------------------- */34 35static void idma64_off(struct idma64 *idma64)36{37 unsigned short count = 100;38 39 dma_writel(idma64, CFG, 0);40 41 channel_clear_bit(idma64, MASK(XFER), idma64->all_chan_mask);42 channel_clear_bit(idma64, MASK(BLOCK), idma64->all_chan_mask);43 channel_clear_bit(idma64, MASK(SRC_TRAN), idma64->all_chan_mask);44 channel_clear_bit(idma64, MASK(DST_TRAN), idma64->all_chan_mask);45 channel_clear_bit(idma64, MASK(ERROR), idma64->all_chan_mask);46 47 do {48 cpu_relax();49 } while (dma_readl(idma64, CFG) & IDMA64_CFG_DMA_EN && --count);50}51 52static void idma64_on(struct idma64 *idma64)53{54 dma_writel(idma64, CFG, IDMA64_CFG_DMA_EN);55}56 57/* ---------------------------------------------------------------------- */58 59static void idma64_chan_init(struct idma64 *idma64, struct idma64_chan *idma64c)60{61 u32 cfghi = IDMA64C_CFGH_SRC_PER(1) | IDMA64C_CFGH_DST_PER(0);62 u32 cfglo = 0;63 64 /* Set default burst alignment */65 cfglo |= IDMA64C_CFGL_DST_BURST_ALIGN | IDMA64C_CFGL_SRC_BURST_ALIGN;66 67 channel_writel(idma64c, CFG_LO, cfglo);68 channel_writel(idma64c, CFG_HI, cfghi);69 70 /* Enable interrupts */71 channel_set_bit(idma64, MASK(XFER), idma64c->mask);72 channel_set_bit(idma64, MASK(ERROR), idma64c->mask);73 74 /*75 * Enforce the controller to be turned on.76 *77 * The iDMA is turned off in ->probe() and looses context during system78 * suspend / resume cycle. That's why we have to enable it each time we79 * use it.80 */81 idma64_on(idma64);82}83 84static void idma64_chan_stop(struct idma64 *idma64, struct idma64_chan *idma64c)85{86 channel_clear_bit(idma64, CH_EN, idma64c->mask);87}88 89static void idma64_chan_start(struct idma64 *idma64, struct idma64_chan *idma64c)90{91 struct idma64_desc *desc = idma64c->desc;92 struct idma64_hw_desc *hw = &desc->hw[0];93 94 channel_writeq(idma64c, SAR, 0);95 channel_writeq(idma64c, DAR, 0);96 97 channel_writel(idma64c, CTL_HI, IDMA64C_CTLH_BLOCK_TS(~0UL));98 channel_writel(idma64c, CTL_LO, IDMA64C_CTLL_LLP_S_EN | IDMA64C_CTLL_LLP_D_EN);99 100 channel_writeq(idma64c, LLP, hw->llp);101 102 channel_set_bit(idma64, CH_EN, idma64c->mask);103}104 105static void idma64_stop_transfer(struct idma64_chan *idma64c)106{107 struct idma64 *idma64 = to_idma64(idma64c->vchan.chan.device);108 109 idma64_chan_stop(idma64, idma64c);110}111 112static void idma64_start_transfer(struct idma64_chan *idma64c)113{114 struct idma64 *idma64 = to_idma64(idma64c->vchan.chan.device);115 struct virt_dma_desc *vdesc;116 117 /* Get the next descriptor */118 vdesc = vchan_next_desc(&idma64c->vchan);119 if (!vdesc) {120 idma64c->desc = NULL;121 return;122 }123 124 list_del(&vdesc->node);125 idma64c->desc = to_idma64_desc(vdesc);126 127 /* Configure the channel */128 idma64_chan_init(idma64, idma64c);129 130 /* Start the channel with a new descriptor */131 idma64_chan_start(idma64, idma64c);132}133 134/* ---------------------------------------------------------------------- */135 136static void idma64_chan_irq(struct idma64 *idma64, unsigned short c,137 u32 status_err, u32 status_xfer)138{139 struct idma64_chan *idma64c = &idma64->chan[c];140 struct dma_chan_percpu *stat;141 struct idma64_desc *desc;142 143 stat = this_cpu_ptr(idma64c->vchan.chan.local);144 145 spin_lock(&idma64c->vchan.lock);146 desc = idma64c->desc;147 if (desc) {148 if (status_err & (1 << c)) {149 dma_writel(idma64, CLEAR(ERROR), idma64c->mask);150 desc->status = DMA_ERROR;151 } else if (status_xfer & (1 << c)) {152 dma_writel(idma64, CLEAR(XFER), idma64c->mask);153 desc->status = DMA_COMPLETE;154 vchan_cookie_complete(&desc->vdesc);155 stat->bytes_transferred += desc->length;156 idma64_start_transfer(idma64c);157 }158 159 /* idma64_start_transfer() updates idma64c->desc */160 if (idma64c->desc == NULL || desc->status == DMA_ERROR)161 idma64_stop_transfer(idma64c);162 }163 spin_unlock(&idma64c->vchan.lock);164}165 166static irqreturn_t idma64_irq(int irq, void *dev)167{168 struct idma64 *idma64 = dev;169 u32 status = dma_readl(idma64, STATUS_INT);170 u32 status_xfer;171 u32 status_err;172 unsigned short i;173 174 /* Since IRQ may be shared, check if DMA controller is powered on */175 if (status == GENMASK(31, 0))176 return IRQ_NONE;177 178 dev_vdbg(idma64->dma.dev, "%s: status=%#x\n", __func__, status);179 180 /* Check if we have any interrupt from the DMA controller */181 if (!status)182 return IRQ_NONE;183 184 status_xfer = dma_readl(idma64, RAW(XFER));185 status_err = dma_readl(idma64, RAW(ERROR));186 187 for (i = 0; i < idma64->dma.chancnt; i++)188 idma64_chan_irq(idma64, i, status_err, status_xfer);189 190 return IRQ_HANDLED;191}192 193/* ---------------------------------------------------------------------- */194 195static struct idma64_desc *idma64_alloc_desc(unsigned int ndesc)196{197 struct idma64_desc *desc;198 199 desc = kzalloc(sizeof(*desc), GFP_NOWAIT);200 if (!desc)201 return NULL;202 203 desc->hw = kcalloc(ndesc, sizeof(*desc->hw), GFP_NOWAIT);204 if (!desc->hw) {205 kfree(desc);206 return NULL;207 }208 209 return desc;210}211 212static void idma64_desc_free(struct idma64_chan *idma64c,213 struct idma64_desc *desc)214{215 struct idma64_hw_desc *hw;216 217 if (desc->ndesc) {218 unsigned int i = desc->ndesc;219 220 do {221 hw = &desc->hw[--i];222 dma_pool_free(idma64c->pool, hw->lli, hw->llp);223 } while (i);224 }225 226 kfree(desc->hw);227 kfree(desc);228}229 230static void idma64_vdesc_free(struct virt_dma_desc *vdesc)231{232 struct idma64_chan *idma64c = to_idma64_chan(vdesc->tx.chan);233 234 idma64_desc_free(idma64c, to_idma64_desc(vdesc));235}236 237static void idma64_hw_desc_fill(struct idma64_hw_desc *hw,238 struct dma_slave_config *config,239 enum dma_transfer_direction direction, u64 llp)240{241 struct idma64_lli *lli = hw->lli;242 u64 sar, dar;243 u32 ctlhi = IDMA64C_CTLH_BLOCK_TS(hw->len);244 u32 ctllo = IDMA64C_CTLL_LLP_S_EN | IDMA64C_CTLL_LLP_D_EN;245 u32 src_width, dst_width;246 247 if (direction == DMA_MEM_TO_DEV) {248 sar = hw->phys;249 dar = config->dst_addr;250 ctllo |= IDMA64C_CTLL_DST_FIX | IDMA64C_CTLL_SRC_INC |251 IDMA64C_CTLL_FC_M2P;252 src_width = __ffs(sar | hw->len | 4);253 dst_width = __ffs(config->dst_addr_width);254 } else { /* DMA_DEV_TO_MEM */255 sar = config->src_addr;256 dar = hw->phys;257 ctllo |= IDMA64C_CTLL_DST_INC | IDMA64C_CTLL_SRC_FIX |258 IDMA64C_CTLL_FC_P2M;259 src_width = __ffs(config->src_addr_width);260 dst_width = __ffs(dar | hw->len | 4);261 }262 263 lli->sar = sar;264 lli->dar = dar;265 266 lli->ctlhi = ctlhi;267 lli->ctllo = ctllo |268 IDMA64C_CTLL_SRC_MSIZE(config->src_maxburst) |269 IDMA64C_CTLL_DST_MSIZE(config->dst_maxburst) |270 IDMA64C_CTLL_DST_WIDTH(dst_width) |271 IDMA64C_CTLL_SRC_WIDTH(src_width);272 273 lli->llp = llp;274}275 276static void idma64_desc_fill(struct idma64_chan *idma64c,277 struct idma64_desc *desc)278{279 struct dma_slave_config *config = &idma64c->config;280 unsigned int i = desc->ndesc;281 struct idma64_hw_desc *hw = &desc->hw[i - 1];282 struct idma64_lli *lli = hw->lli;283 u64 llp = 0;284 285 /* Fill the hardware descriptors and link them to a list */286 do {287 hw = &desc->hw[--i];288 idma64_hw_desc_fill(hw, config, desc->direction, llp);289 llp = hw->llp;290 desc->length += hw->len;291 } while (i);292 293 /* Trigger an interrupt after the last block is transferred */294 lli->ctllo |= IDMA64C_CTLL_INT_EN;295 296 /* Disable LLP transfer in the last block */297 lli->ctllo &= ~(IDMA64C_CTLL_LLP_S_EN | IDMA64C_CTLL_LLP_D_EN);298}299 300static struct dma_async_tx_descriptor *idma64_prep_slave_sg(301 struct dma_chan *chan, struct scatterlist *sgl,302 unsigned int sg_len, enum dma_transfer_direction direction,303 unsigned long flags, void *context)304{305 struct idma64_chan *idma64c = to_idma64_chan(chan);306 struct idma64_desc *desc;307 struct scatterlist *sg;308 unsigned int i;309 310 desc = idma64_alloc_desc(sg_len);311 if (!desc)312 return NULL;313 314 for_each_sg(sgl, sg, sg_len, i) {315 struct idma64_hw_desc *hw = &desc->hw[i];316 317 /* Allocate DMA capable memory for hardware descriptor */318 hw->lli = dma_pool_alloc(idma64c->pool, GFP_NOWAIT, &hw->llp);319 if (!hw->lli) {320 desc->ndesc = i;321 idma64_desc_free(idma64c, desc);322 return NULL;323 }324 325 hw->phys = sg_dma_address(sg);326 hw->len = sg_dma_len(sg);327 }328 329 desc->ndesc = sg_len;330 desc->direction = direction;331 desc->status = DMA_IN_PROGRESS;332 333 idma64_desc_fill(idma64c, desc);334 return vchan_tx_prep(&idma64c->vchan, &desc->vdesc, flags);335}336 337static void idma64_issue_pending(struct dma_chan *chan)338{339 struct idma64_chan *idma64c = to_idma64_chan(chan);340 unsigned long flags;341 342 spin_lock_irqsave(&idma64c->vchan.lock, flags);343 if (vchan_issue_pending(&idma64c->vchan) && !idma64c->desc)344 idma64_start_transfer(idma64c);345 spin_unlock_irqrestore(&idma64c->vchan.lock, flags);346}347 348static size_t idma64_active_desc_size(struct idma64_chan *idma64c)349{350 struct idma64_desc *desc = idma64c->desc;351 struct idma64_hw_desc *hw;352 size_t bytes = desc->length;353 u64 llp = channel_readq(idma64c, LLP);354 u32 ctlhi = channel_readl(idma64c, CTL_HI);355 unsigned int i = 0;356 357 do {358 hw = &desc->hw[i];359 if (hw->llp == llp)360 break;361 bytes -= hw->len;362 } while (++i < desc->ndesc);363 364 if (!i)365 return bytes;366 367 /* The current chunk is not fully transferred yet */368 bytes += desc->hw[--i].len;369 370 return bytes - IDMA64C_CTLH_BLOCK_TS(ctlhi);371}372 373static enum dma_status idma64_tx_status(struct dma_chan *chan,374 dma_cookie_t cookie, struct dma_tx_state *state)375{376 struct idma64_chan *idma64c = to_idma64_chan(chan);377 struct virt_dma_desc *vdesc;378 enum dma_status status;379 size_t bytes;380 unsigned long flags;381 382 status = dma_cookie_status(chan, cookie, state);383 if (status == DMA_COMPLETE)384 return status;385 386 spin_lock_irqsave(&idma64c->vchan.lock, flags);387 vdesc = vchan_find_desc(&idma64c->vchan, cookie);388 if (idma64c->desc && cookie == idma64c->desc->vdesc.tx.cookie) {389 bytes = idma64_active_desc_size(idma64c);390 dma_set_residue(state, bytes);391 status = idma64c->desc->status;392 } else if (vdesc) {393 bytes = to_idma64_desc(vdesc)->length;394 dma_set_residue(state, bytes);395 }396 spin_unlock_irqrestore(&idma64c->vchan.lock, flags);397 398 return status;399}400 401static void convert_burst(u32 *maxburst)402{403 if (*maxburst)404 *maxburst = __fls(*maxburst);405 else406 *maxburst = 0;407}408 409static int idma64_slave_config(struct dma_chan *chan,410 struct dma_slave_config *config)411{412 struct idma64_chan *idma64c = to_idma64_chan(chan);413 414 memcpy(&idma64c->config, config, sizeof(idma64c->config));415 416 convert_burst(&idma64c->config.src_maxburst);417 convert_burst(&idma64c->config.dst_maxburst);418 419 return 0;420}421 422static void idma64_chan_deactivate(struct idma64_chan *idma64c, bool drain)423{424 unsigned short count = 100;425 u32 cfglo;426 427 cfglo = channel_readl(idma64c, CFG_LO);428 if (drain)429 cfglo |= IDMA64C_CFGL_CH_DRAIN;430 else431 cfglo &= ~IDMA64C_CFGL_CH_DRAIN;432 433 channel_writel(idma64c, CFG_LO, cfglo | IDMA64C_CFGL_CH_SUSP);434 do {435 udelay(1);436 cfglo = channel_readl(idma64c, CFG_LO);437 } while (!(cfglo & IDMA64C_CFGL_FIFO_EMPTY) && --count);438}439 440static void idma64_chan_activate(struct idma64_chan *idma64c)441{442 u32 cfglo;443 444 cfglo = channel_readl(idma64c, CFG_LO);445 channel_writel(idma64c, CFG_LO, cfglo & ~IDMA64C_CFGL_CH_SUSP);446}447 448static int idma64_pause(struct dma_chan *chan)449{450 struct idma64_chan *idma64c = to_idma64_chan(chan);451 unsigned long flags;452 453 spin_lock_irqsave(&idma64c->vchan.lock, flags);454 if (idma64c->desc && idma64c->desc->status == DMA_IN_PROGRESS) {455 idma64_chan_deactivate(idma64c, false);456 idma64c->desc->status = DMA_PAUSED;457 }458 spin_unlock_irqrestore(&idma64c->vchan.lock, flags);459 460 return 0;461}462 463static int idma64_resume(struct dma_chan *chan)464{465 struct idma64_chan *idma64c = to_idma64_chan(chan);466 unsigned long flags;467 468 spin_lock_irqsave(&idma64c->vchan.lock, flags);469 if (idma64c->desc && idma64c->desc->status == DMA_PAUSED) {470 idma64c->desc->status = DMA_IN_PROGRESS;471 idma64_chan_activate(idma64c);472 }473 spin_unlock_irqrestore(&idma64c->vchan.lock, flags);474 475 return 0;476}477 478static int idma64_terminate_all(struct dma_chan *chan)479{480 struct idma64_chan *idma64c = to_idma64_chan(chan);481 unsigned long flags;482 LIST_HEAD(head);483 484 spin_lock_irqsave(&idma64c->vchan.lock, flags);485 idma64_chan_deactivate(idma64c, true);486 idma64_stop_transfer(idma64c);487 if (idma64c->desc) {488 idma64_vdesc_free(&idma64c->desc->vdesc);489 idma64c->desc = NULL;490 }491 vchan_get_all_descriptors(&idma64c->vchan, &head);492 spin_unlock_irqrestore(&idma64c->vchan.lock, flags);493 494 vchan_dma_desc_free_list(&idma64c->vchan, &head);495 return 0;496}497 498static void idma64_synchronize(struct dma_chan *chan)499{500 struct idma64_chan *idma64c = to_idma64_chan(chan);501 502 vchan_synchronize(&idma64c->vchan);503}504 505static int idma64_alloc_chan_resources(struct dma_chan *chan)506{507 struct idma64_chan *idma64c = to_idma64_chan(chan);508 509 /* Create a pool of consistent memory blocks for hardware descriptors */510 idma64c->pool = dma_pool_create(dev_name(chan2dev(chan)),511 chan->device->dev,512 sizeof(struct idma64_lli), 8, 0);513 if (!idma64c->pool) {514 dev_err(chan2dev(chan), "No memory for descriptors\n");515 return -ENOMEM;516 }517 518 return 0;519}520 521static void idma64_free_chan_resources(struct dma_chan *chan)522{523 struct idma64_chan *idma64c = to_idma64_chan(chan);524 525 vchan_free_chan_resources(to_virt_chan(chan));526 dma_pool_destroy(idma64c->pool);527 idma64c->pool = NULL;528}529 530/* ---------------------------------------------------------------------- */531 532#define IDMA64_BUSWIDTHS \533 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \534 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \535 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)536 537static int idma64_probe(struct idma64_chip *chip)538{539 struct idma64 *idma64;540 unsigned short nr_chan = IDMA64_NR_CHAN;541 unsigned short i;542 int ret;543 544 idma64 = devm_kzalloc(chip->dev, sizeof(*idma64), GFP_KERNEL);545 if (!idma64)546 return -ENOMEM;547 548 idma64->regs = chip->regs;549 chip->idma64 = idma64;550 551 idma64->chan = devm_kcalloc(chip->dev, nr_chan, sizeof(*idma64->chan),552 GFP_KERNEL);553 if (!idma64->chan)554 return -ENOMEM;555 556 idma64->all_chan_mask = (1 << nr_chan) - 1;557 558 /* Turn off iDMA controller */559 idma64_off(idma64);560 561 ret = devm_request_irq(chip->dev, chip->irq, idma64_irq, IRQF_SHARED,562 dev_name(chip->dev), idma64);563 if (ret)564 return ret;565 566 INIT_LIST_HEAD(&idma64->dma.channels);567 for (i = 0; i < nr_chan; i++) {568 struct idma64_chan *idma64c = &idma64->chan[i];569 570 idma64c->vchan.desc_free = idma64_vdesc_free;571 vchan_init(&idma64c->vchan, &idma64->dma);572 573 idma64c->regs = idma64->regs + i * IDMA64_CH_LENGTH;574 idma64c->mask = BIT(i);575 }576 577 dma_cap_set(DMA_SLAVE, idma64->dma.cap_mask);578 dma_cap_set(DMA_PRIVATE, idma64->dma.cap_mask);579 580 idma64->dma.device_alloc_chan_resources = idma64_alloc_chan_resources;581 idma64->dma.device_free_chan_resources = idma64_free_chan_resources;582 583 idma64->dma.device_prep_slave_sg = idma64_prep_slave_sg;584 585 idma64->dma.device_issue_pending = idma64_issue_pending;586 idma64->dma.device_tx_status = idma64_tx_status;587 588 idma64->dma.device_config = idma64_slave_config;589 idma64->dma.device_pause = idma64_pause;590 idma64->dma.device_resume = idma64_resume;591 idma64->dma.device_terminate_all = idma64_terminate_all;592 idma64->dma.device_synchronize = idma64_synchronize;593 594 idma64->dma.src_addr_widths = IDMA64_BUSWIDTHS;595 idma64->dma.dst_addr_widths = IDMA64_BUSWIDTHS;596 idma64->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);597 idma64->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;598 599 idma64->dma.dev = chip->sysdev;600 601 dma_set_max_seg_size(idma64->dma.dev, IDMA64C_CTLH_BLOCK_TS_MASK);602 603 ret = dma_async_device_register(&idma64->dma);604 if (ret)605 return ret;606 607 dev_info(chip->dev, "Found Intel integrated DMA 64-bit\n");608 return 0;609}610 611static void idma64_remove(struct idma64_chip *chip)612{613 struct idma64 *idma64 = chip->idma64;614 unsigned short i;615 616 dma_async_device_unregister(&idma64->dma);617 618 /*619 * Explicitly call devm_request_irq() to avoid the side effects with620 * the scheduled tasklets.621 */622 devm_free_irq(chip->dev, chip->irq, idma64);623 624 for (i = 0; i < idma64->dma.chancnt; i++) {625 struct idma64_chan *idma64c = &idma64->chan[i];626 627 tasklet_kill(&idma64c->vchan.task);628 }629}630 631/* ---------------------------------------------------------------------- */632 633static int idma64_platform_probe(struct platform_device *pdev)634{635 struct idma64_chip *chip;636 struct device *dev = &pdev->dev;637 struct device *sysdev = dev->parent;638 int ret;639 640 chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL);641 if (!chip)642 return -ENOMEM;643 644 chip->irq = platform_get_irq(pdev, 0);645 if (chip->irq < 0)646 return chip->irq;647 648 chip->regs = devm_platform_ioremap_resource(pdev, 0);649 if (IS_ERR(chip->regs))650 return PTR_ERR(chip->regs);651 652 ret = dma_coerce_mask_and_coherent(sysdev, DMA_BIT_MASK(64));653 if (ret)654 return ret;655 656 chip->dev = dev;657 chip->sysdev = sysdev;658 659 ret = idma64_probe(chip);660 if (ret)661 return ret;662 663 platform_set_drvdata(pdev, chip);664 return 0;665}666 667static void idma64_platform_remove(struct platform_device *pdev)668{669 struct idma64_chip *chip = platform_get_drvdata(pdev);670 671 idma64_remove(chip);672}673 674static int __maybe_unused idma64_pm_suspend(struct device *dev)675{676 struct idma64_chip *chip = dev_get_drvdata(dev);677 678 idma64_off(chip->idma64);679 return 0;680}681 682static int __maybe_unused idma64_pm_resume(struct device *dev)683{684 struct idma64_chip *chip = dev_get_drvdata(dev);685 686 idma64_on(chip->idma64);687 return 0;688}689 690static const struct dev_pm_ops idma64_dev_pm_ops = {691 SET_SYSTEM_SLEEP_PM_OPS(idma64_pm_suspend, idma64_pm_resume)692};693 694static struct platform_driver idma64_platform_driver = {695 .probe = idma64_platform_probe,696 .remove_new = idma64_platform_remove,697 .driver = {698 .name = LPSS_IDMA64_DRIVER_NAME,699 .pm = &idma64_dev_pm_ops,700 },701};702 703module_platform_driver(idma64_platform_driver);704 705MODULE_LICENSE("GPL v2");706MODULE_DESCRIPTION("iDMA64 core driver");707MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");708MODULE_ALIAS("platform:" LPSS_IDMA64_DRIVER_NAME);709