1015 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Renesas RZ/G2L DMA Controller Driver4 *5 * Based on imx-dma.c6 *7 * Copyright (C) 2021 Renesas Electronics Corp.8 * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>9 * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com>10 */11 12#include <linux/bitfield.h>13#include <linux/dma-mapping.h>14#include <linux/dmaengine.h>15#include <linux/interrupt.h>16#include <linux/iopoll.h>17#include <linux/list.h>18#include <linux/module.h>19#include <linux/of.h>20#include <linux/of_dma.h>21#include <linux/of_platform.h>22#include <linux/platform_device.h>23#include <linux/pm_runtime.h>24#include <linux/reset.h>25#include <linux/slab.h>26#include <linux/spinlock.h>27 28#include "../dmaengine.h"29#include "../virt-dma.h"30 31enum rz_dmac_prep_type {32 RZ_DMAC_DESC_MEMCPY,33 RZ_DMAC_DESC_SLAVE_SG,34};35 36struct rz_lmdesc {37 u32 header;38 u32 sa;39 u32 da;40 u32 tb;41 u32 chcfg;42 u32 chitvl;43 u32 chext;44 u32 nxla;45};46 47struct rz_dmac_desc {48 struct virt_dma_desc vd;49 dma_addr_t src;50 dma_addr_t dest;51 size_t len;52 struct list_head node;53 enum dma_transfer_direction direction;54 enum rz_dmac_prep_type type;55 /* For slave sg */56 struct scatterlist *sg;57 unsigned int sgcount;58};59 60#define to_rz_dmac_desc(d) container_of(d, struct rz_dmac_desc, vd)61 62struct rz_dmac_chan {63 struct virt_dma_chan vc;64 void __iomem *ch_base;65 void __iomem *ch_cmn_base;66 unsigned int index;67 int irq;68 struct rz_dmac_desc *desc;69 int descs_allocated;70 71 dma_addr_t src_per_address;72 dma_addr_t dst_per_address;73 74 u32 chcfg;75 u32 chctrl;76 int mid_rid;77 78 struct list_head ld_free;79 struct list_head ld_queue;80 struct list_head ld_active;81 82 struct {83 struct rz_lmdesc *base;84 struct rz_lmdesc *head;85 struct rz_lmdesc *tail;86 dma_addr_t base_dma;87 } lmdesc;88};89 90#define to_rz_dmac_chan(c) container_of(c, struct rz_dmac_chan, vc.chan)91 92struct rz_dmac {93 struct dma_device engine;94 struct device *dev;95 struct reset_control *rstc;96 void __iomem *base;97 void __iomem *ext_base;98 99 unsigned int n_channels;100 struct rz_dmac_chan *channels;101 102 DECLARE_BITMAP(modules, 1024);103};104 105#define to_rz_dmac(d) container_of(d, struct rz_dmac, engine)106 107/*108 * -----------------------------------------------------------------------------109 * Registers110 */111 112#define CHSTAT 0x0024113#define CHCTRL 0x0028114#define CHCFG 0x002c115#define NXLA 0x0038116 117#define DCTRL 0x0000118 119#define EACH_CHANNEL_OFFSET 0x0040120#define CHANNEL_0_7_OFFSET 0x0000121#define CHANNEL_0_7_COMMON_BASE 0x0300122#define CHANNEL_8_15_OFFSET 0x0400123#define CHANNEL_8_15_COMMON_BASE 0x0700124 125#define CHSTAT_ER BIT(4)126#define CHSTAT_EN BIT(0)127 128#define CHCTRL_CLRINTMSK BIT(17)129#define CHCTRL_CLRSUS BIT(9)130#define CHCTRL_CLRTC BIT(6)131#define CHCTRL_CLREND BIT(5)132#define CHCTRL_CLRRQ BIT(4)133#define CHCTRL_SWRST BIT(3)134#define CHCTRL_STG BIT(2)135#define CHCTRL_CLREN BIT(1)136#define CHCTRL_SETEN BIT(0)137#define CHCTRL_DEFAULT (CHCTRL_CLRINTMSK | CHCTRL_CLRSUS | \138 CHCTRL_CLRTC | CHCTRL_CLREND | \139 CHCTRL_CLRRQ | CHCTRL_SWRST | \140 CHCTRL_CLREN)141 142#define CHCFG_DMS BIT(31)143#define CHCFG_DEM BIT(24)144#define CHCFG_DAD BIT(21)145#define CHCFG_SAD BIT(20)146#define CHCFG_REQD BIT(3)147#define CHCFG_SEL(bits) ((bits) & 0x07)148#define CHCFG_MEM_COPY (0x80400008)149#define CHCFG_FILL_DDS_MASK GENMASK(19, 16)150#define CHCFG_FILL_SDS_MASK GENMASK(15, 12)151#define CHCFG_FILL_TM(a) (((a) & BIT(5)) << 22)152#define CHCFG_FILL_AM(a) (((a) & GENMASK(4, 2)) << 6)153#define CHCFG_FILL_LVL(a) (((a) & BIT(1)) << 5)154#define CHCFG_FILL_HIEN(a) (((a) & BIT(0)) << 5)155 156#define MID_RID_MASK GENMASK(9, 0)157#define CHCFG_MASK GENMASK(15, 10)158#define CHCFG_DS_INVALID 0xFF159#define DCTRL_LVINT BIT(1)160#define DCTRL_PR BIT(0)161#define DCTRL_DEFAULT (DCTRL_LVINT | DCTRL_PR)162 163/* LINK MODE DESCRIPTOR */164#define HEADER_LV BIT(0)165 166#define RZ_DMAC_MAX_CHAN_DESCRIPTORS 16167#define RZ_DMAC_MAX_CHANNELS 16168#define DMAC_NR_LMDESC 64169 170/*171 * -----------------------------------------------------------------------------172 * Device access173 */174 175static void rz_dmac_writel(struct rz_dmac *dmac, unsigned int val,176 unsigned int offset)177{178 writel(val, dmac->base + offset);179}180 181static void rz_dmac_ext_writel(struct rz_dmac *dmac, unsigned int val,182 unsigned int offset)183{184 writel(val, dmac->ext_base + offset);185}186 187static u32 rz_dmac_ext_readl(struct rz_dmac *dmac, unsigned int offset)188{189 return readl(dmac->ext_base + offset);190}191 192static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val,193 unsigned int offset, int which)194{195 if (which)196 writel(val, channel->ch_base + offset);197 else198 writel(val, channel->ch_cmn_base + offset);199}200 201static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel,202 unsigned int offset, int which)203{204 if (which)205 return readl(channel->ch_base + offset);206 else207 return readl(channel->ch_cmn_base + offset);208}209 210/*211 * -----------------------------------------------------------------------------212 * Initialization213 */214 215static void rz_lmdesc_setup(struct rz_dmac_chan *channel,216 struct rz_lmdesc *lmdesc)217{218 u32 nxla;219 220 channel->lmdesc.base = lmdesc;221 channel->lmdesc.head = lmdesc;222 channel->lmdesc.tail = lmdesc;223 nxla = channel->lmdesc.base_dma;224 while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) {225 lmdesc->header = 0;226 nxla += sizeof(*lmdesc);227 lmdesc->nxla = nxla;228 lmdesc++;229 }230 231 lmdesc->header = 0;232 lmdesc->nxla = channel->lmdesc.base_dma;233}234 235/*236 * -----------------------------------------------------------------------------237 * Descriptors preparation238 */239 240static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel)241{242 struct rz_lmdesc *lmdesc = channel->lmdesc.head;243 244 while (!(lmdesc->header & HEADER_LV)) {245 lmdesc->header = 0;246 lmdesc++;247 if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))248 lmdesc = channel->lmdesc.base;249 }250 channel->lmdesc.head = lmdesc;251}252 253static void rz_dmac_enable_hw(struct rz_dmac_chan *channel)254{255 struct dma_chan *chan = &channel->vc.chan;256 struct rz_dmac *dmac = to_rz_dmac(chan->device);257 unsigned long flags;258 u32 nxla;259 u32 chctrl;260 u32 chstat;261 262 dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);263 264 local_irq_save(flags);265 266 rz_dmac_lmdesc_recycle(channel);267 268 nxla = channel->lmdesc.base_dma +269 (sizeof(struct rz_lmdesc) * (channel->lmdesc.head -270 channel->lmdesc.base));271 272 chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);273 if (!(chstat & CHSTAT_EN)) {274 chctrl = (channel->chctrl | CHCTRL_SETEN);275 rz_dmac_ch_writel(channel, nxla, NXLA, 1);276 rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);277 rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);278 rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1);279 }280 281 local_irq_restore(flags);282}283 284static void rz_dmac_disable_hw(struct rz_dmac_chan *channel)285{286 struct dma_chan *chan = &channel->vc.chan;287 struct rz_dmac *dmac = to_rz_dmac(chan->device);288 unsigned long flags;289 290 dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);291 292 local_irq_save(flags);293 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);294 local_irq_restore(flags);295}296 297static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars)298{299 u32 dmars_offset = (nr / 2) * 4;300 u32 shift = (nr % 2) * 16;301 u32 dmars32;302 303 dmars32 = rz_dmac_ext_readl(dmac, dmars_offset);304 dmars32 &= ~(0xffff << shift);305 dmars32 |= dmars << shift;306 307 rz_dmac_ext_writel(dmac, dmars32, dmars_offset);308}309 310static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel)311{312 struct dma_chan *chan = &channel->vc.chan;313 struct rz_dmac *dmac = to_rz_dmac(chan->device);314 struct rz_lmdesc *lmdesc = channel->lmdesc.tail;315 struct rz_dmac_desc *d = channel->desc;316 u32 chcfg = CHCFG_MEM_COPY;317 318 /* prepare descriptor */319 lmdesc->sa = d->src;320 lmdesc->da = d->dest;321 lmdesc->tb = d->len;322 lmdesc->chcfg = chcfg;323 lmdesc->chitvl = 0;324 lmdesc->chext = 0;325 lmdesc->header = HEADER_LV;326 327 rz_dmac_set_dmars_register(dmac, channel->index, 0);328 329 channel->chcfg = chcfg;330 channel->chctrl = CHCTRL_STG | CHCTRL_SETEN;331}332 333static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel)334{335 struct dma_chan *chan = &channel->vc.chan;336 struct rz_dmac *dmac = to_rz_dmac(chan->device);337 struct rz_dmac_desc *d = channel->desc;338 struct scatterlist *sg, *sgl = d->sg;339 struct rz_lmdesc *lmdesc;340 unsigned int i, sg_len = d->sgcount;341 342 channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS;343 344 if (d->direction == DMA_DEV_TO_MEM) {345 channel->chcfg |= CHCFG_SAD;346 channel->chcfg &= ~CHCFG_REQD;347 } else {348 channel->chcfg |= CHCFG_DAD | CHCFG_REQD;349 }350 351 lmdesc = channel->lmdesc.tail;352 353 for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) {354 if (d->direction == DMA_DEV_TO_MEM) {355 lmdesc->sa = channel->src_per_address;356 lmdesc->da = sg_dma_address(sg);357 } else {358 lmdesc->sa = sg_dma_address(sg);359 lmdesc->da = channel->dst_per_address;360 }361 362 lmdesc->tb = sg_dma_len(sg);363 lmdesc->chitvl = 0;364 lmdesc->chext = 0;365 if (i == (sg_len - 1)) {366 lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM);367 lmdesc->header = HEADER_LV;368 } else {369 lmdesc->chcfg = channel->chcfg;370 lmdesc->header = HEADER_LV;371 }372 if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))373 lmdesc = channel->lmdesc.base;374 }375 376 channel->lmdesc.tail = lmdesc;377 378 rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid);379 channel->chctrl = CHCTRL_SETEN;380}381 382static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan)383{384 struct rz_dmac_desc *d = chan->desc;385 struct virt_dma_desc *vd;386 387 vd = vchan_next_desc(&chan->vc);388 if (!vd)389 return 0;390 391 list_del(&vd->node);392 393 switch (d->type) {394 case RZ_DMAC_DESC_MEMCPY:395 rz_dmac_prepare_desc_for_memcpy(chan);396 break;397 398 case RZ_DMAC_DESC_SLAVE_SG:399 rz_dmac_prepare_descs_for_slave_sg(chan);400 break;401 402 default:403 return -EINVAL;404 }405 406 rz_dmac_enable_hw(chan);407 408 return 0;409}410 411/*412 * -----------------------------------------------------------------------------413 * DMA engine operations414 */415 416static int rz_dmac_alloc_chan_resources(struct dma_chan *chan)417{418 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);419 420 while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) {421 struct rz_dmac_desc *desc;422 423 desc = kzalloc(sizeof(*desc), GFP_KERNEL);424 if (!desc)425 break;426 427 list_add_tail(&desc->node, &channel->ld_free);428 channel->descs_allocated++;429 }430 431 if (!channel->descs_allocated)432 return -ENOMEM;433 434 return channel->descs_allocated;435}436 437static void rz_dmac_free_chan_resources(struct dma_chan *chan)438{439 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);440 struct rz_dmac *dmac = to_rz_dmac(chan->device);441 struct rz_lmdesc *lmdesc = channel->lmdesc.base;442 struct rz_dmac_desc *desc, *_desc;443 unsigned long flags;444 unsigned int i;445 446 spin_lock_irqsave(&channel->vc.lock, flags);447 448 for (i = 0; i < DMAC_NR_LMDESC; i++)449 lmdesc[i].header = 0;450 451 rz_dmac_disable_hw(channel);452 list_splice_tail_init(&channel->ld_active, &channel->ld_free);453 list_splice_tail_init(&channel->ld_queue, &channel->ld_free);454 455 if (channel->mid_rid >= 0) {456 clear_bit(channel->mid_rid, dmac->modules);457 channel->mid_rid = -EINVAL;458 }459 460 spin_unlock_irqrestore(&channel->vc.lock, flags);461 462 list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) {463 kfree(desc);464 channel->descs_allocated--;465 }466 467 INIT_LIST_HEAD(&channel->ld_free);468 vchan_free_chan_resources(&channel->vc);469}470 471static struct dma_async_tx_descriptor *472rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,473 size_t len, unsigned long flags)474{475 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);476 struct rz_dmac *dmac = to_rz_dmac(chan->device);477 struct rz_dmac_desc *desc;478 479 dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n",480 __func__, channel->index, &src, &dest, len);481 482 if (list_empty(&channel->ld_free))483 return NULL;484 485 desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);486 487 desc->type = RZ_DMAC_DESC_MEMCPY;488 desc->src = src;489 desc->dest = dest;490 desc->len = len;491 desc->direction = DMA_MEM_TO_MEM;492 493 list_move_tail(channel->ld_free.next, &channel->ld_queue);494 return vchan_tx_prep(&channel->vc, &desc->vd, flags);495}496 497static struct dma_async_tx_descriptor *498rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,499 unsigned int sg_len,500 enum dma_transfer_direction direction,501 unsigned long flags, void *context)502{503 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);504 struct rz_dmac_desc *desc;505 struct scatterlist *sg;506 int dma_length = 0;507 int i = 0;508 509 if (list_empty(&channel->ld_free))510 return NULL;511 512 desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);513 514 for_each_sg(sgl, sg, sg_len, i) {515 dma_length += sg_dma_len(sg);516 }517 518 desc->type = RZ_DMAC_DESC_SLAVE_SG;519 desc->sg = sgl;520 desc->sgcount = sg_len;521 desc->len = dma_length;522 desc->direction = direction;523 524 if (direction == DMA_DEV_TO_MEM)525 desc->src = channel->src_per_address;526 else527 desc->dest = channel->dst_per_address;528 529 list_move_tail(channel->ld_free.next, &channel->ld_queue);530 return vchan_tx_prep(&channel->vc, &desc->vd, flags);531}532 533static int rz_dmac_terminate_all(struct dma_chan *chan)534{535 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);536 unsigned long flags;537 LIST_HEAD(head);538 539 rz_dmac_disable_hw(channel);540 spin_lock_irqsave(&channel->vc.lock, flags);541 list_splice_tail_init(&channel->ld_active, &channel->ld_free);542 list_splice_tail_init(&channel->ld_queue, &channel->ld_free);543 vchan_get_all_descriptors(&channel->vc, &head);544 spin_unlock_irqrestore(&channel->vc.lock, flags);545 vchan_dma_desc_free_list(&channel->vc, &head);546 547 return 0;548}549 550static void rz_dmac_issue_pending(struct dma_chan *chan)551{552 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);553 struct rz_dmac *dmac = to_rz_dmac(chan->device);554 struct rz_dmac_desc *desc;555 unsigned long flags;556 557 spin_lock_irqsave(&channel->vc.lock, flags);558 559 if (!list_empty(&channel->ld_queue)) {560 desc = list_first_entry(&channel->ld_queue,561 struct rz_dmac_desc, node);562 channel->desc = desc;563 if (vchan_issue_pending(&channel->vc)) {564 if (rz_dmac_xfer_desc(channel) < 0)565 dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n",566 channel->index);567 else568 list_move_tail(channel->ld_queue.next,569 &channel->ld_active);570 }571 }572 573 spin_unlock_irqrestore(&channel->vc.lock, flags);574}575 576static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds)577{578 u8 i;579 static const enum dma_slave_buswidth ds_lut[] = {580 DMA_SLAVE_BUSWIDTH_1_BYTE,581 DMA_SLAVE_BUSWIDTH_2_BYTES,582 DMA_SLAVE_BUSWIDTH_4_BYTES,583 DMA_SLAVE_BUSWIDTH_8_BYTES,584 DMA_SLAVE_BUSWIDTH_16_BYTES,585 DMA_SLAVE_BUSWIDTH_32_BYTES,586 DMA_SLAVE_BUSWIDTH_64_BYTES,587 DMA_SLAVE_BUSWIDTH_128_BYTES,588 };589 590 for (i = 0; i < ARRAY_SIZE(ds_lut); i++) {591 if (ds_lut[i] == ds)592 return i;593 }594 595 return CHCFG_DS_INVALID;596}597 598static int rz_dmac_config(struct dma_chan *chan,599 struct dma_slave_config *config)600{601 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);602 u32 val;603 604 channel->dst_per_address = config->dst_addr;605 channel->chcfg &= ~CHCFG_FILL_DDS_MASK;606 if (channel->dst_per_address) {607 val = rz_dmac_ds_to_val_mapping(config->dst_addr_width);608 if (val == CHCFG_DS_INVALID)609 return -EINVAL;610 611 channel->chcfg |= FIELD_PREP(CHCFG_FILL_DDS_MASK, val);612 }613 614 channel->src_per_address = config->src_addr;615 channel->chcfg &= ~CHCFG_FILL_SDS_MASK;616 if (channel->src_per_address) {617 val = rz_dmac_ds_to_val_mapping(config->src_addr_width);618 if (val == CHCFG_DS_INVALID)619 return -EINVAL;620 621 channel->chcfg |= FIELD_PREP(CHCFG_FILL_SDS_MASK, val);622 }623 624 return 0;625}626 627static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd)628{629 /*630 * Place holder631 * Descriptor allocation is done during alloc_chan_resources and632 * get freed during free_chan_resources.633 * list is used to manage the descriptors and avoid any memory634 * allocation/free during DMA read/write.635 */636}637 638static void rz_dmac_device_synchronize(struct dma_chan *chan)639{640 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);641 struct rz_dmac *dmac = to_rz_dmac(chan->device);642 u32 chstat;643 int ret;644 645 ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN),646 100, 100000, false, channel, CHSTAT, 1);647 if (ret < 0)648 dev_warn(dmac->dev, "DMA Timeout");649 650 rz_dmac_set_dmars_register(dmac, channel->index, 0);651}652 653/*654 * -----------------------------------------------------------------------------655 * IRQ handling656 */657 658static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel)659{660 struct dma_chan *chan = &channel->vc.chan;661 struct rz_dmac *dmac = to_rz_dmac(chan->device);662 u32 chstat, chctrl;663 664 chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);665 if (chstat & CHSTAT_ER) {666 dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n",667 channel->index, chstat);668 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);669 goto done;670 }671 672 chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1);673 rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1);674done:675 return;676}677 678static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id)679{680 struct rz_dmac_chan *channel = dev_id;681 682 if (channel) {683 rz_dmac_irq_handle_channel(channel);684 return IRQ_WAKE_THREAD;685 }686 /* handle DMAERR irq */687 return IRQ_HANDLED;688}689 690static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id)691{692 struct rz_dmac_chan *channel = dev_id;693 struct rz_dmac_desc *desc = NULL;694 unsigned long flags;695 696 spin_lock_irqsave(&channel->vc.lock, flags);697 698 if (list_empty(&channel->ld_active)) {699 /* Someone might have called terminate all */700 goto out;701 }702 703 desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node);704 vchan_cookie_complete(&desc->vd);705 list_move_tail(channel->ld_active.next, &channel->ld_free);706 if (!list_empty(&channel->ld_queue)) {707 desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc,708 node);709 channel->desc = desc;710 if (rz_dmac_xfer_desc(channel) == 0)711 list_move_tail(channel->ld_queue.next, &channel->ld_active);712 }713out:714 spin_unlock_irqrestore(&channel->vc.lock, flags);715 716 return IRQ_HANDLED;717}718 719/*720 * -----------------------------------------------------------------------------721 * OF xlate and channel filter722 */723 724static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg)725{726 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);727 struct rz_dmac *dmac = to_rz_dmac(chan->device);728 struct of_phandle_args *dma_spec = arg;729 u32 ch_cfg;730 731 channel->mid_rid = dma_spec->args[0] & MID_RID_MASK;732 ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10;733 channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) |734 CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg);735 736 return !test_and_set_bit(channel->mid_rid, dmac->modules);737}738 739static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec,740 struct of_dma *ofdma)741{742 dma_cap_mask_t mask;743 744 if (dma_spec->args_count != 1)745 return NULL;746 747 /* Only slave DMA channels can be allocated via DT */748 dma_cap_zero(mask);749 dma_cap_set(DMA_SLAVE, mask);750 751 return dma_request_channel(mask, rz_dmac_chan_filter, dma_spec);752}753 754/*755 * -----------------------------------------------------------------------------756 * Probe and remove757 */758 759static int rz_dmac_chan_probe(struct rz_dmac *dmac,760 struct rz_dmac_chan *channel,761 u8 index)762{763 struct platform_device *pdev = to_platform_device(dmac->dev);764 struct rz_lmdesc *lmdesc;765 char pdev_irqname[6];766 char *irqname;767 int ret;768 769 channel->index = index;770 channel->mid_rid = -EINVAL;771 772 /* Request the channel interrupt. */773 scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index);774 channel->irq = platform_get_irq_byname(pdev, pdev_irqname);775 if (channel->irq < 0)776 return channel->irq;777 778 irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",779 dev_name(dmac->dev), index);780 if (!irqname)781 return -ENOMEM;782 783 ret = devm_request_threaded_irq(dmac->dev, channel->irq,784 rz_dmac_irq_handler,785 rz_dmac_irq_handler_thread, 0,786 irqname, channel);787 if (ret) {788 dev_err(dmac->dev, "failed to request IRQ %u (%d)\n",789 channel->irq, ret);790 return ret;791 }792 793 /* Set io base address for each channel */794 if (index < 8) {795 channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET +796 EACH_CHANNEL_OFFSET * index;797 channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE;798 } else {799 channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET +800 EACH_CHANNEL_OFFSET * (index - 8);801 channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE;802 }803 804 /* Allocate descriptors */805 lmdesc = dma_alloc_coherent(&pdev->dev,806 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,807 &channel->lmdesc.base_dma, GFP_KERNEL);808 if (!lmdesc) {809 dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n");810 return -ENOMEM;811 }812 rz_lmdesc_setup(channel, lmdesc);813 814 /* Initialize register for each channel */815 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);816 817 channel->vc.desc_free = rz_dmac_virt_desc_free;818 vchan_init(&channel->vc, &dmac->engine);819 INIT_LIST_HEAD(&channel->ld_queue);820 INIT_LIST_HEAD(&channel->ld_free);821 INIT_LIST_HEAD(&channel->ld_active);822 823 return 0;824}825 826static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac)827{828 struct device_node *np = dev->of_node;829 int ret;830 831 ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels);832 if (ret < 0) {833 dev_err(dev, "unable to read dma-channels property\n");834 return ret;835 }836 837 if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) {838 dev_err(dev, "invalid number of channels %u\n", dmac->n_channels);839 return -EINVAL;840 }841 842 return 0;843}844 845static int rz_dmac_probe(struct platform_device *pdev)846{847 const char *irqname = "error";848 struct dma_device *engine;849 struct rz_dmac *dmac;850 int channel_num;851 int ret;852 int irq;853 u8 i;854 855 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);856 if (!dmac)857 return -ENOMEM;858 859 dmac->dev = &pdev->dev;860 platform_set_drvdata(pdev, dmac);861 862 ret = rz_dmac_parse_of(&pdev->dev, dmac);863 if (ret < 0)864 return ret;865 866 dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels,867 sizeof(*dmac->channels), GFP_KERNEL);868 if (!dmac->channels)869 return -ENOMEM;870 871 /* Request resources */872 dmac->base = devm_platform_ioremap_resource(pdev, 0);873 if (IS_ERR(dmac->base))874 return PTR_ERR(dmac->base);875 876 dmac->ext_base = devm_platform_ioremap_resource(pdev, 1);877 if (IS_ERR(dmac->ext_base))878 return PTR_ERR(dmac->ext_base);879 880 /* Register interrupt handler for error */881 irq = platform_get_irq_byname(pdev, irqname);882 if (irq < 0)883 return irq;884 885 ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0,886 irqname, NULL);887 if (ret) {888 dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",889 irq, ret);890 return ret;891 }892 893 /* Initialize the channels. */894 INIT_LIST_HEAD(&dmac->engine.channels);895 896 dmac->rstc = devm_reset_control_array_get_exclusive(&pdev->dev);897 if (IS_ERR(dmac->rstc))898 return dev_err_probe(&pdev->dev, PTR_ERR(dmac->rstc),899 "failed to get resets\n");900 901 pm_runtime_enable(&pdev->dev);902 ret = pm_runtime_resume_and_get(&pdev->dev);903 if (ret < 0) {904 dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n");905 goto err_pm_disable;906 }907 908 ret = reset_control_deassert(dmac->rstc);909 if (ret)910 goto err_pm_runtime_put;911 912 for (i = 0; i < dmac->n_channels; i++) {913 ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i);914 if (ret < 0)915 goto err;916 }917 918 /* Register the DMAC as a DMA provider for DT. */919 ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate,920 NULL);921 if (ret < 0)922 goto err;923 924 /* Register the DMA engine device. */925 engine = &dmac->engine;926 dma_cap_set(DMA_SLAVE, engine->cap_mask);927 dma_cap_set(DMA_MEMCPY, engine->cap_mask);928 rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL);929 rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL);930 931 engine->dev = &pdev->dev;932 933 engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources;934 engine->device_free_chan_resources = rz_dmac_free_chan_resources;935 engine->device_tx_status = dma_cookie_status;936 engine->device_prep_slave_sg = rz_dmac_prep_slave_sg;937 engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy;938 engine->device_config = rz_dmac_config;939 engine->device_terminate_all = rz_dmac_terminate_all;940 engine->device_issue_pending = rz_dmac_issue_pending;941 engine->device_synchronize = rz_dmac_device_synchronize;942 943 engine->copy_align = DMAENGINE_ALIGN_1_BYTE;944 dma_set_max_seg_size(engine->dev, U32_MAX);945 946 ret = dma_async_device_register(engine);947 if (ret < 0) {948 dev_err(&pdev->dev, "unable to register\n");949 goto dma_register_err;950 }951 return 0;952 953dma_register_err:954 of_dma_controller_free(pdev->dev.of_node);955err:956 channel_num = i ? i - 1 : 0;957 for (i = 0; i < channel_num; i++) {958 struct rz_dmac_chan *channel = &dmac->channels[i];959 960 dma_free_coherent(&pdev->dev,961 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,962 channel->lmdesc.base,963 channel->lmdesc.base_dma);964 }965 966 reset_control_assert(dmac->rstc);967err_pm_runtime_put:968 pm_runtime_put(&pdev->dev);969err_pm_disable:970 pm_runtime_disable(&pdev->dev);971 972 return ret;973}974 975static void rz_dmac_remove(struct platform_device *pdev)976{977 struct rz_dmac *dmac = platform_get_drvdata(pdev);978 unsigned int i;979 980 dma_async_device_unregister(&dmac->engine);981 of_dma_controller_free(pdev->dev.of_node);982 for (i = 0; i < dmac->n_channels; i++) {983 struct rz_dmac_chan *channel = &dmac->channels[i];984 985 dma_free_coherent(&pdev->dev,986 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,987 channel->lmdesc.base,988 channel->lmdesc.base_dma);989 }990 reset_control_assert(dmac->rstc);991 pm_runtime_put(&pdev->dev);992 pm_runtime_disable(&pdev->dev);993}994 995static const struct of_device_id of_rz_dmac_match[] = {996 { .compatible = "renesas,rz-dmac", },997 { /* Sentinel */ }998};999MODULE_DEVICE_TABLE(of, of_rz_dmac_match);1000 1001static struct platform_driver rz_dmac_driver = {1002 .driver = {1003 .name = "rz-dmac",1004 .of_match_table = of_rz_dmac_match,1005 },1006 .probe = rz_dmac_probe,1007 .remove_new = rz_dmac_remove,1008};1009 1010module_platform_driver(rz_dmac_driver);1011 1012MODULE_DESCRIPTION("Renesas RZ/G2L DMA Controller Driver");1013MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>");1014MODULE_LICENSE("GPL v2");1015