1278 lines · c
1// SPDX-License-Identifier: GPL-2.0+2//3// Actions Semi Owl SoCs DMA driver4//5// Copyright (c) 2014 Actions Semi Inc.6// Author: David Liu <liuwei@actions-semi.com>7//8// Copyright (c) 2018 Linaro Ltd.9// Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>10 11#include <linux/bitops.h>12#include <linux/clk.h>13#include <linux/delay.h>14#include <linux/dmaengine.h>15#include <linux/dma-mapping.h>16#include <linux/dmapool.h>17#include <linux/err.h>18#include <linux/init.h>19#include <linux/interrupt.h>20#include <linux/io.h>21#include <linux/mm.h>22#include <linux/module.h>23#include <linux/of.h>24#include <linux/of_dma.h>25#include <linux/platform_device.h>26#include <linux/slab.h>27#include "virt-dma.h"28 29#define OWL_DMA_FRAME_MAX_LENGTH 0xfffff30 31/* Global DMA Controller Registers */32#define OWL_DMA_IRQ_PD0 0x0033#define OWL_DMA_IRQ_PD1 0x0434#define OWL_DMA_IRQ_PD2 0x0835#define OWL_DMA_IRQ_PD3 0x0C36#define OWL_DMA_IRQ_EN0 0x1037#define OWL_DMA_IRQ_EN1 0x1438#define OWL_DMA_IRQ_EN2 0x1839#define OWL_DMA_IRQ_EN3 0x1C40#define OWL_DMA_SECURE_ACCESS_CTL 0x2041#define OWL_DMA_NIC_QOS 0x2442#define OWL_DMA_DBGSEL 0x2843#define OWL_DMA_IDLE_STAT 0x2C44 45/* Channel Registers */46#define OWL_DMA_CHAN_BASE(i) (0x100 + (i) * 0x100)47#define OWL_DMAX_MODE 0x0048#define OWL_DMAX_SOURCE 0x0449#define OWL_DMAX_DESTINATION 0x0850#define OWL_DMAX_FRAME_LEN 0x0C51#define OWL_DMAX_FRAME_CNT 0x1052#define OWL_DMAX_REMAIN_FRAME_CNT 0x1453#define OWL_DMAX_REMAIN_CNT 0x1854#define OWL_DMAX_SOURCE_STRIDE 0x1C55#define OWL_DMAX_DESTINATION_STRIDE 0x2056#define OWL_DMAX_START 0x2457#define OWL_DMAX_PAUSE 0x2858#define OWL_DMAX_CHAINED_CTL 0x2C59#define OWL_DMAX_CONSTANT 0x3060#define OWL_DMAX_LINKLIST_CTL 0x3461#define OWL_DMAX_NEXT_DESCRIPTOR 0x3862#define OWL_DMAX_CURRENT_DESCRIPTOR_NUM 0x3C63#define OWL_DMAX_INT_CTL 0x4064#define OWL_DMAX_INT_STATUS 0x4465#define OWL_DMAX_CURRENT_SOURCE_POINTER 0x4866#define OWL_DMAX_CURRENT_DESTINATION_POINTER 0x4C67 68/* OWL_DMAX_MODE Bits */69#define OWL_DMA_MODE_TS(x) (((x) & GENMASK(5, 0)) << 0)70#define OWL_DMA_MODE_ST(x) (((x) & GENMASK(1, 0)) << 8)71#define OWL_DMA_MODE_ST_DEV OWL_DMA_MODE_ST(0)72#define OWL_DMA_MODE_ST_DCU OWL_DMA_MODE_ST(2)73#define OWL_DMA_MODE_ST_SRAM OWL_DMA_MODE_ST(3)74#define OWL_DMA_MODE_DT(x) (((x) & GENMASK(1, 0)) << 10)75#define OWL_DMA_MODE_DT_DEV OWL_DMA_MODE_DT(0)76#define OWL_DMA_MODE_DT_DCU OWL_DMA_MODE_DT(2)77#define OWL_DMA_MODE_DT_SRAM OWL_DMA_MODE_DT(3)78#define OWL_DMA_MODE_SAM(x) (((x) & GENMASK(1, 0)) << 16)79#define OWL_DMA_MODE_SAM_CONST OWL_DMA_MODE_SAM(0)80#define OWL_DMA_MODE_SAM_INC OWL_DMA_MODE_SAM(1)81#define OWL_DMA_MODE_SAM_STRIDE OWL_DMA_MODE_SAM(2)82#define OWL_DMA_MODE_DAM(x) (((x) & GENMASK(1, 0)) << 18)83#define OWL_DMA_MODE_DAM_CONST OWL_DMA_MODE_DAM(0)84#define OWL_DMA_MODE_DAM_INC OWL_DMA_MODE_DAM(1)85#define OWL_DMA_MODE_DAM_STRIDE OWL_DMA_MODE_DAM(2)86#define OWL_DMA_MODE_PW(x) (((x) & GENMASK(2, 0)) << 20)87#define OWL_DMA_MODE_CB BIT(23)88#define OWL_DMA_MODE_NDDBW(x) (((x) & 0x1) << 28)89#define OWL_DMA_MODE_NDDBW_32BIT OWL_DMA_MODE_NDDBW(0)90#define OWL_DMA_MODE_NDDBW_8BIT OWL_DMA_MODE_NDDBW(1)91#define OWL_DMA_MODE_CFE BIT(29)92#define OWL_DMA_MODE_LME BIT(30)93#define OWL_DMA_MODE_CME BIT(31)94 95/* OWL_DMAX_LINKLIST_CTL Bits */96#define OWL_DMA_LLC_SAV(x) (((x) & GENMASK(1, 0)) << 8)97#define OWL_DMA_LLC_SAV_INC OWL_DMA_LLC_SAV(0)98#define OWL_DMA_LLC_SAV_LOAD_NEXT OWL_DMA_LLC_SAV(1)99#define OWL_DMA_LLC_SAV_LOAD_PREV OWL_DMA_LLC_SAV(2)100#define OWL_DMA_LLC_DAV(x) (((x) & GENMASK(1, 0)) << 10)101#define OWL_DMA_LLC_DAV_INC OWL_DMA_LLC_DAV(0)102#define OWL_DMA_LLC_DAV_LOAD_NEXT OWL_DMA_LLC_DAV(1)103#define OWL_DMA_LLC_DAV_LOAD_PREV OWL_DMA_LLC_DAV(2)104#define OWL_DMA_LLC_SUSPEND BIT(16)105 106/* OWL_DMAX_INT_CTL Bits */107#define OWL_DMA_INTCTL_BLOCK BIT(0)108#define OWL_DMA_INTCTL_SUPER_BLOCK BIT(1)109#define OWL_DMA_INTCTL_FRAME BIT(2)110#define OWL_DMA_INTCTL_HALF_FRAME BIT(3)111#define OWL_DMA_INTCTL_LAST_FRAME BIT(4)112 113/* OWL_DMAX_INT_STATUS Bits */114#define OWL_DMA_INTSTAT_BLOCK BIT(0)115#define OWL_DMA_INTSTAT_SUPER_BLOCK BIT(1)116#define OWL_DMA_INTSTAT_FRAME BIT(2)117#define OWL_DMA_INTSTAT_HALF_FRAME BIT(3)118#define OWL_DMA_INTSTAT_LAST_FRAME BIT(4)119 120/* Pack shift and newshift in a single word */121#define BIT_FIELD(val, width, shift, newshift) \122 ((((val) >> (shift)) & ((BIT(width)) - 1)) << (newshift))123 124/* Frame count value is fixed as 1 */125#define FCNT_VAL 0x1126 127/**128 * enum owl_dmadesc_offsets - Describe DMA descriptor, hardware link129 * list for dma transfer130 * @OWL_DMADESC_NEXT_LLI: physical address of the next link list131 * @OWL_DMADESC_SADDR: source physical address132 * @OWL_DMADESC_DADDR: destination physical address133 * @OWL_DMADESC_FLEN: frame length134 * @OWL_DMADESC_SRC_STRIDE: source stride135 * @OWL_DMADESC_DST_STRIDE: destination stride136 * @OWL_DMADESC_CTRLA: dma_mode and linklist ctrl config137 * @OWL_DMADESC_CTRLB: interrupt config138 * @OWL_DMADESC_CONST_NUM: data for constant fill139 * @OWL_DMADESC_SIZE: max size of this enum140 */141enum owl_dmadesc_offsets {142 OWL_DMADESC_NEXT_LLI = 0,143 OWL_DMADESC_SADDR,144 OWL_DMADESC_DADDR,145 OWL_DMADESC_FLEN,146 OWL_DMADESC_SRC_STRIDE,147 OWL_DMADESC_DST_STRIDE,148 OWL_DMADESC_CTRLA,149 OWL_DMADESC_CTRLB,150 OWL_DMADESC_CONST_NUM,151 OWL_DMADESC_SIZE152};153 154enum owl_dma_id {155 S900_DMA,156 S700_DMA,157};158 159/**160 * struct owl_dma_lli - Link list for dma transfer161 * @hw: hardware link list162 * @phys: physical address of hardware link list163 * @node: node for txd's lli_list164 */165struct owl_dma_lli {166 u32 hw[OWL_DMADESC_SIZE];167 dma_addr_t phys;168 struct list_head node;169};170 171/**172 * struct owl_dma_txd - Wrapper for struct dma_async_tx_descriptor173 * @vd: virtual DMA descriptor174 * @lli_list: link list of lli nodes175 * @cyclic: flag to indicate cyclic transfers176 */177struct owl_dma_txd {178 struct virt_dma_desc vd;179 struct list_head lli_list;180 bool cyclic;181};182 183/**184 * struct owl_dma_pchan - Holder for the physical channels185 * @id: physical index to this channel186 * @base: virtual memory base for the dma channel187 * @vchan: the virtual channel currently being served by this physical channel188 */189struct owl_dma_pchan {190 u32 id;191 void __iomem *base;192 struct owl_dma_vchan *vchan;193};194 195/**196 * struct owl_dma_vchan - Wrapper for DMA ENGINE channel197 * @vc: wrapped virtual channel198 * @pchan: the physical channel utilized by this channel199 * @txd: active transaction on this channel200 * @cfg: slave configuration for this channel201 * @drq: physical DMA request ID for this channel202 */203struct owl_dma_vchan {204 struct virt_dma_chan vc;205 struct owl_dma_pchan *pchan;206 struct owl_dma_txd *txd;207 struct dma_slave_config cfg;208 u8 drq;209};210 211/**212 * struct owl_dma - Holder for the Owl DMA controller213 * @dma: dma engine for this instance214 * @base: virtual memory base for the DMA controller215 * @clk: clock for the DMA controller216 * @lock: a lock to use when change DMA controller global register217 * @lli_pool: a pool for the LLI descriptors218 * @irq: interrupt ID for the DMA controller219 * @nr_pchans: the number of physical channels220 * @pchans: array of data for the physical channels221 * @nr_vchans: the number of physical channels222 * @vchans: array of data for the physical channels223 * @devid: device id based on OWL SoC224 */225struct owl_dma {226 struct dma_device dma;227 void __iomem *base;228 struct clk *clk;229 spinlock_t lock;230 struct dma_pool *lli_pool;231 int irq;232 233 unsigned int nr_pchans;234 struct owl_dma_pchan *pchans;235 236 unsigned int nr_vchans;237 struct owl_dma_vchan *vchans;238 enum owl_dma_id devid;239};240 241static void pchan_update(struct owl_dma_pchan *pchan, u32 reg,242 u32 val, bool state)243{244 u32 regval;245 246 regval = readl(pchan->base + reg);247 248 if (state)249 regval |= val;250 else251 regval &= ~val;252 253 writel(regval, pchan->base + reg);254}255 256static void pchan_writel(struct owl_dma_pchan *pchan, u32 reg, u32 data)257{258 writel(data, pchan->base + reg);259}260 261static u32 pchan_readl(struct owl_dma_pchan *pchan, u32 reg)262{263 return readl(pchan->base + reg);264}265 266static void dma_update(struct owl_dma *od, u32 reg, u32 val, bool state)267{268 u32 regval;269 270 regval = readl(od->base + reg);271 272 if (state)273 regval |= val;274 else275 regval &= ~val;276 277 writel(regval, od->base + reg);278}279 280static void dma_writel(struct owl_dma *od, u32 reg, u32 data)281{282 writel(data, od->base + reg);283}284 285static u32 dma_readl(struct owl_dma *od, u32 reg)286{287 return readl(od->base + reg);288}289 290static inline struct owl_dma *to_owl_dma(struct dma_device *dd)291{292 return container_of(dd, struct owl_dma, dma);293}294 295static struct device *chan2dev(struct dma_chan *chan)296{297 return &chan->dev->device;298}299 300static inline struct owl_dma_vchan *to_owl_vchan(struct dma_chan *chan)301{302 return container_of(chan, struct owl_dma_vchan, vc.chan);303}304 305static inline struct owl_dma_txd *to_owl_txd(struct dma_async_tx_descriptor *tx)306{307 return container_of(tx, struct owl_dma_txd, vd.tx);308}309 310static inline u32 llc_hw_ctrla(u32 mode, u32 llc_ctl)311{312 u32 ctl;313 314 ctl = BIT_FIELD(mode, 4, 28, 28) |315 BIT_FIELD(mode, 8, 16, 20) |316 BIT_FIELD(mode, 4, 8, 16) |317 BIT_FIELD(mode, 6, 0, 10) |318 BIT_FIELD(llc_ctl, 2, 10, 8) |319 BIT_FIELD(llc_ctl, 2, 8, 6);320 321 return ctl;322}323 324static inline u32 llc_hw_ctrlb(u32 int_ctl)325{326 u32 ctl;327 328 /*329 * Irrespective of the SoC, ctrlb value starts filling from330 * bit 18.331 */332 ctl = BIT_FIELD(int_ctl, 7, 0, 18);333 334 return ctl;335}336 337static u32 llc_hw_flen(struct owl_dma_lli *lli)338{339 return lli->hw[OWL_DMADESC_FLEN] & GENMASK(19, 0);340}341 342static void owl_dma_free_lli(struct owl_dma *od,343 struct owl_dma_lli *lli)344{345 list_del(&lli->node);346 dma_pool_free(od->lli_pool, lli, lli->phys);347}348 349static struct owl_dma_lli *owl_dma_alloc_lli(struct owl_dma *od)350{351 struct owl_dma_lli *lli;352 dma_addr_t phys;353 354 lli = dma_pool_alloc(od->lli_pool, GFP_NOWAIT, &phys);355 if (!lli)356 return NULL;357 358 INIT_LIST_HEAD(&lli->node);359 lli->phys = phys;360 361 return lli;362}363 364static struct owl_dma_lli *owl_dma_add_lli(struct owl_dma_txd *txd,365 struct owl_dma_lli *prev,366 struct owl_dma_lli *next,367 bool is_cyclic)368{369 if (!is_cyclic)370 list_add_tail(&next->node, &txd->lli_list);371 372 if (prev) {373 prev->hw[OWL_DMADESC_NEXT_LLI] = next->phys;374 prev->hw[OWL_DMADESC_CTRLA] |=375 llc_hw_ctrla(OWL_DMA_MODE_LME, 0);376 }377 378 return next;379}380 381static inline int owl_dma_cfg_lli(struct owl_dma_vchan *vchan,382 struct owl_dma_lli *lli,383 dma_addr_t src, dma_addr_t dst,384 u32 len, enum dma_transfer_direction dir,385 struct dma_slave_config *sconfig,386 bool is_cyclic)387{388 struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);389 u32 mode, ctrlb;390 391 mode = OWL_DMA_MODE_PW(0);392 393 switch (dir) {394 case DMA_MEM_TO_MEM:395 mode |= OWL_DMA_MODE_TS(0) | OWL_DMA_MODE_ST_DCU |396 OWL_DMA_MODE_DT_DCU | OWL_DMA_MODE_SAM_INC |397 OWL_DMA_MODE_DAM_INC;398 399 break;400 case DMA_MEM_TO_DEV:401 mode |= OWL_DMA_MODE_TS(vchan->drq)402 | OWL_DMA_MODE_ST_DCU | OWL_DMA_MODE_DT_DEV403 | OWL_DMA_MODE_SAM_INC | OWL_DMA_MODE_DAM_CONST;404 405 /*406 * Hardware only supports 32bit and 8bit buswidth. Since the407 * default is 32bit, select 8bit only when requested.408 */409 if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE)410 mode |= OWL_DMA_MODE_NDDBW_8BIT;411 412 break;413 case DMA_DEV_TO_MEM:414 mode |= OWL_DMA_MODE_TS(vchan->drq)415 | OWL_DMA_MODE_ST_DEV | OWL_DMA_MODE_DT_DCU416 | OWL_DMA_MODE_SAM_CONST | OWL_DMA_MODE_DAM_INC;417 418 /*419 * Hardware only supports 32bit and 8bit buswidth. Since the420 * default is 32bit, select 8bit only when requested.421 */422 if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE)423 mode |= OWL_DMA_MODE_NDDBW_8BIT;424 425 break;426 default:427 return -EINVAL;428 }429 430 lli->hw[OWL_DMADESC_CTRLA] = llc_hw_ctrla(mode,431 OWL_DMA_LLC_SAV_LOAD_NEXT |432 OWL_DMA_LLC_DAV_LOAD_NEXT);433 434 if (is_cyclic)435 ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_BLOCK);436 else437 ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_SUPER_BLOCK);438 439 lli->hw[OWL_DMADESC_NEXT_LLI] = 0; /* One link list by default */440 lli->hw[OWL_DMADESC_SADDR] = src;441 lli->hw[OWL_DMADESC_DADDR] = dst;442 lli->hw[OWL_DMADESC_SRC_STRIDE] = 0;443 lli->hw[OWL_DMADESC_DST_STRIDE] = 0;444 445 if (od->devid == S700_DMA) {446 /* Max frame length is 1MB */447 lli->hw[OWL_DMADESC_FLEN] = len;448 /*449 * On S700, word starts from offset 0x1C is shared between450 * frame count and ctrlb, where first 12 bits are for frame451 * count and rest of 20 bits are for ctrlb.452 */453 lli->hw[OWL_DMADESC_CTRLB] = FCNT_VAL | ctrlb;454 } else {455 /*456 * On S900, word starts from offset 0xC is shared between457 * frame length (max frame length is 1MB) and frame count,458 * where first 20 bits are for frame length and rest of459 * 12 bits are for frame count.460 */461 lli->hw[OWL_DMADESC_FLEN] = len | FCNT_VAL << 20;462 lli->hw[OWL_DMADESC_CTRLB] = ctrlb;463 }464 465 return 0;466}467 468static struct owl_dma_pchan *owl_dma_get_pchan(struct owl_dma *od,469 struct owl_dma_vchan *vchan)470{471 struct owl_dma_pchan *pchan = NULL;472 unsigned long flags;473 int i;474 475 for (i = 0; i < od->nr_pchans; i++) {476 pchan = &od->pchans[i];477 478 spin_lock_irqsave(&od->lock, flags);479 if (!pchan->vchan) {480 pchan->vchan = vchan;481 spin_unlock_irqrestore(&od->lock, flags);482 break;483 }484 485 spin_unlock_irqrestore(&od->lock, flags);486 }487 488 return pchan;489}490 491static int owl_dma_pchan_busy(struct owl_dma *od, struct owl_dma_pchan *pchan)492{493 unsigned int val;494 495 val = dma_readl(od, OWL_DMA_IDLE_STAT);496 497 return !(val & (1 << pchan->id));498}499 500static void owl_dma_terminate_pchan(struct owl_dma *od,501 struct owl_dma_pchan *pchan)502{503 unsigned long flags;504 u32 irq_pd;505 506 pchan_writel(pchan, OWL_DMAX_START, 0);507 pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);508 509 spin_lock_irqsave(&od->lock, flags);510 dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), false);511 512 irq_pd = dma_readl(od, OWL_DMA_IRQ_PD0);513 if (irq_pd & (1 << pchan->id)) {514 dev_warn(od->dma.dev,515 "terminating pchan %d that still has pending irq\n",516 pchan->id);517 dma_writel(od, OWL_DMA_IRQ_PD0, (1 << pchan->id));518 }519 520 pchan->vchan = NULL;521 522 spin_unlock_irqrestore(&od->lock, flags);523}524 525static void owl_dma_pause_pchan(struct owl_dma_pchan *pchan)526{527 pchan_writel(pchan, 1, OWL_DMAX_PAUSE);528}529 530static void owl_dma_resume_pchan(struct owl_dma_pchan *pchan)531{532 pchan_writel(pchan, 0, OWL_DMAX_PAUSE);533}534 535static int owl_dma_start_next_txd(struct owl_dma_vchan *vchan)536{537 struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);538 struct virt_dma_desc *vd = vchan_next_desc(&vchan->vc);539 struct owl_dma_pchan *pchan = vchan->pchan;540 struct owl_dma_txd *txd = to_owl_txd(&vd->tx);541 struct owl_dma_lli *lli;542 unsigned long flags;543 u32 int_ctl;544 545 list_del(&vd->node);546 547 vchan->txd = txd;548 549 /* Wait for channel inactive */550 while (owl_dma_pchan_busy(od, pchan))551 cpu_relax();552 553 lli = list_first_entry(&txd->lli_list,554 struct owl_dma_lli, node);555 556 if (txd->cyclic)557 int_ctl = OWL_DMA_INTCTL_BLOCK;558 else559 int_ctl = OWL_DMA_INTCTL_SUPER_BLOCK;560 561 pchan_writel(pchan, OWL_DMAX_MODE, OWL_DMA_MODE_LME);562 pchan_writel(pchan, OWL_DMAX_LINKLIST_CTL,563 OWL_DMA_LLC_SAV_LOAD_NEXT | OWL_DMA_LLC_DAV_LOAD_NEXT);564 pchan_writel(pchan, OWL_DMAX_NEXT_DESCRIPTOR, lli->phys);565 pchan_writel(pchan, OWL_DMAX_INT_CTL, int_ctl);566 567 /* Clear IRQ status for this pchan */568 pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);569 570 spin_lock_irqsave(&od->lock, flags);571 572 dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), true);573 574 spin_unlock_irqrestore(&od->lock, flags);575 576 dev_dbg(chan2dev(&vchan->vc.chan), "starting pchan %d\n", pchan->id);577 578 /* Start DMA transfer for this pchan */579 pchan_writel(pchan, OWL_DMAX_START, 0x1);580 581 return 0;582}583 584static void owl_dma_phy_free(struct owl_dma *od, struct owl_dma_vchan *vchan)585{586 /* Ensure that the physical channel is stopped */587 owl_dma_terminate_pchan(od, vchan->pchan);588 589 vchan->pchan = NULL;590}591 592static irqreturn_t owl_dma_interrupt(int irq, void *dev_id)593{594 struct owl_dma *od = dev_id;595 struct owl_dma_vchan *vchan;596 struct owl_dma_pchan *pchan;597 unsigned long pending;598 int i;599 unsigned int global_irq_pending, chan_irq_pending;600 601 spin_lock(&od->lock);602 603 pending = dma_readl(od, OWL_DMA_IRQ_PD0);604 605 /* Clear IRQ status for each pchan */606 for_each_set_bit(i, &pending, od->nr_pchans) {607 pchan = &od->pchans[i];608 pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);609 }610 611 /* Clear pending IRQ */612 dma_writel(od, OWL_DMA_IRQ_PD0, pending);613 614 /* Check missed pending IRQ */615 for (i = 0; i < od->nr_pchans; i++) {616 pchan = &od->pchans[i];617 chan_irq_pending = pchan_readl(pchan, OWL_DMAX_INT_CTL) &618 pchan_readl(pchan, OWL_DMAX_INT_STATUS);619 620 /* Dummy read to ensure OWL_DMA_IRQ_PD0 value is updated */621 dma_readl(od, OWL_DMA_IRQ_PD0);622 623 global_irq_pending = dma_readl(od, OWL_DMA_IRQ_PD0);624 625 if (chan_irq_pending && !(global_irq_pending & BIT(i))) {626 dev_dbg(od->dma.dev,627 "global and channel IRQ pending match err\n");628 629 /* Clear IRQ status for this pchan */630 pchan_update(pchan, OWL_DMAX_INT_STATUS,631 0xff, false);632 633 /* Update global IRQ pending */634 pending |= BIT(i);635 }636 }637 638 spin_unlock(&od->lock);639 640 for_each_set_bit(i, &pending, od->nr_pchans) {641 struct owl_dma_txd *txd;642 643 pchan = &od->pchans[i];644 645 vchan = pchan->vchan;646 if (!vchan) {647 dev_warn(od->dma.dev, "no vchan attached on pchan %d\n",648 pchan->id);649 continue;650 }651 652 spin_lock(&vchan->vc.lock);653 654 txd = vchan->txd;655 if (txd) {656 vchan->txd = NULL;657 658 vchan_cookie_complete(&txd->vd);659 660 /*661 * Start the next descriptor (if any),662 * otherwise free this channel.663 */664 if (vchan_next_desc(&vchan->vc))665 owl_dma_start_next_txd(vchan);666 else667 owl_dma_phy_free(od, vchan);668 }669 670 spin_unlock(&vchan->vc.lock);671 }672 673 return IRQ_HANDLED;674}675 676static void owl_dma_free_txd(struct owl_dma *od, struct owl_dma_txd *txd)677{678 struct owl_dma_lli *lli, *_lli;679 680 if (unlikely(!txd))681 return;682 683 list_for_each_entry_safe(lli, _lli, &txd->lli_list, node)684 owl_dma_free_lli(od, lli);685 686 kfree(txd);687}688 689static void owl_dma_desc_free(struct virt_dma_desc *vd)690{691 struct owl_dma *od = to_owl_dma(vd->tx.chan->device);692 struct owl_dma_txd *txd = to_owl_txd(&vd->tx);693 694 owl_dma_free_txd(od, txd);695}696 697static int owl_dma_terminate_all(struct dma_chan *chan)698{699 struct owl_dma *od = to_owl_dma(chan->device);700 struct owl_dma_vchan *vchan = to_owl_vchan(chan);701 unsigned long flags;702 LIST_HEAD(head);703 704 spin_lock_irqsave(&vchan->vc.lock, flags);705 706 if (vchan->pchan)707 owl_dma_phy_free(od, vchan);708 709 if (vchan->txd) {710 owl_dma_desc_free(&vchan->txd->vd);711 vchan->txd = NULL;712 }713 714 vchan_get_all_descriptors(&vchan->vc, &head);715 716 spin_unlock_irqrestore(&vchan->vc.lock, flags);717 718 vchan_dma_desc_free_list(&vchan->vc, &head);719 720 return 0;721}722 723static int owl_dma_config(struct dma_chan *chan,724 struct dma_slave_config *config)725{726 struct owl_dma_vchan *vchan = to_owl_vchan(chan);727 728 /* Reject definitely invalid configurations */729 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||730 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)731 return -EINVAL;732 733 memcpy(&vchan->cfg, config, sizeof(struct dma_slave_config));734 735 return 0;736}737 738static int owl_dma_pause(struct dma_chan *chan)739{740 struct owl_dma_vchan *vchan = to_owl_vchan(chan);741 unsigned long flags;742 743 spin_lock_irqsave(&vchan->vc.lock, flags);744 745 owl_dma_pause_pchan(vchan->pchan);746 747 spin_unlock_irqrestore(&vchan->vc.lock, flags);748 749 return 0;750}751 752static int owl_dma_resume(struct dma_chan *chan)753{754 struct owl_dma_vchan *vchan = to_owl_vchan(chan);755 unsigned long flags;756 757 if (!vchan->pchan && !vchan->txd)758 return 0;759 760 dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc);761 762 spin_lock_irqsave(&vchan->vc.lock, flags);763 764 owl_dma_resume_pchan(vchan->pchan);765 766 spin_unlock_irqrestore(&vchan->vc.lock, flags);767 768 return 0;769}770 771static u32 owl_dma_getbytes_chan(struct owl_dma_vchan *vchan)772{773 struct owl_dma_pchan *pchan;774 struct owl_dma_txd *txd;775 struct owl_dma_lli *lli;776 unsigned int next_lli_phy;777 size_t bytes;778 779 pchan = vchan->pchan;780 txd = vchan->txd;781 782 if (!pchan || !txd)783 return 0;784 785 /* Get remain count of current node in link list */786 bytes = pchan_readl(pchan, OWL_DMAX_REMAIN_CNT);787 788 /* Loop through the preceding nodes to get total remaining bytes */789 if (pchan_readl(pchan, OWL_DMAX_MODE) & OWL_DMA_MODE_LME) {790 next_lli_phy = pchan_readl(pchan, OWL_DMAX_NEXT_DESCRIPTOR);791 list_for_each_entry(lli, &txd->lli_list, node) {792 /* Start from the next active node */793 if (lli->phys == next_lli_phy) {794 list_for_each_entry(lli, &txd->lli_list, node)795 bytes += llc_hw_flen(lli);796 break;797 }798 }799 }800 801 return bytes;802}803 804static enum dma_status owl_dma_tx_status(struct dma_chan *chan,805 dma_cookie_t cookie,806 struct dma_tx_state *state)807{808 struct owl_dma_vchan *vchan = to_owl_vchan(chan);809 struct owl_dma_lli *lli;810 struct virt_dma_desc *vd;811 struct owl_dma_txd *txd;812 enum dma_status ret;813 unsigned long flags;814 size_t bytes = 0;815 816 ret = dma_cookie_status(chan, cookie, state);817 if (ret == DMA_COMPLETE || !state)818 return ret;819 820 spin_lock_irqsave(&vchan->vc.lock, flags);821 822 vd = vchan_find_desc(&vchan->vc, cookie);823 if (vd) {824 txd = to_owl_txd(&vd->tx);825 list_for_each_entry(lli, &txd->lli_list, node)826 bytes += llc_hw_flen(lli);827 } else {828 bytes = owl_dma_getbytes_chan(vchan);829 }830 831 spin_unlock_irqrestore(&vchan->vc.lock, flags);832 833 dma_set_residue(state, bytes);834 835 return ret;836}837 838static void owl_dma_phy_alloc_and_start(struct owl_dma_vchan *vchan)839{840 struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);841 struct owl_dma_pchan *pchan;842 843 pchan = owl_dma_get_pchan(od, vchan);844 if (!pchan)845 return;846 847 dev_dbg(od->dma.dev, "allocated pchan %d\n", pchan->id);848 849 vchan->pchan = pchan;850 owl_dma_start_next_txd(vchan);851}852 853static void owl_dma_issue_pending(struct dma_chan *chan)854{855 struct owl_dma_vchan *vchan = to_owl_vchan(chan);856 unsigned long flags;857 858 spin_lock_irqsave(&vchan->vc.lock, flags);859 if (vchan_issue_pending(&vchan->vc)) {860 if (!vchan->pchan)861 owl_dma_phy_alloc_and_start(vchan);862 }863 spin_unlock_irqrestore(&vchan->vc.lock, flags);864}865 866static struct dma_async_tx_descriptor867 *owl_dma_prep_memcpy(struct dma_chan *chan,868 dma_addr_t dst, dma_addr_t src,869 size_t len, unsigned long flags)870{871 struct owl_dma *od = to_owl_dma(chan->device);872 struct owl_dma_vchan *vchan = to_owl_vchan(chan);873 struct owl_dma_txd *txd;874 struct owl_dma_lli *lli, *prev = NULL;875 size_t offset, bytes;876 int ret;877 878 if (!len)879 return NULL;880 881 txd = kzalloc(sizeof(*txd), GFP_NOWAIT);882 if (!txd)883 return NULL;884 885 INIT_LIST_HEAD(&txd->lli_list);886 887 /* Process the transfer as frame by frame */888 for (offset = 0; offset < len; offset += bytes) {889 lli = owl_dma_alloc_lli(od);890 if (!lli) {891 dev_warn(chan2dev(chan), "failed to allocate lli\n");892 goto err_txd_free;893 }894 895 bytes = min_t(size_t, (len - offset), OWL_DMA_FRAME_MAX_LENGTH);896 897 ret = owl_dma_cfg_lli(vchan, lli, src + offset, dst + offset,898 bytes, DMA_MEM_TO_MEM,899 &vchan->cfg, txd->cyclic);900 if (ret) {901 dev_warn(chan2dev(chan), "failed to config lli\n");902 goto err_txd_free;903 }904 905 prev = owl_dma_add_lli(txd, prev, lli, false);906 }907 908 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);909 910err_txd_free:911 owl_dma_free_txd(od, txd);912 return NULL;913}914 915static struct dma_async_tx_descriptor916 *owl_dma_prep_slave_sg(struct dma_chan *chan,917 struct scatterlist *sgl,918 unsigned int sg_len,919 enum dma_transfer_direction dir,920 unsigned long flags, void *context)921{922 struct owl_dma *od = to_owl_dma(chan->device);923 struct owl_dma_vchan *vchan = to_owl_vchan(chan);924 struct dma_slave_config *sconfig = &vchan->cfg;925 struct owl_dma_txd *txd;926 struct owl_dma_lli *lli, *prev = NULL;927 struct scatterlist *sg;928 dma_addr_t addr, src = 0, dst = 0;929 size_t len;930 int ret, i;931 932 txd = kzalloc(sizeof(*txd), GFP_NOWAIT);933 if (!txd)934 return NULL;935 936 INIT_LIST_HEAD(&txd->lli_list);937 938 for_each_sg(sgl, sg, sg_len, i) {939 addr = sg_dma_address(sg);940 len = sg_dma_len(sg);941 942 if (len > OWL_DMA_FRAME_MAX_LENGTH) {943 dev_err(od->dma.dev,944 "frame length exceeds max supported length");945 goto err_txd_free;946 }947 948 lli = owl_dma_alloc_lli(od);949 if (!lli) {950 dev_err(chan2dev(chan), "failed to allocate lli");951 goto err_txd_free;952 }953 954 if (dir == DMA_MEM_TO_DEV) {955 src = addr;956 dst = sconfig->dst_addr;957 } else {958 src = sconfig->src_addr;959 dst = addr;960 }961 962 ret = owl_dma_cfg_lli(vchan, lli, src, dst, len, dir, sconfig,963 txd->cyclic);964 if (ret) {965 dev_warn(chan2dev(chan), "failed to config lli");966 goto err_txd_free;967 }968 969 prev = owl_dma_add_lli(txd, prev, lli, false);970 }971 972 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);973 974err_txd_free:975 owl_dma_free_txd(od, txd);976 977 return NULL;978}979 980static struct dma_async_tx_descriptor981 *owl_prep_dma_cyclic(struct dma_chan *chan,982 dma_addr_t buf_addr, size_t buf_len,983 size_t period_len,984 enum dma_transfer_direction dir,985 unsigned long flags)986{987 struct owl_dma *od = to_owl_dma(chan->device);988 struct owl_dma_vchan *vchan = to_owl_vchan(chan);989 struct dma_slave_config *sconfig = &vchan->cfg;990 struct owl_dma_txd *txd;991 struct owl_dma_lli *lli, *prev = NULL, *first = NULL;992 dma_addr_t src = 0, dst = 0;993 unsigned int periods = buf_len / period_len;994 int ret, i;995 996 txd = kzalloc(sizeof(*txd), GFP_NOWAIT);997 if (!txd)998 return NULL;999 1000 INIT_LIST_HEAD(&txd->lli_list);1001 txd->cyclic = true;1002 1003 for (i = 0; i < periods; i++) {1004 lli = owl_dma_alloc_lli(od);1005 if (!lli) {1006 dev_warn(chan2dev(chan), "failed to allocate lli");1007 goto err_txd_free;1008 }1009 1010 if (dir == DMA_MEM_TO_DEV) {1011 src = buf_addr + (period_len * i);1012 dst = sconfig->dst_addr;1013 } else if (dir == DMA_DEV_TO_MEM) {1014 src = sconfig->src_addr;1015 dst = buf_addr + (period_len * i);1016 }1017 1018 ret = owl_dma_cfg_lli(vchan, lli, src, dst, period_len,1019 dir, sconfig, txd->cyclic);1020 if (ret) {1021 dev_warn(chan2dev(chan), "failed to config lli");1022 goto err_txd_free;1023 }1024 1025 if (!first)1026 first = lli;1027 1028 prev = owl_dma_add_lli(txd, prev, lli, false);1029 }1030 1031 /* close the cyclic list */1032 owl_dma_add_lli(txd, prev, first, true);1033 1034 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);1035 1036err_txd_free:1037 owl_dma_free_txd(od, txd);1038 1039 return NULL;1040}1041 1042static void owl_dma_free_chan_resources(struct dma_chan *chan)1043{1044 struct owl_dma_vchan *vchan = to_owl_vchan(chan);1045 1046 /* Ensure all queued descriptors are freed */1047 vchan_free_chan_resources(&vchan->vc);1048}1049 1050static inline void owl_dma_free(struct owl_dma *od)1051{1052 struct owl_dma_vchan *vchan = NULL;1053 struct owl_dma_vchan *next;1054 1055 list_for_each_entry_safe(vchan,1056 next, &od->dma.channels, vc.chan.device_node) {1057 list_del(&vchan->vc.chan.device_node);1058 tasklet_kill(&vchan->vc.task);1059 }1060}1061 1062static struct dma_chan *owl_dma_of_xlate(struct of_phandle_args *dma_spec,1063 struct of_dma *ofdma)1064{1065 struct owl_dma *od = ofdma->of_dma_data;1066 struct owl_dma_vchan *vchan;1067 struct dma_chan *chan;1068 u8 drq = dma_spec->args[0];1069 1070 if (drq > od->nr_vchans)1071 return NULL;1072 1073 chan = dma_get_any_slave_channel(&od->dma);1074 if (!chan)1075 return NULL;1076 1077 vchan = to_owl_vchan(chan);1078 vchan->drq = drq;1079 1080 return chan;1081}1082 1083static const struct of_device_id owl_dma_match[] = {1084 { .compatible = "actions,s500-dma", .data = (void *)S900_DMA,},1085 { .compatible = "actions,s700-dma", .data = (void *)S700_DMA,},1086 { .compatible = "actions,s900-dma", .data = (void *)S900_DMA,},1087 { /* sentinel */ },1088};1089MODULE_DEVICE_TABLE(of, owl_dma_match);1090 1091static int owl_dma_probe(struct platform_device *pdev)1092{1093 struct device_node *np = pdev->dev.of_node;1094 struct owl_dma *od;1095 int ret, i, nr_channels, nr_requests;1096 1097 od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);1098 if (!od)1099 return -ENOMEM;1100 1101 od->base = devm_platform_ioremap_resource(pdev, 0);1102 if (IS_ERR(od->base))1103 return PTR_ERR(od->base);1104 1105 ret = of_property_read_u32(np, "dma-channels", &nr_channels);1106 if (ret) {1107 dev_err(&pdev->dev, "can't get dma-channels\n");1108 return ret;1109 }1110 1111 ret = of_property_read_u32(np, "dma-requests", &nr_requests);1112 if (ret) {1113 dev_err(&pdev->dev, "can't get dma-requests\n");1114 return ret;1115 }1116 1117 dev_info(&pdev->dev, "dma-channels %d, dma-requests %d\n",1118 nr_channels, nr_requests);1119 1120 od->devid = (uintptr_t)of_device_get_match_data(&pdev->dev);1121 1122 od->nr_pchans = nr_channels;1123 od->nr_vchans = nr_requests;1124 1125 pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);1126 1127 platform_set_drvdata(pdev, od);1128 spin_lock_init(&od->lock);1129 1130 dma_cap_set(DMA_MEMCPY, od->dma.cap_mask);1131 dma_cap_set(DMA_SLAVE, od->dma.cap_mask);1132 dma_cap_set(DMA_CYCLIC, od->dma.cap_mask);1133 1134 od->dma.dev = &pdev->dev;1135 od->dma.device_free_chan_resources = owl_dma_free_chan_resources;1136 od->dma.device_tx_status = owl_dma_tx_status;1137 od->dma.device_issue_pending = owl_dma_issue_pending;1138 od->dma.device_prep_dma_memcpy = owl_dma_prep_memcpy;1139 od->dma.device_prep_slave_sg = owl_dma_prep_slave_sg;1140 od->dma.device_prep_dma_cyclic = owl_prep_dma_cyclic;1141 od->dma.device_config = owl_dma_config;1142 od->dma.device_pause = owl_dma_pause;1143 od->dma.device_resume = owl_dma_resume;1144 od->dma.device_terminate_all = owl_dma_terminate_all;1145 od->dma.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);1146 od->dma.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);1147 od->dma.directions = BIT(DMA_MEM_TO_MEM);1148 od->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;1149 1150 INIT_LIST_HEAD(&od->dma.channels);1151 1152 od->clk = devm_clk_get(&pdev->dev, NULL);1153 if (IS_ERR(od->clk)) {1154 dev_err(&pdev->dev, "unable to get clock\n");1155 return PTR_ERR(od->clk);1156 }1157 1158 /*1159 * Even though the DMA controller is capable of generating 41160 * IRQ's for DMA priority feature, we only use 1 IRQ for1161 * simplification.1162 */1163 od->irq = platform_get_irq(pdev, 0);1164 ret = devm_request_irq(&pdev->dev, od->irq, owl_dma_interrupt, 0,1165 dev_name(&pdev->dev), od);1166 if (ret) {1167 dev_err(&pdev->dev, "unable to request IRQ\n");1168 return ret;1169 }1170 1171 /* Init physical channel */1172 od->pchans = devm_kcalloc(&pdev->dev, od->nr_pchans,1173 sizeof(struct owl_dma_pchan), GFP_KERNEL);1174 if (!od->pchans)1175 return -ENOMEM;1176 1177 for (i = 0; i < od->nr_pchans; i++) {1178 struct owl_dma_pchan *pchan = &od->pchans[i];1179 1180 pchan->id = i;1181 pchan->base = od->base + OWL_DMA_CHAN_BASE(i);1182 }1183 1184 /* Init virtual channel */1185 od->vchans = devm_kcalloc(&pdev->dev, od->nr_vchans,1186 sizeof(struct owl_dma_vchan), GFP_KERNEL);1187 if (!od->vchans)1188 return -ENOMEM;1189 1190 for (i = 0; i < od->nr_vchans; i++) {1191 struct owl_dma_vchan *vchan = &od->vchans[i];1192 1193 vchan->vc.desc_free = owl_dma_desc_free;1194 vchan_init(&vchan->vc, &od->dma);1195 }1196 1197 /* Create a pool of consistent memory blocks for hardware descriptors */1198 od->lli_pool = dma_pool_create(dev_name(od->dma.dev), od->dma.dev,1199 sizeof(struct owl_dma_lli),1200 __alignof__(struct owl_dma_lli),1201 0);1202 if (!od->lli_pool) {1203 dev_err(&pdev->dev, "unable to allocate DMA descriptor pool\n");1204 return -ENOMEM;1205 }1206 1207 clk_prepare_enable(od->clk);1208 1209 ret = dma_async_device_register(&od->dma);1210 if (ret) {1211 dev_err(&pdev->dev, "failed to register DMA engine device\n");1212 goto err_pool_free;1213 }1214 1215 /* Device-tree DMA controller registration */1216 ret = of_dma_controller_register(pdev->dev.of_node,1217 owl_dma_of_xlate, od);1218 if (ret) {1219 dev_err(&pdev->dev, "of_dma_controller_register failed\n");1220 goto err_dma_unregister;1221 }1222 1223 return 0;1224 1225err_dma_unregister:1226 dma_async_device_unregister(&od->dma);1227err_pool_free:1228 clk_disable_unprepare(od->clk);1229 dma_pool_destroy(od->lli_pool);1230 1231 return ret;1232}1233 1234static void owl_dma_remove(struct platform_device *pdev)1235{1236 struct owl_dma *od = platform_get_drvdata(pdev);1237 1238 of_dma_controller_free(pdev->dev.of_node);1239 dma_async_device_unregister(&od->dma);1240 1241 /* Mask all interrupts for this execution environment */1242 dma_writel(od, OWL_DMA_IRQ_EN0, 0x0);1243 1244 /* Make sure we won't have any further interrupts */1245 devm_free_irq(od->dma.dev, od->irq, od);1246 1247 owl_dma_free(od);1248 1249 clk_disable_unprepare(od->clk);1250 dma_pool_destroy(od->lli_pool);1251}1252 1253static struct platform_driver owl_dma_driver = {1254 .probe = owl_dma_probe,1255 .remove_new = owl_dma_remove,1256 .driver = {1257 .name = "dma-owl",1258 .of_match_table = of_match_ptr(owl_dma_match),1259 },1260};1261 1262static int owl_dma_init(void)1263{1264 return platform_driver_register(&owl_dma_driver);1265}1266subsys_initcall(owl_dma_init);1267 1268static void __exit owl_dma_exit(void)1269{1270 platform_driver_unregister(&owl_dma_driver);1271}1272module_exit(owl_dma_exit);1273 1274MODULE_AUTHOR("David Liu <liuwei@actions-semi.com>");1275MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");1276MODULE_DESCRIPTION("Actions Semi Owl SoCs DMA driver");1277MODULE_LICENSE("GPL");1278