492 lines · c
1/* SPDX-License-Identifier: GPL-2.0+ */2/*3 * Copyright 2013-2014 Freescale Semiconductor, Inc.4 * Copyright 2018 Angelo Dureghello <angelo@sysam.it>5 */6#ifndef _FSL_EDMA_COMMON_H_7#define _FSL_EDMA_COMMON_H_8 9#include <linux/dma-direction.h>10#include <linux/platform_device.h>11#include "virt-dma.h"12 13#define EDMA_CR_EDBG BIT(1)14#define EDMA_CR_ERCA BIT(2)15#define EDMA_CR_ERGA BIT(3)16#define EDMA_CR_HOE BIT(4)17#define EDMA_CR_HALT BIT(5)18#define EDMA_CR_CLM BIT(6)19#define EDMA_CR_EMLM BIT(7)20#define EDMA_CR_ECX BIT(16)21#define EDMA_CR_CX BIT(17)22 23#define EDMA_SEEI_SEEI(x) ((x) & GENMASK(4, 0))24#define EDMA_CEEI_CEEI(x) ((x) & GENMASK(4, 0))25#define EDMA_CINT_CINT(x) ((x) & GENMASK(4, 0))26#define EDMA_CERR_CERR(x) ((x) & GENMASK(4, 0))27 28#define EDMA_TCD_ATTR_DSIZE(x) (((x) & GENMASK(2, 0)))29#define EDMA_TCD_ATTR_DMOD(x) (((x) & GENMASK(4, 0)) << 3)30#define EDMA_TCD_ATTR_SSIZE(x) (((x) & GENMASK(2, 0)) << 8)31#define EDMA_TCD_ATTR_SMOD(x) (((x) & GENMASK(4, 0)) << 11)32 33#define EDMA_TCD_ITER_MASK GENMASK(14, 0)34#define EDMA_TCD_CITER_CITER(x) ((x) & EDMA_TCD_ITER_MASK)35#define EDMA_TCD_BITER_BITER(x) ((x) & EDMA_TCD_ITER_MASK)36 37#define EDMA_TCD_CSR_START BIT(0)38#define EDMA_TCD_CSR_INT_MAJOR BIT(1)39#define EDMA_TCD_CSR_INT_HALF BIT(2)40#define EDMA_TCD_CSR_D_REQ BIT(3)41#define EDMA_TCD_CSR_E_SG BIT(4)42#define EDMA_TCD_CSR_E_LINK BIT(5)43#define EDMA_TCD_CSR_ACTIVE BIT(6)44#define EDMA_TCD_CSR_DONE BIT(7)45 46#define EDMA_V3_TCD_NBYTES_MLOFF_NBYTES(x) ((x) & GENMASK(9, 0))47#define EDMA_V3_TCD_NBYTES_MLOFF(x) (x << 10)48#define EDMA_V3_TCD_NBYTES_DMLOE (1 << 30)49#define EDMA_V3_TCD_NBYTES_SMLOE (1 << 31)50 51#define EDMAMUX_CHCFG_DIS 0x052#define EDMAMUX_CHCFG_ENBL 0x8053#define EDMAMUX_CHCFG_SOURCE(n) ((n) & 0x3F)54 55#define DMAMUX_NR 256 57#define EDMA_TCD 0x100058 59#define FSL_EDMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \60 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \61 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \62 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))63 64#define EDMA_V3_CH_SBR_RD BIT(22)65#define EDMA_V3_CH_SBR_WR BIT(21)66#define EDMA_V3_CH_CSR_ERQ BIT(0)67#define EDMA_V3_CH_CSR_EARQ BIT(1)68#define EDMA_V3_CH_CSR_EEI BIT(2)69#define EDMA_V3_CH_CSR_DONE BIT(30)70#define EDMA_V3_CH_CSR_ACTIVE BIT(31)71 72enum fsl_edma_pm_state {73 RUNNING = 0,74 SUSPENDED,75};76 77struct fsl_edma_hw_tcd {78 __le32 saddr;79 __le16 soff;80 __le16 attr;81 __le32 nbytes;82 __le32 slast;83 __le32 daddr;84 __le16 doff;85 __le16 citer;86 __le32 dlast_sga;87 __le16 csr;88 __le16 biter;89};90 91struct fsl_edma_hw_tcd64 {92 __le64 saddr;93 __le16 soff;94 __le16 attr;95 __le32 nbytes;96 __le64 slast;97 __le64 daddr;98 __le64 dlast_sga;99 __le16 doff;100 __le16 citer;101 __le16 csr;102 __le16 biter;103} __packed;104 105struct fsl_edma3_ch_reg {106 __le32 ch_csr;107 __le32 ch_es;108 __le32 ch_int;109 __le32 ch_sbr;110 __le32 ch_pri;111 __le32 ch_mux;112 __le32 ch_mattr; /* edma4, reserved for edma3 */113 __le32 ch_reserved;114 union {115 struct fsl_edma_hw_tcd tcd;116 struct fsl_edma_hw_tcd64 tcd64;117 };118} __packed;119 120/*121 * These are iomem pointers, for both v32 and v64.122 */123struct edma_regs {124 void __iomem *cr;125 void __iomem *es;126 void __iomem *erqh;127 void __iomem *erql; /* aka erq on v32 */128 void __iomem *eeih;129 void __iomem *eeil; /* aka eei on v32 */130 void __iomem *seei;131 void __iomem *ceei;132 void __iomem *serq;133 void __iomem *cerq;134 void __iomem *cint;135 void __iomem *cerr;136 void __iomem *ssrt;137 void __iomem *cdne;138 void __iomem *inth;139 void __iomem *intl;140 void __iomem *errh;141 void __iomem *errl;142};143 144struct fsl_edma_sw_tcd {145 dma_addr_t ptcd;146 void *vtcd;147};148 149struct fsl_edma_chan {150 struct virt_dma_chan vchan;151 enum dma_status status;152 enum fsl_edma_pm_state pm_state;153 struct fsl_edma_engine *edma;154 struct fsl_edma_desc *edesc;155 struct dma_slave_config cfg;156 u32 attr;157 bool is_sw;158 struct dma_pool *tcd_pool;159 dma_addr_t dma_dev_addr;160 u32 dma_dev_size;161 enum dma_data_direction dma_dir;162 char chan_name[32];163 void __iomem *tcd;164 void __iomem *mux_addr;165 u32 real_count;166 struct work_struct issue_worker;167 struct platform_device *pdev;168 struct device *pd_dev;169 u32 srcid;170 struct clk *clk;171 int priority;172 int hw_chanid;173 int txirq;174 irqreturn_t (*irq_handler)(int irq, void *dev_id);175 bool is_rxchan;176 bool is_remote;177 bool is_multi_fifo;178};179 180struct fsl_edma_desc {181 struct virt_dma_desc vdesc;182 struct fsl_edma_chan *echan;183 bool iscyclic;184 enum dma_transfer_direction dirn;185 unsigned int n_tcds;186 struct fsl_edma_sw_tcd tcd[];187};188 189#define FSL_EDMA_DRV_HAS_DMACLK BIT(0)190#define FSL_EDMA_DRV_MUX_SWAP BIT(1)191#define FSL_EDMA_DRV_CONFIG32 BIT(2)192#define FSL_EDMA_DRV_WRAP_IO BIT(3)193#define FSL_EDMA_DRV_EDMA64 BIT(4)194#define FSL_EDMA_DRV_HAS_PD BIT(5)195#define FSL_EDMA_DRV_HAS_CHCLK BIT(6)196#define FSL_EDMA_DRV_HAS_CHMUX BIT(7)197#define FSL_EDMA_DRV_MEM_REMOTE BIT(8)198/* control and status register is in tcd address space, edma3 reg layout */199#define FSL_EDMA_DRV_SPLIT_REG BIT(9)200#define FSL_EDMA_DRV_BUS_8BYTE BIT(10)201#define FSL_EDMA_DRV_DEV_TO_DEV BIT(11)202#define FSL_EDMA_DRV_ALIGN_64BYTE BIT(12)203/* Need clean CHn_CSR DONE before enable TCD's ESG */204#define FSL_EDMA_DRV_CLEAR_DONE_E_SG BIT(13)205/* Need clean CHn_CSR DONE before enable TCD's MAJORELINK */206#define FSL_EDMA_DRV_CLEAR_DONE_E_LINK BIT(14)207#define FSL_EDMA_DRV_TCD64 BIT(15)208 209#define FSL_EDMA_DRV_EDMA3 (FSL_EDMA_DRV_SPLIT_REG | \210 FSL_EDMA_DRV_BUS_8BYTE | \211 FSL_EDMA_DRV_DEV_TO_DEV | \212 FSL_EDMA_DRV_ALIGN_64BYTE | \213 FSL_EDMA_DRV_CLEAR_DONE_E_SG | \214 FSL_EDMA_DRV_CLEAR_DONE_E_LINK)215 216#define FSL_EDMA_DRV_EDMA4 (FSL_EDMA_DRV_SPLIT_REG | \217 FSL_EDMA_DRV_BUS_8BYTE | \218 FSL_EDMA_DRV_DEV_TO_DEV | \219 FSL_EDMA_DRV_ALIGN_64BYTE | \220 FSL_EDMA_DRV_CLEAR_DONE_E_LINK)221 222struct fsl_edma_drvdata {223 u32 dmamuxs; /* only used before v3 */224 u32 chreg_off;225 u32 chreg_space_sz;226 u32 flags;227 u32 mux_off; /* channel mux register offset */228 u32 mux_skip; /* how much skip for each channel */229 int (*setup_irq)(struct platform_device *pdev,230 struct fsl_edma_engine *fsl_edma);231};232 233struct fsl_edma_engine {234 struct dma_device dma_dev;235 void __iomem *membase;236 void __iomem *muxbase[DMAMUX_NR];237 struct clk *muxclk[DMAMUX_NR];238 struct clk *dmaclk;239 struct mutex fsl_edma_mutex;240 const struct fsl_edma_drvdata *drvdata;241 u32 n_chans;242 int txirq;243 int errirq;244 bool big_endian;245 struct edma_regs regs;246 u64 chan_masked;247 struct fsl_edma_chan chans[] __counted_by(n_chans);248};249 250static inline u32 fsl_edma_drvflags(struct fsl_edma_chan *fsl_chan)251{252 return fsl_chan->edma->drvdata->flags;253}254 255#define edma_read_tcdreg_c(chan, _tcd, __name) \256_Generic(((_tcd)->__name), \257 __iomem __le64 : edma_readq(chan->edma, &(_tcd)->__name), \258 __iomem __le32 : edma_readl(chan->edma, &(_tcd)->__name), \259 __iomem __le16 : edma_readw(chan->edma, &(_tcd)->__name) \260 )261 262#define edma_read_tcdreg(chan, __name) \263((fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64) ? \264 edma_read_tcdreg_c(chan, ((struct fsl_edma_hw_tcd64 __iomem *)chan->tcd), __name) : \265 edma_read_tcdreg_c(chan, ((struct fsl_edma_hw_tcd __iomem *)chan->tcd), __name) \266)267 268#define edma_write_tcdreg_c(chan, _tcd, _val, __name) \269_Generic((_tcd->__name), \270 __iomem __le64 : edma_writeq(chan->edma, (u64 __force)(_val), &_tcd->__name), \271 __iomem __le32 : edma_writel(chan->edma, (u32 __force)(_val), &_tcd->__name), \272 __iomem __le16 : edma_writew(chan->edma, (u16 __force)(_val), &_tcd->__name), \273 __iomem u8 : edma_writeb(chan->edma, _val, &_tcd->__name) \274 )275 276#define edma_write_tcdreg(chan, val, __name) \277do { \278 struct fsl_edma_hw_tcd64 __iomem *tcd64_r = (struct fsl_edma_hw_tcd64 __iomem *)chan->tcd; \279 struct fsl_edma_hw_tcd __iomem *tcd_r = (struct fsl_edma_hw_tcd __iomem *)chan->tcd; \280 \281 if (fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64) \282 edma_write_tcdreg_c(chan, tcd64_r, val, __name); \283 else \284 edma_write_tcdreg_c(chan, tcd_r, val, __name); \285} while (0)286 287#define edma_cp_tcd_to_reg(chan, __tcd, __name) \288do { \289 struct fsl_edma_hw_tcd64 __iomem *tcd64_r = (struct fsl_edma_hw_tcd64 __iomem *)chan->tcd; \290 struct fsl_edma_hw_tcd __iomem *tcd_r = (struct fsl_edma_hw_tcd __iomem *)chan->tcd; \291 struct fsl_edma_hw_tcd64 *tcd64_m = (struct fsl_edma_hw_tcd64 *)__tcd; \292 struct fsl_edma_hw_tcd *tcd_m = (struct fsl_edma_hw_tcd *)__tcd; \293 \294 if (fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64) \295 edma_write_tcdreg_c(chan, tcd64_r, tcd64_m->__name, __name); \296 else \297 edma_write_tcdreg_c(chan, tcd_r, tcd_m->__name, __name); \298} while (0)299 300#define edma_readl_chreg(chan, __name) \301 edma_readl(chan->edma, \302 (void __iomem *)&(container_of(((__force void *)chan->tcd),\303 struct fsl_edma3_ch_reg, tcd)->__name))304 305#define edma_writel_chreg(chan, val, __name) \306 edma_writel(chan->edma, val, \307 (void __iomem *)&(container_of(((__force void *)chan->tcd),\308 struct fsl_edma3_ch_reg, tcd)->__name))309 310#define fsl_edma_get_tcd(_chan, _tcd, _field) \311(fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64 ? (((struct fsl_edma_hw_tcd64 *)_tcd)->_field) : \312 (((struct fsl_edma_hw_tcd *)_tcd)->_field))313 314#define fsl_edma_le_to_cpu(x) \315_Generic((x), \316 __le64 : le64_to_cpu((x)), \317 __le32 : le32_to_cpu((x)), \318 __le16 : le16_to_cpu((x)) \319)320 321#define fsl_edma_get_tcd_to_cpu(_chan, _tcd, _field) \322(fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64 ? \323 fsl_edma_le_to_cpu(((struct fsl_edma_hw_tcd64 *)_tcd)->_field) : \324 fsl_edma_le_to_cpu(((struct fsl_edma_hw_tcd *)_tcd)->_field))325 326#define fsl_edma_set_tcd_to_le_c(_tcd, _val, _field) \327_Generic(((_tcd)->_field), \328 __le64 : (_tcd)->_field = cpu_to_le64(_val), \329 __le32 : (_tcd)->_field = cpu_to_le32(_val), \330 __le16 : (_tcd)->_field = cpu_to_le16(_val) \331)332 333#define fsl_edma_set_tcd_to_le(_chan, _tcd, _val, _field) \334do { \335 if (fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64) \336 fsl_edma_set_tcd_to_le_c((struct fsl_edma_hw_tcd64 *)_tcd, _val, _field); \337 else \338 fsl_edma_set_tcd_to_le_c((struct fsl_edma_hw_tcd *)_tcd, _val, _field); \339} while (0)340 341/* Need after struct defination */342#include "fsl-edma-trace.h"343 344/*345 * R/W functions for big- or little-endian registers:346 * The eDMA controller's endian is independent of the CPU core's endian.347 * For the big-endian IP module, the offset for 8-bit or 16-bit registers348 * should also be swapped opposite to that in little-endian IP.349 */350static inline u64 edma_readq(struct fsl_edma_engine *edma, void __iomem *addr)351{352 u64 l, h;353 354 if (edma->big_endian) {355 l = ioread32be(addr);356 h = ioread32be(addr + 4);357 } else {358 l = ioread32(addr);359 h = ioread32(addr + 4);360 }361 362 trace_edma_readl(edma, addr, l);363 trace_edma_readl(edma, addr + 4, h);364 365 return (h << 32) | l;366}367 368static inline u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr)369{370 u32 val;371 372 if (edma->big_endian)373 val = ioread32be(addr);374 else375 val = ioread32(addr);376 377 trace_edma_readl(edma, addr, val);378 379 return val;380}381 382static inline u16 edma_readw(struct fsl_edma_engine *edma, void __iomem *addr)383{384 u16 val;385 386 if (edma->big_endian)387 val = ioread16be(addr);388 else389 val = ioread16(addr);390 391 trace_edma_readw(edma, addr, val);392 393 return val;394}395 396static inline void edma_writeb(struct fsl_edma_engine *edma,397 u8 val, void __iomem *addr)398{399 /* swap the reg offset for these in big-endian mode */400 if (edma->big_endian)401 iowrite8(val, (void __iomem *)((unsigned long)addr ^ 0x3));402 else403 iowrite8(val, addr);404 405 trace_edma_writeb(edma, addr, val);406}407 408static inline void edma_writew(struct fsl_edma_engine *edma,409 u16 val, void __iomem *addr)410{411 /* swap the reg offset for these in big-endian mode */412 if (edma->big_endian)413 iowrite16be(val, (void __iomem *)((unsigned long)addr ^ 0x2));414 else415 iowrite16(val, addr);416 417 trace_edma_writew(edma, addr, val);418}419 420static inline void edma_writel(struct fsl_edma_engine *edma,421 u32 val, void __iomem *addr)422{423 if (edma->big_endian)424 iowrite32be(val, addr);425 else426 iowrite32(val, addr);427 428 trace_edma_writel(edma, addr, val);429}430 431static inline void edma_writeq(struct fsl_edma_engine *edma,432 u64 val, void __iomem *addr)433{434 if (edma->big_endian) {435 iowrite32be(val & 0xFFFFFFFF, addr);436 iowrite32be(val >> 32, addr + 4);437 } else {438 iowrite32(val & 0xFFFFFFFF, addr);439 iowrite32(val >> 32, addr + 4);440 }441 442 trace_edma_writel(edma, addr, val & 0xFFFFFFFF);443 trace_edma_writel(edma, addr + 4, val >> 32);444}445 446static inline struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan)447{448 return container_of(chan, struct fsl_edma_chan, vchan.chan);449}450 451static inline struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd)452{453 return container_of(vd, struct fsl_edma_desc, vdesc);454}455 456static inline void fsl_edma_err_chan_handler(struct fsl_edma_chan *fsl_chan)457{458 fsl_chan->status = DMA_ERROR;459}460 461void fsl_edma_tx_chan_handler(struct fsl_edma_chan *fsl_chan);462void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan);463void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan,464 unsigned int slot, bool enable);465void fsl_edma_free_desc(struct virt_dma_desc *vdesc);466int fsl_edma_terminate_all(struct dma_chan *chan);467int fsl_edma_pause(struct dma_chan *chan);468int fsl_edma_resume(struct dma_chan *chan);469int fsl_edma_slave_config(struct dma_chan *chan,470 struct dma_slave_config *cfg);471enum dma_status fsl_edma_tx_status(struct dma_chan *chan,472 dma_cookie_t cookie, struct dma_tx_state *txstate);473struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic(474 struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,475 size_t period_len, enum dma_transfer_direction direction,476 unsigned long flags);477struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg(478 struct dma_chan *chan, struct scatterlist *sgl,479 unsigned int sg_len, enum dma_transfer_direction direction,480 unsigned long flags, void *context);481struct dma_async_tx_descriptor *fsl_edma_prep_memcpy(482 struct dma_chan *chan, dma_addr_t dma_dst, dma_addr_t dma_src,483 size_t len, unsigned long flags);484void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan);485void fsl_edma_issue_pending(struct dma_chan *chan);486int fsl_edma_alloc_chan_resources(struct dma_chan *chan);487void fsl_edma_free_chan_resources(struct dma_chan *chan);488void fsl_edma_cleanup_vchan(struct dma_device *dmadev);489void fsl_edma_setup_regs(struct fsl_edma_engine *edma);490 491#endif /* _FSL_EDMA_COMMON_H_ */492