1848 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * STM32 DMA3 controller driver4 *5 * Copyright (C) STMicroelectronics 20246 * Author(s): Amelie Delaunay <amelie.delaunay@foss.st.com>7 */8 9#include <linux/bitfield.h>10#include <linux/clk.h>11#include <linux/dma-mapping.h>12#include <linux/dmaengine.h>13#include <linux/dmapool.h>14#include <linux/init.h>15#include <linux/iopoll.h>16#include <linux/list.h>17#include <linux/module.h>18#include <linux/of_dma.h>19#include <linux/platform_device.h>20#include <linux/pm_runtime.h>21#include <linux/reset.h>22#include <linux/slab.h>23 24#include "../virt-dma.h"25 26#define STM32_DMA3_SECCFGR 0x0027#define STM32_DMA3_PRIVCFGR 0x0428#define STM32_DMA3_RCFGLOCKR 0x0829#define STM32_DMA3_MISR 0x0c30#define STM32_DMA3_SMISR 0x1031 32#define STM32_DMA3_CLBAR(x) (0x50 + 0x80 * (x))33#define STM32_DMA3_CCIDCFGR(x) (0x54 + 0x80 * (x))34#define STM32_DMA3_CSEMCR(x) (0x58 + 0x80 * (x))35#define STM32_DMA3_CFCR(x) (0x5c + 0x80 * (x))36#define STM32_DMA3_CSR(x) (0x60 + 0x80 * (x))37#define STM32_DMA3_CCR(x) (0x64 + 0x80 * (x))38#define STM32_DMA3_CTR1(x) (0x90 + 0x80 * (x))39#define STM32_DMA3_CTR2(x) (0x94 + 0x80 * (x))40#define STM32_DMA3_CBR1(x) (0x98 + 0x80 * (x))41#define STM32_DMA3_CSAR(x) (0x9c + 0x80 * (x))42#define STM32_DMA3_CDAR(x) (0xa0 + 0x80 * (x))43#define STM32_DMA3_CLLR(x) (0xcc + 0x80 * (x))44 45#define STM32_DMA3_HWCFGR13 0xfc0 /* G_PER_CTRL(X) x=8..15 */46#define STM32_DMA3_HWCFGR12 0xfc4 /* G_PER_CTRL(X) x=0..7 */47#define STM32_DMA3_HWCFGR4 0xfe4 /* G_FIFO_SIZE(X) x=8..15 */48#define STM32_DMA3_HWCFGR3 0xfe8 /* G_FIFO_SIZE(X) x=0..7 */49#define STM32_DMA3_HWCFGR2 0xfec /* G_MAX_REQ_ID */50#define STM32_DMA3_HWCFGR1 0xff0 /* G_MASTER_PORTS, G_NUM_CHANNELS, G_Mx_DATA_WIDTH */51#define STM32_DMA3_VERR 0xff452 53/* SECCFGR DMA secure configuration register */54#define SECCFGR_SEC(x) BIT(x)55 56/* MISR DMA non-secure/secure masked interrupt status register */57#define MISR_MIS(x) BIT(x)58 59/* CxLBAR DMA channel x linked_list base address register */60#define CLBAR_LBA GENMASK(31, 16)61 62/* CxCIDCFGR DMA channel x CID register */63#define CCIDCFGR_CFEN BIT(0)64#define CCIDCFGR_SEM_EN BIT(1)65#define CCIDCFGR_SCID GENMASK(5, 4)66#define CCIDCFGR_SEM_WLIST_CID0 BIT(16)67#define CCIDCFGR_SEM_WLIST_CID1 BIT(17)68#define CCIDCFGR_SEM_WLIST_CID2 BIT(18)69 70enum ccidcfgr_cid {71 CCIDCFGR_CID0,72 CCIDCFGR_CID1,73 CCIDCFGR_CID2,74};75 76/* CxSEMCR DMA channel x semaphore control register */77#define CSEMCR_SEM_MUTEX BIT(0)78#define CSEMCR_SEM_CCID GENMASK(5, 4)79 80/* CxFCR DMA channel x flag clear register */81#define CFCR_TCF BIT(8)82#define CFCR_HTF BIT(9)83#define CFCR_DTEF BIT(10)84#define CFCR_ULEF BIT(11)85#define CFCR_USEF BIT(12)86#define CFCR_SUSPF BIT(13)87 88/* CxSR DMA channel x status register */89#define CSR_IDLEF BIT(0)90#define CSR_TCF BIT(8)91#define CSR_HTF BIT(9)92#define CSR_DTEF BIT(10)93#define CSR_ULEF BIT(11)94#define CSR_USEF BIT(12)95#define CSR_SUSPF BIT(13)96#define CSR_ALL_F GENMASK(13, 8)97#define CSR_FIFOL GENMASK(24, 16)98 99/* CxCR DMA channel x control register */100#define CCR_EN BIT(0)101#define CCR_RESET BIT(1)102#define CCR_SUSP BIT(2)103#define CCR_TCIE BIT(8)104#define CCR_HTIE BIT(9)105#define CCR_DTEIE BIT(10)106#define CCR_ULEIE BIT(11)107#define CCR_USEIE BIT(12)108#define CCR_SUSPIE BIT(13)109#define CCR_ALLIE GENMASK(13, 8)110#define CCR_LSM BIT(16)111#define CCR_LAP BIT(17)112#define CCR_PRIO GENMASK(23, 22)113 114enum ccr_prio {115 CCR_PRIO_LOW,116 CCR_PRIO_MID,117 CCR_PRIO_HIGH,118 CCR_PRIO_VERY_HIGH,119};120 121/* CxTR1 DMA channel x transfer register 1 */122#define CTR1_SINC BIT(3)123#define CTR1_SBL_1 GENMASK(9, 4)124#define CTR1_DINC BIT(19)125#define CTR1_DBL_1 GENMASK(25, 20)126#define CTR1_SDW_LOG2 GENMASK(1, 0)127#define CTR1_PAM GENMASK(12, 11)128#define CTR1_SAP BIT(14)129#define CTR1_DDW_LOG2 GENMASK(17, 16)130#define CTR1_DAP BIT(30)131 132enum ctr1_dw {133 CTR1_DW_BYTE,134 CTR1_DW_HWORD,135 CTR1_DW_WORD,136 CTR1_DW_DWORD, /* Depends on HWCFGR1.G_M0_DATA_WIDTH_ENC and .G_M1_DATA_WIDTH_ENC */137};138 139enum ctr1_pam {140 CTR1_PAM_0S_LT, /* if DDW > SDW, padded with 0s else left-truncated */141 CTR1_PAM_SE_RT, /* if DDW > SDW, sign extended else right-truncated */142 CTR1_PAM_PACK_UNPACK, /* FIFO queued */143};144 145/* CxTR2 DMA channel x transfer register 2 */146#define CTR2_REQSEL GENMASK(7, 0)147#define CTR2_SWREQ BIT(9)148#define CTR2_DREQ BIT(10)149#define CTR2_BREQ BIT(11)150#define CTR2_PFREQ BIT(12)151#define CTR2_TCEM GENMASK(31, 30)152 153enum ctr2_tcem {154 CTR2_TCEM_BLOCK,155 CTR2_TCEM_REPEAT_BLOCK,156 CTR2_TCEM_LLI,157 CTR2_TCEM_CHANNEL,158};159 160/* CxBR1 DMA channel x block register 1 */161#define CBR1_BNDT GENMASK(15, 0)162 163/* CxLLR DMA channel x linked-list address register */164#define CLLR_LA GENMASK(15, 2)165#define CLLR_ULL BIT(16)166#define CLLR_UDA BIT(27)167#define CLLR_USA BIT(28)168#define CLLR_UB1 BIT(29)169#define CLLR_UT2 BIT(30)170#define CLLR_UT1 BIT(31)171 172/* HWCFGR13 DMA hardware configuration register 13 x=8..15 */173/* HWCFGR12 DMA hardware configuration register 12 x=0..7 */174#define G_PER_CTRL(x) (ULL(0x1) << (4 * (x)))175 176/* HWCFGR4 DMA hardware configuration register 4 x=8..15 */177/* HWCFGR3 DMA hardware configuration register 3 x=0..7 */178#define G_FIFO_SIZE(x) (ULL(0x7) << (4 * (x)))179 180#define get_chan_hwcfg(x, mask, reg) (((reg) & (mask)) >> (4 * (x)))181 182/* HWCFGR2 DMA hardware configuration register 2 */183#define G_MAX_REQ_ID GENMASK(7, 0)184 185/* HWCFGR1 DMA hardware configuration register 1 */186#define G_MASTER_PORTS GENMASK(2, 0)187#define G_NUM_CHANNELS GENMASK(12, 8)188#define G_M0_DATA_WIDTH_ENC GENMASK(25, 24)189#define G_M1_DATA_WIDTH_ENC GENMASK(29, 28)190 191enum stm32_dma3_master_ports {192 AXI64, /* 1x AXI: 64-bit port 0 */193 AHB32, /* 1x AHB: 32-bit port 0 */194 AHB32_AHB32, /* 2x AHB: 32-bit port 0 and 32-bit port 1 */195 AXI64_AHB32, /* 1x AXI 64-bit port 0 and 1x AHB 32-bit port 1 */196 AXI64_AXI64, /* 2x AXI: 64-bit port 0 and 64-bit port 1 */197 AXI128_AHB32, /* 1x AXI 128-bit port 0 and 1x AHB 32-bit port 1 */198};199 200enum stm32_dma3_port_data_width {201 DW_32, /* 32-bit, for AHB */202 DW_64, /* 64-bit, for AXI */203 DW_128, /* 128-bit, for AXI */204 DW_INVALID,205};206 207/* VERR DMA version register */208#define VERR_MINREV GENMASK(3, 0)209#define VERR_MAJREV GENMASK(7, 4)210 211/* Device tree */212/* struct stm32_dma3_dt_conf */213/* .ch_conf */214#define STM32_DMA3_DT_PRIO GENMASK(1, 0) /* CCR_PRIO */215#define STM32_DMA3_DT_FIFO GENMASK(7, 4)216/* .tr_conf */217#define STM32_DMA3_DT_SINC BIT(0) /* CTR1_SINC */218#define STM32_DMA3_DT_SAP BIT(1) /* CTR1_SAP */219#define STM32_DMA3_DT_DINC BIT(4) /* CTR1_DINC */220#define STM32_DMA3_DT_DAP BIT(5) /* CTR1_DAP */221#define STM32_DMA3_DT_BREQ BIT(8) /* CTR2_BREQ */222#define STM32_DMA3_DT_PFREQ BIT(9) /* CTR2_PFREQ */223#define STM32_DMA3_DT_TCEM GENMASK(13, 12) /* CTR2_TCEM */224 225/* struct stm32_dma3_chan .config_set bitfield */226#define STM32_DMA3_CFG_SET_DT BIT(0)227#define STM32_DMA3_CFG_SET_DMA BIT(1)228#define STM32_DMA3_CFG_SET_BOTH (STM32_DMA3_CFG_SET_DT | STM32_DMA3_CFG_SET_DMA)229 230#define STM32_DMA3_MAX_BLOCK_SIZE ALIGN_DOWN(CBR1_BNDT, 64)231#define port_is_ahb(maxdw) ({ typeof(maxdw) (_maxdw) = (maxdw); \232 ((_maxdw) != DW_INVALID) && ((_maxdw) == DW_32); })233#define port_is_axi(maxdw) ({ typeof(maxdw) (_maxdw) = (maxdw); \234 ((_maxdw) != DW_INVALID) && ((_maxdw) != DW_32); })235#define get_chan_max_dw(maxdw, maxburst)((port_is_ahb(maxdw) || \236 (maxburst) < DMA_SLAVE_BUSWIDTH_8_BYTES) ? \237 DMA_SLAVE_BUSWIDTH_4_BYTES : DMA_SLAVE_BUSWIDTH_8_BYTES)238 239/* Static linked-list data structure (depends on update bits UT1/UT2/UB1/USA/UDA/ULL) */240struct stm32_dma3_hwdesc {241 u32 ctr1;242 u32 ctr2;243 u32 cbr1;244 u32 csar;245 u32 cdar;246 u32 cllr;247} __packed __aligned(32);248 249/*250 * CLLR_LA / sizeof(struct stm32_dma3_hwdesc) represents the number of hdwdesc that can be addressed251 * by the pointer to the next linked-list data structure. The __aligned forces the 32-byte252 * alignment. So use hardcoded 32. Multiplied by the max block size of each item, it represents253 * the sg size limitation.254 */255#define STM32_DMA3_MAX_SEG_SIZE ((CLLR_LA / 32) * STM32_DMA3_MAX_BLOCK_SIZE)256 257/*258 * Linked-list items259 */260struct stm32_dma3_lli {261 struct stm32_dma3_hwdesc *hwdesc;262 dma_addr_t hwdesc_addr;263};264 265struct stm32_dma3_swdesc {266 struct virt_dma_desc vdesc;267 u32 ccr;268 bool cyclic;269 u32 lli_size;270 struct stm32_dma3_lli lli[] __counted_by(lli_size);271};272 273struct stm32_dma3_dt_conf {274 u32 ch_id;275 u32 req_line;276 u32 ch_conf;277 u32 tr_conf;278};279 280struct stm32_dma3_chan {281 struct virt_dma_chan vchan;282 u32 id;283 int irq;284 u32 fifo_size;285 u32 max_burst;286 bool semaphore_mode;287 struct stm32_dma3_dt_conf dt_config;288 struct dma_slave_config dma_config;289 u8 config_set;290 struct dma_pool *lli_pool;291 struct stm32_dma3_swdesc *swdesc;292 enum ctr2_tcem tcem;293 u32 dma_status;294};295 296struct stm32_dma3_ddata {297 struct dma_device dma_dev;298 void __iomem *base;299 struct clk *clk;300 struct stm32_dma3_chan *chans;301 u32 dma_channels;302 u32 dma_requests;303 enum stm32_dma3_port_data_width ports_max_dw[2];304};305 306static inline struct stm32_dma3_ddata *to_stm32_dma3_ddata(struct stm32_dma3_chan *chan)307{308 return container_of(chan->vchan.chan.device, struct stm32_dma3_ddata, dma_dev);309}310 311static inline struct stm32_dma3_chan *to_stm32_dma3_chan(struct dma_chan *c)312{313 return container_of(c, struct stm32_dma3_chan, vchan.chan);314}315 316static inline struct stm32_dma3_swdesc *to_stm32_dma3_swdesc(struct virt_dma_desc *vdesc)317{318 return container_of(vdesc, struct stm32_dma3_swdesc, vdesc);319}320 321static struct device *chan2dev(struct stm32_dma3_chan *chan)322{323 return &chan->vchan.chan.dev->device;324}325 326static void stm32_dma3_chan_dump_reg(struct stm32_dma3_chan *chan)327{328 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);329 struct device *dev = chan2dev(chan);330 u32 id = chan->id, offset;331 332 offset = STM32_DMA3_SECCFGR;333 dev_dbg(dev, "SECCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset));334 offset = STM32_DMA3_PRIVCFGR;335 dev_dbg(dev, "PRIVCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset));336 offset = STM32_DMA3_CCIDCFGR(id);337 dev_dbg(dev, "C%dCIDCFGR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));338 offset = STM32_DMA3_CSEMCR(id);339 dev_dbg(dev, "C%dSEMCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));340 offset = STM32_DMA3_CSR(id);341 dev_dbg(dev, "C%dSR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));342 offset = STM32_DMA3_CCR(id);343 dev_dbg(dev, "C%dCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));344 offset = STM32_DMA3_CTR1(id);345 dev_dbg(dev, "C%dTR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));346 offset = STM32_DMA3_CTR2(id);347 dev_dbg(dev, "C%dTR2(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));348 offset = STM32_DMA3_CBR1(id);349 dev_dbg(dev, "C%dBR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));350 offset = STM32_DMA3_CSAR(id);351 dev_dbg(dev, "C%dSAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));352 offset = STM32_DMA3_CDAR(id);353 dev_dbg(dev, "C%dDAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));354 offset = STM32_DMA3_CLLR(id);355 dev_dbg(dev, "C%dLLR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));356 offset = STM32_DMA3_CLBAR(id);357 dev_dbg(dev, "C%dLBAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));358}359 360static void stm32_dma3_chan_dump_hwdesc(struct stm32_dma3_chan *chan,361 struct stm32_dma3_swdesc *swdesc)362{363 struct stm32_dma3_hwdesc *hwdesc;364 int i;365 366 for (i = 0; i < swdesc->lli_size; i++) {367 hwdesc = swdesc->lli[i].hwdesc;368 if (i)369 dev_dbg(chan2dev(chan), "V\n");370 dev_dbg(chan2dev(chan), "[%d]@%pad\n", i, &swdesc->lli[i].hwdesc_addr);371 dev_dbg(chan2dev(chan), "| C%dTR1: %08x\n", chan->id, hwdesc->ctr1);372 dev_dbg(chan2dev(chan), "| C%dTR2: %08x\n", chan->id, hwdesc->ctr2);373 dev_dbg(chan2dev(chan), "| C%dBR1: %08x\n", chan->id, hwdesc->cbr1);374 dev_dbg(chan2dev(chan), "| C%dSAR: %08x\n", chan->id, hwdesc->csar);375 dev_dbg(chan2dev(chan), "| C%dDAR: %08x\n", chan->id, hwdesc->cdar);376 dev_dbg(chan2dev(chan), "| C%dLLR: %08x\n", chan->id, hwdesc->cllr);377 }378 379 if (swdesc->cyclic) {380 dev_dbg(chan2dev(chan), "|\n");381 dev_dbg(chan2dev(chan), "-->[0]@%pad\n", &swdesc->lli[0].hwdesc_addr);382 } else {383 dev_dbg(chan2dev(chan), "X\n");384 }385}386 387static struct stm32_dma3_swdesc *stm32_dma3_chan_desc_alloc(struct stm32_dma3_chan *chan, u32 count)388{389 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);390 struct stm32_dma3_swdesc *swdesc;391 int i;392 393 /*394 * If the memory to be allocated for the number of hwdesc (6 u32 members but 32-bytes395 * aligned) is greater than the maximum address of CLLR_LA, then the last items can't be396 * addressed, so abort the allocation.397 */398 if ((count * 32) > CLLR_LA) {399 dev_err(chan2dev(chan), "Transfer is too big (> %luB)\n", STM32_DMA3_MAX_SEG_SIZE);400 return NULL;401 }402 403 swdesc = kzalloc(struct_size(swdesc, lli, count), GFP_NOWAIT);404 if (!swdesc)405 return NULL;406 swdesc->lli_size = count;407 408 for (i = 0; i < count; i++) {409 swdesc->lli[i].hwdesc = dma_pool_zalloc(chan->lli_pool, GFP_NOWAIT,410 &swdesc->lli[i].hwdesc_addr);411 if (!swdesc->lli[i].hwdesc)412 goto err_pool_free;413 }414 swdesc->ccr = 0;415 416 /* Set LL base address */417 writel_relaxed(swdesc->lli[0].hwdesc_addr & CLBAR_LBA,418 ddata->base + STM32_DMA3_CLBAR(chan->id));419 420 /* Set LL allocated port */421 swdesc->ccr &= ~CCR_LAP;422 423 return swdesc;424 425err_pool_free:426 dev_err(chan2dev(chan), "Failed to alloc descriptors\n");427 while (--i >= 0)428 dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr);429 kfree(swdesc);430 431 return NULL;432}433 434static void stm32_dma3_chan_desc_free(struct stm32_dma3_chan *chan,435 struct stm32_dma3_swdesc *swdesc)436{437 int i;438 439 for (i = 0; i < swdesc->lli_size; i++)440 dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr);441 442 kfree(swdesc);443}444 445static void stm32_dma3_chan_vdesc_free(struct virt_dma_desc *vdesc)446{447 struct stm32_dma3_swdesc *swdesc = to_stm32_dma3_swdesc(vdesc);448 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(vdesc->tx.chan);449 450 stm32_dma3_chan_desc_free(chan, swdesc);451}452 453static void stm32_dma3_check_user_setting(struct stm32_dma3_chan *chan)454{455 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);456 struct device *dev = chan2dev(chan);457 u32 ctr1 = readl_relaxed(ddata->base + STM32_DMA3_CTR1(chan->id));458 u32 cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id));459 u32 csar = readl_relaxed(ddata->base + STM32_DMA3_CSAR(chan->id));460 u32 cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id));461 u32 cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id));462 u32 bndt = FIELD_GET(CBR1_BNDT, cbr1);463 u32 sdw = 1 << FIELD_GET(CTR1_SDW_LOG2, ctr1);464 u32 ddw = 1 << FIELD_GET(CTR1_DDW_LOG2, ctr1);465 u32 sap = FIELD_GET(CTR1_SAP, ctr1);466 u32 dap = FIELD_GET(CTR1_DAP, ctr1);467 468 if (!bndt && !FIELD_GET(CLLR_UB1, cllr))469 dev_err(dev, "null source block size and no update of this value\n");470 if (bndt % sdw)471 dev_err(dev, "source block size not multiple of src data width\n");472 if (FIELD_GET(CTR1_PAM, ctr1) == CTR1_PAM_PACK_UNPACK && bndt % ddw)473 dev_err(dev, "(un)packing mode w/ src block size not multiple of dst data width\n");474 if (csar % sdw)475 dev_err(dev, "unaligned source address not multiple of src data width\n");476 if (cdar % ddw)477 dev_err(dev, "unaligned destination address not multiple of dst data width\n");478 if (sdw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[sap]))479 dev_err(dev, "double-word source data width not supported on port %u\n", sap);480 if (ddw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[dap]))481 dev_err(dev, "double-word destination data width not supported on port %u\n", dap);482}483 484static void stm32_dma3_chan_prep_hwdesc(struct stm32_dma3_chan *chan,485 struct stm32_dma3_swdesc *swdesc,486 u32 curr, dma_addr_t src, dma_addr_t dst, u32 len,487 u32 ctr1, u32 ctr2, bool is_last, bool is_cyclic)488{489 struct stm32_dma3_hwdesc *hwdesc;490 dma_addr_t next_lli;491 u32 next = curr + 1;492 493 hwdesc = swdesc->lli[curr].hwdesc;494 hwdesc->ctr1 = ctr1;495 hwdesc->ctr2 = ctr2;496 hwdesc->cbr1 = FIELD_PREP(CBR1_BNDT, len);497 hwdesc->csar = src;498 hwdesc->cdar = dst;499 500 if (is_last) {501 if (is_cyclic)502 next_lli = swdesc->lli[0].hwdesc_addr;503 else504 next_lli = 0;505 } else {506 next_lli = swdesc->lli[next].hwdesc_addr;507 }508 509 hwdesc->cllr = 0;510 if (next_lli) {511 hwdesc->cllr |= CLLR_UT1 | CLLR_UT2 | CLLR_UB1;512 hwdesc->cllr |= CLLR_USA | CLLR_UDA | CLLR_ULL;513 hwdesc->cllr |= (next_lli & CLLR_LA);514 }515 516 /*517 * Make sure to flush the CPU's write buffers so that the descriptors are ready to be read518 * by DMA3. By explicitly using a write memory barrier here, instead of doing it with writel519 * to enable the channel, we avoid an unnecessary barrier in the case where the descriptors520 * are reused (DMA_CTRL_REUSE).521 */522 if (is_last)523 dma_wmb();524}525 526static enum dma_slave_buswidth stm32_dma3_get_max_dw(u32 chan_max_burst,527 enum stm32_dma3_port_data_width port_max_dw,528 u32 len, dma_addr_t addr)529{530 enum dma_slave_buswidth max_dw = get_chan_max_dw(port_max_dw, chan_max_burst);531 532 /* len and addr must be a multiple of dw */533 return 1 << __ffs(len | addr | max_dw);534}535 536static u32 stm32_dma3_get_max_burst(u32 len, enum dma_slave_buswidth dw, u32 chan_max_burst)537{538 u32 max_burst = chan_max_burst ? chan_max_burst / dw : 1;539 540 /* len is a multiple of dw, so if len is < chan_max_burst, shorten burst */541 if (len < chan_max_burst)542 max_burst = len / dw;543 544 /*545 * HW doesn't modify the burst if burst size <= half of the fifo size.546 * If len is not a multiple of burst size, last burst is shortened by HW.547 */548 return max_burst;549}550 551static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transfer_direction dir,552 u32 *ccr, u32 *ctr1, u32 *ctr2,553 dma_addr_t src_addr, dma_addr_t dst_addr, u32 len)554{555 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);556 struct dma_device dma_device = ddata->dma_dev;557 u32 sdw, ddw, sbl_max, dbl_max, tcem, init_dw, init_bl_max;558 u32 _ctr1 = 0, _ctr2 = 0;559 u32 ch_conf = chan->dt_config.ch_conf;560 u32 tr_conf = chan->dt_config.tr_conf;561 u32 sap = FIELD_GET(STM32_DMA3_DT_SAP, tr_conf), sap_max_dw;562 u32 dap = FIELD_GET(STM32_DMA3_DT_DAP, tr_conf), dap_max_dw;563 564 dev_dbg(chan2dev(chan), "%s from %pad to %pad\n",565 dmaengine_get_direction_text(dir), &src_addr, &dst_addr);566 567 sdw = chan->dma_config.src_addr_width ? : get_chan_max_dw(sap, chan->max_burst);568 ddw = chan->dma_config.dst_addr_width ? : get_chan_max_dw(dap, chan->max_burst);569 sbl_max = chan->dma_config.src_maxburst ? : 1;570 dbl_max = chan->dma_config.dst_maxburst ? : 1;571 572 /* Following conditions would raise User Setting Error interrupt */573 if (!(dma_device.src_addr_widths & BIT(sdw)) || !(dma_device.dst_addr_widths & BIT(ddw))) {574 dev_err(chan2dev(chan), "Bus width (src=%u, dst=%u) not supported\n", sdw, ddw);575 return -EINVAL;576 }577 578 if (ddata->ports_max_dw[1] == DW_INVALID && (sap || dap)) {579 dev_err(chan2dev(chan), "Only one master port, port 1 is not supported\n");580 return -EINVAL;581 }582 583 sap_max_dw = ddata->ports_max_dw[sap];584 dap_max_dw = ddata->ports_max_dw[dap];585 if ((port_is_ahb(sap_max_dw) && sdw == DMA_SLAVE_BUSWIDTH_8_BYTES) ||586 (port_is_ahb(dap_max_dw) && ddw == DMA_SLAVE_BUSWIDTH_8_BYTES)) {587 dev_err(chan2dev(chan),588 "8 bytes buswidth (src=%u, dst=%u) not supported on port (sap=%u, dap=%u\n",589 sdw, ddw, sap, dap);590 return -EINVAL;591 }592 593 if (FIELD_GET(STM32_DMA3_DT_SINC, tr_conf))594 _ctr1 |= CTR1_SINC;595 if (sap)596 _ctr1 |= CTR1_SAP;597 if (FIELD_GET(STM32_DMA3_DT_DINC, tr_conf))598 _ctr1 |= CTR1_DINC;599 if (dap)600 _ctr1 |= CTR1_DAP;601 602 _ctr2 |= FIELD_PREP(CTR2_REQSEL, chan->dt_config.req_line) & ~CTR2_SWREQ;603 if (FIELD_GET(STM32_DMA3_DT_BREQ, tr_conf))604 _ctr2 |= CTR2_BREQ;605 if (dir == DMA_DEV_TO_MEM && FIELD_GET(STM32_DMA3_DT_PFREQ, tr_conf))606 _ctr2 |= CTR2_PFREQ;607 tcem = FIELD_GET(STM32_DMA3_DT_TCEM, tr_conf);608 _ctr2 |= FIELD_PREP(CTR2_TCEM, tcem);609 610 /* Store TCEM to know on which event TC flag occurred */611 chan->tcem = tcem;612 /* Store direction for residue computation */613 chan->dma_config.direction = dir;614 615 switch (dir) {616 case DMA_MEM_TO_DEV:617 /* Set destination (device) data width and burst */618 ddw = min_t(u32, ddw, stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw,619 len, dst_addr));620 dbl_max = min_t(u32, dbl_max, stm32_dma3_get_max_burst(len, ddw, chan->max_burst));621 622 /* Set source (memory) data width and burst */623 sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr);624 sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst);625 626 _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw));627 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1);628 _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw));629 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1);630 631 if (ddw != sdw) {632 _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK);633 /* Should never reach this case as ddw is clamped down */634 if (len & (ddw - 1)) {635 dev_err(chan2dev(chan),636 "Packing mode is enabled and len is not multiple of ddw");637 return -EINVAL;638 }639 }640 641 /* dst = dev */642 _ctr2 |= CTR2_DREQ;643 644 break;645 646 case DMA_DEV_TO_MEM:647 /* Set source (device) data width and burst */648 sdw = min_t(u32, sdw, stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw,649 len, src_addr));650 sbl_max = min_t(u32, sbl_max, stm32_dma3_get_max_burst(len, sdw, chan->max_burst));651 652 /* Set destination (memory) data width and burst */653 ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr);654 dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst);655 656 _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw));657 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1);658 _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw));659 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1);660 661 if (ddw != sdw) {662 _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK);663 /* Should never reach this case as ddw is clamped down */664 if (len & (ddw - 1)) {665 dev_err(chan2dev(chan),666 "Packing mode is enabled and len is not multiple of ddw\n");667 return -EINVAL;668 }669 }670 671 /* dst = mem */672 _ctr2 &= ~CTR2_DREQ;673 674 break;675 676 case DMA_MEM_TO_MEM:677 /* Set source (memory) data width and burst */678 init_dw = sdw;679 init_bl_max = sbl_max;680 sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr);681 sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst);682 if (chan->config_set & STM32_DMA3_CFG_SET_DMA) {683 sdw = min_t(u32, init_dw, sdw);684 sbl_max = min_t(u32, init_bl_max,685 stm32_dma3_get_max_burst(len, sdw, chan->max_burst));686 }687 688 /* Set destination (memory) data width and burst */689 init_dw = ddw;690 init_bl_max = dbl_max;691 ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr);692 dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst);693 if (chan->config_set & STM32_DMA3_CFG_SET_DMA) {694 ddw = min_t(u32, init_dw, ddw);695 dbl_max = min_t(u32, init_bl_max,696 stm32_dma3_get_max_burst(len, ddw, chan->max_burst));697 }698 699 _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw));700 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1);701 _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw));702 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1);703 704 if (ddw != sdw) {705 _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK);706 /* Should never reach this case as ddw is clamped down */707 if (len & (ddw - 1)) {708 dev_err(chan2dev(chan),709 "Packing mode is enabled and len is not multiple of ddw");710 return -EINVAL;711 }712 }713 714 /* CTR2_REQSEL/DREQ/BREQ/PFREQ are ignored with CTR2_SWREQ=1 */715 _ctr2 |= CTR2_SWREQ;716 717 break;718 719 default:720 dev_err(chan2dev(chan), "Direction %s not supported\n",721 dmaengine_get_direction_text(dir));722 return -EINVAL;723 }724 725 *ccr |= FIELD_PREP(CCR_PRIO, FIELD_GET(STM32_DMA3_DT_PRIO, ch_conf));726 *ctr1 = _ctr1;727 *ctr2 = _ctr2;728 729 dev_dbg(chan2dev(chan), "%s: sdw=%u bytes sbl=%u beats ddw=%u bytes dbl=%u beats\n",730 __func__, sdw, sbl_max, ddw, dbl_max);731 732 return 0;733}734 735static void stm32_dma3_chan_start(struct stm32_dma3_chan *chan)736{737 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);738 struct virt_dma_desc *vdesc;739 struct stm32_dma3_hwdesc *hwdesc;740 u32 id = chan->id;741 u32 csr, ccr;742 743 vdesc = vchan_next_desc(&chan->vchan);744 if (!vdesc) {745 chan->swdesc = NULL;746 return;747 }748 list_del(&vdesc->node);749 750 chan->swdesc = to_stm32_dma3_swdesc(vdesc);751 hwdesc = chan->swdesc->lli[0].hwdesc;752 753 stm32_dma3_chan_dump_hwdesc(chan, chan->swdesc);754 755 writel_relaxed(chan->swdesc->ccr, ddata->base + STM32_DMA3_CCR(id));756 writel_relaxed(hwdesc->ctr1, ddata->base + STM32_DMA3_CTR1(id));757 writel_relaxed(hwdesc->ctr2, ddata->base + STM32_DMA3_CTR2(id));758 writel_relaxed(hwdesc->cbr1, ddata->base + STM32_DMA3_CBR1(id));759 writel_relaxed(hwdesc->csar, ddata->base + STM32_DMA3_CSAR(id));760 writel_relaxed(hwdesc->cdar, ddata->base + STM32_DMA3_CDAR(id));761 writel_relaxed(hwdesc->cllr, ddata->base + STM32_DMA3_CLLR(id));762 763 /* Clear any pending interrupts */764 csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(id));765 if (csr & CSR_ALL_F)766 writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(id));767 768 stm32_dma3_chan_dump_reg(chan);769 770 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(id));771 writel_relaxed(ccr | CCR_EN, ddata->base + STM32_DMA3_CCR(id));772 773 chan->dma_status = DMA_IN_PROGRESS;774 775 dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan);776}777 778static int stm32_dma3_chan_suspend(struct stm32_dma3_chan *chan, bool susp)779{780 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);781 u32 csr, ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN;782 int ret = 0;783 784 if (susp)785 ccr |= CCR_SUSP;786 else787 ccr &= ~CCR_SUSP;788 789 writel_relaxed(ccr, ddata->base + STM32_DMA3_CCR(chan->id));790 791 if (susp) {792 ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr,793 csr & CSR_SUSPF, 1, 10);794 if (!ret)795 writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id));796 797 stm32_dma3_chan_dump_reg(chan);798 }799 800 return ret;801}802 803static void stm32_dma3_chan_reset(struct stm32_dma3_chan *chan)804{805 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);806 u32 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN;807 808 writel_relaxed(ccr |= CCR_RESET, ddata->base + STM32_DMA3_CCR(chan->id));809}810 811static int stm32_dma3_chan_get_curr_hwdesc(struct stm32_dma3_swdesc *swdesc, u32 cllr, u32 *residue)812{813 u32 i, lli_offset, next_lli_offset = cllr & CLLR_LA;814 815 /* If cllr is null, it means it is either the last or single item */816 if (!cllr)817 return swdesc->lli_size - 1;818 819 /* In cyclic mode, go fast and first check we are not on the last item */820 if (swdesc->cyclic && next_lli_offset == (swdesc->lli[0].hwdesc_addr & CLLR_LA))821 return swdesc->lli_size - 1;822 823 /* As transfer is in progress, look backward from the last item */824 for (i = swdesc->lli_size - 1; i > 0; i--) {825 *residue += FIELD_GET(CBR1_BNDT, swdesc->lli[i].hwdesc->cbr1);826 lli_offset = swdesc->lli[i].hwdesc_addr & CLLR_LA;827 if (lli_offset == next_lli_offset)828 return i - 1;829 }830 831 return -EINVAL;832}833 834static void stm32_dma3_chan_set_residue(struct stm32_dma3_chan *chan,835 struct stm32_dma3_swdesc *swdesc,836 struct dma_tx_state *txstate)837{838 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);839 struct device *dev = chan2dev(chan);840 struct stm32_dma3_hwdesc *hwdesc;841 u32 residue, curr_lli, csr, cdar, cbr1, cllr, bndt, fifol;842 bool pack_unpack;843 int ret;844 845 csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id));846 if (!(csr & CSR_IDLEF) && chan->dma_status != DMA_PAUSED) {847 /* Suspend current transfer to read registers for a snapshot */848 writel_relaxed(swdesc->ccr | CCR_SUSP, ddata->base + STM32_DMA3_CCR(chan->id));849 ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr,850 csr & (CSR_SUSPF | CSR_IDLEF), 1, 10);851 852 if (ret || ((csr & CSR_TCF) && (csr & CSR_IDLEF))) {853 writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id));854 writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id));855 if (ret)856 dev_err(dev, "Channel suspension timeout, csr=%08x\n", csr);857 }858 }859 860 /* If channel is still active (CSR_IDLEF is not set), can't get a reliable residue */861 if (!(csr & CSR_IDLEF))862 dev_warn(dev, "Can't get residue: channel still active, csr=%08x\n", csr);863 864 /*865 * If channel is not suspended, but Idle and Transfer Complete are set,866 * linked-list is over, no residue867 */868 if (!(csr & CSR_SUSPF) && (csr & CSR_TCF) && (csr & CSR_IDLEF))869 return;870 871 /* Read registers to have a snapshot */872 cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id));873 cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id));874 cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id));875 876 /* Resume current transfer */877 if (csr & CSR_SUSPF) {878 writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id));879 writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id));880 }881 882 /* Add current BNDT */883 bndt = FIELD_GET(CBR1_BNDT, cbr1);884 residue = bndt;885 886 /* Get current hwdesc and cumulate residue of pending hwdesc BNDT */887 ret = stm32_dma3_chan_get_curr_hwdesc(swdesc, cllr, &residue);888 if (ret < 0) {889 dev_err(chan2dev(chan), "Can't get residue: current hwdesc not found\n");890 return;891 }892 curr_lli = ret;893 894 /* Read current FIFO level - in units of programmed destination data width */895 hwdesc = swdesc->lli[curr_lli].hwdesc;896 fifol = FIELD_GET(CSR_FIFOL, csr) * (1 << FIELD_GET(CTR1_DDW_LOG2, hwdesc->ctr1));897 /* If the FIFO contains as many bytes as its size, it can't contain more */898 if (fifol == (1 << (chan->fifo_size + 1)))899 goto skip_fifol_update;900 901 /*902 * In case of PACKING (Destination burst length > Source burst length) or UNPACKING903 * (Source burst length > Destination burst length), bytes could be pending in the FIFO904 * (to be packed up to Destination burst length or unpacked into Destination burst length905 * chunks).906 * BNDT is not reliable, as it reflects the number of bytes read from the source but not the907 * number of bytes written to the destination.908 * FIFOL is also not sufficient, because it reflects the number of available write beats in909 * units of Destination data width but not the bytes not yet packed or unpacked.910 * In case of Destination increment DINC, it is possible to compute the number of bytes in911 * the FIFO:912 * fifol_in_bytes = bytes_read - bytes_written.913 */914 pack_unpack = !!(FIELD_GET(CTR1_PAM, hwdesc->ctr1) == CTR1_PAM_PACK_UNPACK);915 if (pack_unpack && (hwdesc->ctr1 & CTR1_DINC)) {916 int bytes_read = FIELD_GET(CBR1_BNDT, hwdesc->cbr1) - bndt;917 int bytes_written = cdar - hwdesc->cdar;918 919 if (bytes_read > 0)920 fifol = bytes_read - bytes_written;921 }922 923skip_fifol_update:924 if (fifol) {925 dev_dbg(chan2dev(chan), "%u byte(s) in the FIFO\n", fifol);926 dma_set_in_flight_bytes(txstate, fifol);927 /*928 * Residue is already accurate for DMA_MEM_TO_DEV as BNDT reflects data read from929 * the source memory buffer, so just need to add fifol to residue in case of930 * DMA_DEV_TO_MEM transfer because these bytes are not yet written in destination931 * memory buffer.932 */933 if (chan->dma_config.direction == DMA_DEV_TO_MEM)934 residue += fifol;935 }936 dma_set_residue(txstate, residue);937}938 939static int stm32_dma3_chan_stop(struct stm32_dma3_chan *chan)940{941 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);942 u32 ccr;943 int ret = 0;944 945 chan->dma_status = DMA_COMPLETE;946 947 /* Disable interrupts */948 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id));949 writel_relaxed(ccr & ~(CCR_ALLIE | CCR_EN), ddata->base + STM32_DMA3_CCR(chan->id));950 951 if (!(ccr & CCR_SUSP) && (ccr & CCR_EN)) {952 /* Suspend the channel */953 ret = stm32_dma3_chan_suspend(chan, true);954 if (ret)955 dev_warn(chan2dev(chan), "%s: timeout, data might be lost\n", __func__);956 }957 958 /*959 * Reset the channel: this causes the reset of the FIFO and the reset of the channel960 * internal state, the reset of CCR_EN and CCR_SUSP bits.961 */962 stm32_dma3_chan_reset(chan);963 964 return ret;965}966 967static void stm32_dma3_chan_complete(struct stm32_dma3_chan *chan)968{969 if (!chan->swdesc)970 return;971 972 vchan_cookie_complete(&chan->swdesc->vdesc);973 chan->swdesc = NULL;974 stm32_dma3_chan_start(chan);975}976 977static irqreturn_t stm32_dma3_chan_irq(int irq, void *devid)978{979 struct stm32_dma3_chan *chan = devid;980 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);981 u32 misr, csr, ccr;982 983 spin_lock(&chan->vchan.lock);984 985 misr = readl_relaxed(ddata->base + STM32_DMA3_MISR);986 if (!(misr & MISR_MIS(chan->id))) {987 spin_unlock(&chan->vchan.lock);988 return IRQ_NONE;989 }990 991 csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id));992 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & CCR_ALLIE;993 994 if (csr & CSR_TCF && ccr & CCR_TCIE) {995 if (chan->swdesc->cyclic)996 vchan_cyclic_callback(&chan->swdesc->vdesc);997 else998 stm32_dma3_chan_complete(chan);999 }1000 1001 if (csr & CSR_USEF && ccr & CCR_USEIE) {1002 dev_err(chan2dev(chan), "User setting error\n");1003 chan->dma_status = DMA_ERROR;1004 /* CCR.EN automatically cleared by HW */1005 stm32_dma3_check_user_setting(chan);1006 stm32_dma3_chan_reset(chan);1007 }1008 1009 if (csr & CSR_ULEF && ccr & CCR_ULEIE) {1010 dev_err(chan2dev(chan), "Update link transfer error\n");1011 chan->dma_status = DMA_ERROR;1012 /* CCR.EN automatically cleared by HW */1013 stm32_dma3_chan_reset(chan);1014 }1015 1016 if (csr & CSR_DTEF && ccr & CCR_DTEIE) {1017 dev_err(chan2dev(chan), "Data transfer error\n");1018 chan->dma_status = DMA_ERROR;1019 /* CCR.EN automatically cleared by HW */1020 stm32_dma3_chan_reset(chan);1021 }1022 1023 /*1024 * Half Transfer Interrupt may be disabled but Half Transfer Flag can be set,1025 * ensure HTF flag to be cleared, with other flags.1026 */1027 csr &= (ccr | CCR_HTIE);1028 1029 if (csr)1030 writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(chan->id));1031 1032 spin_unlock(&chan->vchan.lock);1033 1034 return IRQ_HANDLED;1035}1036 1037static int stm32_dma3_alloc_chan_resources(struct dma_chan *c)1038{1039 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);1040 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);1041 u32 id = chan->id, csemcr, ccid;1042 int ret;1043 1044 ret = pm_runtime_resume_and_get(ddata->dma_dev.dev);1045 if (ret < 0)1046 return ret;1047 1048 /* Ensure the channel is free */1049 if (chan->semaphore_mode &&1050 readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id)) & CSEMCR_SEM_MUTEX) {1051 ret = -EBUSY;1052 goto err_put_sync;1053 }1054 1055 chan->lli_pool = dmam_pool_create(dev_name(&c->dev->device), c->device->dev,1056 sizeof(struct stm32_dma3_hwdesc),1057 __alignof__(struct stm32_dma3_hwdesc), SZ_64K);1058 if (!chan->lli_pool) {1059 dev_err(chan2dev(chan), "Failed to create LLI pool\n");1060 ret = -ENOMEM;1061 goto err_put_sync;1062 }1063 1064 /* Take the channel semaphore */1065 if (chan->semaphore_mode) {1066 writel_relaxed(CSEMCR_SEM_MUTEX, ddata->base + STM32_DMA3_CSEMCR(id));1067 csemcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(id));1068 ccid = FIELD_GET(CSEMCR_SEM_CCID, csemcr);1069 /* Check that the channel is well taken */1070 if (ccid != CCIDCFGR_CID1) {1071 dev_err(chan2dev(chan), "Not under CID1 control (in-use by CID%d)\n", ccid);1072 ret = -EPERM;1073 goto err_pool_destroy;1074 }1075 dev_dbg(chan2dev(chan), "Under CID1 control (semcr=0x%08x)\n", csemcr);1076 }1077 1078 return 0;1079 1080err_pool_destroy:1081 dmam_pool_destroy(chan->lli_pool);1082 chan->lli_pool = NULL;1083 1084err_put_sync:1085 pm_runtime_put_sync(ddata->dma_dev.dev);1086 1087 return ret;1088}1089 1090static void stm32_dma3_free_chan_resources(struct dma_chan *c)1091{1092 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);1093 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);1094 unsigned long flags;1095 1096 /* Ensure channel is in idle state */1097 spin_lock_irqsave(&chan->vchan.lock, flags);1098 stm32_dma3_chan_stop(chan);1099 chan->swdesc = NULL;1100 spin_unlock_irqrestore(&chan->vchan.lock, flags);1101 1102 vchan_free_chan_resources(to_virt_chan(c));1103 1104 dmam_pool_destroy(chan->lli_pool);1105 chan->lli_pool = NULL;1106 1107 /* Release the channel semaphore */1108 if (chan->semaphore_mode)1109 writel_relaxed(0, ddata->base + STM32_DMA3_CSEMCR(chan->id));1110 1111 pm_runtime_put_sync(ddata->dma_dev.dev);1112 1113 /* Reset configuration */1114 memset(&chan->dt_config, 0, sizeof(chan->dt_config));1115 memset(&chan->dma_config, 0, sizeof(chan->dma_config));1116 chan->config_set = 0;1117}1118 1119static void stm32_dma3_init_chan_config_for_memcpy(struct stm32_dma3_chan *chan,1120 dma_addr_t dst, dma_addr_t src)1121{1122 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);1123 u32 dw = get_chan_max_dw(ddata->ports_max_dw[0], chan->max_burst); /* port 0 by default */1124 u32 burst = chan->max_burst / dw;1125 1126 /* Initialize dt_config if channel not pre-configured through DT */1127 if (!(chan->config_set & STM32_DMA3_CFG_SET_DT)) {1128 chan->dt_config.ch_conf = FIELD_PREP(STM32_DMA3_DT_PRIO, CCR_PRIO_VERY_HIGH);1129 chan->dt_config.ch_conf |= FIELD_PREP(STM32_DMA3_DT_FIFO, chan->fifo_size);1130 chan->dt_config.tr_conf = STM32_DMA3_DT_SINC | STM32_DMA3_DT_DINC;1131 chan->dt_config.tr_conf |= FIELD_PREP(STM32_DMA3_DT_TCEM, CTR2_TCEM_CHANNEL);1132 }1133 1134 /* Initialize dma_config if dmaengine_slave_config() not used */1135 if (!(chan->config_set & STM32_DMA3_CFG_SET_DMA)) {1136 chan->dma_config.src_addr_width = dw;1137 chan->dma_config.dst_addr_width = dw;1138 chan->dma_config.src_maxburst = burst;1139 chan->dma_config.dst_maxburst = burst;1140 chan->dma_config.src_addr = src;1141 chan->dma_config.dst_addr = dst;1142 }1143}1144 1145static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_memcpy(struct dma_chan *c,1146 dma_addr_t dst, dma_addr_t src,1147 size_t len, unsigned long flags)1148{1149 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);1150 struct stm32_dma3_swdesc *swdesc;1151 size_t next_size, offset;1152 u32 count, i, ctr1, ctr2;1153 1154 count = DIV_ROUND_UP(len, STM32_DMA3_MAX_BLOCK_SIZE);1155 1156 swdesc = stm32_dma3_chan_desc_alloc(chan, count);1157 if (!swdesc)1158 return NULL;1159 1160 if (chan->config_set != STM32_DMA3_CFG_SET_BOTH)1161 stm32_dma3_init_chan_config_for_memcpy(chan, dst, src);1162 1163 for (i = 0, offset = 0; offset < len; i++, offset += next_size) {1164 size_t remaining;1165 int ret;1166 1167 remaining = len - offset;1168 next_size = min_t(size_t, remaining, STM32_DMA3_MAX_BLOCK_SIZE);1169 1170 ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_MEM, &swdesc->ccr, &ctr1, &ctr2,1171 src + offset, dst + offset, next_size);1172 if (ret)1173 goto err_desc_free;1174 1175 stm32_dma3_chan_prep_hwdesc(chan, swdesc, i, src + offset, dst + offset, next_size,1176 ctr1, ctr2, next_size == remaining, false);1177 }1178 1179 /* Enable Errors interrupts */1180 swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE;1181 /* Enable Transfer state interrupts */1182 swdesc->ccr |= CCR_TCIE;1183 1184 swdesc->cyclic = false;1185 1186 return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags);1187 1188err_desc_free:1189 stm32_dma3_chan_desc_free(chan, swdesc);1190 1191 return NULL;1192}1193 1194static struct dma_async_tx_descriptor *stm32_dma3_prep_slave_sg(struct dma_chan *c,1195 struct scatterlist *sgl,1196 unsigned int sg_len,1197 enum dma_transfer_direction dir,1198 unsigned long flags, void *context)1199{1200 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);1201 struct stm32_dma3_swdesc *swdesc;1202 struct scatterlist *sg;1203 size_t len;1204 dma_addr_t sg_addr, dev_addr, src, dst;1205 u32 i, j, count, ctr1, ctr2;1206 int ret;1207 1208 count = sg_len;1209 for_each_sg(sgl, sg, sg_len, i) {1210 len = sg_dma_len(sg);1211 if (len > STM32_DMA3_MAX_BLOCK_SIZE)1212 count += DIV_ROUND_UP(len, STM32_DMA3_MAX_BLOCK_SIZE) - 1;1213 }1214 1215 swdesc = stm32_dma3_chan_desc_alloc(chan, count);1216 if (!swdesc)1217 return NULL;1218 1219 /* sg_len and i correspond to the initial sgl; count and j correspond to the hwdesc LL */1220 j = 0;1221 for_each_sg(sgl, sg, sg_len, i) {1222 sg_addr = sg_dma_address(sg);1223 dev_addr = (dir == DMA_MEM_TO_DEV) ? chan->dma_config.dst_addr :1224 chan->dma_config.src_addr;1225 len = sg_dma_len(sg);1226 1227 do {1228 size_t chunk = min_t(size_t, len, STM32_DMA3_MAX_BLOCK_SIZE);1229 1230 if (dir == DMA_MEM_TO_DEV) {1231 src = sg_addr;1232 dst = dev_addr;1233 1234 ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2,1235 src, dst, chunk);1236 1237 if (FIELD_GET(CTR1_DINC, ctr1))1238 dev_addr += chunk;1239 } else { /* (dir == DMA_DEV_TO_MEM || dir == DMA_MEM_TO_MEM) */1240 src = dev_addr;1241 dst = sg_addr;1242 1243 ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2,1244 src, dst, chunk);1245 1246 if (FIELD_GET(CTR1_SINC, ctr1))1247 dev_addr += chunk;1248 }1249 1250 if (ret)1251 goto err_desc_free;1252 1253 stm32_dma3_chan_prep_hwdesc(chan, swdesc, j, src, dst, chunk,1254 ctr1, ctr2, j == (count - 1), false);1255 1256 sg_addr += chunk;1257 len -= chunk;1258 j++;1259 } while (len);1260 }1261 1262 /* Enable Error interrupts */1263 swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE;1264 /* Enable Transfer state interrupts */1265 swdesc->ccr |= CCR_TCIE;1266 1267 swdesc->cyclic = false;1268 1269 return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags);1270 1271err_desc_free:1272 stm32_dma3_chan_desc_free(chan, swdesc);1273 1274 return NULL;1275}1276 1277static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_cyclic(struct dma_chan *c,1278 dma_addr_t buf_addr,1279 size_t buf_len, size_t period_len,1280 enum dma_transfer_direction dir,1281 unsigned long flags)1282{1283 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);1284 struct stm32_dma3_swdesc *swdesc;1285 dma_addr_t src, dst;1286 u32 count, i, ctr1, ctr2;1287 int ret;1288 1289 if (!buf_len || !period_len || period_len > STM32_DMA3_MAX_BLOCK_SIZE) {1290 dev_err(chan2dev(chan), "Invalid buffer/period length\n");1291 return NULL;1292 }1293 1294 if (buf_len % period_len) {1295 dev_err(chan2dev(chan), "Buffer length not multiple of period length\n");1296 return NULL;1297 }1298 1299 count = buf_len / period_len;1300 swdesc = stm32_dma3_chan_desc_alloc(chan, count);1301 if (!swdesc)1302 return NULL;1303 1304 if (dir == DMA_MEM_TO_DEV) {1305 src = buf_addr;1306 dst = chan->dma_config.dst_addr;1307 1308 ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_DEV, &swdesc->ccr, &ctr1, &ctr2,1309 src, dst, period_len);1310 } else if (dir == DMA_DEV_TO_MEM) {1311 src = chan->dma_config.src_addr;1312 dst = buf_addr;1313 1314 ret = stm32_dma3_chan_prep_hw(chan, DMA_DEV_TO_MEM, &swdesc->ccr, &ctr1, &ctr2,1315 src, dst, period_len);1316 } else {1317 dev_err(chan2dev(chan), "Invalid direction\n");1318 ret = -EINVAL;1319 }1320 1321 if (ret)1322 goto err_desc_free;1323 1324 for (i = 0; i < count; i++) {1325 if (dir == DMA_MEM_TO_DEV) {1326 src = buf_addr + i * period_len;1327 dst = chan->dma_config.dst_addr;1328 } else { /* (dir == DMA_DEV_TO_MEM) */1329 src = chan->dma_config.src_addr;1330 dst = buf_addr + i * period_len;1331 }1332 1333 stm32_dma3_chan_prep_hwdesc(chan, swdesc, i, src, dst, period_len,1334 ctr1, ctr2, i == (count - 1), true);1335 }1336 1337 /* Enable Error interrupts */1338 swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE;1339 /* Enable Transfer state interrupts */1340 swdesc->ccr |= CCR_TCIE;1341 1342 swdesc->cyclic = true;1343 1344 return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags);1345 1346err_desc_free:1347 stm32_dma3_chan_desc_free(chan, swdesc);1348 1349 return NULL;1350}1351 1352static void stm32_dma3_caps(struct dma_chan *c, struct dma_slave_caps *caps)1353{1354 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);1355 1356 if (!chan->fifo_size) {1357 caps->max_burst = 0;1358 caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);1359 caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);1360 } else {1361 /* Burst transfer should not exceed half of the fifo size */1362 caps->max_burst = chan->max_burst;1363 if (caps->max_burst < DMA_SLAVE_BUSWIDTH_8_BYTES) {1364 caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);1365 caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);1366 }1367 }1368}1369 1370static int stm32_dma3_config(struct dma_chan *c, struct dma_slave_config *config)1371{1372 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);1373 1374 memcpy(&chan->dma_config, config, sizeof(*config));1375 chan->config_set |= STM32_DMA3_CFG_SET_DMA;1376 1377 return 0;1378}1379 1380static int stm32_dma3_pause(struct dma_chan *c)1381{1382 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);1383 int ret;1384 1385 ret = stm32_dma3_chan_suspend(chan, true);1386 if (ret)1387 return ret;1388 1389 chan->dma_status = DMA_PAUSED;1390 1391 dev_dbg(chan2dev(chan), "vchan %pK: paused\n", &chan->vchan);1392 1393 return 0;1394}1395 1396static int stm32_dma3_resume(struct dma_chan *c)1397{1398 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);1399 1400 stm32_dma3_chan_suspend(chan, false);1401 1402 chan->dma_status = DMA_IN_PROGRESS;1403 1404 dev_dbg(chan2dev(chan), "vchan %pK: resumed\n", &chan->vchan);1405 1406 return 0;1407}1408 1409static int stm32_dma3_terminate_all(struct dma_chan *c)1410{1411 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);1412 unsigned long flags;1413 LIST_HEAD(head);1414 1415 spin_lock_irqsave(&chan->vchan.lock, flags);1416 1417 if (chan->swdesc) {1418 vchan_terminate_vdesc(&chan->swdesc->vdesc);1419 chan->swdesc = NULL;1420 }1421 1422 stm32_dma3_chan_stop(chan);1423 1424 vchan_get_all_descriptors(&chan->vchan, &head);1425 1426 spin_unlock_irqrestore(&chan->vchan.lock, flags);1427 vchan_dma_desc_free_list(&chan->vchan, &head);1428 1429 dev_dbg(chan2dev(chan), "vchan %pK: terminated\n", &chan->vchan);1430 1431 return 0;1432}1433 1434static void stm32_dma3_synchronize(struct dma_chan *c)1435{1436 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);1437 1438 vchan_synchronize(&chan->vchan);1439}1440 1441static enum dma_status stm32_dma3_tx_status(struct dma_chan *c, dma_cookie_t cookie,1442 struct dma_tx_state *txstate)1443{1444 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);1445 struct stm32_dma3_swdesc *swdesc = NULL;1446 enum dma_status status;1447 unsigned long flags;1448 struct virt_dma_desc *vd;1449 1450 status = dma_cookie_status(c, cookie, txstate);1451 if (status == DMA_COMPLETE)1452 return status;1453 1454 if (!txstate)1455 return chan->dma_status;1456 1457 spin_lock_irqsave(&chan->vchan.lock, flags);1458 1459 vd = vchan_find_desc(&chan->vchan, cookie);1460 if (vd)1461 swdesc = to_stm32_dma3_swdesc(vd);1462 else if (chan->swdesc && chan->swdesc->vdesc.tx.cookie == cookie)1463 swdesc = chan->swdesc;1464 1465 /* Get residue/in_flight_bytes only if a transfer is currently running (swdesc != NULL) */1466 if (swdesc)1467 stm32_dma3_chan_set_residue(chan, swdesc, txstate);1468 1469 spin_unlock_irqrestore(&chan->vchan.lock, flags);1470 1471 return chan->dma_status;1472}1473 1474static void stm32_dma3_issue_pending(struct dma_chan *c)1475{1476 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);1477 unsigned long flags;1478 1479 spin_lock_irqsave(&chan->vchan.lock, flags);1480 1481 if (vchan_issue_pending(&chan->vchan) && !chan->swdesc) {1482 dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan);1483 stm32_dma3_chan_start(chan);1484 }1485 1486 spin_unlock_irqrestore(&chan->vchan.lock, flags);1487}1488 1489static bool stm32_dma3_filter_fn(struct dma_chan *c, void *fn_param)1490{1491 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);1492 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);1493 struct stm32_dma3_dt_conf *conf = fn_param;1494 u32 mask, semcr;1495 int ret;1496 1497 dev_dbg(c->device->dev, "%s(%s): req_line=%d ch_conf=%08x tr_conf=%08x\n",1498 __func__, dma_chan_name(c), conf->req_line, conf->ch_conf, conf->tr_conf);1499 1500 if (!of_property_read_u32(c->device->dev->of_node, "dma-channel-mask", &mask))1501 if (!(mask & BIT(chan->id)))1502 return false;1503 1504 ret = pm_runtime_resume_and_get(ddata->dma_dev.dev);1505 if (ret < 0)1506 return false;1507 semcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id));1508 pm_runtime_put_sync(ddata->dma_dev.dev);1509 1510 /* Check if chan is free */1511 if (semcr & CSEMCR_SEM_MUTEX)1512 return false;1513 1514 /* Check if chan fifo fits well */1515 if (FIELD_GET(STM32_DMA3_DT_FIFO, conf->ch_conf) != chan->fifo_size)1516 return false;1517 1518 return true;1519}1520 1521static struct dma_chan *stm32_dma3_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma)1522{1523 struct stm32_dma3_ddata *ddata = ofdma->of_dma_data;1524 dma_cap_mask_t mask = ddata->dma_dev.cap_mask;1525 struct stm32_dma3_dt_conf conf;1526 struct stm32_dma3_chan *chan;1527 struct dma_chan *c;1528 1529 if (dma_spec->args_count < 3) {1530 dev_err(ddata->dma_dev.dev, "Invalid args count\n");1531 return NULL;1532 }1533 1534 conf.req_line = dma_spec->args[0];1535 conf.ch_conf = dma_spec->args[1];1536 conf.tr_conf = dma_spec->args[2];1537 1538 if (conf.req_line >= ddata->dma_requests) {1539 dev_err(ddata->dma_dev.dev, "Invalid request line\n");1540 return NULL;1541 }1542 1543 /* Request dma channel among the generic dma controller list */1544 c = dma_request_channel(mask, stm32_dma3_filter_fn, &conf);1545 if (!c) {1546 dev_err(ddata->dma_dev.dev, "No suitable channel found\n");1547 return NULL;1548 }1549 1550 chan = to_stm32_dma3_chan(c);1551 chan->dt_config = conf;1552 chan->config_set |= STM32_DMA3_CFG_SET_DT;1553 1554 return c;1555}1556 1557static u32 stm32_dma3_check_rif(struct stm32_dma3_ddata *ddata)1558{1559 u32 chan_reserved, mask = 0, i, ccidcfgr, invalid_cid = 0;1560 1561 /* Reserve Secure channels */1562 chan_reserved = readl_relaxed(ddata->base + STM32_DMA3_SECCFGR);1563 1564 /*1565 * CID filtering must be configured to ensure that the DMA3 channel will inherit the CID of1566 * the processor which is configuring and using the given channel.1567 * In case CID filtering is not configured, dma-channel-mask property can be used to1568 * specify available DMA channels to the kernel.1569 */1570 of_property_read_u32(ddata->dma_dev.dev->of_node, "dma-channel-mask", &mask);1571 1572 /* Reserve !CID-filtered not in dma-channel-mask, static CID != CID1, CID1 not allowed */1573 for (i = 0; i < ddata->dma_channels; i++) {1574 ccidcfgr = readl_relaxed(ddata->base + STM32_DMA3_CCIDCFGR(i));1575 1576 if (!(ccidcfgr & CCIDCFGR_CFEN)) { /* !CID-filtered */1577 invalid_cid |= BIT(i);1578 if (!(mask & BIT(i))) /* Not in dma-channel-mask */1579 chan_reserved |= BIT(i);1580 } else { /* CID-filtered */1581 if (!(ccidcfgr & CCIDCFGR_SEM_EN)) { /* Static CID mode */1582 if (FIELD_GET(CCIDCFGR_SCID, ccidcfgr) != CCIDCFGR_CID1)1583 chan_reserved |= BIT(i);1584 } else { /* Semaphore mode */1585 if (!FIELD_GET(CCIDCFGR_SEM_WLIST_CID1, ccidcfgr))1586 chan_reserved |= BIT(i);1587 ddata->chans[i].semaphore_mode = true;1588 }1589 }1590 dev_dbg(ddata->dma_dev.dev, "chan%d: %s mode, %s\n", i,1591 !(ccidcfgr & CCIDCFGR_CFEN) ? "!CID-filtered" :1592 ddata->chans[i].semaphore_mode ? "Semaphore" : "Static CID",1593 (chan_reserved & BIT(i)) ? "denied" :1594 mask & BIT(i) ? "force allowed" : "allowed");1595 }1596 1597 if (invalid_cid)1598 dev_warn(ddata->dma_dev.dev, "chan%*pbl have invalid CID configuration\n",1599 ddata->dma_channels, &invalid_cid);1600 1601 return chan_reserved;1602}1603 1604static const struct of_device_id stm32_dma3_of_match[] = {1605 { .compatible = "st,stm32mp25-dma3", },1606 { /* sentinel */ },1607};1608MODULE_DEVICE_TABLE(of, stm32_dma3_of_match);1609 1610static int stm32_dma3_probe(struct platform_device *pdev)1611{1612 struct device_node *np = pdev->dev.of_node;1613 struct stm32_dma3_ddata *ddata;1614 struct reset_control *reset;1615 struct stm32_dma3_chan *chan;1616 struct dma_device *dma_dev;1617 u32 master_ports, chan_reserved, i, verr;1618 u64 hwcfgr;1619 int ret;1620 1621 ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);1622 if (!ddata)1623 return -ENOMEM;1624 platform_set_drvdata(pdev, ddata);1625 1626 dma_dev = &ddata->dma_dev;1627 1628 ddata->base = devm_platform_ioremap_resource(pdev, 0);1629 if (IS_ERR(ddata->base))1630 return PTR_ERR(ddata->base);1631 1632 ddata->clk = devm_clk_get(&pdev->dev, NULL);1633 if (IS_ERR(ddata->clk))1634 return dev_err_probe(&pdev->dev, PTR_ERR(ddata->clk), "Failed to get clk\n");1635 1636 reset = devm_reset_control_get_optional(&pdev->dev, NULL);1637 if (IS_ERR(reset))1638 return dev_err_probe(&pdev->dev, PTR_ERR(reset), "Failed to get reset\n");1639 1640 ret = clk_prepare_enable(ddata->clk);1641 if (ret)1642 return dev_err_probe(&pdev->dev, ret, "Failed to enable clk\n");1643 1644 reset_control_reset(reset);1645 1646 INIT_LIST_HEAD(&dma_dev->channels);1647 1648 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);1649 dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);1650 dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);1651 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);1652 dma_dev->dev = &pdev->dev;1653 /*1654 * This controller supports up to 8-byte buswidth depending on the port used and the1655 * channel, and can only access address at even boundaries, multiple of the buswidth.1656 */1657 dma_dev->copy_align = DMAENGINE_ALIGN_8_BYTES;1658 dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |1659 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |1660 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |1661 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);1662 dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |1663 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |1664 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |1665 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);1666 dma_dev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | BIT(DMA_MEM_TO_MEM);1667 1668 dma_dev->descriptor_reuse = true;1669 dma_dev->max_sg_burst = STM32_DMA3_MAX_SEG_SIZE;1670 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;1671 dma_dev->device_alloc_chan_resources = stm32_dma3_alloc_chan_resources;1672 dma_dev->device_free_chan_resources = stm32_dma3_free_chan_resources;1673 dma_dev->device_prep_dma_memcpy = stm32_dma3_prep_dma_memcpy;1674 dma_dev->device_prep_slave_sg = stm32_dma3_prep_slave_sg;1675 dma_dev->device_prep_dma_cyclic = stm32_dma3_prep_dma_cyclic;1676 dma_dev->device_caps = stm32_dma3_caps;1677 dma_dev->device_config = stm32_dma3_config;1678 dma_dev->device_pause = stm32_dma3_pause;1679 dma_dev->device_resume = stm32_dma3_resume;1680 dma_dev->device_terminate_all = stm32_dma3_terminate_all;1681 dma_dev->device_synchronize = stm32_dma3_synchronize;1682 dma_dev->device_tx_status = stm32_dma3_tx_status;1683 dma_dev->device_issue_pending = stm32_dma3_issue_pending;1684 1685 /* if dma_channels is not modified, get it from hwcfgr1 */1686 if (of_property_read_u32(np, "dma-channels", &ddata->dma_channels)) {1687 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1);1688 ddata->dma_channels = FIELD_GET(G_NUM_CHANNELS, hwcfgr);1689 }1690 1691 /* if dma_requests is not modified, get it from hwcfgr2 */1692 if (of_property_read_u32(np, "dma-requests", &ddata->dma_requests)) {1693 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR2);1694 ddata->dma_requests = FIELD_GET(G_MAX_REQ_ID, hwcfgr) + 1;1695 }1696 1697 /* G_MASTER_PORTS, G_M0_DATA_WIDTH_ENC, G_M1_DATA_WIDTH_ENC in HWCFGR1 */1698 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1);1699 master_ports = FIELD_GET(G_MASTER_PORTS, hwcfgr);1700 1701 ddata->ports_max_dw[0] = FIELD_GET(G_M0_DATA_WIDTH_ENC, hwcfgr);1702 if (master_ports == AXI64 || master_ports == AHB32) /* Single master port */1703 ddata->ports_max_dw[1] = DW_INVALID;1704 else /* Dual master ports */1705 ddata->ports_max_dw[1] = FIELD_GET(G_M1_DATA_WIDTH_ENC, hwcfgr);1706 1707 ddata->chans = devm_kcalloc(&pdev->dev, ddata->dma_channels, sizeof(*ddata->chans),1708 GFP_KERNEL);1709 if (!ddata->chans) {1710 ret = -ENOMEM;1711 goto err_clk_disable;1712 }1713 1714 chan_reserved = stm32_dma3_check_rif(ddata);1715 1716 if (chan_reserved == GENMASK(ddata->dma_channels - 1, 0)) {1717 ret = -ENODEV;1718 dev_err_probe(&pdev->dev, ret, "No channel available, abort registration\n");1719 goto err_clk_disable;1720 }1721 1722 /* G_FIFO_SIZE x=0..7 in HWCFGR3 and G_FIFO_SIZE x=8..15 in HWCFGR4 */1723 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR3);1724 hwcfgr |= ((u64)readl_relaxed(ddata->base + STM32_DMA3_HWCFGR4)) << 32;1725 1726 for (i = 0; i < ddata->dma_channels; i++) {1727 if (chan_reserved & BIT(i))1728 continue;1729 1730 chan = &ddata->chans[i];1731 chan->id = i;1732 chan->fifo_size = get_chan_hwcfg(i, G_FIFO_SIZE(i), hwcfgr);1733 /* If chan->fifo_size > 0 then half of the fifo size, else no burst when no FIFO */1734 chan->max_burst = (chan->fifo_size) ? (1 << (chan->fifo_size + 1)) / 2 : 0;1735 }1736 1737 ret = dmaenginem_async_device_register(dma_dev);1738 if (ret)1739 goto err_clk_disable;1740 1741 for (i = 0; i < ddata->dma_channels; i++) {1742 char name[12];1743 1744 if (chan_reserved & BIT(i))1745 continue;1746 1747 chan = &ddata->chans[i];1748 snprintf(name, sizeof(name), "dma%dchan%d", ddata->dma_dev.dev_id, chan->id);1749 1750 chan->vchan.desc_free = stm32_dma3_chan_vdesc_free;1751 vchan_init(&chan->vchan, dma_dev);1752 1753 ret = dma_async_device_channel_register(&ddata->dma_dev, &chan->vchan.chan, name);1754 if (ret) {1755 dev_err_probe(&pdev->dev, ret, "Failed to register channel %s\n", name);1756 goto err_clk_disable;1757 }1758 1759 ret = platform_get_irq(pdev, i);1760 if (ret < 0)1761 goto err_clk_disable;1762 chan->irq = ret;1763 1764 ret = devm_request_irq(&pdev->dev, chan->irq, stm32_dma3_chan_irq, 0,1765 dev_name(chan2dev(chan)), chan);1766 if (ret) {1767 dev_err_probe(&pdev->dev, ret, "Failed to request channel %s IRQ\n",1768 dev_name(chan2dev(chan)));1769 goto err_clk_disable;1770 }1771 }1772 1773 ret = of_dma_controller_register(np, stm32_dma3_of_xlate, ddata);1774 if (ret) {1775 dev_err_probe(&pdev->dev, ret, "Failed to register controller\n");1776 goto err_clk_disable;1777 }1778 1779 verr = readl_relaxed(ddata->base + STM32_DMA3_VERR);1780 1781 pm_runtime_set_active(&pdev->dev);1782 pm_runtime_enable(&pdev->dev);1783 pm_runtime_get_noresume(&pdev->dev);1784 pm_runtime_put(&pdev->dev);1785 1786 dev_info(&pdev->dev, "STM32 DMA3 registered rev:%lu.%lu\n",1787 FIELD_GET(VERR_MAJREV, verr), FIELD_GET(VERR_MINREV, verr));1788 1789 return 0;1790 1791err_clk_disable:1792 clk_disable_unprepare(ddata->clk);1793 1794 return ret;1795}1796 1797static void stm32_dma3_remove(struct platform_device *pdev)1798{1799 pm_runtime_disable(&pdev->dev);1800}1801 1802static int stm32_dma3_runtime_suspend(struct device *dev)1803{1804 struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev);1805 1806 clk_disable_unprepare(ddata->clk);1807 1808 return 0;1809}1810 1811static int stm32_dma3_runtime_resume(struct device *dev)1812{1813 struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev);1814 int ret;1815 1816 ret = clk_prepare_enable(ddata->clk);1817 if (ret)1818 dev_err(dev, "Failed to enable clk: %d\n", ret);1819 1820 return ret;1821}1822 1823static const struct dev_pm_ops stm32_dma3_pm_ops = {1824 SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)1825 RUNTIME_PM_OPS(stm32_dma3_runtime_suspend, stm32_dma3_runtime_resume, NULL)1826};1827 1828static struct platform_driver stm32_dma3_driver = {1829 .probe = stm32_dma3_probe,1830 .remove_new = stm32_dma3_remove,1831 .driver = {1832 .name = "stm32-dma3",1833 .of_match_table = stm32_dma3_of_match,1834 .pm = pm_ptr(&stm32_dma3_pm_ops),1835 },1836};1837 1838static int __init stm32_dma3_init(void)1839{1840 return platform_driver_register(&stm32_dma3_driver);1841}1842 1843subsys_initcall(stm32_dma3_init);1844 1845MODULE_DESCRIPTION("STM32 DMA3 controller driver");1846MODULE_AUTHOR("Amelie Delaunay <amelie.delaunay@foss.st.com>");1847MODULE_LICENSE("GPL");1848