1830 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 *4 * Copyright (C) STMicroelectronics SA 20175 * Author(s): M'boumba Cedric Madianga <cedric.madianga@gmail.com>6 * Pierre-Yves Mordret <pierre-yves.mordret@st.com>7 *8 * Driver for STM32 MDMA controller9 *10 * Inspired by stm32-dma.c and dma-jz4780.c11 */12 13#include <linux/bitfield.h>14#include <linux/clk.h>15#include <linux/delay.h>16#include <linux/dmaengine.h>17#include <linux/dma-mapping.h>18#include <linux/dmapool.h>19#include <linux/err.h>20#include <linux/init.h>21#include <linux/iopoll.h>22#include <linux/jiffies.h>23#include <linux/list.h>24#include <linux/log2.h>25#include <linux/module.h>26#include <linux/of.h>27#include <linux/of_dma.h>28#include <linux/platform_device.h>29#include <linux/pm_runtime.h>30#include <linux/reset.h>31#include <linux/slab.h>32 33#include "../virt-dma.h"34 35#define STM32_MDMA_GISR0 0x0000 /* MDMA Int Status Reg 1 */36 37/* MDMA Channel x interrupt/status register */38#define STM32_MDMA_CISR(x) (0x40 + 0x40 * (x)) /* x = 0..62 */39#define STM32_MDMA_CISR_CRQA BIT(16)40#define STM32_MDMA_CISR_TCIF BIT(4)41#define STM32_MDMA_CISR_BTIF BIT(3)42#define STM32_MDMA_CISR_BRTIF BIT(2)43#define STM32_MDMA_CISR_CTCIF BIT(1)44#define STM32_MDMA_CISR_TEIF BIT(0)45 46/* MDMA Channel x interrupt flag clear register */47#define STM32_MDMA_CIFCR(x) (0x44 + 0x40 * (x))48#define STM32_MDMA_CIFCR_CLTCIF BIT(4)49#define STM32_MDMA_CIFCR_CBTIF BIT(3)50#define STM32_MDMA_CIFCR_CBRTIF BIT(2)51#define STM32_MDMA_CIFCR_CCTCIF BIT(1)52#define STM32_MDMA_CIFCR_CTEIF BIT(0)53#define STM32_MDMA_CIFCR_CLEAR_ALL (STM32_MDMA_CIFCR_CLTCIF \54 | STM32_MDMA_CIFCR_CBTIF \55 | STM32_MDMA_CIFCR_CBRTIF \56 | STM32_MDMA_CIFCR_CCTCIF \57 | STM32_MDMA_CIFCR_CTEIF)58 59/* MDMA Channel x error status register */60#define STM32_MDMA_CESR(x) (0x48 + 0x40 * (x))61#define STM32_MDMA_CESR_BSE BIT(11)62#define STM32_MDMA_CESR_ASR BIT(10)63#define STM32_MDMA_CESR_TEMD BIT(9)64#define STM32_MDMA_CESR_TELD BIT(8)65#define STM32_MDMA_CESR_TED BIT(7)66#define STM32_MDMA_CESR_TEA_MASK GENMASK(6, 0)67 68/* MDMA Channel x control register */69#define STM32_MDMA_CCR(x) (0x4C + 0x40 * (x))70#define STM32_MDMA_CCR_SWRQ BIT(16)71#define STM32_MDMA_CCR_WEX BIT(14)72#define STM32_MDMA_CCR_HEX BIT(13)73#define STM32_MDMA_CCR_BEX BIT(12)74#define STM32_MDMA_CCR_SM BIT(8)75#define STM32_MDMA_CCR_PL_MASK GENMASK(7, 6)76#define STM32_MDMA_CCR_PL(n) FIELD_PREP(STM32_MDMA_CCR_PL_MASK, (n))77#define STM32_MDMA_CCR_TCIE BIT(5)78#define STM32_MDMA_CCR_BTIE BIT(4)79#define STM32_MDMA_CCR_BRTIE BIT(3)80#define STM32_MDMA_CCR_CTCIE BIT(2)81#define STM32_MDMA_CCR_TEIE BIT(1)82#define STM32_MDMA_CCR_EN BIT(0)83#define STM32_MDMA_CCR_IRQ_MASK (STM32_MDMA_CCR_TCIE \84 | STM32_MDMA_CCR_BTIE \85 | STM32_MDMA_CCR_BRTIE \86 | STM32_MDMA_CCR_CTCIE \87 | STM32_MDMA_CCR_TEIE)88 89/* MDMA Channel x transfer configuration register */90#define STM32_MDMA_CTCR(x) (0x50 + 0x40 * (x))91#define STM32_MDMA_CTCR_BWM BIT(31)92#define STM32_MDMA_CTCR_SWRM BIT(30)93#define STM32_MDMA_CTCR_TRGM_MSK GENMASK(29, 28)94#define STM32_MDMA_CTCR_TRGM(n) FIELD_PREP(STM32_MDMA_CTCR_TRGM_MSK, (n))95#define STM32_MDMA_CTCR_TRGM_GET(n) FIELD_GET(STM32_MDMA_CTCR_TRGM_MSK, (n))96#define STM32_MDMA_CTCR_PAM_MASK GENMASK(27, 26)97#define STM32_MDMA_CTCR_PAM(n) FIELD_PREP(STM32_MDMA_CTCR_PAM_MASK, (n))98#define STM32_MDMA_CTCR_PKE BIT(25)99#define STM32_MDMA_CTCR_TLEN_MSK GENMASK(24, 18)100#define STM32_MDMA_CTCR_TLEN(n) FIELD_PREP(STM32_MDMA_CTCR_TLEN_MSK, (n))101#define STM32_MDMA_CTCR_TLEN_GET(n) FIELD_GET(STM32_MDMA_CTCR_TLEN_MSK, (n))102#define STM32_MDMA_CTCR_LEN2_MSK GENMASK(25, 18)103#define STM32_MDMA_CTCR_LEN2(n) FIELD_PREP(STM32_MDMA_CTCR_LEN2_MSK, (n))104#define STM32_MDMA_CTCR_LEN2_GET(n) FIELD_GET(STM32_MDMA_CTCR_LEN2_MSK, (n))105#define STM32_MDMA_CTCR_DBURST_MASK GENMASK(17, 15)106#define STM32_MDMA_CTCR_DBURST(n) FIELD_PREP(STM32_MDMA_CTCR_DBURST_MASK, (n))107#define STM32_MDMA_CTCR_SBURST_MASK GENMASK(14, 12)108#define STM32_MDMA_CTCR_SBURST(n) FIELD_PREP(STM32_MDMA_CTCR_SBURST_MASK, (n))109#define STM32_MDMA_CTCR_DINCOS_MASK GENMASK(11, 10)110#define STM32_MDMA_CTCR_DINCOS(n) FIELD_PREP(STM32_MDMA_CTCR_DINCOS_MASK, (n))111#define STM32_MDMA_CTCR_SINCOS_MASK GENMASK(9, 8)112#define STM32_MDMA_CTCR_SINCOS(n) FIELD_PREP(STM32_MDMA_CTCR_SINCOS_MASK, (n))113#define STM32_MDMA_CTCR_DSIZE_MASK GENMASK(7, 6)114#define STM32_MDMA_CTCR_DSIZE(n) FIELD_PREP(STM32_MDMA_CTCR_DSIZE_MASK, (n))115#define STM32_MDMA_CTCR_SSIZE_MASK GENMASK(5, 4)116#define STM32_MDMA_CTCR_SSIZE(n) FIELD_PREP(STM32_MDMA_CTCR_SSIZE_MASK, (n))117#define STM32_MDMA_CTCR_DINC_MASK GENMASK(3, 2)118#define STM32_MDMA_CTCR_DINC(n) FIELD_PREP(STM32_MDMA_CTCR_DINC_MASK, (n))119#define STM32_MDMA_CTCR_SINC_MASK GENMASK(1, 0)120#define STM32_MDMA_CTCR_SINC(n) FIELD_PREP(STM32_MDMA_CTCR_SINC_MASK, (n))121#define STM32_MDMA_CTCR_CFG_MASK (STM32_MDMA_CTCR_SINC_MASK \122 | STM32_MDMA_CTCR_DINC_MASK \123 | STM32_MDMA_CTCR_SINCOS_MASK \124 | STM32_MDMA_CTCR_DINCOS_MASK \125 | STM32_MDMA_CTCR_LEN2_MSK \126 | STM32_MDMA_CTCR_TRGM_MSK)127 128/* MDMA Channel x block number of data register */129#define STM32_MDMA_CBNDTR(x) (0x54 + 0x40 * (x))130#define STM32_MDMA_CBNDTR_BRC_MK GENMASK(31, 20)131#define STM32_MDMA_CBNDTR_BRC(n) FIELD_PREP(STM32_MDMA_CBNDTR_BRC_MK, (n))132#define STM32_MDMA_CBNDTR_BRC_GET(n) FIELD_GET(STM32_MDMA_CBNDTR_BRC_MK, (n))133 134#define STM32_MDMA_CBNDTR_BRDUM BIT(19)135#define STM32_MDMA_CBNDTR_BRSUM BIT(18)136#define STM32_MDMA_CBNDTR_BNDT_MASK GENMASK(16, 0)137#define STM32_MDMA_CBNDTR_BNDT(n) FIELD_PREP(STM32_MDMA_CBNDTR_BNDT_MASK, (n))138 139/* MDMA Channel x source address register */140#define STM32_MDMA_CSAR(x) (0x58 + 0x40 * (x))141 142/* MDMA Channel x destination address register */143#define STM32_MDMA_CDAR(x) (0x5C + 0x40 * (x))144 145/* MDMA Channel x block repeat address update register */146#define STM32_MDMA_CBRUR(x) (0x60 + 0x40 * (x))147#define STM32_MDMA_CBRUR_DUV_MASK GENMASK(31, 16)148#define STM32_MDMA_CBRUR_DUV(n) FIELD_PREP(STM32_MDMA_CBRUR_DUV_MASK, (n))149#define STM32_MDMA_CBRUR_SUV_MASK GENMASK(15, 0)150#define STM32_MDMA_CBRUR_SUV(n) FIELD_PREP(STM32_MDMA_CBRUR_SUV_MASK, (n))151 152/* MDMA Channel x link address register */153#define STM32_MDMA_CLAR(x) (0x64 + 0x40 * (x))154 155/* MDMA Channel x trigger and bus selection register */156#define STM32_MDMA_CTBR(x) (0x68 + 0x40 * (x))157#define STM32_MDMA_CTBR_DBUS BIT(17)158#define STM32_MDMA_CTBR_SBUS BIT(16)159#define STM32_MDMA_CTBR_TSEL_MASK GENMASK(5, 0)160#define STM32_MDMA_CTBR_TSEL(n) FIELD_PREP(STM32_MDMA_CTBR_TSEL_MASK, (n))161 162/* MDMA Channel x mask address register */163#define STM32_MDMA_CMAR(x) (0x70 + 0x40 * (x))164 165/* MDMA Channel x mask data register */166#define STM32_MDMA_CMDR(x) (0x74 + 0x40 * (x))167 168#define STM32_MDMA_MAX_BUF_LEN 128169#define STM32_MDMA_MAX_BLOCK_LEN 65536170#define STM32_MDMA_MAX_CHANNELS 32171#define STM32_MDMA_MAX_REQUESTS 256172#define STM32_MDMA_MAX_BURST 128173#define STM32_MDMA_VERY_HIGH_PRIORITY 0x3174 175enum stm32_mdma_trigger_mode {176 STM32_MDMA_BUFFER,177 STM32_MDMA_BLOCK,178 STM32_MDMA_BLOCK_REP,179 STM32_MDMA_LINKED_LIST,180};181 182enum stm32_mdma_width {183 STM32_MDMA_BYTE,184 STM32_MDMA_HALF_WORD,185 STM32_MDMA_WORD,186 STM32_MDMA_DOUBLE_WORD,187};188 189enum stm32_mdma_inc_mode {190 STM32_MDMA_FIXED = 0,191 STM32_MDMA_INC = 2,192 STM32_MDMA_DEC = 3,193};194 195struct stm32_mdma_chan_config {196 u32 request;197 u32 priority_level;198 u32 transfer_config;199 u32 mask_addr;200 u32 mask_data;201 bool m2m_hw; /* True when MDMA is triggered by STM32 DMA */202};203 204struct stm32_mdma_hwdesc {205 u32 ctcr;206 u32 cbndtr;207 u32 csar;208 u32 cdar;209 u32 cbrur;210 u32 clar;211 u32 ctbr;212 u32 dummy;213 u32 cmar;214 u32 cmdr;215} __aligned(64);216 217struct stm32_mdma_desc_node {218 struct stm32_mdma_hwdesc *hwdesc;219 dma_addr_t hwdesc_phys;220};221 222struct stm32_mdma_desc {223 struct virt_dma_desc vdesc;224 u32 ccr;225 bool cyclic;226 u32 count;227 struct stm32_mdma_desc_node node[] __counted_by(count);228};229 230struct stm32_mdma_dma_config {231 u32 request; /* STM32 DMA channel stream id, triggering MDMA */232 u32 cmar; /* STM32 DMA interrupt flag clear register address */233 u32 cmdr; /* STM32 DMA Transfer Complete flag */234};235 236struct stm32_mdma_chan {237 struct virt_dma_chan vchan;238 struct dma_pool *desc_pool;239 u32 id;240 struct stm32_mdma_desc *desc;241 u32 curr_hwdesc;242 struct dma_slave_config dma_config;243 struct stm32_mdma_chan_config chan_config;244 bool busy;245 u32 mem_burst;246 u32 mem_width;247};248 249struct stm32_mdma_device {250 struct dma_device ddev;251 void __iomem *base;252 struct clk *clk;253 int irq;254 u32 nr_channels;255 u32 nr_requests;256 u32 nr_ahb_addr_masks;257 u32 chan_reserved;258 struct stm32_mdma_chan chan[STM32_MDMA_MAX_CHANNELS];259 u32 ahb_addr_masks[] __counted_by(nr_ahb_addr_masks);260};261 262static struct stm32_mdma_device *stm32_mdma_get_dev(263 struct stm32_mdma_chan *chan)264{265 return container_of(chan->vchan.chan.device, struct stm32_mdma_device,266 ddev);267}268 269static struct stm32_mdma_chan *to_stm32_mdma_chan(struct dma_chan *c)270{271 return container_of(c, struct stm32_mdma_chan, vchan.chan);272}273 274static struct stm32_mdma_desc *to_stm32_mdma_desc(struct virt_dma_desc *vdesc)275{276 return container_of(vdesc, struct stm32_mdma_desc, vdesc);277}278 279static struct device *chan2dev(struct stm32_mdma_chan *chan)280{281 return &chan->vchan.chan.dev->device;282}283 284static struct device *mdma2dev(struct stm32_mdma_device *mdma_dev)285{286 return mdma_dev->ddev.dev;287}288 289static u32 stm32_mdma_read(struct stm32_mdma_device *dmadev, u32 reg)290{291 return readl_relaxed(dmadev->base + reg);292}293 294static void stm32_mdma_write(struct stm32_mdma_device *dmadev, u32 reg, u32 val)295{296 writel_relaxed(val, dmadev->base + reg);297}298 299static void stm32_mdma_set_bits(struct stm32_mdma_device *dmadev, u32 reg,300 u32 mask)301{302 void __iomem *addr = dmadev->base + reg;303 304 writel_relaxed(readl_relaxed(addr) | mask, addr);305}306 307static void stm32_mdma_clr_bits(struct stm32_mdma_device *dmadev, u32 reg,308 u32 mask)309{310 void __iomem *addr = dmadev->base + reg;311 312 writel_relaxed(readl_relaxed(addr) & ~mask, addr);313}314 315static struct stm32_mdma_desc *stm32_mdma_alloc_desc(316 struct stm32_mdma_chan *chan, u32 count)317{318 struct stm32_mdma_desc *desc;319 int i;320 321 desc = kzalloc(struct_size(desc, node, count), GFP_NOWAIT);322 if (!desc)323 return NULL;324 desc->count = count;325 326 for (i = 0; i < count; i++) {327 desc->node[i].hwdesc =328 dma_pool_alloc(chan->desc_pool, GFP_NOWAIT,329 &desc->node[i].hwdesc_phys);330 if (!desc->node[i].hwdesc)331 goto err;332 }333 334 return desc;335 336err:337 dev_err(chan2dev(chan), "Failed to allocate descriptor\n");338 while (--i >= 0)339 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,340 desc->node[i].hwdesc_phys);341 kfree(desc);342 return NULL;343}344 345static void stm32_mdma_desc_free(struct virt_dma_desc *vdesc)346{347 struct stm32_mdma_desc *desc = to_stm32_mdma_desc(vdesc);348 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(vdesc->tx.chan);349 int i;350 351 for (i = 0; i < desc->count; i++)352 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,353 desc->node[i].hwdesc_phys);354 kfree(desc);355}356 357static int stm32_mdma_get_width(struct stm32_mdma_chan *chan,358 enum dma_slave_buswidth width)359{360 switch (width) {361 case DMA_SLAVE_BUSWIDTH_1_BYTE:362 case DMA_SLAVE_BUSWIDTH_2_BYTES:363 case DMA_SLAVE_BUSWIDTH_4_BYTES:364 case DMA_SLAVE_BUSWIDTH_8_BYTES:365 return ffs(width) - 1;366 default:367 dev_err(chan2dev(chan), "Dma bus width %i not supported\n",368 width);369 return -EINVAL;370 }371}372 373static enum dma_slave_buswidth stm32_mdma_get_max_width(dma_addr_t addr,374 u32 buf_len, u32 tlen)375{376 enum dma_slave_buswidth max_width = DMA_SLAVE_BUSWIDTH_8_BYTES;377 378 for (max_width = DMA_SLAVE_BUSWIDTH_8_BYTES;379 max_width > DMA_SLAVE_BUSWIDTH_1_BYTE;380 max_width >>= 1) {381 /*382 * Address and buffer length both have to be aligned on383 * bus width384 */385 if ((((buf_len | addr) & (max_width - 1)) == 0) &&386 tlen >= max_width)387 break;388 }389 390 return max_width;391}392 393static u32 stm32_mdma_get_best_burst(u32 buf_len, u32 tlen, u32 max_burst,394 enum dma_slave_buswidth width)395{396 u32 best_burst;397 398 best_burst = min((u32)1 << __ffs(tlen | buf_len),399 max_burst * width) / width;400 401 return (best_burst > 0) ? best_burst : 1;402}403 404static int stm32_mdma_disable_chan(struct stm32_mdma_chan *chan)405{406 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);407 u32 ccr, cisr, id, reg;408 int ret;409 410 id = chan->id;411 reg = STM32_MDMA_CCR(id);412 413 /* Disable interrupts */414 stm32_mdma_clr_bits(dmadev, reg, STM32_MDMA_CCR_IRQ_MASK);415 416 ccr = stm32_mdma_read(dmadev, reg);417 if (ccr & STM32_MDMA_CCR_EN) {418 stm32_mdma_clr_bits(dmadev, reg, STM32_MDMA_CCR_EN);419 420 /* Ensure that any ongoing transfer has been completed */421 ret = readl_relaxed_poll_timeout_atomic(422 dmadev->base + STM32_MDMA_CISR(id), cisr,423 (cisr & STM32_MDMA_CISR_CTCIF), 10, 1000);424 if (ret) {425 dev_err(chan2dev(chan), "%s: timeout!\n", __func__);426 return -EBUSY;427 }428 }429 430 return 0;431}432 433static void stm32_mdma_stop(struct stm32_mdma_chan *chan)434{435 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);436 u32 status;437 int ret;438 439 /* Disable DMA */440 ret = stm32_mdma_disable_chan(chan);441 if (ret < 0)442 return;443 444 /* Clear interrupt status if it is there */445 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));446 if (status) {447 dev_dbg(chan2dev(chan), "%s(): clearing interrupt: 0x%08x\n",448 __func__, status);449 stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(chan->id), status);450 }451 452 chan->busy = false;453}454 455static void stm32_mdma_set_bus(struct stm32_mdma_device *dmadev, u32 *ctbr,456 u32 ctbr_mask, u32 src_addr)457{458 u32 mask;459 int i;460 461 /* Check if memory device is on AHB or AXI */462 *ctbr &= ~ctbr_mask;463 mask = src_addr & 0xF0000000;464 for (i = 0; i < dmadev->nr_ahb_addr_masks; i++) {465 if (mask == dmadev->ahb_addr_masks[i]) {466 *ctbr |= ctbr_mask;467 break;468 }469 }470}471 472static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan,473 enum dma_transfer_direction direction,474 u32 *mdma_ccr, u32 *mdma_ctcr,475 u32 *mdma_ctbr, dma_addr_t addr,476 u32 buf_len)477{478 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);479 struct stm32_mdma_chan_config *chan_config = &chan->chan_config;480 enum dma_slave_buswidth src_addr_width, dst_addr_width;481 phys_addr_t src_addr, dst_addr;482 int src_bus_width, dst_bus_width;483 u32 src_maxburst, dst_maxburst, src_best_burst, dst_best_burst;484 u32 ccr, ctcr, ctbr, tlen;485 486 src_addr_width = chan->dma_config.src_addr_width;487 dst_addr_width = chan->dma_config.dst_addr_width;488 src_maxburst = chan->dma_config.src_maxburst;489 dst_maxburst = chan->dma_config.dst_maxburst;490 491 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)) & ~STM32_MDMA_CCR_EN;492 ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id));493 ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id));494 495 /* Enable HW request mode */496 ctcr &= ~STM32_MDMA_CTCR_SWRM;497 498 /* Set DINC, SINC, DINCOS, SINCOS, TRGM and TLEN retrieve from DT */499 ctcr &= ~STM32_MDMA_CTCR_CFG_MASK;500 ctcr |= chan_config->transfer_config & STM32_MDMA_CTCR_CFG_MASK;501 502 /*503 * For buffer transfer length (TLEN) we have to set504 * the number of bytes - 1 in CTCR register505 */506 tlen = STM32_MDMA_CTCR_LEN2_GET(ctcr);507 ctcr &= ~STM32_MDMA_CTCR_LEN2_MSK;508 ctcr |= STM32_MDMA_CTCR_TLEN((tlen - 1));509 510 /* Disable Pack Enable */511 ctcr &= ~STM32_MDMA_CTCR_PKE;512 513 /* Check burst size constraints */514 if (src_maxburst * src_addr_width > STM32_MDMA_MAX_BURST ||515 dst_maxburst * dst_addr_width > STM32_MDMA_MAX_BURST) {516 dev_err(chan2dev(chan),517 "burst size * bus width higher than %d bytes\n",518 STM32_MDMA_MAX_BURST);519 return -EINVAL;520 }521 522 if ((!is_power_of_2(src_maxburst) && src_maxburst > 0) ||523 (!is_power_of_2(dst_maxburst) && dst_maxburst > 0)) {524 dev_err(chan2dev(chan), "burst size must be a power of 2\n");525 return -EINVAL;526 }527 528 /*529 * Configure channel control:530 * - Clear SW request as in this case this is a HW one531 * - Clear WEX, HEX and BEX bits532 * - Set priority level533 */534 ccr &= ~(STM32_MDMA_CCR_SWRQ | STM32_MDMA_CCR_WEX | STM32_MDMA_CCR_HEX |535 STM32_MDMA_CCR_BEX | STM32_MDMA_CCR_PL_MASK);536 ccr |= STM32_MDMA_CCR_PL(chan_config->priority_level);537 538 /* Configure Trigger selection */539 ctbr &= ~STM32_MDMA_CTBR_TSEL_MASK;540 ctbr |= STM32_MDMA_CTBR_TSEL(chan_config->request);541 542 switch (direction) {543 case DMA_MEM_TO_DEV:544 dst_addr = chan->dma_config.dst_addr;545 546 /* Set device data size */547 if (chan_config->m2m_hw)548 dst_addr_width = stm32_mdma_get_max_width(dst_addr, buf_len,549 STM32_MDMA_MAX_BUF_LEN);550 dst_bus_width = stm32_mdma_get_width(chan, dst_addr_width);551 if (dst_bus_width < 0)552 return dst_bus_width;553 ctcr &= ~STM32_MDMA_CTCR_DSIZE_MASK;554 ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width);555 if (chan_config->m2m_hw) {556 ctcr &= ~STM32_MDMA_CTCR_DINCOS_MASK;557 ctcr |= STM32_MDMA_CTCR_DINCOS(dst_bus_width);558 }559 560 /* Set device burst value */561 if (chan_config->m2m_hw)562 dst_maxburst = STM32_MDMA_MAX_BUF_LEN / dst_addr_width;563 564 dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,565 dst_maxburst,566 dst_addr_width);567 chan->mem_burst = dst_best_burst;568 ctcr &= ~STM32_MDMA_CTCR_DBURST_MASK;569 ctcr |= STM32_MDMA_CTCR_DBURST((ilog2(dst_best_burst)));570 571 /* Set memory data size */572 src_addr_width = stm32_mdma_get_max_width(addr, buf_len, tlen);573 chan->mem_width = src_addr_width;574 src_bus_width = stm32_mdma_get_width(chan, src_addr_width);575 if (src_bus_width < 0)576 return src_bus_width;577 ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK |578 STM32_MDMA_CTCR_SINCOS_MASK;579 ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width) |580 STM32_MDMA_CTCR_SINCOS(src_bus_width);581 582 /* Set memory burst value */583 src_maxburst = STM32_MDMA_MAX_BUF_LEN / src_addr_width;584 src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,585 src_maxburst,586 src_addr_width);587 chan->mem_burst = src_best_burst;588 ctcr &= ~STM32_MDMA_CTCR_SBURST_MASK;589 ctcr |= STM32_MDMA_CTCR_SBURST((ilog2(src_best_burst)));590 591 /* Select bus */592 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS,593 dst_addr);594 595 if (dst_bus_width != src_bus_width)596 ctcr |= STM32_MDMA_CTCR_PKE;597 598 /* Set destination address */599 stm32_mdma_write(dmadev, STM32_MDMA_CDAR(chan->id), dst_addr);600 break;601 602 case DMA_DEV_TO_MEM:603 src_addr = chan->dma_config.src_addr;604 605 /* Set device data size */606 if (chan_config->m2m_hw)607 src_addr_width = stm32_mdma_get_max_width(src_addr, buf_len,608 STM32_MDMA_MAX_BUF_LEN);609 610 src_bus_width = stm32_mdma_get_width(chan, src_addr_width);611 if (src_bus_width < 0)612 return src_bus_width;613 ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK;614 ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width);615 if (chan_config->m2m_hw) {616 ctcr &= ~STM32_MDMA_CTCR_SINCOS_MASK;617 ctcr |= STM32_MDMA_CTCR_SINCOS(src_bus_width);618 }619 620 /* Set device burst value */621 if (chan_config->m2m_hw)622 src_maxburst = STM32_MDMA_MAX_BUF_LEN / src_addr_width;623 624 src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,625 src_maxburst,626 src_addr_width);627 ctcr &= ~STM32_MDMA_CTCR_SBURST_MASK;628 ctcr |= STM32_MDMA_CTCR_SBURST((ilog2(src_best_burst)));629 630 /* Set memory data size */631 dst_addr_width = stm32_mdma_get_max_width(addr, buf_len, tlen);632 chan->mem_width = dst_addr_width;633 dst_bus_width = stm32_mdma_get_width(chan, dst_addr_width);634 if (dst_bus_width < 0)635 return dst_bus_width;636 ctcr &= ~(STM32_MDMA_CTCR_DSIZE_MASK |637 STM32_MDMA_CTCR_DINCOS_MASK);638 ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width) |639 STM32_MDMA_CTCR_DINCOS(dst_bus_width);640 641 /* Set memory burst value */642 dst_maxburst = STM32_MDMA_MAX_BUF_LEN / dst_addr_width;643 dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,644 dst_maxburst,645 dst_addr_width);646 ctcr &= ~STM32_MDMA_CTCR_DBURST_MASK;647 ctcr |= STM32_MDMA_CTCR_DBURST((ilog2(dst_best_burst)));648 649 /* Select bus */650 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS,651 src_addr);652 653 if (dst_bus_width != src_bus_width)654 ctcr |= STM32_MDMA_CTCR_PKE;655 656 /* Set source address */657 stm32_mdma_write(dmadev, STM32_MDMA_CSAR(chan->id), src_addr);658 break;659 660 default:661 dev_err(chan2dev(chan), "Dma direction is not supported\n");662 return -EINVAL;663 }664 665 *mdma_ccr = ccr;666 *mdma_ctcr = ctcr;667 *mdma_ctbr = ctbr;668 669 return 0;670}671 672static void stm32_mdma_dump_hwdesc(struct stm32_mdma_chan *chan,673 struct stm32_mdma_desc_node *node)674{675 dev_dbg(chan2dev(chan), "hwdesc: %pad\n", &node->hwdesc_phys);676 dev_dbg(chan2dev(chan), "CTCR: 0x%08x\n", node->hwdesc->ctcr);677 dev_dbg(chan2dev(chan), "CBNDTR: 0x%08x\n", node->hwdesc->cbndtr);678 dev_dbg(chan2dev(chan), "CSAR: 0x%08x\n", node->hwdesc->csar);679 dev_dbg(chan2dev(chan), "CDAR: 0x%08x\n", node->hwdesc->cdar);680 dev_dbg(chan2dev(chan), "CBRUR: 0x%08x\n", node->hwdesc->cbrur);681 dev_dbg(chan2dev(chan), "CLAR: 0x%08x\n", node->hwdesc->clar);682 dev_dbg(chan2dev(chan), "CTBR: 0x%08x\n", node->hwdesc->ctbr);683 dev_dbg(chan2dev(chan), "CMAR: 0x%08x\n", node->hwdesc->cmar);684 dev_dbg(chan2dev(chan), "CMDR: 0x%08x\n\n", node->hwdesc->cmdr);685}686 687static void stm32_mdma_setup_hwdesc(struct stm32_mdma_chan *chan,688 struct stm32_mdma_desc *desc,689 enum dma_transfer_direction dir, u32 count,690 dma_addr_t src_addr, dma_addr_t dst_addr,691 u32 len, u32 ctcr, u32 ctbr, bool is_last,692 bool is_first, bool is_cyclic)693{694 struct stm32_mdma_chan_config *config = &chan->chan_config;695 struct stm32_mdma_hwdesc *hwdesc;696 u32 next = count + 1;697 698 hwdesc = desc->node[count].hwdesc;699 hwdesc->ctcr = ctcr;700 hwdesc->cbndtr &= ~(STM32_MDMA_CBNDTR_BRC_MK |701 STM32_MDMA_CBNDTR_BRDUM |702 STM32_MDMA_CBNDTR_BRSUM |703 STM32_MDMA_CBNDTR_BNDT_MASK);704 hwdesc->cbndtr |= STM32_MDMA_CBNDTR_BNDT(len);705 hwdesc->csar = src_addr;706 hwdesc->cdar = dst_addr;707 hwdesc->cbrur = 0;708 hwdesc->ctbr = ctbr;709 hwdesc->cmar = config->mask_addr;710 hwdesc->cmdr = config->mask_data;711 712 if (is_last) {713 if (is_cyclic)714 hwdesc->clar = desc->node[0].hwdesc_phys;715 else716 hwdesc->clar = 0;717 } else {718 hwdesc->clar = desc->node[next].hwdesc_phys;719 }720 721 stm32_mdma_dump_hwdesc(chan, &desc->node[count]);722}723 724static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan,725 struct stm32_mdma_desc *desc,726 struct scatterlist *sgl, u32 sg_len,727 enum dma_transfer_direction direction)728{729 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);730 struct dma_slave_config *dma_config = &chan->dma_config;731 struct stm32_mdma_chan_config *chan_config = &chan->chan_config;732 struct scatterlist *sg;733 dma_addr_t src_addr, dst_addr;734 u32 m2m_hw_period, ccr, ctcr, ctbr;735 int i, ret = 0;736 737 if (chan_config->m2m_hw)738 m2m_hw_period = sg_dma_len(sgl);739 740 for_each_sg(sgl, sg, sg_len, i) {741 if (sg_dma_len(sg) > STM32_MDMA_MAX_BLOCK_LEN) {742 dev_err(chan2dev(chan), "Invalid block len\n");743 return -EINVAL;744 }745 746 if (direction == DMA_MEM_TO_DEV) {747 src_addr = sg_dma_address(sg);748 dst_addr = dma_config->dst_addr;749 if (chan_config->m2m_hw && (i & 1))750 dst_addr += m2m_hw_period;751 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr,752 &ctcr, &ctbr, src_addr,753 sg_dma_len(sg));754 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS,755 src_addr);756 } else {757 src_addr = dma_config->src_addr;758 if (chan_config->m2m_hw && (i & 1))759 src_addr += m2m_hw_period;760 dst_addr = sg_dma_address(sg);761 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr,762 &ctcr, &ctbr, dst_addr,763 sg_dma_len(sg));764 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS,765 dst_addr);766 }767 768 if (ret < 0)769 return ret;770 771 stm32_mdma_setup_hwdesc(chan, desc, direction, i, src_addr,772 dst_addr, sg_dma_len(sg), ctcr, ctbr,773 i == sg_len - 1, i == 0, false);774 }775 776 /* Enable interrupts */777 ccr &= ~STM32_MDMA_CCR_IRQ_MASK;778 ccr |= STM32_MDMA_CCR_TEIE | STM32_MDMA_CCR_CTCIE;779 desc->ccr = ccr;780 781 return 0;782}783 784static struct dma_async_tx_descriptor *785stm32_mdma_prep_slave_sg(struct dma_chan *c, struct scatterlist *sgl,786 u32 sg_len, enum dma_transfer_direction direction,787 unsigned long flags, void *context)788{789 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);790 struct stm32_mdma_chan_config *chan_config = &chan->chan_config;791 struct stm32_mdma_desc *desc;792 int i, ret;793 794 /*795 * Once DMA is in setup cyclic mode the channel we cannot assign this796 * channel anymore. The DMA channel needs to be aborted or terminated797 * for allowing another request.798 */799 if (chan->desc && chan->desc->cyclic) {800 dev_err(chan2dev(chan),801 "Request not allowed when dma in cyclic mode\n");802 return NULL;803 }804 805 desc = stm32_mdma_alloc_desc(chan, sg_len);806 if (!desc)807 return NULL;808 809 ret = stm32_mdma_setup_xfer(chan, desc, sgl, sg_len, direction);810 if (ret < 0)811 goto xfer_setup_err;812 813 /*814 * In case of M2M HW transfer triggered by STM32 DMA, we do not have to clear the815 * transfer complete flag by hardware in order to let the CPU rearm the STM32 DMA816 * with the next sg element and update some data in dmaengine framework.817 */818 if (chan_config->m2m_hw && direction == DMA_MEM_TO_DEV) {819 struct stm32_mdma_hwdesc *hwdesc;820 821 for (i = 0; i < sg_len; i++) {822 hwdesc = desc->node[i].hwdesc;823 hwdesc->cmar = 0;824 hwdesc->cmdr = 0;825 }826 }827 828 desc->cyclic = false;829 830 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);831 832xfer_setup_err:833 for (i = 0; i < desc->count; i++)834 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,835 desc->node[i].hwdesc_phys);836 kfree(desc);837 return NULL;838}839 840static struct dma_async_tx_descriptor *841stm32_mdma_prep_dma_cyclic(struct dma_chan *c, dma_addr_t buf_addr,842 size_t buf_len, size_t period_len,843 enum dma_transfer_direction direction,844 unsigned long flags)845{846 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);847 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);848 struct dma_slave_config *dma_config = &chan->dma_config;849 struct stm32_mdma_chan_config *chan_config = &chan->chan_config;850 struct stm32_mdma_desc *desc;851 dma_addr_t src_addr, dst_addr;852 u32 ccr, ctcr, ctbr, count;853 int i, ret;854 855 /*856 * Once DMA is in setup cyclic mode the channel we cannot assign this857 * channel anymore. The DMA channel needs to be aborted or terminated858 * for allowing another request.859 */860 if (chan->desc && chan->desc->cyclic) {861 dev_err(chan2dev(chan),862 "Request not allowed when dma in cyclic mode\n");863 return NULL;864 }865 866 if (!buf_len || !period_len || period_len > STM32_MDMA_MAX_BLOCK_LEN) {867 dev_err(chan2dev(chan), "Invalid buffer/period len\n");868 return NULL;869 }870 871 if (buf_len % period_len) {872 dev_err(chan2dev(chan), "buf_len not multiple of period_len\n");873 return NULL;874 }875 876 count = buf_len / period_len;877 878 desc = stm32_mdma_alloc_desc(chan, count);879 if (!desc)880 return NULL;881 882 /* Select bus */883 if (direction == DMA_MEM_TO_DEV) {884 src_addr = buf_addr;885 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, &ctcr,886 &ctbr, src_addr, period_len);887 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS,888 src_addr);889 } else {890 dst_addr = buf_addr;891 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, &ctcr,892 &ctbr, dst_addr, period_len);893 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS,894 dst_addr);895 }896 897 if (ret < 0)898 goto xfer_setup_err;899 900 /* Enable interrupts */901 ccr &= ~STM32_MDMA_CCR_IRQ_MASK;902 ccr |= STM32_MDMA_CCR_TEIE | STM32_MDMA_CCR_CTCIE | STM32_MDMA_CCR_BTIE;903 desc->ccr = ccr;904 905 /* Configure hwdesc list */906 for (i = 0; i < count; i++) {907 if (direction == DMA_MEM_TO_DEV) {908 src_addr = buf_addr + i * period_len;909 dst_addr = dma_config->dst_addr;910 if (chan_config->m2m_hw && (i & 1))911 dst_addr += period_len;912 } else {913 src_addr = dma_config->src_addr;914 if (chan_config->m2m_hw && (i & 1))915 src_addr += period_len;916 dst_addr = buf_addr + i * period_len;917 }918 919 stm32_mdma_setup_hwdesc(chan, desc, direction, i, src_addr,920 dst_addr, period_len, ctcr, ctbr,921 i == count - 1, i == 0, true);922 }923 924 desc->cyclic = true;925 926 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);927 928xfer_setup_err:929 for (i = 0; i < desc->count; i++)930 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,931 desc->node[i].hwdesc_phys);932 kfree(desc);933 return NULL;934}935 936static struct dma_async_tx_descriptor *937stm32_mdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, dma_addr_t src,938 size_t len, unsigned long flags)939{940 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);941 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);942 enum dma_slave_buswidth max_width;943 struct stm32_mdma_desc *desc;944 struct stm32_mdma_hwdesc *hwdesc;945 u32 ccr, ctcr, ctbr, cbndtr, count, max_burst, mdma_burst;946 u32 best_burst, tlen;947 size_t xfer_count, offset;948 int src_bus_width, dst_bus_width;949 int i;950 951 /*952 * Once DMA is in setup cyclic mode the channel we cannot assign this953 * channel anymore. The DMA channel needs to be aborted or terminated954 * to allow another request955 */956 if (chan->desc && chan->desc->cyclic) {957 dev_err(chan2dev(chan),958 "Request not allowed when dma in cyclic mode\n");959 return NULL;960 }961 962 count = DIV_ROUND_UP(len, STM32_MDMA_MAX_BLOCK_LEN);963 desc = stm32_mdma_alloc_desc(chan, count);964 if (!desc)965 return NULL;966 967 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)) & ~STM32_MDMA_CCR_EN;968 ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id));969 ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id));970 cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id));971 972 /* Enable sw req, some interrupts and clear other bits */973 ccr &= ~(STM32_MDMA_CCR_WEX | STM32_MDMA_CCR_HEX |974 STM32_MDMA_CCR_BEX | STM32_MDMA_CCR_PL_MASK |975 STM32_MDMA_CCR_IRQ_MASK);976 ccr |= STM32_MDMA_CCR_TEIE;977 978 /* Enable SW request mode, dest/src inc and clear other bits */979 ctcr &= ~(STM32_MDMA_CTCR_BWM | STM32_MDMA_CTCR_TRGM_MSK |980 STM32_MDMA_CTCR_PAM_MASK | STM32_MDMA_CTCR_PKE |981 STM32_MDMA_CTCR_TLEN_MSK | STM32_MDMA_CTCR_DBURST_MASK |982 STM32_MDMA_CTCR_SBURST_MASK | STM32_MDMA_CTCR_DINCOS_MASK |983 STM32_MDMA_CTCR_SINCOS_MASK | STM32_MDMA_CTCR_DSIZE_MASK |984 STM32_MDMA_CTCR_SSIZE_MASK | STM32_MDMA_CTCR_DINC_MASK |985 STM32_MDMA_CTCR_SINC_MASK);986 ctcr |= STM32_MDMA_CTCR_SWRM | STM32_MDMA_CTCR_SINC(STM32_MDMA_INC) |987 STM32_MDMA_CTCR_DINC(STM32_MDMA_INC);988 989 /* Reset HW request */990 ctbr &= ~STM32_MDMA_CTBR_TSEL_MASK;991 992 /* Select bus */993 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS, src);994 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS, dest);995 996 /* Clear CBNDTR registers */997 cbndtr &= ~(STM32_MDMA_CBNDTR_BRC_MK | STM32_MDMA_CBNDTR_BRDUM |998 STM32_MDMA_CBNDTR_BRSUM | STM32_MDMA_CBNDTR_BNDT_MASK);999 1000 if (len <= STM32_MDMA_MAX_BLOCK_LEN) {1001 cbndtr |= STM32_MDMA_CBNDTR_BNDT(len);1002 if (len <= STM32_MDMA_MAX_BUF_LEN) {1003 /* Setup a buffer transfer */1004 ccr |= STM32_MDMA_CCR_TCIE | STM32_MDMA_CCR_CTCIE;1005 ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_BUFFER);1006 } else {1007 /* Setup a block transfer */1008 ccr |= STM32_MDMA_CCR_BTIE | STM32_MDMA_CCR_CTCIE;1009 ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_BLOCK);1010 }1011 1012 tlen = STM32_MDMA_MAX_BUF_LEN;1013 ctcr |= STM32_MDMA_CTCR_TLEN((tlen - 1));1014 1015 /* Set source best burst size */1016 max_width = stm32_mdma_get_max_width(src, len, tlen);1017 src_bus_width = stm32_mdma_get_width(chan, max_width);1018 1019 max_burst = tlen / max_width;1020 best_burst = stm32_mdma_get_best_burst(len, tlen, max_burst,1021 max_width);1022 mdma_burst = ilog2(best_burst);1023 1024 ctcr |= STM32_MDMA_CTCR_SBURST(mdma_burst) |1025 STM32_MDMA_CTCR_SSIZE(src_bus_width) |1026 STM32_MDMA_CTCR_SINCOS(src_bus_width);1027 1028 /* Set destination best burst size */1029 max_width = stm32_mdma_get_max_width(dest, len, tlen);1030 dst_bus_width = stm32_mdma_get_width(chan, max_width);1031 1032 max_burst = tlen / max_width;1033 best_burst = stm32_mdma_get_best_burst(len, tlen, max_burst,1034 max_width);1035 mdma_burst = ilog2(best_burst);1036 1037 ctcr |= STM32_MDMA_CTCR_DBURST(mdma_burst) |1038 STM32_MDMA_CTCR_DSIZE(dst_bus_width) |1039 STM32_MDMA_CTCR_DINCOS(dst_bus_width);1040 1041 if (dst_bus_width != src_bus_width)1042 ctcr |= STM32_MDMA_CTCR_PKE;1043 1044 /* Prepare hardware descriptor */1045 hwdesc = desc->node[0].hwdesc;1046 hwdesc->ctcr = ctcr;1047 hwdesc->cbndtr = cbndtr;1048 hwdesc->csar = src;1049 hwdesc->cdar = dest;1050 hwdesc->cbrur = 0;1051 hwdesc->clar = 0;1052 hwdesc->ctbr = ctbr;1053 hwdesc->cmar = 0;1054 hwdesc->cmdr = 0;1055 1056 stm32_mdma_dump_hwdesc(chan, &desc->node[0]);1057 } else {1058 /* Setup a LLI transfer */1059 ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_LINKED_LIST) |1060 STM32_MDMA_CTCR_TLEN((STM32_MDMA_MAX_BUF_LEN - 1));1061 ccr |= STM32_MDMA_CCR_BTIE | STM32_MDMA_CCR_CTCIE;1062 tlen = STM32_MDMA_MAX_BUF_LEN;1063 1064 for (i = 0, offset = 0; offset < len;1065 i++, offset += xfer_count) {1066 xfer_count = min_t(size_t, len - offset,1067 STM32_MDMA_MAX_BLOCK_LEN);1068 1069 /* Set source best burst size */1070 max_width = stm32_mdma_get_max_width(src, len, tlen);1071 src_bus_width = stm32_mdma_get_width(chan, max_width);1072 1073 max_burst = tlen / max_width;1074 best_burst = stm32_mdma_get_best_burst(len, tlen,1075 max_burst,1076 max_width);1077 mdma_burst = ilog2(best_burst);1078 1079 ctcr |= STM32_MDMA_CTCR_SBURST(mdma_burst) |1080 STM32_MDMA_CTCR_SSIZE(src_bus_width) |1081 STM32_MDMA_CTCR_SINCOS(src_bus_width);1082 1083 /* Set destination best burst size */1084 max_width = stm32_mdma_get_max_width(dest, len, tlen);1085 dst_bus_width = stm32_mdma_get_width(chan, max_width);1086 1087 max_burst = tlen / max_width;1088 best_burst = stm32_mdma_get_best_burst(len, tlen,1089 max_burst,1090 max_width);1091 mdma_burst = ilog2(best_burst);1092 1093 ctcr |= STM32_MDMA_CTCR_DBURST(mdma_burst) |1094 STM32_MDMA_CTCR_DSIZE(dst_bus_width) |1095 STM32_MDMA_CTCR_DINCOS(dst_bus_width);1096 1097 if (dst_bus_width != src_bus_width)1098 ctcr |= STM32_MDMA_CTCR_PKE;1099 1100 /* Prepare hardware descriptor */1101 stm32_mdma_setup_hwdesc(chan, desc, DMA_MEM_TO_MEM, i,1102 src + offset, dest + offset,1103 xfer_count, ctcr, ctbr,1104 i == count - 1, i == 0, false);1105 }1106 }1107 1108 desc->ccr = ccr;1109 1110 desc->cyclic = false;1111 1112 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);1113}1114 1115static void stm32_mdma_dump_reg(struct stm32_mdma_chan *chan)1116{1117 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);1118 1119 dev_dbg(chan2dev(chan), "CCR: 0x%08x\n",1120 stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)));1121 dev_dbg(chan2dev(chan), "CTCR: 0x%08x\n",1122 stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id)));1123 dev_dbg(chan2dev(chan), "CBNDTR: 0x%08x\n",1124 stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id)));1125 dev_dbg(chan2dev(chan), "CSAR: 0x%08x\n",1126 stm32_mdma_read(dmadev, STM32_MDMA_CSAR(chan->id)));1127 dev_dbg(chan2dev(chan), "CDAR: 0x%08x\n",1128 stm32_mdma_read(dmadev, STM32_MDMA_CDAR(chan->id)));1129 dev_dbg(chan2dev(chan), "CBRUR: 0x%08x\n",1130 stm32_mdma_read(dmadev, STM32_MDMA_CBRUR(chan->id)));1131 dev_dbg(chan2dev(chan), "CLAR: 0x%08x\n",1132 stm32_mdma_read(dmadev, STM32_MDMA_CLAR(chan->id)));1133 dev_dbg(chan2dev(chan), "CTBR: 0x%08x\n",1134 stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id)));1135 dev_dbg(chan2dev(chan), "CMAR: 0x%08x\n",1136 stm32_mdma_read(dmadev, STM32_MDMA_CMAR(chan->id)));1137 dev_dbg(chan2dev(chan), "CMDR: 0x%08x\n",1138 stm32_mdma_read(dmadev, STM32_MDMA_CMDR(chan->id)));1139}1140 1141static void stm32_mdma_start_transfer(struct stm32_mdma_chan *chan)1142{1143 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);1144 struct virt_dma_desc *vdesc;1145 struct stm32_mdma_hwdesc *hwdesc;1146 u32 id = chan->id;1147 u32 status, reg;1148 1149 vdesc = vchan_next_desc(&chan->vchan);1150 if (!vdesc) {1151 chan->desc = NULL;1152 return;1153 }1154 1155 list_del(&vdesc->node);1156 1157 chan->desc = to_stm32_mdma_desc(vdesc);1158 hwdesc = chan->desc->node[0].hwdesc;1159 chan->curr_hwdesc = 0;1160 1161 stm32_mdma_write(dmadev, STM32_MDMA_CCR(id), chan->desc->ccr);1162 stm32_mdma_write(dmadev, STM32_MDMA_CTCR(id), hwdesc->ctcr);1163 stm32_mdma_write(dmadev, STM32_MDMA_CBNDTR(id), hwdesc->cbndtr);1164 stm32_mdma_write(dmadev, STM32_MDMA_CSAR(id), hwdesc->csar);1165 stm32_mdma_write(dmadev, STM32_MDMA_CDAR(id), hwdesc->cdar);1166 stm32_mdma_write(dmadev, STM32_MDMA_CBRUR(id), hwdesc->cbrur);1167 stm32_mdma_write(dmadev, STM32_MDMA_CLAR(id), hwdesc->clar);1168 stm32_mdma_write(dmadev, STM32_MDMA_CTBR(id), hwdesc->ctbr);1169 stm32_mdma_write(dmadev, STM32_MDMA_CMAR(id), hwdesc->cmar);1170 stm32_mdma_write(dmadev, STM32_MDMA_CMDR(id), hwdesc->cmdr);1171 1172 /* Clear interrupt status if it is there */1173 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(id));1174 if (status)1175 stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(id), status);1176 1177 stm32_mdma_dump_reg(chan);1178 1179 /* Start DMA */1180 stm32_mdma_set_bits(dmadev, STM32_MDMA_CCR(id), STM32_MDMA_CCR_EN);1181 1182 /* Set SW request in case of MEM2MEM transfer */1183 if (hwdesc->ctcr & STM32_MDMA_CTCR_SWRM) {1184 reg = STM32_MDMA_CCR(id);1185 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_SWRQ);1186 }1187 1188 chan->busy = true;1189 1190 dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan);1191}1192 1193static void stm32_mdma_issue_pending(struct dma_chan *c)1194{1195 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);1196 unsigned long flags;1197 1198 spin_lock_irqsave(&chan->vchan.lock, flags);1199 1200 if (!vchan_issue_pending(&chan->vchan))1201 goto end;1202 1203 dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan);1204 1205 if (!chan->desc && !chan->busy)1206 stm32_mdma_start_transfer(chan);1207 1208end:1209 spin_unlock_irqrestore(&chan->vchan.lock, flags);1210}1211 1212static int stm32_mdma_pause(struct dma_chan *c)1213{1214 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);1215 unsigned long flags;1216 int ret;1217 1218 spin_lock_irqsave(&chan->vchan.lock, flags);1219 ret = stm32_mdma_disable_chan(chan);1220 spin_unlock_irqrestore(&chan->vchan.lock, flags);1221 1222 if (!ret)1223 dev_dbg(chan2dev(chan), "vchan %pK: pause\n", &chan->vchan);1224 1225 return ret;1226}1227 1228static int stm32_mdma_resume(struct dma_chan *c)1229{1230 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);1231 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);1232 struct stm32_mdma_hwdesc *hwdesc;1233 unsigned long flags;1234 u32 status, reg;1235 1236 /* Transfer can be terminated */1237 if (!chan->desc || (stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)) & STM32_MDMA_CCR_EN))1238 return -EPERM;1239 1240 hwdesc = chan->desc->node[chan->curr_hwdesc].hwdesc;1241 1242 spin_lock_irqsave(&chan->vchan.lock, flags);1243 1244 /* Re-configure control register */1245 stm32_mdma_write(dmadev, STM32_MDMA_CCR(chan->id), chan->desc->ccr);1246 1247 /* Clear interrupt status if it is there */1248 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));1249 if (status)1250 stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(chan->id), status);1251 1252 stm32_mdma_dump_reg(chan);1253 1254 /* Re-start DMA */1255 reg = STM32_MDMA_CCR(chan->id);1256 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_EN);1257 1258 /* Set SW request in case of MEM2MEM transfer */1259 if (hwdesc->ctcr & STM32_MDMA_CTCR_SWRM)1260 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_SWRQ);1261 1262 spin_unlock_irqrestore(&chan->vchan.lock, flags);1263 1264 dev_dbg(chan2dev(chan), "vchan %pK: resume\n", &chan->vchan);1265 1266 return 0;1267}1268 1269static int stm32_mdma_terminate_all(struct dma_chan *c)1270{1271 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);1272 unsigned long flags;1273 LIST_HEAD(head);1274 1275 spin_lock_irqsave(&chan->vchan.lock, flags);1276 if (chan->desc) {1277 vchan_terminate_vdesc(&chan->desc->vdesc);1278 if (chan->busy)1279 stm32_mdma_stop(chan);1280 chan->desc = NULL;1281 }1282 vchan_get_all_descriptors(&chan->vchan, &head);1283 spin_unlock_irqrestore(&chan->vchan.lock, flags);1284 1285 vchan_dma_desc_free_list(&chan->vchan, &head);1286 1287 return 0;1288}1289 1290static void stm32_mdma_synchronize(struct dma_chan *c)1291{1292 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);1293 1294 vchan_synchronize(&chan->vchan);1295}1296 1297static int stm32_mdma_slave_config(struct dma_chan *c,1298 struct dma_slave_config *config)1299{1300 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);1301 1302 memcpy(&chan->dma_config, config, sizeof(*config));1303 1304 /* Check if user is requesting STM32 DMA to trigger MDMA */1305 if (config->peripheral_size) {1306 struct stm32_mdma_dma_config *mdma_config;1307 1308 mdma_config = (struct stm32_mdma_dma_config *)chan->dma_config.peripheral_config;1309 chan->chan_config.request = mdma_config->request;1310 chan->chan_config.mask_addr = mdma_config->cmar;1311 chan->chan_config.mask_data = mdma_config->cmdr;1312 chan->chan_config.m2m_hw = true;1313 }1314 1315 return 0;1316}1317 1318static size_t stm32_mdma_desc_residue(struct stm32_mdma_chan *chan,1319 struct stm32_mdma_desc *desc,1320 u32 curr_hwdesc,1321 struct dma_tx_state *state)1322{1323 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);1324 struct stm32_mdma_hwdesc *hwdesc;1325 u32 cisr, clar, cbndtr, residue, modulo, burst_size;1326 int i;1327 1328 cisr = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));1329 1330 residue = 0;1331 /* Get the next hw descriptor to process from current transfer */1332 clar = stm32_mdma_read(dmadev, STM32_MDMA_CLAR(chan->id));1333 for (i = desc->count - 1; i >= 0; i--) {1334 hwdesc = desc->node[i].hwdesc;1335 1336 if (hwdesc->clar == clar)1337 break;/* Current transfer found, stop cumulating */1338 1339 /* Cumulate residue of unprocessed hw descriptors */1340 residue += STM32_MDMA_CBNDTR_BNDT(hwdesc->cbndtr);1341 }1342 cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id));1343 residue += cbndtr & STM32_MDMA_CBNDTR_BNDT_MASK;1344 1345 state->in_flight_bytes = 0;1346 if (chan->chan_config.m2m_hw && (cisr & STM32_MDMA_CISR_CRQA))1347 state->in_flight_bytes = cbndtr & STM32_MDMA_CBNDTR_BNDT_MASK;1348 1349 if (!chan->mem_burst)1350 return residue;1351 1352 burst_size = chan->mem_burst * chan->mem_width;1353 modulo = residue % burst_size;1354 if (modulo)1355 residue = residue - modulo + burst_size;1356 1357 return residue;1358}1359 1360static enum dma_status stm32_mdma_tx_status(struct dma_chan *c,1361 dma_cookie_t cookie,1362 struct dma_tx_state *state)1363{1364 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);1365 struct virt_dma_desc *vdesc;1366 enum dma_status status;1367 unsigned long flags;1368 u32 residue = 0;1369 1370 status = dma_cookie_status(c, cookie, state);1371 if ((status == DMA_COMPLETE) || (!state))1372 return status;1373 1374 spin_lock_irqsave(&chan->vchan.lock, flags);1375 1376 vdesc = vchan_find_desc(&chan->vchan, cookie);1377 if (chan->desc && cookie == chan->desc->vdesc.tx.cookie)1378 residue = stm32_mdma_desc_residue(chan, chan->desc, chan->curr_hwdesc, state);1379 else if (vdesc)1380 residue = stm32_mdma_desc_residue(chan, to_stm32_mdma_desc(vdesc), 0, state);1381 1382 dma_set_residue(state, residue);1383 1384 spin_unlock_irqrestore(&chan->vchan.lock, flags);1385 1386 return status;1387}1388 1389static void stm32_mdma_xfer_end(struct stm32_mdma_chan *chan)1390{1391 vchan_cookie_complete(&chan->desc->vdesc);1392 chan->desc = NULL;1393 chan->busy = false;1394 1395 /* Start the next transfer if this driver has a next desc */1396 stm32_mdma_start_transfer(chan);1397}1398 1399static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid)1400{1401 struct stm32_mdma_device *dmadev = devid;1402 struct stm32_mdma_chan *chan;1403 u32 reg, id, ccr, ien, status;1404 1405 /* Find out which channel generates the interrupt */1406 status = readl_relaxed(dmadev->base + STM32_MDMA_GISR0);1407 if (!status) {1408 dev_dbg(mdma2dev(dmadev), "spurious it\n");1409 return IRQ_NONE;1410 }1411 id = __ffs(status);1412 chan = &dmadev->chan[id];1413 1414 /* Handle interrupt for the channel */1415 spin_lock(&chan->vchan.lock);1416 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(id));1417 /* Mask Channel ReQuest Active bit which can be set in case of MEM2MEM */1418 status &= ~STM32_MDMA_CISR_CRQA;1419 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(id));1420 ien = (ccr & STM32_MDMA_CCR_IRQ_MASK) >> 1;1421 1422 if (!(status & ien)) {1423 spin_unlock(&chan->vchan.lock);1424 if (chan->busy)1425 dev_warn(chan2dev(chan),1426 "spurious it (status=0x%04x, ien=0x%04x)\n", status, ien);1427 else1428 dev_dbg(chan2dev(chan),1429 "spurious it (status=0x%04x, ien=0x%04x)\n", status, ien);1430 return IRQ_NONE;1431 }1432 1433 reg = STM32_MDMA_CIFCR(id);1434 1435 if (status & STM32_MDMA_CISR_TEIF) {1436 dev_err(chan2dev(chan), "Transfer Err: stat=0x%08x\n",1437 readl_relaxed(dmadev->base + STM32_MDMA_CESR(id)));1438 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CTEIF);1439 status &= ~STM32_MDMA_CISR_TEIF;1440 }1441 1442 if (status & STM32_MDMA_CISR_CTCIF) {1443 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CCTCIF);1444 status &= ~STM32_MDMA_CISR_CTCIF;1445 stm32_mdma_xfer_end(chan);1446 }1447 1448 if (status & STM32_MDMA_CISR_BRTIF) {1449 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBRTIF);1450 status &= ~STM32_MDMA_CISR_BRTIF;1451 }1452 1453 if (status & STM32_MDMA_CISR_BTIF) {1454 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBTIF);1455 status &= ~STM32_MDMA_CISR_BTIF;1456 chan->curr_hwdesc++;1457 if (chan->desc && chan->desc->cyclic) {1458 if (chan->curr_hwdesc == chan->desc->count)1459 chan->curr_hwdesc = 0;1460 vchan_cyclic_callback(&chan->desc->vdesc);1461 }1462 }1463 1464 if (status & STM32_MDMA_CISR_TCIF) {1465 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CLTCIF);1466 status &= ~STM32_MDMA_CISR_TCIF;1467 }1468 1469 if (status) {1470 stm32_mdma_set_bits(dmadev, reg, status);1471 dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status);1472 if (!(ccr & STM32_MDMA_CCR_EN))1473 dev_err(chan2dev(chan), "chan disabled by HW\n");1474 }1475 1476 spin_unlock(&chan->vchan.lock);1477 1478 return IRQ_HANDLED;1479}1480 1481static int stm32_mdma_alloc_chan_resources(struct dma_chan *c)1482{1483 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);1484 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);1485 int ret;1486 1487 chan->desc_pool = dmam_pool_create(dev_name(&c->dev->device),1488 c->device->dev,1489 sizeof(struct stm32_mdma_hwdesc),1490 __alignof__(struct stm32_mdma_hwdesc),1491 0);1492 if (!chan->desc_pool) {1493 dev_err(chan2dev(chan), "failed to allocate descriptor pool\n");1494 return -ENOMEM;1495 }1496 1497 ret = pm_runtime_resume_and_get(dmadev->ddev.dev);1498 if (ret < 0)1499 return ret;1500 1501 ret = stm32_mdma_disable_chan(chan);1502 if (ret < 0)1503 pm_runtime_put(dmadev->ddev.dev);1504 1505 return ret;1506}1507 1508static void stm32_mdma_free_chan_resources(struct dma_chan *c)1509{1510 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);1511 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);1512 unsigned long flags;1513 1514 dev_dbg(chan2dev(chan), "Freeing channel %d\n", chan->id);1515 1516 if (chan->busy) {1517 spin_lock_irqsave(&chan->vchan.lock, flags);1518 stm32_mdma_stop(chan);1519 chan->desc = NULL;1520 spin_unlock_irqrestore(&chan->vchan.lock, flags);1521 }1522 1523 pm_runtime_put(dmadev->ddev.dev);1524 vchan_free_chan_resources(to_virt_chan(c));1525 dmam_pool_destroy(chan->desc_pool);1526 chan->desc_pool = NULL;1527}1528 1529static bool stm32_mdma_filter_fn(struct dma_chan *c, void *fn_param)1530{1531 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);1532 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);1533 1534 /* Check if chan is marked Secure */1535 if (dmadev->chan_reserved & BIT(chan->id))1536 return false;1537 1538 return true;1539}1540 1541static struct dma_chan *stm32_mdma_of_xlate(struct of_phandle_args *dma_spec,1542 struct of_dma *ofdma)1543{1544 struct stm32_mdma_device *dmadev = ofdma->of_dma_data;1545 dma_cap_mask_t mask = dmadev->ddev.cap_mask;1546 struct stm32_mdma_chan *chan;1547 struct dma_chan *c;1548 struct stm32_mdma_chan_config config;1549 1550 if (dma_spec->args_count < 5) {1551 dev_err(mdma2dev(dmadev), "Bad number of args\n");1552 return NULL;1553 }1554 1555 memset(&config, 0, sizeof(config));1556 config.request = dma_spec->args[0];1557 config.priority_level = dma_spec->args[1];1558 config.transfer_config = dma_spec->args[2];1559 config.mask_addr = dma_spec->args[3];1560 config.mask_data = dma_spec->args[4];1561 1562 if (config.request >= dmadev->nr_requests) {1563 dev_err(mdma2dev(dmadev), "Bad request line\n");1564 return NULL;1565 }1566 1567 if (config.priority_level > STM32_MDMA_VERY_HIGH_PRIORITY) {1568 dev_err(mdma2dev(dmadev), "Priority level not supported\n");1569 return NULL;1570 }1571 1572 c = __dma_request_channel(&mask, stm32_mdma_filter_fn, &config, ofdma->of_node);1573 if (!c) {1574 dev_err(mdma2dev(dmadev), "No more channels available\n");1575 return NULL;1576 }1577 1578 chan = to_stm32_mdma_chan(c);1579 chan->chan_config = config;1580 1581 return c;1582}1583 1584static const struct of_device_id stm32_mdma_of_match[] = {1585 { .compatible = "st,stm32h7-mdma", },1586 { /* sentinel */ },1587};1588MODULE_DEVICE_TABLE(of, stm32_mdma_of_match);1589 1590static int stm32_mdma_probe(struct platform_device *pdev)1591{1592 struct stm32_mdma_chan *chan;1593 struct stm32_mdma_device *dmadev;1594 struct dma_device *dd;1595 struct device_node *of_node;1596 struct reset_control *rst;1597 u32 nr_channels, nr_requests;1598 int i, count, ret;1599 1600 of_node = pdev->dev.of_node;1601 if (!of_node)1602 return -ENODEV;1603 1604 ret = device_property_read_u32(&pdev->dev, "dma-channels",1605 &nr_channels);1606 if (ret) {1607 nr_channels = STM32_MDMA_MAX_CHANNELS;1608 dev_warn(&pdev->dev, "MDMA defaulting on %i channels\n",1609 nr_channels);1610 }1611 1612 ret = device_property_read_u32(&pdev->dev, "dma-requests",1613 &nr_requests);1614 if (ret) {1615 nr_requests = STM32_MDMA_MAX_REQUESTS;1616 dev_warn(&pdev->dev, "MDMA defaulting on %i request lines\n",1617 nr_requests);1618 }1619 1620 count = device_property_count_u32(&pdev->dev, "st,ahb-addr-masks");1621 if (count < 0)1622 count = 0;1623 1624 dmadev = devm_kzalloc(&pdev->dev,1625 struct_size(dmadev, ahb_addr_masks, count),1626 GFP_KERNEL);1627 if (!dmadev)1628 return -ENOMEM;1629 dmadev->nr_ahb_addr_masks = count;1630 1631 dmadev->nr_channels = nr_channels;1632 dmadev->nr_requests = nr_requests;1633 device_property_read_u32_array(&pdev->dev, "st,ahb-addr-masks",1634 dmadev->ahb_addr_masks,1635 count);1636 1637 dmadev->base = devm_platform_ioremap_resource(pdev, 0);1638 if (IS_ERR(dmadev->base))1639 return PTR_ERR(dmadev->base);1640 1641 dmadev->clk = devm_clk_get(&pdev->dev, NULL);1642 if (IS_ERR(dmadev->clk))1643 return dev_err_probe(&pdev->dev, PTR_ERR(dmadev->clk),1644 "Missing clock controller\n");1645 1646 ret = clk_prepare_enable(dmadev->clk);1647 if (ret < 0) {1648 dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret);1649 return ret;1650 }1651 1652 rst = devm_reset_control_get(&pdev->dev, NULL);1653 if (IS_ERR(rst)) {1654 ret = PTR_ERR(rst);1655 if (ret == -EPROBE_DEFER)1656 goto err_clk;1657 } else {1658 reset_control_assert(rst);1659 udelay(2);1660 reset_control_deassert(rst);1661 }1662 1663 dd = &dmadev->ddev;1664 dma_cap_set(DMA_SLAVE, dd->cap_mask);1665 dma_cap_set(DMA_PRIVATE, dd->cap_mask);1666 dma_cap_set(DMA_CYCLIC, dd->cap_mask);1667 dma_cap_set(DMA_MEMCPY, dd->cap_mask);1668 dd->device_alloc_chan_resources = stm32_mdma_alloc_chan_resources;1669 dd->device_free_chan_resources = stm32_mdma_free_chan_resources;1670 dd->device_tx_status = stm32_mdma_tx_status;1671 dd->device_issue_pending = stm32_mdma_issue_pending;1672 dd->device_prep_slave_sg = stm32_mdma_prep_slave_sg;1673 dd->device_prep_dma_cyclic = stm32_mdma_prep_dma_cyclic;1674 dd->device_prep_dma_memcpy = stm32_mdma_prep_dma_memcpy;1675 dd->device_config = stm32_mdma_slave_config;1676 dd->device_pause = stm32_mdma_pause;1677 dd->device_resume = stm32_mdma_resume;1678 dd->device_terminate_all = stm32_mdma_terminate_all;1679 dd->device_synchronize = stm32_mdma_synchronize;1680 dd->descriptor_reuse = true;1681 1682 dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |1683 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |1684 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |1685 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);1686 dd->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |1687 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |1688 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |1689 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);1690 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |1691 BIT(DMA_MEM_TO_MEM);1692 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;1693 dd->max_burst = STM32_MDMA_MAX_BURST;1694 dd->dev = &pdev->dev;1695 INIT_LIST_HEAD(&dd->channels);1696 1697 for (i = 0; i < dmadev->nr_channels; i++) {1698 chan = &dmadev->chan[i];1699 chan->id = i;1700 1701 if (stm32_mdma_read(dmadev, STM32_MDMA_CCR(i)) & STM32_MDMA_CCR_SM)1702 dmadev->chan_reserved |= BIT(i);1703 1704 chan->vchan.desc_free = stm32_mdma_desc_free;1705 vchan_init(&chan->vchan, dd);1706 }1707 1708 dmadev->irq = platform_get_irq(pdev, 0);1709 if (dmadev->irq < 0) {1710 ret = dmadev->irq;1711 goto err_clk;1712 }1713 1714 ret = devm_request_irq(&pdev->dev, dmadev->irq, stm32_mdma_irq_handler,1715 0, dev_name(&pdev->dev), dmadev);1716 if (ret) {1717 dev_err(&pdev->dev, "failed to request IRQ\n");1718 goto err_clk;1719 }1720 1721 ret = dmaenginem_async_device_register(dd);1722 if (ret)1723 goto err_clk;1724 1725 ret = of_dma_controller_register(of_node, stm32_mdma_of_xlate, dmadev);1726 if (ret < 0) {1727 dev_err(&pdev->dev,1728 "STM32 MDMA DMA OF registration failed %d\n", ret);1729 goto err_clk;1730 }1731 1732 platform_set_drvdata(pdev, dmadev);1733 pm_runtime_set_active(&pdev->dev);1734 pm_runtime_enable(&pdev->dev);1735 pm_runtime_get_noresume(&pdev->dev);1736 pm_runtime_put(&pdev->dev);1737 1738 dev_info(&pdev->dev, "STM32 MDMA driver registered\n");1739 1740 return 0;1741 1742err_clk:1743 clk_disable_unprepare(dmadev->clk);1744 1745 return ret;1746}1747 1748#ifdef CONFIG_PM1749static int stm32_mdma_runtime_suspend(struct device *dev)1750{1751 struct stm32_mdma_device *dmadev = dev_get_drvdata(dev);1752 1753 clk_disable_unprepare(dmadev->clk);1754 1755 return 0;1756}1757 1758static int stm32_mdma_runtime_resume(struct device *dev)1759{1760 struct stm32_mdma_device *dmadev = dev_get_drvdata(dev);1761 int ret;1762 1763 ret = clk_prepare_enable(dmadev->clk);1764 if (ret) {1765 dev_err(dev, "failed to prepare_enable clock\n");1766 return ret;1767 }1768 1769 return 0;1770}1771#endif1772 1773#ifdef CONFIG_PM_SLEEP1774static int stm32_mdma_pm_suspend(struct device *dev)1775{1776 struct stm32_mdma_device *dmadev = dev_get_drvdata(dev);1777 u32 ccr, id;1778 int ret;1779 1780 ret = pm_runtime_resume_and_get(dev);1781 if (ret < 0)1782 return ret;1783 1784 for (id = 0; id < dmadev->nr_channels; id++) {1785 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(id));1786 if (ccr & STM32_MDMA_CCR_EN) {1787 dev_warn(dev, "Suspend is prevented by Chan %i\n", id);1788 return -EBUSY;1789 }1790 }1791 1792 pm_runtime_put_sync(dev);1793 1794 pm_runtime_force_suspend(dev);1795 1796 return 0;1797}1798 1799static int stm32_mdma_pm_resume(struct device *dev)1800{1801 return pm_runtime_force_resume(dev);1802}1803#endif1804 1805static const struct dev_pm_ops stm32_mdma_pm_ops = {1806 SET_SYSTEM_SLEEP_PM_OPS(stm32_mdma_pm_suspend, stm32_mdma_pm_resume)1807 SET_RUNTIME_PM_OPS(stm32_mdma_runtime_suspend,1808 stm32_mdma_runtime_resume, NULL)1809};1810 1811static struct platform_driver stm32_mdma_driver = {1812 .probe = stm32_mdma_probe,1813 .driver = {1814 .name = "stm32-mdma",1815 .of_match_table = stm32_mdma_of_match,1816 .pm = &stm32_mdma_pm_ops,1817 },1818};1819 1820static int __init stm32_mdma_init(void)1821{1822 return platform_driver_register(&stm32_mdma_driver);1823}1824 1825subsys_initcall(stm32_mdma_init);1826 1827MODULE_DESCRIPTION("Driver for STM32 MDMA controller");1828MODULE_AUTHOR("M'boumba Cedric Madianga <cedric.madianga@gmail.com>");1829MODULE_AUTHOR("Pierre-Yves Mordret <pierre-yves.mordret@st.com>");1830