1783 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Driver for STM32 DMA controller4 *5 * Inspired by dma-jz4740.c and tegra20-apb-dma.c6 *7 * Copyright (C) M'boumba Cedric Madianga 20158 * Author: M'boumba Cedric Madianga <cedric.madianga@gmail.com>9 * Pierre-Yves Mordret <pierre-yves.mordret@st.com>10 */11 12#include <linux/bitfield.h>13#include <linux/clk.h>14#include <linux/delay.h>15#include <linux/dmaengine.h>16#include <linux/dma-mapping.h>17#include <linux/err.h>18#include <linux/init.h>19#include <linux/iopoll.h>20#include <linux/jiffies.h>21#include <linux/list.h>22#include <linux/module.h>23#include <linux/of.h>24#include <linux/of_dma.h>25#include <linux/platform_device.h>26#include <linux/pm_runtime.h>27#include <linux/reset.h>28#include <linux/sched.h>29#include <linux/slab.h>30 31#include "../virt-dma.h"32 33#define STM32_DMA_LISR 0x0000 /* DMA Low Int Status Reg */34#define STM32_DMA_HISR 0x0004 /* DMA High Int Status Reg */35#define STM32_DMA_ISR(n) (((n) & 4) ? STM32_DMA_HISR : STM32_DMA_LISR)36#define STM32_DMA_LIFCR 0x0008 /* DMA Low Int Flag Clear Reg */37#define STM32_DMA_HIFCR 0x000c /* DMA High Int Flag Clear Reg */38#define STM32_DMA_IFCR(n) (((n) & 4) ? STM32_DMA_HIFCR : STM32_DMA_LIFCR)39#define STM32_DMA_TCI BIT(5) /* Transfer Complete Interrupt */40#define STM32_DMA_HTI BIT(4) /* Half Transfer Interrupt */41#define STM32_DMA_TEI BIT(3) /* Transfer Error Interrupt */42#define STM32_DMA_DMEI BIT(2) /* Direct Mode Error Interrupt */43#define STM32_DMA_FEI BIT(0) /* FIFO Error Interrupt */44#define STM32_DMA_MASKI (STM32_DMA_TCI \45 | STM32_DMA_TEI \46 | STM32_DMA_DMEI \47 | STM32_DMA_FEI)48/*49 * If (chan->id % 4) is 2 or 3, left shift the mask by 16 bits;50 * if (ch % 4) is 1 or 3, additionally left shift the mask by 6 bits.51 */52#define STM32_DMA_FLAGS_SHIFT(n) ({ typeof(n) (_n) = (n); \53 (((_n) & 2) << 3) | (((_n) & 1) * 6); })54 55/* DMA Stream x Configuration Register */56#define STM32_DMA_SCR(x) (0x0010 + 0x18 * (x)) /* x = 0..7 */57#define STM32_DMA_SCR_REQ_MASK GENMASK(27, 25)58#define STM32_DMA_SCR_MBURST_MASK GENMASK(24, 23)59#define STM32_DMA_SCR_PBURST_MASK GENMASK(22, 21)60#define STM32_DMA_SCR_PL_MASK GENMASK(17, 16)61#define STM32_DMA_SCR_MSIZE_MASK GENMASK(14, 13)62#define STM32_DMA_SCR_PSIZE_MASK GENMASK(12, 11)63#define STM32_DMA_SCR_DIR_MASK GENMASK(7, 6)64#define STM32_DMA_SCR_TRBUFF BIT(20) /* Bufferable transfer for USART/UART */65#define STM32_DMA_SCR_CT BIT(19) /* Target in double buffer */66#define STM32_DMA_SCR_DBM BIT(18) /* Double Buffer Mode */67#define STM32_DMA_SCR_PINCOS BIT(15) /* Peripheral inc offset size */68#define STM32_DMA_SCR_MINC BIT(10) /* Memory increment mode */69#define STM32_DMA_SCR_PINC BIT(9) /* Peripheral increment mode */70#define STM32_DMA_SCR_CIRC BIT(8) /* Circular mode */71#define STM32_DMA_SCR_PFCTRL BIT(5) /* Peripheral Flow Controller */72#define STM32_DMA_SCR_TCIE BIT(4) /* Transfer Complete Int Enable73 */74#define STM32_DMA_SCR_TEIE BIT(2) /* Transfer Error Int Enable */75#define STM32_DMA_SCR_DMEIE BIT(1) /* Direct Mode Err Int Enable */76#define STM32_DMA_SCR_EN BIT(0) /* Stream Enable */77#define STM32_DMA_SCR_CFG_MASK (STM32_DMA_SCR_PINC \78 | STM32_DMA_SCR_MINC \79 | STM32_DMA_SCR_PINCOS \80 | STM32_DMA_SCR_PL_MASK)81#define STM32_DMA_SCR_IRQ_MASK (STM32_DMA_SCR_TCIE \82 | STM32_DMA_SCR_TEIE \83 | STM32_DMA_SCR_DMEIE)84 85/* DMA Stream x number of data register */86#define STM32_DMA_SNDTR(x) (0x0014 + 0x18 * (x))87 88/* DMA stream peripheral address register */89#define STM32_DMA_SPAR(x) (0x0018 + 0x18 * (x))90 91/* DMA stream x memory 0 address register */92#define STM32_DMA_SM0AR(x) (0x001c + 0x18 * (x))93 94/* DMA stream x memory 1 address register */95#define STM32_DMA_SM1AR(x) (0x0020 + 0x18 * (x))96 97/* DMA stream x FIFO control register */98#define STM32_DMA_SFCR(x) (0x0024 + 0x18 * (x))99#define STM32_DMA_SFCR_FTH_MASK GENMASK(1, 0)100#define STM32_DMA_SFCR_FEIE BIT(7) /* FIFO error interrupt enable */101#define STM32_DMA_SFCR_DMDIS BIT(2) /* Direct mode disable */102#define STM32_DMA_SFCR_MASK (STM32_DMA_SFCR_FEIE \103 | STM32_DMA_SFCR_DMDIS)104 105/* DMA direction */106#define STM32_DMA_DEV_TO_MEM 0x00107#define STM32_DMA_MEM_TO_DEV 0x01108#define STM32_DMA_MEM_TO_MEM 0x02109 110/* DMA priority level */111#define STM32_DMA_PRIORITY_LOW 0x00112#define STM32_DMA_PRIORITY_MEDIUM 0x01113#define STM32_DMA_PRIORITY_HIGH 0x02114#define STM32_DMA_PRIORITY_VERY_HIGH 0x03115 116/* DMA FIFO threshold selection */117#define STM32_DMA_FIFO_THRESHOLD_1QUARTERFULL 0x00118#define STM32_DMA_FIFO_THRESHOLD_HALFFULL 0x01119#define STM32_DMA_FIFO_THRESHOLD_3QUARTERSFULL 0x02120#define STM32_DMA_FIFO_THRESHOLD_FULL 0x03121#define STM32_DMA_FIFO_THRESHOLD_NONE 0x04122 123#define STM32_DMA_MAX_DATA_ITEMS 0xffff124/*125 * Valid transfer starts from @0 to @0xFFFE leading to unaligned scatter126 * gather at boundary. Thus it's safer to round down this value on FIFO127 * size (16 Bytes)128 */129#define STM32_DMA_ALIGNED_MAX_DATA_ITEMS \130 ALIGN_DOWN(STM32_DMA_MAX_DATA_ITEMS, 16)131#define STM32_DMA_MAX_CHANNELS 0x08132#define STM32_DMA_MAX_REQUEST_ID 0x08133#define STM32_DMA_MAX_DATA_PARAM 0x03134#define STM32_DMA_FIFO_SIZE 16 /* FIFO is 16 bytes */135#define STM32_DMA_MIN_BURST 4136#define STM32_DMA_MAX_BURST 16137 138/* DMA Features */139#define STM32_DMA_THRESHOLD_FTR_MASK GENMASK(1, 0)140#define STM32_DMA_DIRECT_MODE_MASK BIT(2)141#define STM32_DMA_ALT_ACK_MODE_MASK BIT(4)142#define STM32_DMA_MDMA_STREAM_ID_MASK GENMASK(19, 16)143 144enum stm32_dma_width {145 STM32_DMA_BYTE,146 STM32_DMA_HALF_WORD,147 STM32_DMA_WORD,148};149 150enum stm32_dma_burst_size {151 STM32_DMA_BURST_SINGLE,152 STM32_DMA_BURST_INCR4,153 STM32_DMA_BURST_INCR8,154 STM32_DMA_BURST_INCR16,155};156 157/**158 * struct stm32_dma_cfg - STM32 DMA custom configuration159 * @channel_id: channel ID160 * @request_line: DMA request161 * @stream_config: 32bit mask specifying the DMA channel configuration162 * @features: 32bit mask specifying the DMA Feature list163 */164struct stm32_dma_cfg {165 u32 channel_id;166 u32 request_line;167 u32 stream_config;168 u32 features;169};170 171struct stm32_dma_chan_reg {172 u32 dma_lisr;173 u32 dma_hisr;174 u32 dma_lifcr;175 u32 dma_hifcr;176 u32 dma_scr;177 u32 dma_sndtr;178 u32 dma_spar;179 u32 dma_sm0ar;180 u32 dma_sm1ar;181 u32 dma_sfcr;182};183 184struct stm32_dma_sg_req {185 u32 len;186 struct stm32_dma_chan_reg chan_reg;187};188 189struct stm32_dma_desc {190 struct virt_dma_desc vdesc;191 bool cyclic;192 u32 num_sgs;193 struct stm32_dma_sg_req sg_req[] __counted_by(num_sgs);194};195 196/**197 * struct stm32_dma_mdma_config - STM32 DMA MDMA configuration198 * @stream_id: DMA request to trigger STM32 MDMA transfer199 * @ifcr: DMA interrupt flag clear register address,200 * used by STM32 MDMA to clear DMA Transfer Complete flag201 * @tcf: DMA Transfer Complete flag202 */203struct stm32_dma_mdma_config {204 u32 stream_id;205 u32 ifcr;206 u32 tcf;207};208 209struct stm32_dma_chan {210 struct virt_dma_chan vchan;211 bool config_init;212 bool busy;213 u32 id;214 u32 irq;215 struct stm32_dma_desc *desc;216 u32 next_sg;217 struct dma_slave_config dma_sconfig;218 struct stm32_dma_chan_reg chan_reg;219 u32 threshold;220 u32 mem_burst;221 u32 mem_width;222 enum dma_status status;223 bool trig_mdma;224 struct stm32_dma_mdma_config mdma_config;225};226 227struct stm32_dma_device {228 struct dma_device ddev;229 void __iomem *base;230 struct clk *clk;231 bool mem2mem;232 struct stm32_dma_chan chan[STM32_DMA_MAX_CHANNELS];233};234 235static struct stm32_dma_device *stm32_dma_get_dev(struct stm32_dma_chan *chan)236{237 return container_of(chan->vchan.chan.device, struct stm32_dma_device,238 ddev);239}240 241static struct stm32_dma_chan *to_stm32_dma_chan(struct dma_chan *c)242{243 return container_of(c, struct stm32_dma_chan, vchan.chan);244}245 246static struct stm32_dma_desc *to_stm32_dma_desc(struct virt_dma_desc *vdesc)247{248 return container_of(vdesc, struct stm32_dma_desc, vdesc);249}250 251static struct device *chan2dev(struct stm32_dma_chan *chan)252{253 return &chan->vchan.chan.dev->device;254}255 256static u32 stm32_dma_read(struct stm32_dma_device *dmadev, u32 reg)257{258 return readl_relaxed(dmadev->base + reg);259}260 261static void stm32_dma_write(struct stm32_dma_device *dmadev, u32 reg, u32 val)262{263 writel_relaxed(val, dmadev->base + reg);264}265 266static int stm32_dma_get_width(struct stm32_dma_chan *chan,267 enum dma_slave_buswidth width)268{269 switch (width) {270 case DMA_SLAVE_BUSWIDTH_1_BYTE:271 return STM32_DMA_BYTE;272 case DMA_SLAVE_BUSWIDTH_2_BYTES:273 return STM32_DMA_HALF_WORD;274 case DMA_SLAVE_BUSWIDTH_4_BYTES:275 return STM32_DMA_WORD;276 default:277 dev_err(chan2dev(chan), "Dma bus width not supported\n");278 return -EINVAL;279 }280}281 282static enum dma_slave_buswidth stm32_dma_get_max_width(u32 buf_len,283 dma_addr_t buf_addr,284 u32 threshold)285{286 enum dma_slave_buswidth max_width;287 288 if (threshold == STM32_DMA_FIFO_THRESHOLD_FULL)289 max_width = DMA_SLAVE_BUSWIDTH_4_BYTES;290 else291 max_width = DMA_SLAVE_BUSWIDTH_2_BYTES;292 293 while ((buf_len < max_width || buf_len % max_width) &&294 max_width > DMA_SLAVE_BUSWIDTH_1_BYTE)295 max_width = max_width >> 1;296 297 if (buf_addr & (max_width - 1))298 max_width = DMA_SLAVE_BUSWIDTH_1_BYTE;299 300 return max_width;301}302 303static bool stm32_dma_fifo_threshold_is_allowed(u32 burst, u32 threshold,304 enum dma_slave_buswidth width)305{306 u32 remaining;307 308 if (threshold == STM32_DMA_FIFO_THRESHOLD_NONE)309 return false;310 311 if (width != DMA_SLAVE_BUSWIDTH_UNDEFINED) {312 if (burst != 0) {313 /*314 * If number of beats fit in several whole bursts315 * this configuration is allowed.316 */317 remaining = ((STM32_DMA_FIFO_SIZE / width) *318 (threshold + 1) / 4) % burst;319 320 if (remaining == 0)321 return true;322 } else {323 return true;324 }325 }326 327 return false;328}329 330static bool stm32_dma_is_burst_possible(u32 buf_len, u32 threshold)331{332 /* If FIFO direct mode, burst is not possible */333 if (threshold == STM32_DMA_FIFO_THRESHOLD_NONE)334 return false;335 336 /*337 * Buffer or period length has to be aligned on FIFO depth.338 * Otherwise bytes may be stuck within FIFO at buffer or period339 * length.340 */341 return ((buf_len % ((threshold + 1) * 4)) == 0);342}343 344static u32 stm32_dma_get_best_burst(u32 buf_len, u32 max_burst, u32 threshold,345 enum dma_slave_buswidth width)346{347 u32 best_burst = max_burst;348 349 if (best_burst == 1 || !stm32_dma_is_burst_possible(buf_len, threshold))350 return 0;351 352 while ((buf_len < best_burst * width && best_burst > 1) ||353 !stm32_dma_fifo_threshold_is_allowed(best_burst, threshold,354 width)) {355 if (best_burst > STM32_DMA_MIN_BURST)356 best_burst = best_burst >> 1;357 else358 best_burst = 0;359 }360 361 return best_burst;362}363 364static int stm32_dma_get_burst(struct stm32_dma_chan *chan, u32 maxburst)365{366 switch (maxburst) {367 case 0:368 case 1:369 return STM32_DMA_BURST_SINGLE;370 case 4:371 return STM32_DMA_BURST_INCR4;372 case 8:373 return STM32_DMA_BURST_INCR8;374 case 16:375 return STM32_DMA_BURST_INCR16;376 default:377 dev_err(chan2dev(chan), "Dma burst size not supported\n");378 return -EINVAL;379 }380}381 382static void stm32_dma_set_fifo_config(struct stm32_dma_chan *chan,383 u32 src_burst, u32 dst_burst)384{385 chan->chan_reg.dma_sfcr &= ~STM32_DMA_SFCR_MASK;386 chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_DMEIE;387 388 if (!src_burst && !dst_burst) {389 /* Using direct mode */390 chan->chan_reg.dma_scr |= STM32_DMA_SCR_DMEIE;391 } else {392 /* Using FIFO mode */393 chan->chan_reg.dma_sfcr |= STM32_DMA_SFCR_MASK;394 }395}396 397static int stm32_dma_slave_config(struct dma_chan *c,398 struct dma_slave_config *config)399{400 struct stm32_dma_chan *chan = to_stm32_dma_chan(c);401 402 memcpy(&chan->dma_sconfig, config, sizeof(*config));403 404 /* Check if user is requesting DMA to trigger STM32 MDMA */405 if (config->peripheral_size) {406 config->peripheral_config = &chan->mdma_config;407 config->peripheral_size = sizeof(chan->mdma_config);408 chan->trig_mdma = true;409 }410 411 chan->config_init = true;412 413 return 0;414}415 416static u32 stm32_dma_irq_status(struct stm32_dma_chan *chan)417{418 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);419 u32 flags, dma_isr;420 421 /*422 * Read "flags" from DMA_xISR register corresponding to the selected423 * DMA channel at the correct bit offset inside that register.424 */425 426 dma_isr = stm32_dma_read(dmadev, STM32_DMA_ISR(chan->id));427 flags = dma_isr >> STM32_DMA_FLAGS_SHIFT(chan->id);428 429 return flags & STM32_DMA_MASKI;430}431 432static void stm32_dma_irq_clear(struct stm32_dma_chan *chan, u32 flags)433{434 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);435 u32 dma_ifcr;436 437 /*438 * Write "flags" to the DMA_xIFCR register corresponding to the selected439 * DMA channel at the correct bit offset inside that register.440 */441 flags &= STM32_DMA_MASKI;442 dma_ifcr = flags << STM32_DMA_FLAGS_SHIFT(chan->id);443 444 stm32_dma_write(dmadev, STM32_DMA_IFCR(chan->id), dma_ifcr);445}446 447static int stm32_dma_disable_chan(struct stm32_dma_chan *chan)448{449 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);450 u32 dma_scr, id, reg;451 452 id = chan->id;453 reg = STM32_DMA_SCR(id);454 dma_scr = stm32_dma_read(dmadev, reg);455 456 if (dma_scr & STM32_DMA_SCR_EN) {457 dma_scr &= ~STM32_DMA_SCR_EN;458 stm32_dma_write(dmadev, reg, dma_scr);459 460 return readl_relaxed_poll_timeout_atomic(dmadev->base + reg,461 dma_scr, !(dma_scr & STM32_DMA_SCR_EN),462 10, 1000000);463 }464 465 return 0;466}467 468static void stm32_dma_stop(struct stm32_dma_chan *chan)469{470 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);471 u32 dma_scr, dma_sfcr, status;472 int ret;473 474 /* Disable interrupts */475 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id));476 dma_scr &= ~STM32_DMA_SCR_IRQ_MASK;477 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr);478 dma_sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id));479 dma_sfcr &= ~STM32_DMA_SFCR_FEIE;480 stm32_dma_write(dmadev, STM32_DMA_SFCR(chan->id), dma_sfcr);481 482 /* Disable DMA */483 ret = stm32_dma_disable_chan(chan);484 if (ret < 0)485 return;486 487 /* Clear interrupt status if it is there */488 status = stm32_dma_irq_status(chan);489 if (status) {490 dev_dbg(chan2dev(chan), "%s(): clearing interrupt: 0x%08x\n",491 __func__, status);492 stm32_dma_irq_clear(chan, status);493 }494 495 chan->busy = false;496 chan->status = DMA_COMPLETE;497}498 499static int stm32_dma_terminate_all(struct dma_chan *c)500{501 struct stm32_dma_chan *chan = to_stm32_dma_chan(c);502 unsigned long flags;503 LIST_HEAD(head);504 505 spin_lock_irqsave(&chan->vchan.lock, flags);506 507 if (chan->desc) {508 dma_cookie_complete(&chan->desc->vdesc.tx);509 vchan_terminate_vdesc(&chan->desc->vdesc);510 if (chan->busy)511 stm32_dma_stop(chan);512 chan->desc = NULL;513 }514 515 vchan_get_all_descriptors(&chan->vchan, &head);516 spin_unlock_irqrestore(&chan->vchan.lock, flags);517 vchan_dma_desc_free_list(&chan->vchan, &head);518 519 return 0;520}521 522static void stm32_dma_synchronize(struct dma_chan *c)523{524 struct stm32_dma_chan *chan = to_stm32_dma_chan(c);525 526 vchan_synchronize(&chan->vchan);527}528 529static void stm32_dma_dump_reg(struct stm32_dma_chan *chan)530{531 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);532 u32 scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id));533 u32 ndtr = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id));534 u32 spar = stm32_dma_read(dmadev, STM32_DMA_SPAR(chan->id));535 u32 sm0ar = stm32_dma_read(dmadev, STM32_DMA_SM0AR(chan->id));536 u32 sm1ar = stm32_dma_read(dmadev, STM32_DMA_SM1AR(chan->id));537 u32 sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id));538 539 dev_dbg(chan2dev(chan), "SCR: 0x%08x\n", scr);540 dev_dbg(chan2dev(chan), "NDTR: 0x%08x\n", ndtr);541 dev_dbg(chan2dev(chan), "SPAR: 0x%08x\n", spar);542 dev_dbg(chan2dev(chan), "SM0AR: 0x%08x\n", sm0ar);543 dev_dbg(chan2dev(chan), "SM1AR: 0x%08x\n", sm1ar);544 dev_dbg(chan2dev(chan), "SFCR: 0x%08x\n", sfcr);545}546 547static void stm32_dma_sg_inc(struct stm32_dma_chan *chan)548{549 chan->next_sg++;550 if (chan->desc->cyclic && (chan->next_sg == chan->desc->num_sgs))551 chan->next_sg = 0;552}553 554static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan);555 556static void stm32_dma_start_transfer(struct stm32_dma_chan *chan)557{558 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);559 struct virt_dma_desc *vdesc;560 struct stm32_dma_sg_req *sg_req;561 struct stm32_dma_chan_reg *reg;562 u32 status;563 int ret;564 565 ret = stm32_dma_disable_chan(chan);566 if (ret < 0)567 return;568 569 if (!chan->desc) {570 vdesc = vchan_next_desc(&chan->vchan);571 if (!vdesc)572 return;573 574 list_del(&vdesc->node);575 576 chan->desc = to_stm32_dma_desc(vdesc);577 chan->next_sg = 0;578 }579 580 if (chan->next_sg == chan->desc->num_sgs)581 chan->next_sg = 0;582 583 sg_req = &chan->desc->sg_req[chan->next_sg];584 reg = &sg_req->chan_reg;585 586 /* When DMA triggers STM32 MDMA, DMA Transfer Complete is managed by STM32 MDMA */587 if (chan->trig_mdma && chan->dma_sconfig.direction != DMA_MEM_TO_DEV)588 reg->dma_scr &= ~STM32_DMA_SCR_TCIE;589 590 reg->dma_scr &= ~STM32_DMA_SCR_EN;591 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr);592 stm32_dma_write(dmadev, STM32_DMA_SPAR(chan->id), reg->dma_spar);593 stm32_dma_write(dmadev, STM32_DMA_SM0AR(chan->id), reg->dma_sm0ar);594 stm32_dma_write(dmadev, STM32_DMA_SFCR(chan->id), reg->dma_sfcr);595 stm32_dma_write(dmadev, STM32_DMA_SM1AR(chan->id), reg->dma_sm1ar);596 stm32_dma_write(dmadev, STM32_DMA_SNDTR(chan->id), reg->dma_sndtr);597 598 stm32_dma_sg_inc(chan);599 600 /* Clear interrupt status if it is there */601 status = stm32_dma_irq_status(chan);602 if (status)603 stm32_dma_irq_clear(chan, status);604 605 if (chan->desc->cyclic)606 stm32_dma_configure_next_sg(chan);607 608 stm32_dma_dump_reg(chan);609 610 /* Start DMA */611 chan->busy = true;612 chan->status = DMA_IN_PROGRESS;613 reg->dma_scr |= STM32_DMA_SCR_EN;614 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr);615 616 dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan);617}618 619static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan)620{621 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);622 struct stm32_dma_sg_req *sg_req;623 u32 dma_scr, dma_sm0ar, dma_sm1ar, id;624 625 id = chan->id;626 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id));627 628 sg_req = &chan->desc->sg_req[chan->next_sg];629 630 if (dma_scr & STM32_DMA_SCR_CT) {631 dma_sm0ar = sg_req->chan_reg.dma_sm0ar;632 stm32_dma_write(dmadev, STM32_DMA_SM0AR(id), dma_sm0ar);633 dev_dbg(chan2dev(chan), "CT=1 <=> SM0AR: 0x%08x\n",634 stm32_dma_read(dmadev, STM32_DMA_SM0AR(id)));635 } else {636 dma_sm1ar = sg_req->chan_reg.dma_sm1ar;637 stm32_dma_write(dmadev, STM32_DMA_SM1AR(id), dma_sm1ar);638 dev_dbg(chan2dev(chan), "CT=0 <=> SM1AR: 0x%08x\n",639 stm32_dma_read(dmadev, STM32_DMA_SM1AR(id)));640 }641}642 643static void stm32_dma_handle_chan_paused(struct stm32_dma_chan *chan)644{645 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);646 u32 dma_scr;647 648 /*649 * Read and store current remaining data items and peripheral/memory addresses to be650 * updated on resume651 */652 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id));653 /*654 * Transfer can be paused while between a previous resume and reconfiguration on transfer655 * complete. If transfer is cyclic and CIRC and DBM have been deactivated for resume, need656 * to set it here in SCR backup to ensure a good reconfiguration on transfer complete.657 */658 if (chan->desc && chan->desc->cyclic) {659 if (chan->desc->num_sgs == 1)660 dma_scr |= STM32_DMA_SCR_CIRC;661 else662 dma_scr |= STM32_DMA_SCR_DBM;663 }664 chan->chan_reg.dma_scr = dma_scr;665 666 /*667 * Need to temporarily deactivate CIRC/DBM until next Transfer Complete interrupt, otherwise668 * on resume NDTR autoreload value will be wrong (lower than the initial period length)669 */670 if (chan->desc && chan->desc->cyclic) {671 dma_scr &= ~(STM32_DMA_SCR_DBM | STM32_DMA_SCR_CIRC);672 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr);673 }674 675 chan->chan_reg.dma_sndtr = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id));676 677 chan->status = DMA_PAUSED;678 679 dev_dbg(chan2dev(chan), "vchan %pK: paused\n", &chan->vchan);680}681 682static void stm32_dma_post_resume_reconfigure(struct stm32_dma_chan *chan)683{684 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);685 struct stm32_dma_sg_req *sg_req;686 u32 dma_scr, status, id;687 688 id = chan->id;689 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id));690 691 /* Clear interrupt status if it is there */692 status = stm32_dma_irq_status(chan);693 if (status)694 stm32_dma_irq_clear(chan, status);695 696 if (!chan->next_sg)697 sg_req = &chan->desc->sg_req[chan->desc->num_sgs - 1];698 else699 sg_req = &chan->desc->sg_req[chan->next_sg - 1];700 701 /* Reconfigure NDTR with the initial value */702 stm32_dma_write(dmadev, STM32_DMA_SNDTR(chan->id), sg_req->chan_reg.dma_sndtr);703 704 /* Restore SPAR */705 stm32_dma_write(dmadev, STM32_DMA_SPAR(id), sg_req->chan_reg.dma_spar);706 707 /* Restore SM0AR/SM1AR whatever DBM/CT as they may have been modified */708 stm32_dma_write(dmadev, STM32_DMA_SM0AR(id), sg_req->chan_reg.dma_sm0ar);709 stm32_dma_write(dmadev, STM32_DMA_SM1AR(id), sg_req->chan_reg.dma_sm1ar);710 711 /* Reactivate CIRC/DBM if needed */712 if (chan->chan_reg.dma_scr & STM32_DMA_SCR_DBM) {713 dma_scr |= STM32_DMA_SCR_DBM;714 /* Restore CT */715 if (chan->chan_reg.dma_scr & STM32_DMA_SCR_CT)716 dma_scr &= ~STM32_DMA_SCR_CT;717 else718 dma_scr |= STM32_DMA_SCR_CT;719 } else if (chan->chan_reg.dma_scr & STM32_DMA_SCR_CIRC) {720 dma_scr |= STM32_DMA_SCR_CIRC;721 }722 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr);723 724 stm32_dma_configure_next_sg(chan);725 726 stm32_dma_dump_reg(chan);727 728 dma_scr |= STM32_DMA_SCR_EN;729 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr);730 731 dev_dbg(chan2dev(chan), "vchan %pK: reconfigured after pause/resume\n", &chan->vchan);732}733 734static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan, u32 scr)735{736 if (!chan->desc)737 return;738 739 if (chan->desc->cyclic) {740 vchan_cyclic_callback(&chan->desc->vdesc);741 if (chan->trig_mdma)742 return;743 stm32_dma_sg_inc(chan);744 /* cyclic while CIRC/DBM disable => post resume reconfiguration needed */745 if (!(scr & (STM32_DMA_SCR_CIRC | STM32_DMA_SCR_DBM)))746 stm32_dma_post_resume_reconfigure(chan);747 else if (scr & STM32_DMA_SCR_DBM)748 stm32_dma_configure_next_sg(chan);749 } else {750 chan->busy = false;751 chan->status = DMA_COMPLETE;752 if (chan->next_sg == chan->desc->num_sgs) {753 vchan_cookie_complete(&chan->desc->vdesc);754 chan->desc = NULL;755 }756 stm32_dma_start_transfer(chan);757 }758}759 760static irqreturn_t stm32_dma_chan_irq(int irq, void *devid)761{762 struct stm32_dma_chan *chan = devid;763 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);764 u32 status, scr, sfcr;765 766 spin_lock(&chan->vchan.lock);767 768 status = stm32_dma_irq_status(chan);769 scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id));770 sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id));771 772 if (status & STM32_DMA_FEI) {773 stm32_dma_irq_clear(chan, STM32_DMA_FEI);774 status &= ~STM32_DMA_FEI;775 if (sfcr & STM32_DMA_SFCR_FEIE) {776 if (!(scr & STM32_DMA_SCR_EN) &&777 !(status & STM32_DMA_TCI))778 dev_err(chan2dev(chan), "FIFO Error\n");779 else780 dev_dbg(chan2dev(chan), "FIFO over/underrun\n");781 }782 }783 if (status & STM32_DMA_DMEI) {784 stm32_dma_irq_clear(chan, STM32_DMA_DMEI);785 status &= ~STM32_DMA_DMEI;786 if (sfcr & STM32_DMA_SCR_DMEIE)787 dev_dbg(chan2dev(chan), "Direct mode overrun\n");788 }789 790 if (status & STM32_DMA_TCI) {791 stm32_dma_irq_clear(chan, STM32_DMA_TCI);792 if (scr & STM32_DMA_SCR_TCIE) {793 if (chan->status != DMA_PAUSED)794 stm32_dma_handle_chan_done(chan, scr);795 }796 status &= ~STM32_DMA_TCI;797 }798 799 if (status & STM32_DMA_HTI) {800 stm32_dma_irq_clear(chan, STM32_DMA_HTI);801 status &= ~STM32_DMA_HTI;802 }803 804 if (status) {805 stm32_dma_irq_clear(chan, status);806 dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status);807 if (!(scr & STM32_DMA_SCR_EN))808 dev_err(chan2dev(chan), "chan disabled by HW\n");809 }810 811 spin_unlock(&chan->vchan.lock);812 813 return IRQ_HANDLED;814}815 816static void stm32_dma_issue_pending(struct dma_chan *c)817{818 struct stm32_dma_chan *chan = to_stm32_dma_chan(c);819 unsigned long flags;820 821 spin_lock_irqsave(&chan->vchan.lock, flags);822 if (vchan_issue_pending(&chan->vchan) && !chan->desc && !chan->busy) {823 dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan);824 stm32_dma_start_transfer(chan);825 826 }827 spin_unlock_irqrestore(&chan->vchan.lock, flags);828}829 830static int stm32_dma_pause(struct dma_chan *c)831{832 struct stm32_dma_chan *chan = to_stm32_dma_chan(c);833 unsigned long flags;834 int ret;835 836 if (chan->status != DMA_IN_PROGRESS)837 return -EPERM;838 839 spin_lock_irqsave(&chan->vchan.lock, flags);840 841 ret = stm32_dma_disable_chan(chan);842 if (!ret)843 stm32_dma_handle_chan_paused(chan);844 845 spin_unlock_irqrestore(&chan->vchan.lock, flags);846 847 return ret;848}849 850static int stm32_dma_resume(struct dma_chan *c)851{852 struct stm32_dma_chan *chan = to_stm32_dma_chan(c);853 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);854 struct stm32_dma_chan_reg chan_reg = chan->chan_reg;855 u32 id = chan->id, scr, ndtr, offset, spar, sm0ar, sm1ar;856 struct stm32_dma_sg_req *sg_req;857 unsigned long flags;858 859 if (chan->status != DMA_PAUSED)860 return -EPERM;861 862 scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id));863 if (WARN_ON(scr & STM32_DMA_SCR_EN))864 return -EPERM;865 866 spin_lock_irqsave(&chan->vchan.lock, flags);867 868 /* sg_reg[prev_sg] contains original ndtr, sm0ar and sm1ar before pausing the transfer */869 if (!chan->next_sg)870 sg_req = &chan->desc->sg_req[chan->desc->num_sgs - 1];871 else872 sg_req = &chan->desc->sg_req[chan->next_sg - 1];873 874 ndtr = sg_req->chan_reg.dma_sndtr;875 offset = (ndtr - chan_reg.dma_sndtr);876 offset <<= FIELD_GET(STM32_DMA_SCR_PSIZE_MASK, chan_reg.dma_scr);877 spar = sg_req->chan_reg.dma_spar;878 sm0ar = sg_req->chan_reg.dma_sm0ar;879 sm1ar = sg_req->chan_reg.dma_sm1ar;880 881 /*882 * The peripheral and/or memory addresses have to be updated in order to adjust the883 * address pointers. Need to check increment.884 */885 if (chan_reg.dma_scr & STM32_DMA_SCR_PINC)886 stm32_dma_write(dmadev, STM32_DMA_SPAR(id), spar + offset);887 else888 stm32_dma_write(dmadev, STM32_DMA_SPAR(id), spar);889 890 if (!(chan_reg.dma_scr & STM32_DMA_SCR_MINC))891 offset = 0;892 893 /*894 * In case of DBM, the current target could be SM1AR.895 * Need to temporarily deactivate CIRC/DBM to finish the current transfer, so896 * SM0AR becomes the current target and must be updated with SM1AR + offset if CT=1.897 */898 if ((chan_reg.dma_scr & STM32_DMA_SCR_DBM) && (chan_reg.dma_scr & STM32_DMA_SCR_CT))899 stm32_dma_write(dmadev, STM32_DMA_SM1AR(id), sm1ar + offset);900 else901 stm32_dma_write(dmadev, STM32_DMA_SM0AR(id), sm0ar + offset);902 903 /* NDTR must be restored otherwise internal HW counter won't be correctly reset */904 stm32_dma_write(dmadev, STM32_DMA_SNDTR(id), chan_reg.dma_sndtr);905 906 /*907 * Need to temporarily deactivate CIRC/DBM until next Transfer Complete interrupt,908 * otherwise NDTR autoreload value will be wrong (lower than the initial period length)909 */910 if (chan_reg.dma_scr & (STM32_DMA_SCR_CIRC | STM32_DMA_SCR_DBM))911 chan_reg.dma_scr &= ~(STM32_DMA_SCR_CIRC | STM32_DMA_SCR_DBM);912 913 if (chan_reg.dma_scr & STM32_DMA_SCR_DBM)914 stm32_dma_configure_next_sg(chan);915 916 stm32_dma_dump_reg(chan);917 918 /* The stream may then be re-enabled to restart transfer from the point it was stopped */919 chan->status = DMA_IN_PROGRESS;920 chan_reg.dma_scr |= STM32_DMA_SCR_EN;921 stm32_dma_write(dmadev, STM32_DMA_SCR(id), chan_reg.dma_scr);922 923 spin_unlock_irqrestore(&chan->vchan.lock, flags);924 925 dev_dbg(chan2dev(chan), "vchan %pK: resumed\n", &chan->vchan);926 927 return 0;928}929 930static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,931 enum dma_transfer_direction direction,932 enum dma_slave_buswidth *buswidth,933 u32 buf_len, dma_addr_t buf_addr)934{935 enum dma_slave_buswidth src_addr_width, dst_addr_width;936 int src_bus_width, dst_bus_width;937 int src_burst_size, dst_burst_size;938 u32 src_maxburst, dst_maxburst, src_best_burst, dst_best_burst;939 u32 dma_scr, fifoth;940 941 src_addr_width = chan->dma_sconfig.src_addr_width;942 dst_addr_width = chan->dma_sconfig.dst_addr_width;943 src_maxburst = chan->dma_sconfig.src_maxburst;944 dst_maxburst = chan->dma_sconfig.dst_maxburst;945 fifoth = chan->threshold;946 947 switch (direction) {948 case DMA_MEM_TO_DEV:949 /* Set device data size */950 dst_bus_width = stm32_dma_get_width(chan, dst_addr_width);951 if (dst_bus_width < 0)952 return dst_bus_width;953 954 /* Set device burst size */955 dst_best_burst = stm32_dma_get_best_burst(buf_len,956 dst_maxburst,957 fifoth,958 dst_addr_width);959 960 dst_burst_size = stm32_dma_get_burst(chan, dst_best_burst);961 if (dst_burst_size < 0)962 return dst_burst_size;963 964 /* Set memory data size */965 src_addr_width = stm32_dma_get_max_width(buf_len, buf_addr,966 fifoth);967 chan->mem_width = src_addr_width;968 src_bus_width = stm32_dma_get_width(chan, src_addr_width);969 if (src_bus_width < 0)970 return src_bus_width;971 972 /*973 * Set memory burst size - burst not possible if address is not aligned on974 * the address boundary equal to the size of the transfer975 */976 if (buf_addr & (buf_len - 1))977 src_maxburst = 1;978 else979 src_maxburst = STM32_DMA_MAX_BURST;980 src_best_burst = stm32_dma_get_best_burst(buf_len,981 src_maxburst,982 fifoth,983 src_addr_width);984 src_burst_size = stm32_dma_get_burst(chan, src_best_burst);985 if (src_burst_size < 0)986 return src_burst_size;987 988 dma_scr = FIELD_PREP(STM32_DMA_SCR_DIR_MASK, STM32_DMA_MEM_TO_DEV) |989 FIELD_PREP(STM32_DMA_SCR_PSIZE_MASK, dst_bus_width) |990 FIELD_PREP(STM32_DMA_SCR_MSIZE_MASK, src_bus_width) |991 FIELD_PREP(STM32_DMA_SCR_PBURST_MASK, dst_burst_size) |992 FIELD_PREP(STM32_DMA_SCR_MBURST_MASK, src_burst_size);993 994 /* Set FIFO threshold */995 chan->chan_reg.dma_sfcr &= ~STM32_DMA_SFCR_FTH_MASK;996 if (fifoth != STM32_DMA_FIFO_THRESHOLD_NONE)997 chan->chan_reg.dma_sfcr |= FIELD_PREP(STM32_DMA_SFCR_FTH_MASK, fifoth);998 999 /* Set peripheral address */1000 chan->chan_reg.dma_spar = chan->dma_sconfig.dst_addr;1001 *buswidth = dst_addr_width;1002 break;1003 1004 case DMA_DEV_TO_MEM:1005 /* Set device data size */1006 src_bus_width = stm32_dma_get_width(chan, src_addr_width);1007 if (src_bus_width < 0)1008 return src_bus_width;1009 1010 /* Set device burst size */1011 src_best_burst = stm32_dma_get_best_burst(buf_len,1012 src_maxburst,1013 fifoth,1014 src_addr_width);1015 chan->mem_burst = src_best_burst;1016 src_burst_size = stm32_dma_get_burst(chan, src_best_burst);1017 if (src_burst_size < 0)1018 return src_burst_size;1019 1020 /* Set memory data size */1021 dst_addr_width = stm32_dma_get_max_width(buf_len, buf_addr,1022 fifoth);1023 chan->mem_width = dst_addr_width;1024 dst_bus_width = stm32_dma_get_width(chan, dst_addr_width);1025 if (dst_bus_width < 0)1026 return dst_bus_width;1027 1028 /*1029 * Set memory burst size - burst not possible if address is not aligned on1030 * the address boundary equal to the size of the transfer1031 */1032 if (buf_addr & (buf_len - 1))1033 dst_maxburst = 1;1034 else1035 dst_maxburst = STM32_DMA_MAX_BURST;1036 dst_best_burst = stm32_dma_get_best_burst(buf_len,1037 dst_maxburst,1038 fifoth,1039 dst_addr_width);1040 chan->mem_burst = dst_best_burst;1041 dst_burst_size = stm32_dma_get_burst(chan, dst_best_burst);1042 if (dst_burst_size < 0)1043 return dst_burst_size;1044 1045 dma_scr = FIELD_PREP(STM32_DMA_SCR_DIR_MASK, STM32_DMA_DEV_TO_MEM) |1046 FIELD_PREP(STM32_DMA_SCR_PSIZE_MASK, src_bus_width) |1047 FIELD_PREP(STM32_DMA_SCR_MSIZE_MASK, dst_bus_width) |1048 FIELD_PREP(STM32_DMA_SCR_PBURST_MASK, src_burst_size) |1049 FIELD_PREP(STM32_DMA_SCR_MBURST_MASK, dst_burst_size);1050 1051 /* Set FIFO threshold */1052 chan->chan_reg.dma_sfcr &= ~STM32_DMA_SFCR_FTH_MASK;1053 if (fifoth != STM32_DMA_FIFO_THRESHOLD_NONE)1054 chan->chan_reg.dma_sfcr |= FIELD_PREP(STM32_DMA_SFCR_FTH_MASK, fifoth);1055 1056 /* Set peripheral address */1057 chan->chan_reg.dma_spar = chan->dma_sconfig.src_addr;1058 *buswidth = chan->dma_sconfig.src_addr_width;1059 break;1060 1061 default:1062 dev_err(chan2dev(chan), "Dma direction is not supported\n");1063 return -EINVAL;1064 }1065 1066 stm32_dma_set_fifo_config(chan, src_best_burst, dst_best_burst);1067 1068 /* Set DMA control register */1069 chan->chan_reg.dma_scr &= ~(STM32_DMA_SCR_DIR_MASK |1070 STM32_DMA_SCR_PSIZE_MASK | STM32_DMA_SCR_MSIZE_MASK |1071 STM32_DMA_SCR_PBURST_MASK | STM32_DMA_SCR_MBURST_MASK);1072 chan->chan_reg.dma_scr |= dma_scr;1073 1074 return 0;1075}1076 1077static void stm32_dma_clear_reg(struct stm32_dma_chan_reg *regs)1078{1079 memset(regs, 0, sizeof(struct stm32_dma_chan_reg));1080}1081 1082static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg(1083 struct dma_chan *c, struct scatterlist *sgl,1084 u32 sg_len, enum dma_transfer_direction direction,1085 unsigned long flags, void *context)1086{1087 struct stm32_dma_chan *chan = to_stm32_dma_chan(c);1088 struct stm32_dma_desc *desc;1089 struct scatterlist *sg;1090 enum dma_slave_buswidth buswidth;1091 u32 nb_data_items;1092 int i, ret;1093 1094 if (!chan->config_init) {1095 dev_err(chan2dev(chan), "dma channel is not configured\n");1096 return NULL;1097 }1098 1099 if (sg_len < 1) {1100 dev_err(chan2dev(chan), "Invalid segment length %d\n", sg_len);1101 return NULL;1102 }1103 1104 desc = kzalloc(struct_size(desc, sg_req, sg_len), GFP_NOWAIT);1105 if (!desc)1106 return NULL;1107 desc->num_sgs = sg_len;1108 1109 /* Set peripheral flow controller */1110 if (chan->dma_sconfig.device_fc)1111 chan->chan_reg.dma_scr |= STM32_DMA_SCR_PFCTRL;1112 else1113 chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_PFCTRL;1114 1115 /* Activate Double Buffer Mode if DMA triggers STM32 MDMA and more than 1 sg */1116 if (chan->trig_mdma && sg_len > 1) {1117 chan->chan_reg.dma_scr |= STM32_DMA_SCR_DBM;1118 chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_CT;1119 }1120 1121 for_each_sg(sgl, sg, sg_len, i) {1122 ret = stm32_dma_set_xfer_param(chan, direction, &buswidth,1123 sg_dma_len(sg),1124 sg_dma_address(sg));1125 if (ret < 0)1126 goto err;1127 1128 desc->sg_req[i].len = sg_dma_len(sg);1129 1130 nb_data_items = desc->sg_req[i].len / buswidth;1131 if (nb_data_items > STM32_DMA_ALIGNED_MAX_DATA_ITEMS) {1132 dev_err(chan2dev(chan), "nb items not supported\n");1133 goto err;1134 }1135 1136 stm32_dma_clear_reg(&desc->sg_req[i].chan_reg);1137 desc->sg_req[i].chan_reg.dma_scr = chan->chan_reg.dma_scr;1138 desc->sg_req[i].chan_reg.dma_sfcr = chan->chan_reg.dma_sfcr;1139 desc->sg_req[i].chan_reg.dma_spar = chan->chan_reg.dma_spar;1140 desc->sg_req[i].chan_reg.dma_sm0ar = sg_dma_address(sg);1141 desc->sg_req[i].chan_reg.dma_sm1ar = sg_dma_address(sg);1142 if (chan->trig_mdma)1143 desc->sg_req[i].chan_reg.dma_sm1ar += sg_dma_len(sg);1144 desc->sg_req[i].chan_reg.dma_sndtr = nb_data_items;1145 }1146 desc->cyclic = false;1147 1148 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);1149 1150err:1151 kfree(desc);1152 return NULL;1153}1154 1155static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic(1156 struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len,1157 size_t period_len, enum dma_transfer_direction direction,1158 unsigned long flags)1159{1160 struct stm32_dma_chan *chan = to_stm32_dma_chan(c);1161 struct stm32_dma_desc *desc;1162 enum dma_slave_buswidth buswidth;1163 u32 num_periods, nb_data_items;1164 int i, ret;1165 1166 if (!buf_len || !period_len) {1167 dev_err(chan2dev(chan), "Invalid buffer/period len\n");1168 return NULL;1169 }1170 1171 if (!chan->config_init) {1172 dev_err(chan2dev(chan), "dma channel is not configured\n");1173 return NULL;1174 }1175 1176 if (buf_len % period_len) {1177 dev_err(chan2dev(chan), "buf_len not multiple of period_len\n");1178 return NULL;1179 }1180 1181 /*1182 * We allow to take more number of requests till DMA is1183 * not started. The driver will loop over all requests.1184 * Once DMA is started then new requests can be queued only after1185 * terminating the DMA.1186 */1187 if (chan->busy) {1188 dev_err(chan2dev(chan), "Request not allowed when dma busy\n");1189 return NULL;1190 }1191 1192 ret = stm32_dma_set_xfer_param(chan, direction, &buswidth, period_len,1193 buf_addr);1194 if (ret < 0)1195 return NULL;1196 1197 nb_data_items = period_len / buswidth;1198 if (nb_data_items > STM32_DMA_ALIGNED_MAX_DATA_ITEMS) {1199 dev_err(chan2dev(chan), "number of items not supported\n");1200 return NULL;1201 }1202 1203 /* Enable Circular mode or double buffer mode */1204 if (buf_len == period_len) {1205 chan->chan_reg.dma_scr |= STM32_DMA_SCR_CIRC;1206 } else {1207 chan->chan_reg.dma_scr |= STM32_DMA_SCR_DBM;1208 chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_CT;1209 }1210 1211 /* Clear periph ctrl if client set it */1212 chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_PFCTRL;1213 1214 num_periods = buf_len / period_len;1215 1216 desc = kzalloc(struct_size(desc, sg_req, num_periods), GFP_NOWAIT);1217 if (!desc)1218 return NULL;1219 desc->num_sgs = num_periods;1220 1221 for (i = 0; i < num_periods; i++) {1222 desc->sg_req[i].len = period_len;1223 1224 stm32_dma_clear_reg(&desc->sg_req[i].chan_reg);1225 desc->sg_req[i].chan_reg.dma_scr = chan->chan_reg.dma_scr;1226 desc->sg_req[i].chan_reg.dma_sfcr = chan->chan_reg.dma_sfcr;1227 desc->sg_req[i].chan_reg.dma_spar = chan->chan_reg.dma_spar;1228 desc->sg_req[i].chan_reg.dma_sm0ar = buf_addr;1229 desc->sg_req[i].chan_reg.dma_sm1ar = buf_addr;1230 if (chan->trig_mdma)1231 desc->sg_req[i].chan_reg.dma_sm1ar += period_len;1232 desc->sg_req[i].chan_reg.dma_sndtr = nb_data_items;1233 if (!chan->trig_mdma)1234 buf_addr += period_len;1235 }1236 desc->cyclic = true;1237 1238 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);1239}1240 1241static struct dma_async_tx_descriptor *stm32_dma_prep_dma_memcpy(1242 struct dma_chan *c, dma_addr_t dest,1243 dma_addr_t src, size_t len, unsigned long flags)1244{1245 struct stm32_dma_chan *chan = to_stm32_dma_chan(c);1246 enum dma_slave_buswidth max_width;1247 struct stm32_dma_desc *desc;1248 size_t xfer_count, offset;1249 u32 num_sgs, best_burst, threshold;1250 int dma_burst, i;1251 1252 num_sgs = DIV_ROUND_UP(len, STM32_DMA_ALIGNED_MAX_DATA_ITEMS);1253 desc = kzalloc(struct_size(desc, sg_req, num_sgs), GFP_NOWAIT);1254 if (!desc)1255 return NULL;1256 desc->num_sgs = num_sgs;1257 1258 threshold = chan->threshold;1259 1260 for (offset = 0, i = 0; offset < len; offset += xfer_count, i++) {1261 xfer_count = min_t(size_t, len - offset,1262 STM32_DMA_ALIGNED_MAX_DATA_ITEMS);1263 1264 /* Compute best burst size */1265 max_width = DMA_SLAVE_BUSWIDTH_1_BYTE;1266 best_burst = stm32_dma_get_best_burst(len, STM32_DMA_MAX_BURST,1267 threshold, max_width);1268 dma_burst = stm32_dma_get_burst(chan, best_burst);1269 if (dma_burst < 0) {1270 kfree(desc);1271 return NULL;1272 }1273 1274 stm32_dma_clear_reg(&desc->sg_req[i].chan_reg);1275 desc->sg_req[i].chan_reg.dma_scr =1276 FIELD_PREP(STM32_DMA_SCR_DIR_MASK, STM32_DMA_MEM_TO_MEM) |1277 FIELD_PREP(STM32_DMA_SCR_PBURST_MASK, dma_burst) |1278 FIELD_PREP(STM32_DMA_SCR_MBURST_MASK, dma_burst) |1279 STM32_DMA_SCR_MINC |1280 STM32_DMA_SCR_PINC |1281 STM32_DMA_SCR_TCIE |1282 STM32_DMA_SCR_TEIE;1283 desc->sg_req[i].chan_reg.dma_sfcr |= STM32_DMA_SFCR_MASK;1284 desc->sg_req[i].chan_reg.dma_sfcr |= FIELD_PREP(STM32_DMA_SFCR_FTH_MASK, threshold);1285 desc->sg_req[i].chan_reg.dma_spar = src + offset;1286 desc->sg_req[i].chan_reg.dma_sm0ar = dest + offset;1287 desc->sg_req[i].chan_reg.dma_sndtr = xfer_count;1288 desc->sg_req[i].len = xfer_count;1289 }1290 desc->cyclic = false;1291 1292 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);1293}1294 1295static u32 stm32_dma_get_remaining_bytes(struct stm32_dma_chan *chan)1296{1297 u32 dma_scr, width, ndtr;1298 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);1299 1300 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id));1301 width = FIELD_GET(STM32_DMA_SCR_PSIZE_MASK, dma_scr);1302 ndtr = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id));1303 1304 return ndtr << width;1305}1306 1307/**1308 * stm32_dma_is_current_sg - check that expected sg_req is currently transferred1309 * @chan: dma channel1310 *1311 * This function called when IRQ are disable, checks that the hardware has not1312 * switched on the next transfer in double buffer mode. The test is done by1313 * comparing the next_sg memory address with the hardware related register1314 * (based on CT bit value).1315 *1316 * Returns true if expected current transfer is still running or double1317 * buffer mode is not activated.1318 */1319static bool stm32_dma_is_current_sg(struct stm32_dma_chan *chan)1320{1321 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);1322 struct stm32_dma_sg_req *sg_req;1323 u32 dma_scr, dma_smar, id, period_len;1324 1325 id = chan->id;1326 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id));1327 1328 /* In cyclic CIRC but not DBM, CT is not used */1329 if (!(dma_scr & STM32_DMA_SCR_DBM))1330 return true;1331 1332 sg_req = &chan->desc->sg_req[chan->next_sg];1333 period_len = sg_req->len;1334 1335 /* DBM - take care of a previous pause/resume not yet post reconfigured */1336 if (dma_scr & STM32_DMA_SCR_CT) {1337 dma_smar = stm32_dma_read(dmadev, STM32_DMA_SM0AR(id));1338 /*1339 * If transfer has been pause/resumed,1340 * SM0AR is in the range of [SM0AR:SM0AR+period_len]1341 */1342 return (dma_smar >= sg_req->chan_reg.dma_sm0ar &&1343 dma_smar < sg_req->chan_reg.dma_sm0ar + period_len);1344 }1345 1346 dma_smar = stm32_dma_read(dmadev, STM32_DMA_SM1AR(id));1347 /*1348 * If transfer has been pause/resumed,1349 * SM1AR is in the range of [SM1AR:SM1AR+period_len]1350 */1351 return (dma_smar >= sg_req->chan_reg.dma_sm1ar &&1352 dma_smar < sg_req->chan_reg.dma_sm1ar + period_len);1353}1354 1355static size_t stm32_dma_desc_residue(struct stm32_dma_chan *chan,1356 struct stm32_dma_desc *desc,1357 u32 next_sg)1358{1359 u32 modulo, burst_size;1360 u32 residue;1361 u32 n_sg = next_sg;1362 struct stm32_dma_sg_req *sg_req = &chan->desc->sg_req[chan->next_sg];1363 int i;1364 1365 /*1366 * Calculate the residue means compute the descriptors1367 * information:1368 * - the sg_req currently transferred1369 * - the Hardware remaining position in this sg (NDTR bits field).1370 *1371 * A race condition may occur if DMA is running in cyclic or double1372 * buffer mode, since the DMA register are automatically reloaded at end1373 * of period transfer. The hardware may have switched to the next1374 * transfer (CT bit updated) just before the position (SxNDTR reg) is1375 * read.1376 * In this case the SxNDTR reg could (or not) correspond to the new1377 * transfer position, and not the expected one.1378 * The strategy implemented in the stm32 driver is to:1379 * - read the SxNDTR register1380 * - crosscheck that hardware is still in current transfer.1381 * In case of switch, we can assume that the DMA is at the beginning of1382 * the next transfer. So we approximate the residue in consequence, by1383 * pointing on the beginning of next transfer.1384 *1385 * This race condition doesn't apply for none cyclic mode, as double1386 * buffer is not used. In such situation registers are updated by the1387 * software.1388 */1389 1390 residue = stm32_dma_get_remaining_bytes(chan);1391 1392 if ((chan->desc->cyclic || chan->trig_mdma) && !stm32_dma_is_current_sg(chan)) {1393 n_sg++;1394 if (n_sg == chan->desc->num_sgs)1395 n_sg = 0;1396 if (!chan->trig_mdma)1397 residue = sg_req->len;1398 }1399 1400 /*1401 * In cyclic mode, for the last period, residue = remaining bytes1402 * from NDTR,1403 * else for all other periods in cyclic mode, and in sg mode,1404 * residue = remaining bytes from NDTR + remaining1405 * periods/sg to be transferred1406 */1407 if ((!chan->desc->cyclic && !chan->trig_mdma) || n_sg != 0)1408 for (i = n_sg; i < desc->num_sgs; i++)1409 residue += desc->sg_req[i].len;1410 1411 if (!chan->mem_burst)1412 return residue;1413 1414 burst_size = chan->mem_burst * chan->mem_width;1415 modulo = residue % burst_size;1416 if (modulo)1417 residue = residue - modulo + burst_size;1418 1419 return residue;1420}1421 1422static enum dma_status stm32_dma_tx_status(struct dma_chan *c,1423 dma_cookie_t cookie,1424 struct dma_tx_state *state)1425{1426 struct stm32_dma_chan *chan = to_stm32_dma_chan(c);1427 struct virt_dma_desc *vdesc;1428 enum dma_status status;1429 unsigned long flags;1430 u32 residue = 0;1431 1432 status = dma_cookie_status(c, cookie, state);1433 if (status == DMA_COMPLETE)1434 return status;1435 1436 status = chan->status;1437 1438 if (!state)1439 return status;1440 1441 spin_lock_irqsave(&chan->vchan.lock, flags);1442 vdesc = vchan_find_desc(&chan->vchan, cookie);1443 if (chan->desc && cookie == chan->desc->vdesc.tx.cookie)1444 residue = stm32_dma_desc_residue(chan, chan->desc,1445 chan->next_sg);1446 else if (vdesc)1447 residue = stm32_dma_desc_residue(chan,1448 to_stm32_dma_desc(vdesc), 0);1449 dma_set_residue(state, residue);1450 1451 spin_unlock_irqrestore(&chan->vchan.lock, flags);1452 1453 return status;1454}1455 1456static int stm32_dma_alloc_chan_resources(struct dma_chan *c)1457{1458 struct stm32_dma_chan *chan = to_stm32_dma_chan(c);1459 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);1460 int ret;1461 1462 chan->config_init = false;1463 1464 ret = pm_runtime_resume_and_get(dmadev->ddev.dev);1465 if (ret < 0)1466 return ret;1467 1468 ret = stm32_dma_disable_chan(chan);1469 if (ret < 0)1470 pm_runtime_put(dmadev->ddev.dev);1471 1472 return ret;1473}1474 1475static void stm32_dma_free_chan_resources(struct dma_chan *c)1476{1477 struct stm32_dma_chan *chan = to_stm32_dma_chan(c);1478 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);1479 unsigned long flags;1480 1481 dev_dbg(chan2dev(chan), "Freeing channel %d\n", chan->id);1482 1483 if (chan->busy) {1484 spin_lock_irqsave(&chan->vchan.lock, flags);1485 stm32_dma_stop(chan);1486 chan->desc = NULL;1487 spin_unlock_irqrestore(&chan->vchan.lock, flags);1488 }1489 1490 pm_runtime_put(dmadev->ddev.dev);1491 1492 vchan_free_chan_resources(to_virt_chan(c));1493 stm32_dma_clear_reg(&chan->chan_reg);1494 chan->threshold = 0;1495}1496 1497static void stm32_dma_desc_free(struct virt_dma_desc *vdesc)1498{1499 kfree(container_of(vdesc, struct stm32_dma_desc, vdesc));1500}1501 1502static void stm32_dma_set_config(struct stm32_dma_chan *chan,1503 struct stm32_dma_cfg *cfg)1504{1505 stm32_dma_clear_reg(&chan->chan_reg);1506 1507 chan->chan_reg.dma_scr = cfg->stream_config & STM32_DMA_SCR_CFG_MASK;1508 chan->chan_reg.dma_scr |= FIELD_PREP(STM32_DMA_SCR_REQ_MASK, cfg->request_line);1509 1510 /* Enable Interrupts */1511 chan->chan_reg.dma_scr |= STM32_DMA_SCR_TEIE | STM32_DMA_SCR_TCIE;1512 1513 chan->threshold = FIELD_GET(STM32_DMA_THRESHOLD_FTR_MASK, cfg->features);1514 if (FIELD_GET(STM32_DMA_DIRECT_MODE_MASK, cfg->features))1515 chan->threshold = STM32_DMA_FIFO_THRESHOLD_NONE;1516 if (FIELD_GET(STM32_DMA_ALT_ACK_MODE_MASK, cfg->features))1517 chan->chan_reg.dma_scr |= STM32_DMA_SCR_TRBUFF;1518 chan->mdma_config.stream_id = FIELD_GET(STM32_DMA_MDMA_STREAM_ID_MASK, cfg->features);1519}1520 1521static struct dma_chan *stm32_dma_of_xlate(struct of_phandle_args *dma_spec,1522 struct of_dma *ofdma)1523{1524 struct stm32_dma_device *dmadev = ofdma->of_dma_data;1525 struct device *dev = dmadev->ddev.dev;1526 struct stm32_dma_cfg cfg;1527 struct stm32_dma_chan *chan;1528 struct dma_chan *c;1529 1530 if (dma_spec->args_count < 4) {1531 dev_err(dev, "Bad number of cells\n");1532 return NULL;1533 }1534 1535 cfg.channel_id = dma_spec->args[0];1536 cfg.request_line = dma_spec->args[1];1537 cfg.stream_config = dma_spec->args[2];1538 cfg.features = dma_spec->args[3];1539 1540 if (cfg.channel_id >= STM32_DMA_MAX_CHANNELS ||1541 cfg.request_line >= STM32_DMA_MAX_REQUEST_ID) {1542 dev_err(dev, "Bad channel and/or request id\n");1543 return NULL;1544 }1545 1546 chan = &dmadev->chan[cfg.channel_id];1547 1548 c = dma_get_slave_channel(&chan->vchan.chan);1549 if (!c) {1550 dev_err(dev, "No more channels available\n");1551 return NULL;1552 }1553 1554 stm32_dma_set_config(chan, &cfg);1555 1556 return c;1557}1558 1559static const struct of_device_id stm32_dma_of_match[] = {1560 { .compatible = "st,stm32-dma", },1561 { /* sentinel */ },1562};1563MODULE_DEVICE_TABLE(of, stm32_dma_of_match);1564 1565static int stm32_dma_probe(struct platform_device *pdev)1566{1567 struct stm32_dma_chan *chan;1568 struct stm32_dma_device *dmadev;1569 struct dma_device *dd;1570 struct resource *res;1571 struct reset_control *rst;1572 int i, ret;1573 1574 dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL);1575 if (!dmadev)1576 return -ENOMEM;1577 1578 dd = &dmadev->ddev;1579 1580 dmadev->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);1581 if (IS_ERR(dmadev->base))1582 return PTR_ERR(dmadev->base);1583 1584 dmadev->clk = devm_clk_get(&pdev->dev, NULL);1585 if (IS_ERR(dmadev->clk))1586 return dev_err_probe(&pdev->dev, PTR_ERR(dmadev->clk), "Can't get clock\n");1587 1588 ret = clk_prepare_enable(dmadev->clk);1589 if (ret < 0) {1590 dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret);1591 return ret;1592 }1593 1594 dmadev->mem2mem = of_property_read_bool(pdev->dev.of_node,1595 "st,mem2mem");1596 1597 rst = devm_reset_control_get(&pdev->dev, NULL);1598 if (IS_ERR(rst)) {1599 ret = PTR_ERR(rst);1600 if (ret == -EPROBE_DEFER)1601 goto clk_free;1602 } else {1603 reset_control_assert(rst);1604 udelay(2);1605 reset_control_deassert(rst);1606 }1607 1608 dma_set_max_seg_size(&pdev->dev, STM32_DMA_ALIGNED_MAX_DATA_ITEMS);1609 1610 dma_cap_set(DMA_SLAVE, dd->cap_mask);1611 dma_cap_set(DMA_PRIVATE, dd->cap_mask);1612 dma_cap_set(DMA_CYCLIC, dd->cap_mask);1613 dd->device_alloc_chan_resources = stm32_dma_alloc_chan_resources;1614 dd->device_free_chan_resources = stm32_dma_free_chan_resources;1615 dd->device_tx_status = stm32_dma_tx_status;1616 dd->device_issue_pending = stm32_dma_issue_pending;1617 dd->device_prep_slave_sg = stm32_dma_prep_slave_sg;1618 dd->device_prep_dma_cyclic = stm32_dma_prep_dma_cyclic;1619 dd->device_config = stm32_dma_slave_config;1620 dd->device_pause = stm32_dma_pause;1621 dd->device_resume = stm32_dma_resume;1622 dd->device_terminate_all = stm32_dma_terminate_all;1623 dd->device_synchronize = stm32_dma_synchronize;1624 dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |1625 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |1626 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);1627 dd->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |1628 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |1629 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);1630 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);1631 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;1632 dd->copy_align = DMAENGINE_ALIGN_32_BYTES;1633 dd->max_burst = STM32_DMA_MAX_BURST;1634 dd->max_sg_burst = STM32_DMA_ALIGNED_MAX_DATA_ITEMS;1635 dd->descriptor_reuse = true;1636 dd->dev = &pdev->dev;1637 INIT_LIST_HEAD(&dd->channels);1638 1639 if (dmadev->mem2mem) {1640 dma_cap_set(DMA_MEMCPY, dd->cap_mask);1641 dd->device_prep_dma_memcpy = stm32_dma_prep_dma_memcpy;1642 dd->directions |= BIT(DMA_MEM_TO_MEM);1643 }1644 1645 for (i = 0; i < STM32_DMA_MAX_CHANNELS; i++) {1646 chan = &dmadev->chan[i];1647 chan->id = i;1648 chan->vchan.desc_free = stm32_dma_desc_free;1649 vchan_init(&chan->vchan, dd);1650 1651 chan->mdma_config.ifcr = res->start;1652 chan->mdma_config.ifcr += STM32_DMA_IFCR(chan->id);1653 1654 chan->mdma_config.tcf = STM32_DMA_TCI;1655 chan->mdma_config.tcf <<= STM32_DMA_FLAGS_SHIFT(chan->id);1656 }1657 1658 ret = dma_async_device_register(dd);1659 if (ret)1660 goto clk_free;1661 1662 for (i = 0; i < STM32_DMA_MAX_CHANNELS; i++) {1663 chan = &dmadev->chan[i];1664 ret = platform_get_irq(pdev, i);1665 if (ret < 0)1666 goto err_unregister;1667 chan->irq = ret;1668 1669 ret = devm_request_irq(&pdev->dev, chan->irq,1670 stm32_dma_chan_irq, 0,1671 dev_name(chan2dev(chan)), chan);1672 if (ret) {1673 dev_err(&pdev->dev,1674 "request_irq failed with err %d channel %d\n",1675 ret, i);1676 goto err_unregister;1677 }1678 }1679 1680 ret = of_dma_controller_register(pdev->dev.of_node,1681 stm32_dma_of_xlate, dmadev);1682 if (ret < 0) {1683 dev_err(&pdev->dev,1684 "STM32 DMA DMA OF registration failed %d\n", ret);1685 goto err_unregister;1686 }1687 1688 platform_set_drvdata(pdev, dmadev);1689 1690 pm_runtime_set_active(&pdev->dev);1691 pm_runtime_enable(&pdev->dev);1692 pm_runtime_get_noresume(&pdev->dev);1693 pm_runtime_put(&pdev->dev);1694 1695 dev_info(&pdev->dev, "STM32 DMA driver registered\n");1696 1697 return 0;1698 1699err_unregister:1700 dma_async_device_unregister(dd);1701clk_free:1702 clk_disable_unprepare(dmadev->clk);1703 1704 return ret;1705}1706 1707#ifdef CONFIG_PM1708static int stm32_dma_runtime_suspend(struct device *dev)1709{1710 struct stm32_dma_device *dmadev = dev_get_drvdata(dev);1711 1712 clk_disable_unprepare(dmadev->clk);1713 1714 return 0;1715}1716 1717static int stm32_dma_runtime_resume(struct device *dev)1718{1719 struct stm32_dma_device *dmadev = dev_get_drvdata(dev);1720 int ret;1721 1722 ret = clk_prepare_enable(dmadev->clk);1723 if (ret) {1724 dev_err(dev, "failed to prepare_enable clock\n");1725 return ret;1726 }1727 1728 return 0;1729}1730#endif1731 1732#ifdef CONFIG_PM_SLEEP1733static int stm32_dma_pm_suspend(struct device *dev)1734{1735 struct stm32_dma_device *dmadev = dev_get_drvdata(dev);1736 int id, ret, scr;1737 1738 ret = pm_runtime_resume_and_get(dev);1739 if (ret < 0)1740 return ret;1741 1742 for (id = 0; id < STM32_DMA_MAX_CHANNELS; id++) {1743 scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id));1744 if (scr & STM32_DMA_SCR_EN) {1745 dev_warn(dev, "Suspend is prevented by Chan %i\n", id);1746 return -EBUSY;1747 }1748 }1749 1750 pm_runtime_put_sync(dev);1751 1752 pm_runtime_force_suspend(dev);1753 1754 return 0;1755}1756 1757static int stm32_dma_pm_resume(struct device *dev)1758{1759 return pm_runtime_force_resume(dev);1760}1761#endif1762 1763static const struct dev_pm_ops stm32_dma_pm_ops = {1764 SET_SYSTEM_SLEEP_PM_OPS(stm32_dma_pm_suspend, stm32_dma_pm_resume)1765 SET_RUNTIME_PM_OPS(stm32_dma_runtime_suspend,1766 stm32_dma_runtime_resume, NULL)1767};1768 1769static struct platform_driver stm32_dma_driver = {1770 .driver = {1771 .name = "stm32-dma",1772 .of_match_table = stm32_dma_of_match,1773 .pm = &stm32_dma_pm_ops,1774 },1775 .probe = stm32_dma_probe,1776};1777 1778static int __init stm32_dma_init(void)1779{1780 return platform_driver_register(&stm32_dma_driver);1781}1782subsys_initcall(stm32_dma_init);1783