526 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.4 * Synopsys DesignWare eDMA v0 core5 *6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>7 */8 9#include <linux/bitfield.h>10#include <linux/irqreturn.h>11#include <linux/io-64-nonatomic-lo-hi.h>12 13#include "dw-edma-core.h"14#include "dw-edma-v0-core.h"15#include "dw-edma-v0-regs.h"16#include "dw-edma-v0-debugfs.h"17 18enum dw_edma_control {19 DW_EDMA_V0_CB = BIT(0),20 DW_EDMA_V0_TCB = BIT(1),21 DW_EDMA_V0_LLP = BIT(2),22 DW_EDMA_V0_LIE = BIT(3),23 DW_EDMA_V0_RIE = BIT(4),24 DW_EDMA_V0_CCS = BIT(8),25 DW_EDMA_V0_LLE = BIT(9),26};27 28static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw)29{30 return dw->chip->reg_base;31}32 33#define SET_32(dw, name, value) \34 writel(value, &(__dw_regs(dw)->name))35 36#define GET_32(dw, name) \37 readl(&(__dw_regs(dw)->name))38 39#define SET_RW_32(dw, dir, name, value) \40 do { \41 if ((dir) == EDMA_DIR_WRITE) \42 SET_32(dw, wr_##name, value); \43 else \44 SET_32(dw, rd_##name, value); \45 } while (0)46 47#define GET_RW_32(dw, dir, name) \48 ((dir) == EDMA_DIR_WRITE \49 ? GET_32(dw, wr_##name) \50 : GET_32(dw, rd_##name))51 52#define SET_BOTH_32(dw, name, value) \53 do { \54 SET_32(dw, wr_##name, value); \55 SET_32(dw, rd_##name, value); \56 } while (0)57 58#define SET_64(dw, name, value) \59 writeq(value, &(__dw_regs(dw)->name))60 61#define GET_64(dw, name) \62 readq(&(__dw_regs(dw)->name))63 64#define SET_RW_64(dw, dir, name, value) \65 do { \66 if ((dir) == EDMA_DIR_WRITE) \67 SET_64(dw, wr_##name, value); \68 else \69 SET_64(dw, rd_##name, value); \70 } while (0)71 72#define GET_RW_64(dw, dir, name) \73 ((dir) == EDMA_DIR_WRITE \74 ? GET_64(dw, wr_##name) \75 : GET_64(dw, rd_##name))76 77#define SET_BOTH_64(dw, name, value) \78 do { \79 SET_64(dw, wr_##name, value); \80 SET_64(dw, rd_##name, value); \81 } while (0)82 83#define SET_COMPAT(dw, name, value) \84 writel(value, &(__dw_regs(dw)->type.unroll.name))85 86#define SET_RW_COMPAT(dw, dir, name, value) \87 do { \88 if ((dir) == EDMA_DIR_WRITE) \89 SET_COMPAT(dw, wr_##name, value); \90 else \91 SET_COMPAT(dw, rd_##name, value); \92 } while (0)93 94static inline struct dw_edma_v0_ch_regs __iomem *95__dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)96{97 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY)98 return &(__dw_regs(dw)->type.legacy.ch);99 100 if (dir == EDMA_DIR_WRITE)101 return &__dw_regs(dw)->type.unroll.ch[ch].wr;102 103 return &__dw_regs(dw)->type.unroll.ch[ch].rd;104}105 106static inline void writel_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,107 u32 value, void __iomem *addr)108{109 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) {110 u32 viewport_sel;111 unsigned long flags;112 113 raw_spin_lock_irqsave(&dw->lock, flags);114 115 viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);116 if (dir == EDMA_DIR_READ)117 viewport_sel |= BIT(31);118 119 writel(viewport_sel,120 &(__dw_regs(dw)->type.legacy.viewport_sel));121 writel(value, addr);122 123 raw_spin_unlock_irqrestore(&dw->lock, flags);124 } else {125 writel(value, addr);126 }127}128 129static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,130 const void __iomem *addr)131{132 u32 value;133 134 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) {135 u32 viewport_sel;136 unsigned long flags;137 138 raw_spin_lock_irqsave(&dw->lock, flags);139 140 viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);141 if (dir == EDMA_DIR_READ)142 viewport_sel |= BIT(31);143 144 writel(viewport_sel,145 &(__dw_regs(dw)->type.legacy.viewport_sel));146 value = readl(addr);147 148 raw_spin_unlock_irqrestore(&dw->lock, flags);149 } else {150 value = readl(addr);151 }152 153 return value;154}155 156#define SET_CH_32(dw, dir, ch, name, value) \157 writel_ch(dw, dir, ch, value, &(__dw_ch_regs(dw, dir, ch)->name))158 159#define GET_CH_32(dw, dir, ch, name) \160 readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name))161 162/* eDMA management callbacks */163static void dw_edma_v0_core_off(struct dw_edma *dw)164{165 SET_BOTH_32(dw, int_mask,166 EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);167 SET_BOTH_32(dw, int_clear,168 EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);169 SET_BOTH_32(dw, engine_en, 0);170}171 172static u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir)173{174 u32 num_ch;175 176 if (dir == EDMA_DIR_WRITE)177 num_ch = FIELD_GET(EDMA_V0_WRITE_CH_COUNT_MASK,178 GET_32(dw, ctrl));179 else180 num_ch = FIELD_GET(EDMA_V0_READ_CH_COUNT_MASK,181 GET_32(dw, ctrl));182 183 if (num_ch > EDMA_V0_MAX_NR_CH)184 num_ch = EDMA_V0_MAX_NR_CH;185 186 return (u16)num_ch;187}188 189static enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan)190{191 struct dw_edma *dw = chan->dw;192 u32 tmp;193 194 tmp = FIELD_GET(EDMA_V0_CH_STATUS_MASK,195 GET_CH_32(dw, chan->dir, chan->id, ch_control1));196 197 if (tmp == 1)198 return DMA_IN_PROGRESS;199 else if (tmp == 3)200 return DMA_COMPLETE;201 else202 return DMA_ERROR;203}204 205static void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan)206{207 struct dw_edma *dw = chan->dw;208 209 SET_RW_32(dw, chan->dir, int_clear,210 FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)));211}212 213static void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan)214{215 struct dw_edma *dw = chan->dw;216 217 SET_RW_32(dw, chan->dir, int_clear,218 FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)));219}220 221static u32 dw_edma_v0_core_status_done_int(struct dw_edma *dw, enum dw_edma_dir dir)222{223 return FIELD_GET(EDMA_V0_DONE_INT_MASK,224 GET_RW_32(dw, dir, int_status));225}226 227static u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir)228{229 return FIELD_GET(EDMA_V0_ABORT_INT_MASK,230 GET_RW_32(dw, dir, int_status));231}232 233static irqreturn_t234dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,235 dw_edma_handler_t done, dw_edma_handler_t abort)236{237 struct dw_edma *dw = dw_irq->dw;238 unsigned long total, pos, val;239 irqreturn_t ret = IRQ_NONE;240 struct dw_edma_chan *chan;241 unsigned long off;242 u32 mask;243 244 if (dir == EDMA_DIR_WRITE) {245 total = dw->wr_ch_cnt;246 off = 0;247 mask = dw_irq->wr_mask;248 } else {249 total = dw->rd_ch_cnt;250 off = dw->wr_ch_cnt;251 mask = dw_irq->rd_mask;252 }253 254 val = dw_edma_v0_core_status_done_int(dw, dir);255 val &= mask;256 for_each_set_bit(pos, &val, total) {257 chan = &dw->chan[pos + off];258 259 dw_edma_v0_core_clear_done_int(chan);260 done(chan);261 262 ret = IRQ_HANDLED;263 }264 265 val = dw_edma_v0_core_status_abort_int(dw, dir);266 val &= mask;267 for_each_set_bit(pos, &val, total) {268 chan = &dw->chan[pos + off];269 270 dw_edma_v0_core_clear_abort_int(chan);271 abort(chan);272 273 ret = IRQ_HANDLED;274 }275 276 return ret;277}278 279static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i,280 u32 control, u32 size, u64 sar, u64 dar)281{282 ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli);283 284 if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {285 struct dw_edma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs;286 287 lli->control = control;288 lli->transfer_size = size;289 lli->sar.reg = sar;290 lli->dar.reg = dar;291 } else {292 struct dw_edma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs;293 294 writel(control, &lli->control);295 writel(size, &lli->transfer_size);296 writeq(sar, &lli->sar.reg);297 writeq(dar, &lli->dar.reg);298 }299}300 301static void dw_edma_v0_write_ll_link(struct dw_edma_chunk *chunk,302 int i, u32 control, u64 pointer)303{304 ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli);305 306 if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {307 struct dw_edma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs;308 309 llp->control = control;310 llp->llp.reg = pointer;311 } else {312 struct dw_edma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs;313 314 writel(control, &llp->control);315 writeq(pointer, &llp->llp.reg);316 }317}318 319static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk)320{321 struct dw_edma_burst *child;322 struct dw_edma_chan *chan = chunk->chan;323 u32 control = 0, i = 0;324 int j;325 326 if (chunk->cb)327 control = DW_EDMA_V0_CB;328 329 j = chunk->bursts_alloc;330 list_for_each_entry(child, &chunk->burst->list, list) {331 j--;332 if (!j) {333 control |= DW_EDMA_V0_LIE;334 if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))335 control |= DW_EDMA_V0_RIE;336 }337 338 dw_edma_v0_write_ll_data(chunk, i++, control, child->sz,339 child->sar, child->dar);340 }341 342 control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB;343 if (!chunk->cb)344 control |= DW_EDMA_V0_CB;345 346 dw_edma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr);347}348 349static void dw_edma_v0_sync_ll_data(struct dw_edma_chunk *chunk)350{351 /*352 * In case of remote eDMA engine setup, the DW PCIe RP/EP internal353 * configuration registers and application memory are normally accessed354 * over different buses. Ensure LL-data reaches the memory before the355 * doorbell register is toggled by issuing the dummy-read from the remote356 * LL memory in a hope that the MRd TLP will return only after the357 * last MWr TLP is completed358 */359 if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))360 readl(chunk->ll_region.vaddr.io);361}362 363static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first)364{365 struct dw_edma_chan *chan = chunk->chan;366 struct dw_edma *dw = chan->dw;367 u32 tmp;368 369 dw_edma_v0_core_write_chunk(chunk);370 371 if (first) {372 /* Enable engine */373 SET_RW_32(dw, chan->dir, engine_en, BIT(0));374 if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) {375 switch (chan->id) {376 case 0:377 SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en,378 BIT(0));379 break;380 case 1:381 SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en,382 BIT(0));383 break;384 case 2:385 SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en,386 BIT(0));387 break;388 case 3:389 SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en,390 BIT(0));391 break;392 case 4:393 SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en,394 BIT(0));395 break;396 case 5:397 SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en,398 BIT(0));399 break;400 case 6:401 SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en,402 BIT(0));403 break;404 case 7:405 SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en,406 BIT(0));407 break;408 }409 }410 /* Interrupt unmask - done, abort */411 tmp = GET_RW_32(dw, chan->dir, int_mask);412 tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id));413 tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id));414 SET_RW_32(dw, chan->dir, int_mask, tmp);415 /* Linked list error */416 tmp = GET_RW_32(dw, chan->dir, linked_list_err_en);417 tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id));418 SET_RW_32(dw, chan->dir, linked_list_err_en, tmp);419 /* Channel control */420 SET_CH_32(dw, chan->dir, chan->id, ch_control1,421 (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE));422 /* Linked list */423 /* llp is not aligned on 64bit -> keep 32bit accesses */424 SET_CH_32(dw, chan->dir, chan->id, llp.lsb,425 lower_32_bits(chunk->ll_region.paddr));426 SET_CH_32(dw, chan->dir, chan->id, llp.msb,427 upper_32_bits(chunk->ll_region.paddr));428 }429 430 dw_edma_v0_sync_ll_data(chunk);431 432 /* Doorbell */433 SET_RW_32(dw, chan->dir, doorbell,434 FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id));435}436 437static void dw_edma_v0_core_ch_config(struct dw_edma_chan *chan)438{439 struct dw_edma *dw = chan->dw;440 u32 tmp = 0;441 442 /* MSI done addr - low, high */443 SET_RW_32(dw, chan->dir, done_imwr.lsb, chan->msi.address_lo);444 SET_RW_32(dw, chan->dir, done_imwr.msb, chan->msi.address_hi);445 /* MSI abort addr - low, high */446 SET_RW_32(dw, chan->dir, abort_imwr.lsb, chan->msi.address_lo);447 SET_RW_32(dw, chan->dir, abort_imwr.msb, chan->msi.address_hi);448 /* MSI data - low, high */449 switch (chan->id) {450 case 0:451 case 1:452 tmp = GET_RW_32(dw, chan->dir, ch01_imwr_data);453 break;454 455 case 2:456 case 3:457 tmp = GET_RW_32(dw, chan->dir, ch23_imwr_data);458 break;459 460 case 4:461 case 5:462 tmp = GET_RW_32(dw, chan->dir, ch45_imwr_data);463 break;464 465 case 6:466 case 7:467 tmp = GET_RW_32(dw, chan->dir, ch67_imwr_data);468 break;469 }470 471 if (chan->id & BIT(0)) {472 /* Channel odd {1, 3, 5, 7} */473 tmp &= EDMA_V0_CH_EVEN_MSI_DATA_MASK;474 tmp |= FIELD_PREP(EDMA_V0_CH_ODD_MSI_DATA_MASK,475 chan->msi.data);476 } else {477 /* Channel even {0, 2, 4, 6} */478 tmp &= EDMA_V0_CH_ODD_MSI_DATA_MASK;479 tmp |= FIELD_PREP(EDMA_V0_CH_EVEN_MSI_DATA_MASK,480 chan->msi.data);481 }482 483 switch (chan->id) {484 case 0:485 case 1:486 SET_RW_32(dw, chan->dir, ch01_imwr_data, tmp);487 break;488 489 case 2:490 case 3:491 SET_RW_32(dw, chan->dir, ch23_imwr_data, tmp);492 break;493 494 case 4:495 case 5:496 SET_RW_32(dw, chan->dir, ch45_imwr_data, tmp);497 break;498 499 case 6:500 case 7:501 SET_RW_32(dw, chan->dir, ch67_imwr_data, tmp);502 break;503 }504}505 506/* eDMA debugfs callbacks */507static void dw_edma_v0_core_debugfs_on(struct dw_edma *dw)508{509 dw_edma_v0_debugfs_on(dw);510}511 512static const struct dw_edma_core_ops dw_edma_v0_core = {513 .off = dw_edma_v0_core_off,514 .ch_count = dw_edma_v0_core_ch_count,515 .ch_status = dw_edma_v0_core_ch_status,516 .handle_int = dw_edma_v0_core_handle_int,517 .start = dw_edma_v0_core_start,518 .ch_config = dw_edma_v0_core_ch_config,519 .debugfs_on = dw_edma_v0_core_debugfs_on,520};521 522void dw_edma_v0_core_register(struct dw_edma *dw)523{524 dw->core = &dw_edma_v0_core;525}526