1299 lines · c
1// SPDX-License-Identifier: GPL-2.02// Copyright 2014-2015 Freescale3// Copyright 2018 NXP4 5/*6 * Driver for NXP Layerscape Queue Direct Memory Access Controller7 *8 * Author:9 * Wen He <wen.he_1@nxp.com>10 * Jiaheng Fan <jiaheng.fan@nxp.com>11 *12 */13 14#include <linux/module.h>15#include <linux/delay.h>16#include <linux/of.h>17#include <linux/of_dma.h>18#include <linux/dma-mapping.h>19#include <linux/platform_device.h>20 21#include "virt-dma.h"22#include "fsldma.h"23 24/* Register related definition */25#define FSL_QDMA_DMR 0x026#define FSL_QDMA_DSR 0x427#define FSL_QDMA_DEIER 0xe0028#define FSL_QDMA_DEDR 0xe0429#define FSL_QDMA_DECFDW0R 0xe1030#define FSL_QDMA_DECFDW1R 0xe1431#define FSL_QDMA_DECFDW2R 0xe1832#define FSL_QDMA_DECFDW3R 0xe1c33#define FSL_QDMA_DECFQIDR 0xe3034#define FSL_QDMA_DECBR 0xe3435 36#define FSL_QDMA_BCQMR(x) (0xc0 + 0x100 * (x))37#define FSL_QDMA_BCQSR(x) (0xc4 + 0x100 * (x))38#define FSL_QDMA_BCQEDPA_SADDR(x) (0xc8 + 0x100 * (x))39#define FSL_QDMA_BCQDPA_SADDR(x) (0xcc + 0x100 * (x))40#define FSL_QDMA_BCQEEPA_SADDR(x) (0xd0 + 0x100 * (x))41#define FSL_QDMA_BCQEPA_SADDR(x) (0xd4 + 0x100 * (x))42#define FSL_QDMA_BCQIER(x) (0xe0 + 0x100 * (x))43#define FSL_QDMA_BCQIDR(x) (0xe4 + 0x100 * (x))44 45#define FSL_QDMA_SQDPAR 0x80c46#define FSL_QDMA_SQEPAR 0x81447#define FSL_QDMA_BSQMR 0x80048#define FSL_QDMA_BSQSR 0x80449#define FSL_QDMA_BSQICR 0x82850#define FSL_QDMA_CQMR 0xa0051#define FSL_QDMA_CQDSCR1 0xa0852#define FSL_QDMA_CQDSCR2 0xa0c53#define FSL_QDMA_CQIER 0xa1054#define FSL_QDMA_CQEDR 0xa1455#define FSL_QDMA_SQCCMR 0xa2056 57/* Registers for bit and genmask */58#define FSL_QDMA_CQIDR_SQT BIT(15)59#define QDMA_CCDF_FORMAT BIT(29)60#define QDMA_CCDF_SER BIT(30)61#define QDMA_SG_FIN BIT(30)62#define QDMA_SG_LEN_MASK GENMASK(29, 0)63#define QDMA_CCDF_MASK GENMASK(28, 20)64 65#define FSL_QDMA_DEDR_CLEAR GENMASK(31, 0)66#define FSL_QDMA_BCQIDR_CLEAR GENMASK(31, 0)67#define FSL_QDMA_DEIER_CLEAR GENMASK(31, 0)68 69#define FSL_QDMA_BCQIER_CQTIE BIT(15)70#define FSL_QDMA_BCQIER_CQPEIE BIT(23)71#define FSL_QDMA_BSQICR_ICEN BIT(31)72 73#define FSL_QDMA_BSQICR_ICST(x) ((x) << 16)74#define FSL_QDMA_CQIER_MEIE BIT(31)75#define FSL_QDMA_CQIER_TEIE BIT(0)76#define FSL_QDMA_SQCCMR_ENTER_WM BIT(21)77 78#define FSL_QDMA_BCQMR_EN BIT(31)79#define FSL_QDMA_BCQMR_EI BIT(30)80#define FSL_QDMA_BCQMR_CD_THLD(x) ((x) << 20)81#define FSL_QDMA_BCQMR_CQ_SIZE(x) ((x) << 16)82 83#define FSL_QDMA_BCQSR_QF BIT(16)84#define FSL_QDMA_BCQSR_XOFF BIT(0)85 86#define FSL_QDMA_BSQMR_EN BIT(31)87#define FSL_QDMA_BSQMR_DI BIT(30)88#define FSL_QDMA_BSQMR_CQ_SIZE(x) ((x) << 16)89 90#define FSL_QDMA_BSQSR_QE BIT(17)91 92#define FSL_QDMA_DMR_DQD BIT(30)93#define FSL_QDMA_DSR_DB BIT(31)94 95/* Size related definition */96#define FSL_QDMA_QUEUE_MAX 897#define FSL_QDMA_COMMAND_BUFFER_SIZE 6498#define FSL_QDMA_DESCRIPTOR_BUFFER_SIZE 3299#define FSL_QDMA_CIRCULAR_DESC_SIZE_MIN 64100#define FSL_QDMA_CIRCULAR_DESC_SIZE_MAX 16384101#define FSL_QDMA_QUEUE_NUM_MAX 8102 103/* Field definition for CMD */104#define FSL_QDMA_CMD_RWTTYPE 0x4105#define FSL_QDMA_CMD_LWC 0x2106#define FSL_QDMA_CMD_RWTTYPE_OFFSET 28107#define FSL_QDMA_CMD_NS_OFFSET 27108#define FSL_QDMA_CMD_DQOS_OFFSET 24109#define FSL_QDMA_CMD_WTHROTL_OFFSET 20110#define FSL_QDMA_CMD_DSEN_OFFSET 19111#define FSL_QDMA_CMD_LWC_OFFSET 16112#define FSL_QDMA_CMD_PF BIT(17)113 114/* Field definition for Descriptor status */115#define QDMA_CCDF_STATUS_RTE BIT(5)116#define QDMA_CCDF_STATUS_WTE BIT(4)117#define QDMA_CCDF_STATUS_CDE BIT(2)118#define QDMA_CCDF_STATUS_SDE BIT(1)119#define QDMA_CCDF_STATUS_DDE BIT(0)120#define QDMA_CCDF_STATUS_MASK (QDMA_CCDF_STATUS_RTE | \121 QDMA_CCDF_STATUS_WTE | \122 QDMA_CCDF_STATUS_CDE | \123 QDMA_CCDF_STATUS_SDE | \124 QDMA_CCDF_STATUS_DDE)125 126/* Field definition for Descriptor offset */127#define QDMA_CCDF_OFFSET 20128#define QDMA_SDDF_CMD(x) (((u64)(x)) << 32)129 130/* Field definition for safe loop count*/131#define FSL_QDMA_HALT_COUNT 1500132#define FSL_QDMA_MAX_SIZE 16385133#define FSL_QDMA_COMP_TIMEOUT 1000134#define FSL_COMMAND_QUEUE_OVERFLLOW 10135 136#define FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma_engine, x) \137 (((fsl_qdma_engine)->block_offset) * (x))138 139/**140 * struct fsl_qdma_format - This is the struct holding describing compound141 * descriptor format with qDMA.142 * @status: Command status and enqueue status notification.143 * @cfg: Frame offset and frame format.144 * @addr_lo: Holding the compound descriptor of the lower145 * 32-bits address in memory 40-bit address.146 * @addr_hi: Same as above member, but point high 8-bits in147 * memory 40-bit address.148 * @__reserved1: Reserved field.149 * @cfg8b_w1: Compound descriptor command queue origin produced150 * by qDMA and dynamic debug field.151 * @data: Pointer to the memory 40-bit address, describes DMA152 * source information and DMA destination information.153 */154struct fsl_qdma_format {155 __le32 status;156 __le32 cfg;157 union {158 struct {159 __le32 addr_lo;160 u8 addr_hi;161 u8 __reserved1[2];162 u8 cfg8b_w1;163 } __packed;164 struct {165 __le32 __reserved2;166 __le32 cmd;167 } __packed;168 __le64 data;169 };170} __packed;171 172/* qDMA status notification pre information */173struct fsl_pre_status {174 u64 addr;175 u8 queue;176};177 178static DEFINE_PER_CPU(struct fsl_pre_status, pre);179 180struct fsl_qdma_chan {181 struct virt_dma_chan vchan;182 struct virt_dma_desc vdesc;183 enum dma_status status;184 struct fsl_qdma_engine *qdma;185 struct fsl_qdma_queue *queue;186};187 188struct fsl_qdma_queue {189 struct fsl_qdma_format *virt_head;190 struct fsl_qdma_format *virt_tail;191 struct list_head comp_used;192 struct list_head comp_free;193 struct dma_pool *comp_pool;194 struct dma_pool *desc_pool;195 spinlock_t queue_lock;196 dma_addr_t bus_addr;197 u32 n_cq;198 u32 id;199 struct fsl_qdma_format *cq;200 void __iomem *block_base;201};202 203struct fsl_qdma_comp {204 dma_addr_t bus_addr;205 dma_addr_t desc_bus_addr;206 struct fsl_qdma_format *virt_addr;207 struct fsl_qdma_format *desc_virt_addr;208 struct fsl_qdma_chan *qchan;209 struct virt_dma_desc vdesc;210 struct list_head list;211};212 213struct fsl_qdma_engine {214 struct dma_device dma_dev;215 void __iomem *ctrl_base;216 void __iomem *status_base;217 void __iomem *block_base;218 u32 n_chans;219 u32 n_queues;220 struct mutex fsl_qdma_mutex;221 int error_irq;222 int *queue_irq;223 u32 feature;224 struct fsl_qdma_queue *queue;225 struct fsl_qdma_queue **status;226 struct fsl_qdma_chan *chans;227 int block_number;228 int block_offset;229 int irq_base;230 int desc_allocated;231 232};233 234static inline u64235qdma_ccdf_addr_get64(const struct fsl_qdma_format *ccdf)236{237 return le64_to_cpu(ccdf->data) & (U64_MAX >> 24);238}239 240static inline void241qdma_desc_addr_set64(struct fsl_qdma_format *ccdf, u64 addr)242{243 ccdf->addr_hi = upper_32_bits(addr);244 ccdf->addr_lo = cpu_to_le32(lower_32_bits(addr));245}246 247static inline u8248qdma_ccdf_get_queue(const struct fsl_qdma_format *ccdf)249{250 return ccdf->cfg8b_w1 & U8_MAX;251}252 253static inline int254qdma_ccdf_get_offset(const struct fsl_qdma_format *ccdf)255{256 return (le32_to_cpu(ccdf->cfg) & QDMA_CCDF_MASK) >> QDMA_CCDF_OFFSET;257}258 259static inline void260qdma_ccdf_set_format(struct fsl_qdma_format *ccdf, int offset)261{262 ccdf->cfg = cpu_to_le32(QDMA_CCDF_FORMAT |263 (offset << QDMA_CCDF_OFFSET));264}265 266static inline int267qdma_ccdf_get_status(const struct fsl_qdma_format *ccdf)268{269 return (le32_to_cpu(ccdf->status) & QDMA_CCDF_STATUS_MASK);270}271 272static inline void273qdma_ccdf_set_ser(struct fsl_qdma_format *ccdf, int status)274{275 ccdf->status = cpu_to_le32(QDMA_CCDF_SER | status);276}277 278static inline void qdma_csgf_set_len(struct fsl_qdma_format *csgf, int len)279{280 csgf->cfg = cpu_to_le32(len & QDMA_SG_LEN_MASK);281}282 283static inline void qdma_csgf_set_f(struct fsl_qdma_format *csgf, int len)284{285 csgf->cfg = cpu_to_le32(QDMA_SG_FIN | (len & QDMA_SG_LEN_MASK));286}287 288static u32 qdma_readl(struct fsl_qdma_engine *qdma, void __iomem *addr)289{290 return FSL_DMA_IN(qdma, addr, 32);291}292 293static void qdma_writel(struct fsl_qdma_engine *qdma, u32 val,294 void __iomem *addr)295{296 FSL_DMA_OUT(qdma, addr, val, 32);297}298 299static struct fsl_qdma_chan *to_fsl_qdma_chan(struct dma_chan *chan)300{301 return container_of(chan, struct fsl_qdma_chan, vchan.chan);302}303 304static struct fsl_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd)305{306 return container_of(vd, struct fsl_qdma_comp, vdesc);307}308 309static void fsl_qdma_free_chan_resources(struct dma_chan *chan)310{311 struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);312 struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;313 struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma;314 struct fsl_qdma_comp *comp_temp, *_comp_temp;315 unsigned long flags;316 LIST_HEAD(head);317 318 spin_lock_irqsave(&fsl_chan->vchan.lock, flags);319 vchan_get_all_descriptors(&fsl_chan->vchan, &head);320 spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);321 322 vchan_dma_desc_free_list(&fsl_chan->vchan, &head);323 324 if (!fsl_queue->comp_pool && !fsl_queue->desc_pool)325 return;326 327 list_for_each_entry_safe(comp_temp, _comp_temp,328 &fsl_queue->comp_used, list) {329 dma_pool_free(fsl_queue->comp_pool,330 comp_temp->virt_addr,331 comp_temp->bus_addr);332 dma_pool_free(fsl_queue->desc_pool,333 comp_temp->desc_virt_addr,334 comp_temp->desc_bus_addr);335 list_del(&comp_temp->list);336 kfree(comp_temp);337 }338 339 list_for_each_entry_safe(comp_temp, _comp_temp,340 &fsl_queue->comp_free, list) {341 dma_pool_free(fsl_queue->comp_pool,342 comp_temp->virt_addr,343 comp_temp->bus_addr);344 dma_pool_free(fsl_queue->desc_pool,345 comp_temp->desc_virt_addr,346 comp_temp->desc_bus_addr);347 list_del(&comp_temp->list);348 kfree(comp_temp);349 }350 351 dma_pool_destroy(fsl_queue->comp_pool);352 dma_pool_destroy(fsl_queue->desc_pool);353 354 fsl_qdma->desc_allocated--;355 fsl_queue->comp_pool = NULL;356 fsl_queue->desc_pool = NULL;357}358 359static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp,360 dma_addr_t dst, dma_addr_t src, u32 len)361{362 struct fsl_qdma_format *sdf, *ddf;363 struct fsl_qdma_format *ccdf, *csgf_desc, *csgf_src, *csgf_dest;364 365 ccdf = fsl_comp->virt_addr;366 csgf_desc = fsl_comp->virt_addr + 1;367 csgf_src = fsl_comp->virt_addr + 2;368 csgf_dest = fsl_comp->virt_addr + 3;369 sdf = fsl_comp->desc_virt_addr;370 ddf = fsl_comp->desc_virt_addr + 1;371 372 memset(fsl_comp->virt_addr, 0, FSL_QDMA_COMMAND_BUFFER_SIZE);373 memset(fsl_comp->desc_virt_addr, 0, FSL_QDMA_DESCRIPTOR_BUFFER_SIZE);374 /* Head Command Descriptor(Frame Descriptor) */375 qdma_desc_addr_set64(ccdf, fsl_comp->bus_addr + 16);376 qdma_ccdf_set_format(ccdf, qdma_ccdf_get_offset(ccdf));377 qdma_ccdf_set_ser(ccdf, qdma_ccdf_get_status(ccdf));378 /* Status notification is enqueued to status queue. */379 /* Compound Command Descriptor(Frame List Table) */380 qdma_desc_addr_set64(csgf_desc, fsl_comp->desc_bus_addr);381 /* It must be 32 as Compound S/G Descriptor */382 qdma_csgf_set_len(csgf_desc, 32);383 qdma_desc_addr_set64(csgf_src, src);384 qdma_csgf_set_len(csgf_src, len);385 qdma_desc_addr_set64(csgf_dest, dst);386 qdma_csgf_set_len(csgf_dest, len);387 /* This entry is the last entry. */388 qdma_csgf_set_f(csgf_dest, len);389 /* Descriptor Buffer */390 sdf->cmd = cpu_to_le32((FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET) |391 FSL_QDMA_CMD_PF);392 393 ddf->cmd = cpu_to_le32((FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET) |394 (FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET));395}396 397/*398 * Pre-request full command descriptor for enqueue.399 */400static int fsl_qdma_pre_request_enqueue_desc(struct fsl_qdma_queue *queue)401{402 int i;403 struct fsl_qdma_comp *comp_temp, *_comp_temp;404 405 for (i = 0; i < queue->n_cq + FSL_COMMAND_QUEUE_OVERFLLOW; i++) {406 comp_temp = kzalloc(sizeof(*comp_temp), GFP_KERNEL);407 if (!comp_temp)408 goto err_alloc;409 comp_temp->virt_addr =410 dma_pool_alloc(queue->comp_pool, GFP_KERNEL,411 &comp_temp->bus_addr);412 if (!comp_temp->virt_addr)413 goto err_dma_alloc;414 415 comp_temp->desc_virt_addr =416 dma_pool_alloc(queue->desc_pool, GFP_KERNEL,417 &comp_temp->desc_bus_addr);418 if (!comp_temp->desc_virt_addr)419 goto err_desc_dma_alloc;420 421 list_add_tail(&comp_temp->list, &queue->comp_free);422 }423 424 return 0;425 426err_desc_dma_alloc:427 dma_pool_free(queue->comp_pool, comp_temp->virt_addr,428 comp_temp->bus_addr);429 430err_dma_alloc:431 kfree(comp_temp);432 433err_alloc:434 list_for_each_entry_safe(comp_temp, _comp_temp,435 &queue->comp_free, list) {436 if (comp_temp->virt_addr)437 dma_pool_free(queue->comp_pool,438 comp_temp->virt_addr,439 comp_temp->bus_addr);440 if (comp_temp->desc_virt_addr)441 dma_pool_free(queue->desc_pool,442 comp_temp->desc_virt_addr,443 comp_temp->desc_bus_addr);444 445 list_del(&comp_temp->list);446 kfree(comp_temp);447 }448 449 return -ENOMEM;450}451 452/*453 * Request a command descriptor for enqueue.454 */455static struct fsl_qdma_comp456*fsl_qdma_request_enqueue_desc(struct fsl_qdma_chan *fsl_chan)457{458 unsigned long flags;459 struct fsl_qdma_comp *comp_temp;460 int timeout = FSL_QDMA_COMP_TIMEOUT;461 struct fsl_qdma_queue *queue = fsl_chan->queue;462 463 while (timeout--) {464 spin_lock_irqsave(&queue->queue_lock, flags);465 if (!list_empty(&queue->comp_free)) {466 comp_temp = list_first_entry(&queue->comp_free,467 struct fsl_qdma_comp,468 list);469 list_del(&comp_temp->list);470 471 spin_unlock_irqrestore(&queue->queue_lock, flags);472 comp_temp->qchan = fsl_chan;473 return comp_temp;474 }475 spin_unlock_irqrestore(&queue->queue_lock, flags);476 udelay(1);477 }478 479 return NULL;480}481 482static struct fsl_qdma_queue483*fsl_qdma_alloc_queue_resources(struct platform_device *pdev,484 struct fsl_qdma_engine *fsl_qdma)485{486 int ret, len, i, j;487 int queue_num, block_number;488 unsigned int queue_size[FSL_QDMA_QUEUE_MAX];489 struct fsl_qdma_queue *queue_head, *queue_temp;490 491 queue_num = fsl_qdma->n_queues;492 block_number = fsl_qdma->block_number;493 494 if (queue_num > FSL_QDMA_QUEUE_MAX)495 queue_num = FSL_QDMA_QUEUE_MAX;496 len = sizeof(*queue_head) * queue_num * block_number;497 queue_head = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);498 if (!queue_head)499 return NULL;500 501 ret = device_property_read_u32_array(&pdev->dev, "queue-sizes",502 queue_size, queue_num);503 if (ret) {504 dev_err(&pdev->dev, "Can't get queue-sizes.\n");505 return NULL;506 }507 for (j = 0; j < block_number; j++) {508 for (i = 0; i < queue_num; i++) {509 if (queue_size[i] > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX ||510 queue_size[i] < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {511 dev_err(&pdev->dev,512 "Get wrong queue-sizes.\n");513 return NULL;514 }515 queue_temp = queue_head + i + (j * queue_num);516 517 queue_temp->cq =518 dmam_alloc_coherent(&pdev->dev,519 sizeof(struct fsl_qdma_format) *520 queue_size[i],521 &queue_temp->bus_addr,522 GFP_KERNEL);523 if (!queue_temp->cq)524 return NULL;525 queue_temp->block_base = fsl_qdma->block_base +526 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);527 queue_temp->n_cq = queue_size[i];528 queue_temp->id = i;529 queue_temp->virt_head = queue_temp->cq;530 queue_temp->virt_tail = queue_temp->cq;531 /*532 * List for queue command buffer533 */534 INIT_LIST_HEAD(&queue_temp->comp_used);535 spin_lock_init(&queue_temp->queue_lock);536 }537 }538 return queue_head;539}540 541static struct fsl_qdma_queue542*fsl_qdma_prep_status_queue(struct platform_device *pdev)543{544 int ret;545 unsigned int status_size;546 struct fsl_qdma_queue *status_head;547 struct device_node *np = pdev->dev.of_node;548 549 ret = of_property_read_u32(np, "status-sizes", &status_size);550 if (ret) {551 dev_err(&pdev->dev, "Can't get status-sizes.\n");552 return NULL;553 }554 if (status_size > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX ||555 status_size < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {556 dev_err(&pdev->dev, "Get wrong status_size.\n");557 return NULL;558 }559 status_head = devm_kzalloc(&pdev->dev,560 sizeof(*status_head), GFP_KERNEL);561 if (!status_head)562 return NULL;563 564 /*565 * Buffer for queue command566 */567 status_head->cq = dmam_alloc_coherent(&pdev->dev,568 sizeof(struct fsl_qdma_format) *569 status_size,570 &status_head->bus_addr,571 GFP_KERNEL);572 if (!status_head->cq)573 return NULL;574 575 status_head->n_cq = status_size;576 status_head->virt_head = status_head->cq;577 status_head->virt_tail = status_head->cq;578 status_head->comp_pool = NULL;579 580 return status_head;581}582 583static int fsl_qdma_halt(struct fsl_qdma_engine *fsl_qdma)584{585 u32 reg;586 int i, j, count = FSL_QDMA_HALT_COUNT;587 void __iomem *block, *ctrl = fsl_qdma->ctrl_base;588 589 /* Disable the command queue and wait for idle state. */590 reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR);591 reg |= FSL_QDMA_DMR_DQD;592 qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR);593 for (j = 0; j < fsl_qdma->block_number; j++) {594 block = fsl_qdma->block_base +595 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);596 for (i = 0; i < FSL_QDMA_QUEUE_NUM_MAX; i++)597 qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQMR(i));598 }599 while (1) {600 reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DSR);601 if (!(reg & FSL_QDMA_DSR_DB))602 break;603 if (count-- < 0)604 return -EBUSY;605 udelay(100);606 }607 608 for (j = 0; j < fsl_qdma->block_number; j++) {609 block = fsl_qdma->block_base +610 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);611 612 /* Disable status queue. */613 qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BSQMR);614 615 /*616 * clear the command queue interrupt detect register for617 * all queues.618 */619 qdma_writel(fsl_qdma, FSL_QDMA_BCQIDR_CLEAR,620 block + FSL_QDMA_BCQIDR(0));621 }622 623 return 0;624}625 626static int627fsl_qdma_queue_transfer_complete(struct fsl_qdma_engine *fsl_qdma,628 __iomem void *block,629 int id)630{631 bool duplicate;632 u32 reg, i, count;633 u8 completion_status;634 struct fsl_qdma_queue *temp_queue;635 struct fsl_qdma_format *status_addr;636 struct fsl_qdma_comp *fsl_comp = NULL;637 struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue;638 struct fsl_qdma_queue *fsl_status = fsl_qdma->status[id];639 640 count = FSL_QDMA_MAX_SIZE;641 642 while (count--) {643 duplicate = 0;644 reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQSR);645 if (reg & FSL_QDMA_BSQSR_QE)646 return 0;647 648 status_addr = fsl_status->virt_head;649 650 if (qdma_ccdf_get_queue(status_addr) ==651 __this_cpu_read(pre.queue) &&652 qdma_ccdf_addr_get64(status_addr) ==653 __this_cpu_read(pre.addr))654 duplicate = 1;655 i = qdma_ccdf_get_queue(status_addr) +656 id * fsl_qdma->n_queues;657 __this_cpu_write(pre.addr, qdma_ccdf_addr_get64(status_addr));658 __this_cpu_write(pre.queue, qdma_ccdf_get_queue(status_addr));659 temp_queue = fsl_queue + i;660 661 spin_lock(&temp_queue->queue_lock);662 if (list_empty(&temp_queue->comp_used)) {663 if (!duplicate) {664 spin_unlock(&temp_queue->queue_lock);665 return -EAGAIN;666 }667 } else {668 fsl_comp = list_first_entry(&temp_queue->comp_used,669 struct fsl_qdma_comp, list);670 if (fsl_comp->bus_addr + 16 !=671 __this_cpu_read(pre.addr)) {672 if (!duplicate) {673 spin_unlock(&temp_queue->queue_lock);674 return -EAGAIN;675 }676 }677 }678 679 if (duplicate) {680 reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR);681 reg |= FSL_QDMA_BSQMR_DI;682 qdma_desc_addr_set64(status_addr, 0x0);683 fsl_status->virt_head++;684 if (fsl_status->virt_head == fsl_status->cq685 + fsl_status->n_cq)686 fsl_status->virt_head = fsl_status->cq;687 qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR);688 spin_unlock(&temp_queue->queue_lock);689 continue;690 }691 list_del(&fsl_comp->list);692 693 completion_status = qdma_ccdf_get_status(status_addr);694 695 reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR);696 reg |= FSL_QDMA_BSQMR_DI;697 qdma_desc_addr_set64(status_addr, 0x0);698 fsl_status->virt_head++;699 if (fsl_status->virt_head == fsl_status->cq + fsl_status->n_cq)700 fsl_status->virt_head = fsl_status->cq;701 qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR);702 spin_unlock(&temp_queue->queue_lock);703 704 /* The completion_status is evaluated here705 * (outside of spin lock)706 */707 if (completion_status) {708 /* A completion error occurred! */709 if (completion_status & QDMA_CCDF_STATUS_WTE) {710 /* Write transaction error */711 fsl_comp->vdesc.tx_result.result =712 DMA_TRANS_WRITE_FAILED;713 } else if (completion_status & QDMA_CCDF_STATUS_RTE) {714 /* Read transaction error */715 fsl_comp->vdesc.tx_result.result =716 DMA_TRANS_READ_FAILED;717 } else {718 /* Command/source/destination719 * description error720 */721 fsl_comp->vdesc.tx_result.result =722 DMA_TRANS_ABORTED;723 dev_err(fsl_qdma->dma_dev.dev,724 "DMA status descriptor error %x\n",725 completion_status);726 }727 }728 729 spin_lock(&fsl_comp->qchan->vchan.lock);730 vchan_cookie_complete(&fsl_comp->vdesc);731 fsl_comp->qchan->status = DMA_COMPLETE;732 spin_unlock(&fsl_comp->qchan->vchan.lock);733 }734 735 return 0;736}737 738static irqreturn_t fsl_qdma_error_handler(int irq, void *dev_id)739{740 unsigned int intr;741 struct fsl_qdma_engine *fsl_qdma = dev_id;742 void __iomem *status = fsl_qdma->status_base;743 unsigned int decfdw0r;744 unsigned int decfdw1r;745 unsigned int decfdw2r;746 unsigned int decfdw3r;747 748 intr = qdma_readl(fsl_qdma, status + FSL_QDMA_DEDR);749 750 if (intr) {751 decfdw0r = qdma_readl(fsl_qdma, status + FSL_QDMA_DECFDW0R);752 decfdw1r = qdma_readl(fsl_qdma, status + FSL_QDMA_DECFDW1R);753 decfdw2r = qdma_readl(fsl_qdma, status + FSL_QDMA_DECFDW2R);754 decfdw3r = qdma_readl(fsl_qdma, status + FSL_QDMA_DECFDW3R);755 dev_err(fsl_qdma->dma_dev.dev,756 "DMA transaction error! (%x: %x-%x-%x-%x)\n",757 intr, decfdw0r, decfdw1r, decfdw2r, decfdw3r);758 }759 760 qdma_writel(fsl_qdma, FSL_QDMA_DEDR_CLEAR, status + FSL_QDMA_DEDR);761 return IRQ_HANDLED;762}763 764static irqreturn_t fsl_qdma_queue_handler(int irq, void *dev_id)765{766 int id;767 unsigned int intr, reg;768 struct fsl_qdma_engine *fsl_qdma = dev_id;769 void __iomem *block, *ctrl = fsl_qdma->ctrl_base;770 771 id = irq - fsl_qdma->irq_base;772 if (id < 0 && id > fsl_qdma->block_number) {773 dev_err(fsl_qdma->dma_dev.dev,774 "irq %d is wrong irq_base is %d\n",775 irq, fsl_qdma->irq_base);776 }777 778 block = fsl_qdma->block_base +779 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, id);780 781 intr = qdma_readl(fsl_qdma, block + FSL_QDMA_BCQIDR(0));782 783 if ((intr & FSL_QDMA_CQIDR_SQT) != 0)784 intr = fsl_qdma_queue_transfer_complete(fsl_qdma, block, id);785 786 if (intr != 0) {787 reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR);788 reg |= FSL_QDMA_DMR_DQD;789 qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR);790 qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQIER(0));791 dev_err(fsl_qdma->dma_dev.dev, "QDMA: status err!\n");792 }793 794 /* Clear all detected events and interrupts. */795 qdma_writel(fsl_qdma, FSL_QDMA_BCQIDR_CLEAR,796 block + FSL_QDMA_BCQIDR(0));797 798 return IRQ_HANDLED;799}800 801static int802fsl_qdma_irq_init(struct platform_device *pdev,803 struct fsl_qdma_engine *fsl_qdma)804{805 int i;806 int cpu;807 int ret;808 char irq_name[32];809 810 fsl_qdma->error_irq =811 platform_get_irq_byname(pdev, "qdma-error");812 if (fsl_qdma->error_irq < 0)813 return fsl_qdma->error_irq;814 815 ret = devm_request_irq(&pdev->dev, fsl_qdma->error_irq,816 fsl_qdma_error_handler, 0,817 "qDMA error", fsl_qdma);818 if (ret) {819 dev_err(&pdev->dev, "Can't register qDMA controller IRQ.\n");820 return ret;821 }822 823 for (i = 0; i < fsl_qdma->block_number; i++) {824 sprintf(irq_name, "qdma-queue%d", i);825 fsl_qdma->queue_irq[i] =826 platform_get_irq_byname(pdev, irq_name);827 828 if (fsl_qdma->queue_irq[i] < 0)829 return fsl_qdma->queue_irq[i];830 831 ret = devm_request_irq(&pdev->dev,832 fsl_qdma->queue_irq[i],833 fsl_qdma_queue_handler,834 0,835 "qDMA queue",836 fsl_qdma);837 if (ret) {838 dev_err(&pdev->dev,839 "Can't register qDMA queue IRQ.\n");840 return ret;841 }842 843 cpu = i % num_online_cpus();844 ret = irq_set_affinity_hint(fsl_qdma->queue_irq[i],845 get_cpu_mask(cpu));846 if (ret) {847 dev_err(&pdev->dev,848 "Can't set cpu %d affinity to IRQ %d.\n",849 cpu,850 fsl_qdma->queue_irq[i]);851 return ret;852 }853 }854 855 return 0;856}857 858static void fsl_qdma_irq_exit(struct platform_device *pdev,859 struct fsl_qdma_engine *fsl_qdma)860{861 int i;862 863 devm_free_irq(&pdev->dev, fsl_qdma->error_irq, fsl_qdma);864 for (i = 0; i < fsl_qdma->block_number; i++)865 devm_free_irq(&pdev->dev, fsl_qdma->queue_irq[i], fsl_qdma);866}867 868static int fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma)869{870 u32 reg;871 int i, j, ret;872 struct fsl_qdma_queue *temp;873 void __iomem *status = fsl_qdma->status_base;874 void __iomem *block, *ctrl = fsl_qdma->ctrl_base;875 struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue;876 877 /* Try to halt the qDMA engine first. */878 ret = fsl_qdma_halt(fsl_qdma);879 if (ret) {880 dev_err(fsl_qdma->dma_dev.dev, "DMA halt failed!");881 return ret;882 }883 884 for (i = 0; i < fsl_qdma->block_number; i++) {885 /*886 * Clear the command queue interrupt detect register for887 * all queues.888 */889 890 block = fsl_qdma->block_base +891 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, i);892 qdma_writel(fsl_qdma, FSL_QDMA_BCQIDR_CLEAR,893 block + FSL_QDMA_BCQIDR(0));894 }895 896 for (j = 0; j < fsl_qdma->block_number; j++) {897 block = fsl_qdma->block_base +898 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);899 for (i = 0; i < fsl_qdma->n_queues; i++) {900 temp = fsl_queue + i + (j * fsl_qdma->n_queues);901 /*902 * Initialize Command Queue registers to903 * point to the first904 * command descriptor in memory.905 * Dequeue Pointer Address Registers906 * Enqueue Pointer Address Registers907 */908 909 qdma_writel(fsl_qdma, temp->bus_addr,910 block + FSL_QDMA_BCQDPA_SADDR(i));911 qdma_writel(fsl_qdma, temp->bus_addr,912 block + FSL_QDMA_BCQEPA_SADDR(i));913 914 /* Initialize the queue mode. */915 reg = FSL_QDMA_BCQMR_EN;916 reg |= FSL_QDMA_BCQMR_CD_THLD(ilog2(temp->n_cq) - 4);917 reg |= FSL_QDMA_BCQMR_CQ_SIZE(ilog2(temp->n_cq) - 6);918 qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BCQMR(i));919 }920 921 /*922 * Workaround for erratum: ERR010812.923 * We must enable XOFF to avoid the enqueue rejection occurs.924 * Setting SQCCMR ENTER_WM to 0x20.925 */926 927 qdma_writel(fsl_qdma, FSL_QDMA_SQCCMR_ENTER_WM,928 block + FSL_QDMA_SQCCMR);929 930 /*931 * Initialize status queue registers to point to the first932 * command descriptor in memory.933 * Dequeue Pointer Address Registers934 * Enqueue Pointer Address Registers935 */936 937 qdma_writel(fsl_qdma, fsl_qdma->status[j]->bus_addr,938 block + FSL_QDMA_SQEPAR);939 qdma_writel(fsl_qdma, fsl_qdma->status[j]->bus_addr,940 block + FSL_QDMA_SQDPAR);941 /* Initialize status queue interrupt. */942 qdma_writel(fsl_qdma, FSL_QDMA_BCQIER_CQTIE,943 block + FSL_QDMA_BCQIER(0));944 qdma_writel(fsl_qdma, FSL_QDMA_BSQICR_ICEN |945 FSL_QDMA_BSQICR_ICST(5) | 0x8000,946 block + FSL_QDMA_BSQICR);947 qdma_writel(fsl_qdma, FSL_QDMA_CQIER_MEIE |948 FSL_QDMA_CQIER_TEIE,949 block + FSL_QDMA_CQIER);950 951 /* Initialize the status queue mode. */952 reg = FSL_QDMA_BSQMR_EN;953 reg |= FSL_QDMA_BSQMR_CQ_SIZE(ilog2954 (fsl_qdma->status[j]->n_cq) - 6);955 956 qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR);957 reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR);958 }959 960 /* Initialize controller interrupt register. */961 qdma_writel(fsl_qdma, FSL_QDMA_DEDR_CLEAR, status + FSL_QDMA_DEDR);962 qdma_writel(fsl_qdma, FSL_QDMA_DEIER_CLEAR, status + FSL_QDMA_DEIER);963 964 reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR);965 reg &= ~FSL_QDMA_DMR_DQD;966 qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR);967 968 return 0;969}970 971static struct dma_async_tx_descriptor *972fsl_qdma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst,973 dma_addr_t src, size_t len, unsigned long flags)974{975 struct fsl_qdma_comp *fsl_comp;976 struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);977 978 fsl_comp = fsl_qdma_request_enqueue_desc(fsl_chan);979 980 if (!fsl_comp)981 return NULL;982 983 fsl_qdma_comp_fill_memcpy(fsl_comp, dst, src, len);984 985 return vchan_tx_prep(&fsl_chan->vchan, &fsl_comp->vdesc, flags);986}987 988static void fsl_qdma_enqueue_desc(struct fsl_qdma_chan *fsl_chan)989{990 u32 reg;991 struct virt_dma_desc *vdesc;992 struct fsl_qdma_comp *fsl_comp;993 struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;994 void __iomem *block = fsl_queue->block_base;995 996 reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQSR(fsl_queue->id));997 if (reg & (FSL_QDMA_BCQSR_QF | FSL_QDMA_BCQSR_XOFF))998 return;999 vdesc = vchan_next_desc(&fsl_chan->vchan);1000 if (!vdesc)1001 return;1002 list_del(&vdesc->node);1003 fsl_comp = to_fsl_qdma_comp(vdesc);1004 1005 memcpy(fsl_queue->virt_head++,1006 fsl_comp->virt_addr, sizeof(struct fsl_qdma_format));1007 if (fsl_queue->virt_head == fsl_queue->cq + fsl_queue->n_cq)1008 fsl_queue->virt_head = fsl_queue->cq;1009 1010 list_add_tail(&fsl_comp->list, &fsl_queue->comp_used);1011 barrier();1012 reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQMR(fsl_queue->id));1013 reg |= FSL_QDMA_BCQMR_EI;1014 qdma_writel(fsl_chan->qdma, reg, block + FSL_QDMA_BCQMR(fsl_queue->id));1015 fsl_chan->status = DMA_IN_PROGRESS;1016}1017 1018static void fsl_qdma_free_desc(struct virt_dma_desc *vdesc)1019{1020 unsigned long flags;1021 struct fsl_qdma_comp *fsl_comp;1022 struct fsl_qdma_queue *fsl_queue;1023 1024 fsl_comp = to_fsl_qdma_comp(vdesc);1025 fsl_queue = fsl_comp->qchan->queue;1026 1027 spin_lock_irqsave(&fsl_queue->queue_lock, flags);1028 list_add_tail(&fsl_comp->list, &fsl_queue->comp_free);1029 spin_unlock_irqrestore(&fsl_queue->queue_lock, flags);1030}1031 1032static void fsl_qdma_issue_pending(struct dma_chan *chan)1033{1034 unsigned long flags;1035 struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);1036 struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;1037 1038 spin_lock_irqsave(&fsl_queue->queue_lock, flags);1039 spin_lock(&fsl_chan->vchan.lock);1040 if (vchan_issue_pending(&fsl_chan->vchan))1041 fsl_qdma_enqueue_desc(fsl_chan);1042 spin_unlock(&fsl_chan->vchan.lock);1043 spin_unlock_irqrestore(&fsl_queue->queue_lock, flags);1044}1045 1046static void fsl_qdma_synchronize(struct dma_chan *chan)1047{1048 struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);1049 1050 vchan_synchronize(&fsl_chan->vchan);1051}1052 1053static int fsl_qdma_terminate_all(struct dma_chan *chan)1054{1055 LIST_HEAD(head);1056 unsigned long flags;1057 struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);1058 1059 spin_lock_irqsave(&fsl_chan->vchan.lock, flags);1060 vchan_get_all_descriptors(&fsl_chan->vchan, &head);1061 spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);1062 vchan_dma_desc_free_list(&fsl_chan->vchan, &head);1063 return 0;1064}1065 1066static int fsl_qdma_alloc_chan_resources(struct dma_chan *chan)1067{1068 int ret;1069 struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);1070 struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma;1071 struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;1072 1073 if (fsl_queue->comp_pool && fsl_queue->desc_pool)1074 return fsl_qdma->desc_allocated;1075 1076 INIT_LIST_HEAD(&fsl_queue->comp_free);1077 1078 /*1079 * The dma pool for queue command buffer1080 */1081 fsl_queue->comp_pool =1082 dma_pool_create("comp_pool",1083 chan->device->dev,1084 FSL_QDMA_COMMAND_BUFFER_SIZE,1085 64, 0);1086 if (!fsl_queue->comp_pool)1087 return -ENOMEM;1088 1089 /*1090 * The dma pool for Descriptor(SD/DD) buffer1091 */1092 fsl_queue->desc_pool =1093 dma_pool_create("desc_pool",1094 chan->device->dev,1095 FSL_QDMA_DESCRIPTOR_BUFFER_SIZE,1096 32, 0);1097 if (!fsl_queue->desc_pool)1098 goto err_desc_pool;1099 1100 ret = fsl_qdma_pre_request_enqueue_desc(fsl_queue);1101 if (ret) {1102 dev_err(chan->device->dev,1103 "failed to alloc dma buffer for S/G descriptor\n");1104 goto err_mem;1105 }1106 1107 fsl_qdma->desc_allocated++;1108 return fsl_qdma->desc_allocated;1109 1110err_mem:1111 dma_pool_destroy(fsl_queue->desc_pool);1112err_desc_pool:1113 dma_pool_destroy(fsl_queue->comp_pool);1114 return -ENOMEM;1115}1116 1117static int fsl_qdma_probe(struct platform_device *pdev)1118{1119 int ret, i;1120 int blk_num, blk_off;1121 u32 len, chans, queues;1122 struct fsl_qdma_chan *fsl_chan;1123 struct fsl_qdma_engine *fsl_qdma;1124 struct device_node *np = pdev->dev.of_node;1125 1126 ret = of_property_read_u32(np, "dma-channels", &chans);1127 if (ret) {1128 dev_err(&pdev->dev, "Can't get dma-channels.\n");1129 return ret;1130 }1131 1132 ret = of_property_read_u32(np, "block-offset", &blk_off);1133 if (ret) {1134 dev_err(&pdev->dev, "Can't get block-offset.\n");1135 return ret;1136 }1137 1138 ret = of_property_read_u32(np, "block-number", &blk_num);1139 if (ret) {1140 dev_err(&pdev->dev, "Can't get block-number.\n");1141 return ret;1142 }1143 1144 blk_num = min_t(int, blk_num, num_online_cpus());1145 1146 len = sizeof(*fsl_qdma);1147 fsl_qdma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);1148 if (!fsl_qdma)1149 return -ENOMEM;1150 1151 len = sizeof(*fsl_chan) * chans;1152 fsl_qdma->chans = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);1153 if (!fsl_qdma->chans)1154 return -ENOMEM;1155 1156 len = sizeof(struct fsl_qdma_queue *) * blk_num;1157 fsl_qdma->status = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);1158 if (!fsl_qdma->status)1159 return -ENOMEM;1160 1161 len = sizeof(int) * blk_num;1162 fsl_qdma->queue_irq = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);1163 if (!fsl_qdma->queue_irq)1164 return -ENOMEM;1165 1166 ret = of_property_read_u32(np, "fsl,dma-queues", &queues);1167 if (ret) {1168 dev_err(&pdev->dev, "Can't get queues.\n");1169 return ret;1170 }1171 1172 fsl_qdma->desc_allocated = 0;1173 fsl_qdma->n_chans = chans;1174 fsl_qdma->n_queues = queues;1175 fsl_qdma->block_number = blk_num;1176 fsl_qdma->block_offset = blk_off;1177 1178 mutex_init(&fsl_qdma->fsl_qdma_mutex);1179 1180 for (i = 0; i < fsl_qdma->block_number; i++) {1181 fsl_qdma->status[i] = fsl_qdma_prep_status_queue(pdev);1182 if (!fsl_qdma->status[i])1183 return -ENOMEM;1184 }1185 fsl_qdma->ctrl_base = devm_platform_ioremap_resource(pdev, 0);1186 if (IS_ERR(fsl_qdma->ctrl_base))1187 return PTR_ERR(fsl_qdma->ctrl_base);1188 1189 fsl_qdma->status_base = devm_platform_ioremap_resource(pdev, 1);1190 if (IS_ERR(fsl_qdma->status_base))1191 return PTR_ERR(fsl_qdma->status_base);1192 1193 fsl_qdma->block_base = devm_platform_ioremap_resource(pdev, 2);1194 if (IS_ERR(fsl_qdma->block_base))1195 return PTR_ERR(fsl_qdma->block_base);1196 fsl_qdma->queue = fsl_qdma_alloc_queue_resources(pdev, fsl_qdma);1197 if (!fsl_qdma->queue)1198 return -ENOMEM;1199 1200 fsl_qdma->irq_base = platform_get_irq_byname(pdev, "qdma-queue0");1201 if (fsl_qdma->irq_base < 0)1202 return fsl_qdma->irq_base;1203 1204 fsl_qdma->feature = of_property_read_bool(np, "big-endian");1205 INIT_LIST_HEAD(&fsl_qdma->dma_dev.channels);1206 1207 for (i = 0; i < fsl_qdma->n_chans; i++) {1208 struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i];1209 1210 fsl_chan->qdma = fsl_qdma;1211 fsl_chan->queue = fsl_qdma->queue + i % (fsl_qdma->n_queues *1212 fsl_qdma->block_number);1213 fsl_chan->vchan.desc_free = fsl_qdma_free_desc;1214 vchan_init(&fsl_chan->vchan, &fsl_qdma->dma_dev);1215 }1216 1217 dma_cap_set(DMA_MEMCPY, fsl_qdma->dma_dev.cap_mask);1218 1219 fsl_qdma->dma_dev.dev = &pdev->dev;1220 fsl_qdma->dma_dev.device_free_chan_resources =1221 fsl_qdma_free_chan_resources;1222 fsl_qdma->dma_dev.device_alloc_chan_resources =1223 fsl_qdma_alloc_chan_resources;1224 fsl_qdma->dma_dev.device_tx_status = dma_cookie_status;1225 fsl_qdma->dma_dev.device_prep_dma_memcpy = fsl_qdma_prep_memcpy;1226 fsl_qdma->dma_dev.device_issue_pending = fsl_qdma_issue_pending;1227 fsl_qdma->dma_dev.device_synchronize = fsl_qdma_synchronize;1228 fsl_qdma->dma_dev.device_terminate_all = fsl_qdma_terminate_all;1229 1230 ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(40));1231 if (ret) {1232 dev_err(&pdev->dev, "dma_set_mask failure.\n");1233 return ret;1234 }1235 1236 platform_set_drvdata(pdev, fsl_qdma);1237 1238 ret = fsl_qdma_reg_init(fsl_qdma);1239 if (ret) {1240 dev_err(&pdev->dev, "Can't Initialize the qDMA engine.\n");1241 return ret;1242 }1243 1244 ret = fsl_qdma_irq_init(pdev, fsl_qdma);1245 if (ret)1246 return ret;1247 1248 ret = dma_async_device_register(&fsl_qdma->dma_dev);1249 if (ret) {1250 dev_err(&pdev->dev, "Can't register NXP Layerscape qDMA engine.\n");1251 return ret;1252 }1253 1254 return 0;1255}1256 1257static void fsl_qdma_cleanup_vchan(struct dma_device *dmadev)1258{1259 struct fsl_qdma_chan *chan, *_chan;1260 1261 list_for_each_entry_safe(chan, _chan,1262 &dmadev->channels, vchan.chan.device_node) {1263 list_del(&chan->vchan.chan.device_node);1264 tasklet_kill(&chan->vchan.task);1265 }1266}1267 1268static void fsl_qdma_remove(struct platform_device *pdev)1269{1270 struct device_node *np = pdev->dev.of_node;1271 struct fsl_qdma_engine *fsl_qdma = platform_get_drvdata(pdev);1272 1273 fsl_qdma_irq_exit(pdev, fsl_qdma);1274 fsl_qdma_cleanup_vchan(&fsl_qdma->dma_dev);1275 of_dma_controller_free(np);1276 dma_async_device_unregister(&fsl_qdma->dma_dev);1277}1278 1279static const struct of_device_id fsl_qdma_dt_ids[] = {1280 { .compatible = "fsl,ls1021a-qdma", },1281 { /* sentinel */ }1282};1283MODULE_DEVICE_TABLE(of, fsl_qdma_dt_ids);1284 1285static struct platform_driver fsl_qdma_driver = {1286 .driver = {1287 .name = "fsl-qdma",1288 .of_match_table = fsl_qdma_dt_ids,1289 },1290 .probe = fsl_qdma_probe,1291 .remove_new = fsl_qdma_remove,1292};1293 1294module_platform_driver(fsl_qdma_driver);1295 1296MODULE_ALIAS("platform:fsl-qdma");1297MODULE_LICENSE("GPL v2");1298MODULE_DESCRIPTION("NXP Layerscape qDMA engine driver");1299