841 lines · c
1// SPDX-License-Identifier: GPL-2.02// Copyright 2019 NXP3 4#include <linux/init.h>5#include <linux/module.h>6#include <linux/dmapool.h>7#include <linux/of_irq.h>8#include <linux/iommu.h>9#include <linux/sys_soc.h>10#include <linux/fsl/mc.h>11#include <soc/fsl/dpaa2-io.h>12 13#include "../virt-dma.h"14#include "dpdmai.h"15#include "dpaa2-qdma.h"16 17static bool smmu_disable = true;18 19static struct dpaa2_qdma_chan *to_dpaa2_qdma_chan(struct dma_chan *chan)20{21 return container_of(chan, struct dpaa2_qdma_chan, vchan.chan);22}23 24static struct dpaa2_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd)25{26 return container_of(vd, struct dpaa2_qdma_comp, vdesc);27}28 29static int dpaa2_qdma_alloc_chan_resources(struct dma_chan *chan)30{31 struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);32 struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma;33 struct device *dev = &dpaa2_qdma->priv->dpdmai_dev->dev;34 35 dpaa2_chan->fd_pool = dma_pool_create("fd_pool", dev,36 sizeof(struct dpaa2_fd),37 sizeof(struct dpaa2_fd), 0);38 if (!dpaa2_chan->fd_pool)39 goto err;40 41 dpaa2_chan->fl_pool =42 dma_pool_create("fl_pool", dev,43 sizeof(struct dpaa2_fl_entry) * 3,44 sizeof(struct dpaa2_fl_entry), 0);45 46 if (!dpaa2_chan->fl_pool)47 goto err_fd;48 49 dpaa2_chan->sdd_pool =50 dma_pool_create("sdd_pool", dev,51 sizeof(struct dpaa2_qdma_sd_d) * 2,52 sizeof(struct dpaa2_qdma_sd_d), 0);53 if (!dpaa2_chan->sdd_pool)54 goto err_fl;55 56 return dpaa2_qdma->desc_allocated++;57err_fl:58 dma_pool_destroy(dpaa2_chan->fl_pool);59err_fd:60 dma_pool_destroy(dpaa2_chan->fd_pool);61err:62 return -ENOMEM;63}64 65static void dpaa2_qdma_free_chan_resources(struct dma_chan *chan)66{67 struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);68 struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma;69 unsigned long flags;70 71 LIST_HEAD(head);72 73 spin_lock_irqsave(&dpaa2_chan->vchan.lock, flags);74 vchan_get_all_descriptors(&dpaa2_chan->vchan, &head);75 spin_unlock_irqrestore(&dpaa2_chan->vchan.lock, flags);76 77 vchan_dma_desc_free_list(&dpaa2_chan->vchan, &head);78 79 dpaa2_dpdmai_free_comp(dpaa2_chan, &dpaa2_chan->comp_used);80 dpaa2_dpdmai_free_comp(dpaa2_chan, &dpaa2_chan->comp_free);81 82 dma_pool_destroy(dpaa2_chan->fd_pool);83 dma_pool_destroy(dpaa2_chan->fl_pool);84 dma_pool_destroy(dpaa2_chan->sdd_pool);85 dpaa2_qdma->desc_allocated--;86}87 88/*89 * Request a command descriptor for enqueue.90 */91static struct dpaa2_qdma_comp *92dpaa2_qdma_request_desc(struct dpaa2_qdma_chan *dpaa2_chan)93{94 struct dpaa2_qdma_priv *qdma_priv = dpaa2_chan->qdma->priv;95 struct device *dev = &qdma_priv->dpdmai_dev->dev;96 struct dpaa2_qdma_comp *comp_temp = NULL;97 unsigned long flags;98 99 spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);100 if (list_empty(&dpaa2_chan->comp_free)) {101 spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);102 comp_temp = kzalloc(sizeof(*comp_temp), GFP_NOWAIT);103 if (!comp_temp)104 goto err;105 comp_temp->fd_virt_addr =106 dma_pool_alloc(dpaa2_chan->fd_pool, GFP_NOWAIT,107 &comp_temp->fd_bus_addr);108 if (!comp_temp->fd_virt_addr)109 goto err_comp;110 111 comp_temp->fl_virt_addr =112 dma_pool_alloc(dpaa2_chan->fl_pool, GFP_NOWAIT,113 &comp_temp->fl_bus_addr);114 if (!comp_temp->fl_virt_addr)115 goto err_fd_virt;116 117 comp_temp->desc_virt_addr =118 dma_pool_alloc(dpaa2_chan->sdd_pool, GFP_NOWAIT,119 &comp_temp->desc_bus_addr);120 if (!comp_temp->desc_virt_addr)121 goto err_fl_virt;122 123 comp_temp->qchan = dpaa2_chan;124 return comp_temp;125 }126 127 comp_temp = list_first_entry(&dpaa2_chan->comp_free,128 struct dpaa2_qdma_comp, list);129 list_del(&comp_temp->list);130 spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);131 132 comp_temp->qchan = dpaa2_chan;133 134 return comp_temp;135 136err_fl_virt:137 dma_pool_free(dpaa2_chan->fl_pool,138 comp_temp->fl_virt_addr,139 comp_temp->fl_bus_addr);140err_fd_virt:141 dma_pool_free(dpaa2_chan->fd_pool,142 comp_temp->fd_virt_addr,143 comp_temp->fd_bus_addr);144err_comp:145 kfree(comp_temp);146err:147 dev_err(dev, "Failed to request descriptor\n");148 return NULL;149}150 151static void152dpaa2_qdma_populate_fd(u32 format, struct dpaa2_qdma_comp *dpaa2_comp)153{154 struct dpaa2_fd *fd;155 156 fd = dpaa2_comp->fd_virt_addr;157 memset(fd, 0, sizeof(struct dpaa2_fd));158 159 /* fd populated */160 dpaa2_fd_set_addr(fd, dpaa2_comp->fl_bus_addr);161 162 /*163 * Bypass memory translation, Frame list format, short length disable164 * we need to disable BMT if fsl-mc use iova addr165 */166 if (smmu_disable)167 dpaa2_fd_set_bpid(fd, QMAN_FD_BMT_ENABLE);168 dpaa2_fd_set_format(fd, QMAN_FD_FMT_ENABLE | QMAN_FD_SL_DISABLE);169 170 dpaa2_fd_set_frc(fd, format | QDMA_SER_CTX);171}172 173/* first frame list for descriptor buffer */174static void175dpaa2_qdma_populate_first_framel(struct dpaa2_fl_entry *f_list,176 struct dpaa2_qdma_comp *dpaa2_comp,177 bool wrt_changed)178{179 struct dpaa2_qdma_sd_d *sdd;180 181 sdd = dpaa2_comp->desc_virt_addr;182 memset(sdd, 0, 2 * (sizeof(*sdd)));183 184 /* source descriptor CMD */185 sdd->cmd = cpu_to_le32(QDMA_SD_CMD_RDTTYPE_COHERENT);186 sdd++;187 188 /* dest descriptor CMD */189 if (wrt_changed)190 sdd->cmd = cpu_to_le32(LX2160_QDMA_DD_CMD_WRTTYPE_COHERENT);191 else192 sdd->cmd = cpu_to_le32(QDMA_DD_CMD_WRTTYPE_COHERENT);193 194 memset(f_list, 0, sizeof(struct dpaa2_fl_entry));195 196 /* first frame list to source descriptor */197 dpaa2_fl_set_addr(f_list, dpaa2_comp->desc_bus_addr);198 dpaa2_fl_set_len(f_list, 0x20);199 dpaa2_fl_set_format(f_list, QDMA_FL_FMT_SBF | QDMA_FL_SL_LONG);200 201 /* bypass memory translation */202 if (smmu_disable)203 f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE);204}205 206/* source and destination frame list */207static void208dpaa2_qdma_populate_frames(struct dpaa2_fl_entry *f_list,209 dma_addr_t dst, dma_addr_t src,210 size_t len, uint8_t fmt)211{212 /* source frame list to source buffer */213 memset(f_list, 0, sizeof(struct dpaa2_fl_entry));214 215 dpaa2_fl_set_addr(f_list, src);216 dpaa2_fl_set_len(f_list, len);217 218 /* single buffer frame or scatter gather frame */219 dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG));220 221 /* bypass memory translation */222 if (smmu_disable)223 f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE);224 225 f_list++;226 227 /* destination frame list to destination buffer */228 memset(f_list, 0, sizeof(struct dpaa2_fl_entry));229 230 dpaa2_fl_set_addr(f_list, dst);231 dpaa2_fl_set_len(f_list, len);232 dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG));233 /* single buffer frame or scatter gather frame */234 dpaa2_fl_set_final(f_list, QDMA_FL_F);235 /* bypass memory translation */236 if (smmu_disable)237 f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE);238}239 240static struct dma_async_tx_descriptor241*dpaa2_qdma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst,242 dma_addr_t src, size_t len, ulong flags)243{244 struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);245 struct dpaa2_qdma_engine *dpaa2_qdma;246 struct dpaa2_qdma_comp *dpaa2_comp;247 struct dpaa2_fl_entry *f_list;248 bool wrt_changed;249 250 dpaa2_qdma = dpaa2_chan->qdma;251 dpaa2_comp = dpaa2_qdma_request_desc(dpaa2_chan);252 if (!dpaa2_comp)253 return NULL;254 255 wrt_changed = (bool)dpaa2_qdma->qdma_wrtype_fixup;256 257 /* populate Frame descriptor */258 dpaa2_qdma_populate_fd(QDMA_FD_LONG_FORMAT, dpaa2_comp);259 260 f_list = dpaa2_comp->fl_virt_addr;261 262 /* first frame list for descriptor buffer (logn format) */263 dpaa2_qdma_populate_first_framel(f_list, dpaa2_comp, wrt_changed);264 265 f_list++;266 267 dpaa2_qdma_populate_frames(f_list, dst, src, len, QDMA_FL_FMT_SBF);268 269 return vchan_tx_prep(&dpaa2_chan->vchan, &dpaa2_comp->vdesc, flags);270}271 272static void dpaa2_qdma_issue_pending(struct dma_chan *chan)273{274 struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);275 struct dpaa2_qdma_comp *dpaa2_comp;276 struct virt_dma_desc *vdesc;277 struct dpaa2_fd *fd;278 unsigned long flags;279 int err;280 281 spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);282 spin_lock(&dpaa2_chan->vchan.lock);283 if (vchan_issue_pending(&dpaa2_chan->vchan)) {284 vdesc = vchan_next_desc(&dpaa2_chan->vchan);285 if (!vdesc)286 goto err_enqueue;287 dpaa2_comp = to_fsl_qdma_comp(vdesc);288 289 fd = dpaa2_comp->fd_virt_addr;290 291 list_del(&vdesc->node);292 list_add_tail(&dpaa2_comp->list, &dpaa2_chan->comp_used);293 294 err = dpaa2_io_service_enqueue_fq(NULL, dpaa2_chan->fqid, fd);295 if (err) {296 list_move_tail(&dpaa2_comp->list,297 &dpaa2_chan->comp_free);298 }299 }300err_enqueue:301 spin_unlock(&dpaa2_chan->vchan.lock);302 spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);303}304 305static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev)306{307 struct dpaa2_qdma_priv_per_prio *ppriv;308 struct device *dev = &ls_dev->dev;309 struct dpaa2_qdma_priv *priv;310 u8 prio_def = DPDMAI_PRIO_NUM;311 int err = -EINVAL;312 int i;313 314 priv = dev_get_drvdata(dev);315 316 priv->dev = dev;317 priv->dpqdma_id = ls_dev->obj_desc.id;318 319 /* Get the handle for the DPDMAI this interface is associate with */320 err = dpdmai_open(priv->mc_io, 0, priv->dpqdma_id, &ls_dev->mc_handle);321 if (err) {322 dev_err(dev, "dpdmai_open() failed\n");323 return err;324 }325 326 dev_dbg(dev, "Opened dpdmai object successfully\n");327 328 err = dpdmai_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,329 &priv->dpdmai_attr);330 if (err) {331 dev_err(dev, "dpdmai_get_attributes() failed\n");332 goto exit;333 }334 335 if (priv->dpdmai_attr.version.major > DPDMAI_VER_MAJOR) {336 err = -EINVAL;337 dev_err(dev, "DPDMAI major version mismatch\n"338 "Found %u.%u, supported version is %u.%u\n",339 priv->dpdmai_attr.version.major,340 priv->dpdmai_attr.version.minor,341 DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR);342 goto exit;343 }344 345 if (priv->dpdmai_attr.version.minor > DPDMAI_VER_MINOR) {346 err = -EINVAL;347 dev_err(dev, "DPDMAI minor version mismatch\n"348 "Found %u.%u, supported version is %u.%u\n",349 priv->dpdmai_attr.version.major,350 priv->dpdmai_attr.version.minor,351 DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR);352 goto exit;353 }354 355 priv->num_pairs = min(priv->dpdmai_attr.num_of_priorities, prio_def);356 ppriv = kcalloc(priv->num_pairs, sizeof(*ppriv), GFP_KERNEL);357 if (!ppriv) {358 err = -ENOMEM;359 goto exit;360 }361 priv->ppriv = ppriv;362 363 for (i = 0; i < priv->num_pairs; i++) {364 err = dpdmai_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle,365 i, 0, &priv->rx_queue_attr[i]);366 if (err) {367 dev_err(dev, "dpdmai_get_rx_queue() failed\n");368 goto exit;369 }370 ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid;371 372 err = dpdmai_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle,373 i, 0, &priv->tx_queue_attr[i]);374 if (err) {375 dev_err(dev, "dpdmai_get_tx_queue() failed\n");376 goto exit;377 }378 ppriv->req_fqid = priv->tx_queue_attr[i].fqid;379 ppriv->prio = DPAA2_QDMA_DEFAULT_PRIORITY;380 ppriv->priv = priv;381 ppriv++;382 }383 384 return 0;385exit:386 dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle);387 return err;388}389 390static void dpaa2_qdma_fqdan_cb(struct dpaa2_io_notification_ctx *ctx)391{392 struct dpaa2_qdma_priv_per_prio *ppriv = container_of(ctx,393 struct dpaa2_qdma_priv_per_prio, nctx);394 struct dpaa2_qdma_comp *dpaa2_comp, *_comp_tmp;395 struct dpaa2_qdma_priv *priv = ppriv->priv;396 u32 n_chans = priv->dpaa2_qdma->n_chans;397 struct dpaa2_qdma_chan *qchan;398 const struct dpaa2_fd *fd_eq;399 const struct dpaa2_fd *fd;400 struct dpaa2_dq *dq;401 int is_last = 0;402 int found;403 u8 status;404 int err;405 int i;406 407 do {408 err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid,409 ppriv->store);410 } while (err);411 412 while (!is_last) {413 do {414 dq = dpaa2_io_store_next(ppriv->store, &is_last);415 } while (!is_last && !dq);416 if (!dq) {417 dev_err(priv->dev, "FQID returned no valid frames!\n");418 continue;419 }420 421 /* obtain FD and process the error */422 fd = dpaa2_dq_fd(dq);423 424 status = dpaa2_fd_get_ctrl(fd) & 0xff;425 if (status)426 dev_err(priv->dev, "FD error occurred\n");427 found = 0;428 for (i = 0; i < n_chans; i++) {429 qchan = &priv->dpaa2_qdma->chans[i];430 spin_lock(&qchan->queue_lock);431 if (list_empty(&qchan->comp_used)) {432 spin_unlock(&qchan->queue_lock);433 continue;434 }435 list_for_each_entry_safe(dpaa2_comp, _comp_tmp,436 &qchan->comp_used, list) {437 fd_eq = dpaa2_comp->fd_virt_addr;438 439 if (le64_to_cpu(fd_eq->simple.addr) ==440 le64_to_cpu(fd->simple.addr)) {441 spin_lock(&qchan->vchan.lock);442 vchan_cookie_complete(&443 dpaa2_comp->vdesc);444 spin_unlock(&qchan->vchan.lock);445 found = 1;446 break;447 }448 }449 spin_unlock(&qchan->queue_lock);450 if (found)451 break;452 }453 }454 455 dpaa2_io_service_rearm(NULL, ctx);456}457 458static int __cold dpaa2_qdma_dpio_setup(struct dpaa2_qdma_priv *priv)459{460 struct dpaa2_qdma_priv_per_prio *ppriv;461 struct device *dev = priv->dev;462 int err = -EINVAL;463 int i, num;464 465 num = priv->num_pairs;466 ppriv = priv->ppriv;467 for (i = 0; i < num; i++) {468 ppriv->nctx.is_cdan = 0;469 ppriv->nctx.desired_cpu = DPAA2_IO_ANY_CPU;470 ppriv->nctx.id = ppriv->rsp_fqid;471 ppriv->nctx.cb = dpaa2_qdma_fqdan_cb;472 err = dpaa2_io_service_register(NULL, &ppriv->nctx, dev);473 if (err) {474 dev_err(dev, "Notification register failed\n");475 goto err_service;476 }477 478 ppriv->store =479 dpaa2_io_store_create(DPAA2_QDMA_STORE_SIZE, dev);480 if (!ppriv->store) {481 err = -ENOMEM;482 dev_err(dev, "dpaa2_io_store_create() failed\n");483 goto err_store;484 }485 486 ppriv++;487 }488 return 0;489 490err_store:491 dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev);492err_service:493 ppriv--;494 while (ppriv >= priv->ppriv) {495 dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev);496 dpaa2_io_store_destroy(ppriv->store);497 ppriv--;498 }499 return err;500}501 502static void dpaa2_dpmai_store_free(struct dpaa2_qdma_priv *priv)503{504 struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;505 int i;506 507 for (i = 0; i < priv->num_pairs; i++) {508 dpaa2_io_store_destroy(ppriv->store);509 ppriv++;510 }511}512 513static void dpaa2_dpdmai_dpio_free(struct dpaa2_qdma_priv *priv)514{515 struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;516 struct device *dev = priv->dev;517 int i;518 519 for (i = 0; i < priv->num_pairs; i++) {520 dpaa2_io_service_deregister(NULL, &ppriv->nctx, dev);521 ppriv++;522 }523}524 525static int __cold dpaa2_dpdmai_bind(struct dpaa2_qdma_priv *priv)526{527 struct dpdmai_rx_queue_cfg rx_queue_cfg;528 struct dpaa2_qdma_priv_per_prio *ppriv;529 struct device *dev = priv->dev;530 struct fsl_mc_device *ls_dev;531 int i, num;532 int err;533 534 ls_dev = to_fsl_mc_device(dev);535 num = priv->num_pairs;536 ppriv = priv->ppriv;537 for (i = 0; i < num; i++) {538 rx_queue_cfg.options = DPDMAI_QUEUE_OPT_USER_CTX |539 DPDMAI_QUEUE_OPT_DEST;540 rx_queue_cfg.user_ctx = ppriv->nctx.qman64;541 rx_queue_cfg.dest_cfg.dest_type = DPDMAI_DEST_DPIO;542 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;543 rx_queue_cfg.dest_cfg.priority = ppriv->prio;544 err = dpdmai_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle,545 rx_queue_cfg.dest_cfg.priority, 0,546 &rx_queue_cfg);547 if (err) {548 dev_err(dev, "dpdmai_set_rx_queue() failed\n");549 return err;550 }551 552 ppriv++;553 }554 555 return 0;556}557 558static int __cold dpaa2_dpdmai_dpio_unbind(struct dpaa2_qdma_priv *priv)559{560 struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;561 struct device *dev = priv->dev;562 struct fsl_mc_device *ls_dev;563 int err = 0;564 int i;565 566 ls_dev = to_fsl_mc_device(dev);567 568 for (i = 0; i < priv->num_pairs; i++) {569 ppriv->nctx.qman64 = 0;570 ppriv->nctx.dpio_id = 0;571 ppriv++;572 }573 574 err = dpdmai_reset(priv->mc_io, 0, ls_dev->mc_handle);575 if (err)576 dev_err(dev, "dpdmai_reset() failed\n");577 578 return err;579}580 581static void dpaa2_dpdmai_free_comp(struct dpaa2_qdma_chan *qchan,582 struct list_head *head)583{584 struct dpaa2_qdma_comp *comp_tmp, *_comp_tmp;585 unsigned long flags;586 587 list_for_each_entry_safe(comp_tmp, _comp_tmp,588 head, list) {589 spin_lock_irqsave(&qchan->queue_lock, flags);590 list_del(&comp_tmp->list);591 spin_unlock_irqrestore(&qchan->queue_lock, flags);592 dma_pool_free(qchan->fd_pool,593 comp_tmp->fd_virt_addr,594 comp_tmp->fd_bus_addr);595 dma_pool_free(qchan->fl_pool,596 comp_tmp->fl_virt_addr,597 comp_tmp->fl_bus_addr);598 dma_pool_free(qchan->sdd_pool,599 comp_tmp->desc_virt_addr,600 comp_tmp->desc_bus_addr);601 kfree(comp_tmp);602 }603}604 605static void dpaa2_dpdmai_free_channels(struct dpaa2_qdma_engine *dpaa2_qdma)606{607 struct dpaa2_qdma_chan *qchan;608 int num, i;609 610 num = dpaa2_qdma->n_chans;611 for (i = 0; i < num; i++) {612 qchan = &dpaa2_qdma->chans[i];613 dpaa2_dpdmai_free_comp(qchan, &qchan->comp_used);614 dpaa2_dpdmai_free_comp(qchan, &qchan->comp_free);615 dma_pool_destroy(qchan->fd_pool);616 dma_pool_destroy(qchan->fl_pool);617 dma_pool_destroy(qchan->sdd_pool);618 }619}620 621static void dpaa2_qdma_free_desc(struct virt_dma_desc *vdesc)622{623 struct dpaa2_qdma_comp *dpaa2_comp;624 struct dpaa2_qdma_chan *qchan;625 unsigned long flags;626 627 dpaa2_comp = to_fsl_qdma_comp(vdesc);628 qchan = dpaa2_comp->qchan;629 spin_lock_irqsave(&qchan->queue_lock, flags);630 list_move_tail(&dpaa2_comp->list, &qchan->comp_free);631 spin_unlock_irqrestore(&qchan->queue_lock, flags);632}633 634static int dpaa2_dpdmai_init_channels(struct dpaa2_qdma_engine *dpaa2_qdma)635{636 struct dpaa2_qdma_priv *priv = dpaa2_qdma->priv;637 struct dpaa2_qdma_chan *dpaa2_chan;638 int num = priv->num_pairs;639 int i;640 641 INIT_LIST_HEAD(&dpaa2_qdma->dma_dev.channels);642 for (i = 0; i < dpaa2_qdma->n_chans; i++) {643 dpaa2_chan = &dpaa2_qdma->chans[i];644 dpaa2_chan->qdma = dpaa2_qdma;645 dpaa2_chan->fqid = priv->tx_queue_attr[i % num].fqid;646 dpaa2_chan->vchan.desc_free = dpaa2_qdma_free_desc;647 vchan_init(&dpaa2_chan->vchan, &dpaa2_qdma->dma_dev);648 spin_lock_init(&dpaa2_chan->queue_lock);649 INIT_LIST_HEAD(&dpaa2_chan->comp_used);650 INIT_LIST_HEAD(&dpaa2_chan->comp_free);651 }652 return 0;653}654 655static int dpaa2_qdma_probe(struct fsl_mc_device *dpdmai_dev)656{657 struct device *dev = &dpdmai_dev->dev;658 struct dpaa2_qdma_engine *dpaa2_qdma;659 struct dpaa2_qdma_priv *priv;660 int err;661 662 priv = kzalloc(sizeof(*priv), GFP_KERNEL);663 if (!priv)664 return -ENOMEM;665 dev_set_drvdata(dev, priv);666 priv->dpdmai_dev = dpdmai_dev;667 668 priv->iommu_domain = iommu_get_domain_for_dev(dev);669 if (priv->iommu_domain)670 smmu_disable = false;671 672 /* obtain a MC portal */673 err = fsl_mc_portal_allocate(dpdmai_dev, 0, &priv->mc_io);674 if (err) {675 if (err == -ENXIO)676 err = -EPROBE_DEFER;677 else678 dev_err(dev, "MC portal allocation failed\n");679 goto err_mcportal;680 }681 682 /* DPDMAI initialization */683 err = dpaa2_qdma_setup(dpdmai_dev);684 if (err) {685 dev_err(dev, "dpaa2_dpdmai_setup() failed\n");686 goto err_dpdmai_setup;687 }688 689 /* DPIO */690 err = dpaa2_qdma_dpio_setup(priv);691 if (err) {692 dev_err(dev, "dpaa2_dpdmai_dpio_setup() failed\n");693 goto err_dpio_setup;694 }695 696 /* DPDMAI binding to DPIO */697 err = dpaa2_dpdmai_bind(priv);698 if (err) {699 dev_err(dev, "dpaa2_dpdmai_bind() failed\n");700 goto err_bind;701 }702 703 /* DPDMAI enable */704 err = dpdmai_enable(priv->mc_io, 0, dpdmai_dev->mc_handle);705 if (err) {706 dev_err(dev, "dpdmai_enable() failed\n");707 goto err_enable;708 }709 710 dpaa2_qdma = kzalloc(sizeof(*dpaa2_qdma), GFP_KERNEL);711 if (!dpaa2_qdma) {712 err = -ENOMEM;713 goto err_eng;714 }715 716 priv->dpaa2_qdma = dpaa2_qdma;717 dpaa2_qdma->priv = priv;718 719 dpaa2_qdma->desc_allocated = 0;720 dpaa2_qdma->n_chans = NUM_CH;721 722 dpaa2_dpdmai_init_channels(dpaa2_qdma);723 724 if (soc_device_match(soc_fixup_tuning))725 dpaa2_qdma->qdma_wrtype_fixup = true;726 else727 dpaa2_qdma->qdma_wrtype_fixup = false;728 729 dma_cap_set(DMA_PRIVATE, dpaa2_qdma->dma_dev.cap_mask);730 dma_cap_set(DMA_SLAVE, dpaa2_qdma->dma_dev.cap_mask);731 dma_cap_set(DMA_MEMCPY, dpaa2_qdma->dma_dev.cap_mask);732 733 dpaa2_qdma->dma_dev.dev = dev;734 dpaa2_qdma->dma_dev.device_alloc_chan_resources =735 dpaa2_qdma_alloc_chan_resources;736 dpaa2_qdma->dma_dev.device_free_chan_resources =737 dpaa2_qdma_free_chan_resources;738 dpaa2_qdma->dma_dev.device_tx_status = dma_cookie_status;739 dpaa2_qdma->dma_dev.device_prep_dma_memcpy = dpaa2_qdma_prep_memcpy;740 dpaa2_qdma->dma_dev.device_issue_pending = dpaa2_qdma_issue_pending;741 742 err = dma_async_device_register(&dpaa2_qdma->dma_dev);743 if (err) {744 dev_err(dev, "Can't register NXP QDMA engine.\n");745 goto err_dpaa2_qdma;746 }747 748 return 0;749 750err_dpaa2_qdma:751 kfree(dpaa2_qdma);752err_eng:753 dpdmai_disable(priv->mc_io, 0, dpdmai_dev->mc_handle);754err_enable:755 dpaa2_dpdmai_dpio_unbind(priv);756err_bind:757 dpaa2_dpmai_store_free(priv);758 dpaa2_dpdmai_dpio_free(priv);759err_dpio_setup:760 kfree(priv->ppriv);761 dpdmai_close(priv->mc_io, 0, dpdmai_dev->mc_handle);762err_dpdmai_setup:763 fsl_mc_portal_free(priv->mc_io);764err_mcportal:765 kfree(priv);766 dev_set_drvdata(dev, NULL);767 return err;768}769 770static void dpaa2_qdma_remove(struct fsl_mc_device *ls_dev)771{772 struct dpaa2_qdma_engine *dpaa2_qdma;773 struct dpaa2_qdma_priv *priv;774 struct device *dev;775 776 dev = &ls_dev->dev;777 priv = dev_get_drvdata(dev);778 dpaa2_qdma = priv->dpaa2_qdma;779 780 dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle);781 dpaa2_dpdmai_dpio_unbind(priv);782 dpaa2_dpmai_store_free(priv);783 dpaa2_dpdmai_dpio_free(priv);784 dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle);785 fsl_mc_portal_free(priv->mc_io);786 dev_set_drvdata(dev, NULL);787 dpaa2_dpdmai_free_channels(dpaa2_qdma);788 789 dma_async_device_unregister(&dpaa2_qdma->dma_dev);790 kfree(priv);791 kfree(dpaa2_qdma);792}793 794static void dpaa2_qdma_shutdown(struct fsl_mc_device *ls_dev)795{796 struct dpaa2_qdma_priv *priv;797 struct device *dev;798 799 dev = &ls_dev->dev;800 priv = dev_get_drvdata(dev);801 802 dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle);803 dpaa2_dpdmai_dpio_unbind(priv);804 dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle);805 dpdmai_destroy(priv->mc_io, 0, priv->dpqdma_id, ls_dev->mc_handle);806}807 808static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = {809 {810 .vendor = FSL_MC_VENDOR_FREESCALE,811 .obj_type = "dpdmai",812 },813 { .vendor = 0x0 }814};815 816static struct fsl_mc_driver dpaa2_qdma_driver = {817 .driver = {818 .name = "dpaa2-qdma",819 },820 .probe = dpaa2_qdma_probe,821 .remove = dpaa2_qdma_remove,822 .shutdown = dpaa2_qdma_shutdown,823 .match_id_table = dpaa2_qdma_id_table824};825 826static int __init dpaa2_qdma_driver_init(void)827{828 return fsl_mc_driver_register(&(dpaa2_qdma_driver));829}830late_initcall(dpaa2_qdma_driver_init);831 832static void __exit fsl_qdma_exit(void)833{834 fsl_mc_driver_unregister(&(dpaa2_qdma_driver));835}836module_exit(fsl_qdma_exit);837 838MODULE_ALIAS("platform:fsl-dpaa2-qdma");839MODULE_LICENSE("GPL v2");840MODULE_DESCRIPTION("NXP Layerscape DPAA2 qDMA engine driver");841