899 lines · c
1// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)2/*3 * Copyright 2014-2016 Freescale Semiconductor Inc.4 * Copyright 2016-2019 NXP5 *6 */7#include <linux/types.h>8#include <linux/fsl/mc.h>9#include <soc/fsl/dpaa2-io.h>10#include <linux/init.h>11#include <linux/module.h>12#include <linux/platform_device.h>13#include <linux/interrupt.h>14#include <linux/dma-mapping.h>15#include <linux/dim.h>16#include <linux/slab.h>17 18#include "dpio.h"19#include "qbman-portal.h"20 21struct dpaa2_io {22 struct dpaa2_io_desc dpio_desc;23 struct qbman_swp_desc swp_desc;24 struct qbman_swp *swp;25 struct list_head node;26 /* protect against multiple management commands */27 spinlock_t lock_mgmt_cmd;28 /* protect notifications list */29 spinlock_t lock_notifications;30 struct list_head notifications;31 struct device *dev;32 33 /* Net DIM */34 struct dim rx_dim;35 /* protect against concurrent Net DIM updates */36 spinlock_t dim_lock;37 u16 event_ctr;38 u64 bytes;39 u64 frames;40};41 42struct dpaa2_io_store {43 unsigned int max;44 dma_addr_t paddr;45 struct dpaa2_dq *vaddr;46 void *alloced_addr; /* unaligned value from kmalloc() */47 unsigned int idx; /* position of the next-to-be-returned entry */48 struct qbman_swp *swp; /* portal used to issue VDQCR */49 struct device *dev; /* device used for DMA mapping */50};51 52/* keep a per cpu array of DPIOs for fast access */53static struct dpaa2_io *dpio_by_cpu[NR_CPUS];54static struct list_head dpio_list = LIST_HEAD_INIT(dpio_list);55static DEFINE_SPINLOCK(dpio_list_lock);56 57static inline struct dpaa2_io *service_select_by_cpu(struct dpaa2_io *d,58 int cpu)59{60 if (d)61 return d;62 63 if (cpu != DPAA2_IO_ANY_CPU && cpu >= num_possible_cpus())64 return NULL;65 66 /*67 * If cpu == -1, choose the current cpu, with no guarantees about68 * potentially being migrated away.69 */70 if (cpu < 0)71 cpu = raw_smp_processor_id();72 73 /* If a specific cpu was requested, pick it up immediately */74 return dpio_by_cpu[cpu];75}76 77static inline struct dpaa2_io *service_select(struct dpaa2_io *d)78{79 if (d)80 return d;81 82 d = service_select_by_cpu(d, -1);83 if (d)84 return d;85 86 spin_lock(&dpio_list_lock);87 d = list_entry(dpio_list.next, struct dpaa2_io, node);88 list_del(&d->node);89 list_add_tail(&d->node, &dpio_list);90 spin_unlock(&dpio_list_lock);91 92 return d;93}94 95/**96 * dpaa2_io_service_select() - return a dpaa2_io service affined to this cpu97 * @cpu: the cpu id98 *99 * Return the affine dpaa2_io service, or NULL if there is no service affined100 * to the specified cpu. If DPAA2_IO_ANY_CPU is used, return the next available101 * service.102 */103struct dpaa2_io *dpaa2_io_service_select(int cpu)104{105 if (cpu == DPAA2_IO_ANY_CPU)106 return service_select(NULL);107 108 return service_select_by_cpu(NULL, cpu);109}110EXPORT_SYMBOL_GPL(dpaa2_io_service_select);111 112static void dpaa2_io_dim_work(struct work_struct *w)113{114 struct dim *dim = container_of(w, struct dim, work);115 struct dim_cq_moder moder =116 net_dim_get_rx_moderation(dim->mode, dim->profile_ix);117 struct dpaa2_io *d = container_of(dim, struct dpaa2_io, rx_dim);118 119 dpaa2_io_set_irq_coalescing(d, moder.usec);120 dim->state = DIM_START_MEASURE;121}122 123/**124 * dpaa2_io_create() - create a dpaa2_io object.125 * @desc: the dpaa2_io descriptor126 * @dev: the actual DPIO device127 *128 * Activates a "struct dpaa2_io" corresponding to the given config of an actual129 * DPIO object.130 *131 * Return a valid dpaa2_io object for success, or NULL for failure.132 */133struct dpaa2_io *dpaa2_io_create(const struct dpaa2_io_desc *desc,134 struct device *dev)135{136 struct dpaa2_io *obj = kmalloc(sizeof(*obj), GFP_KERNEL);137 u32 qman_256_cycles_per_ns;138 139 if (!obj)140 return NULL;141 142 /* check if CPU is out of range (-1 means any cpu) */143 if (desc->cpu != DPAA2_IO_ANY_CPU && desc->cpu >= num_possible_cpus()) {144 kfree(obj);145 return NULL;146 }147 148 obj->dpio_desc = *desc;149 obj->swp_desc.cena_bar = obj->dpio_desc.regs_cena;150 obj->swp_desc.cinh_bar = obj->dpio_desc.regs_cinh;151 obj->swp_desc.qman_clk = obj->dpio_desc.qman_clk;152 obj->swp_desc.qman_version = obj->dpio_desc.qman_version;153 154 /* Compute how many 256 QBMAN cycles fit into one ns. This is because155 * the interrupt timeout period register needs to be specified in QBMAN156 * clock cycles in increments of 256.157 */158 qman_256_cycles_per_ns = 256000 / (obj->swp_desc.qman_clk / 1000000);159 obj->swp_desc.qman_256_cycles_per_ns = qman_256_cycles_per_ns;160 obj->swp = qbman_swp_init(&obj->swp_desc);161 162 if (!obj->swp) {163 kfree(obj);164 return NULL;165 }166 167 INIT_LIST_HEAD(&obj->node);168 spin_lock_init(&obj->lock_mgmt_cmd);169 spin_lock_init(&obj->lock_notifications);170 spin_lock_init(&obj->dim_lock);171 INIT_LIST_HEAD(&obj->notifications);172 173 /* For now only enable DQRR interrupts */174 qbman_swp_interrupt_set_trigger(obj->swp,175 QBMAN_SWP_INTERRUPT_DQRI);176 qbman_swp_interrupt_clear_status(obj->swp, 0xffffffff);177 if (obj->dpio_desc.receives_notifications)178 qbman_swp_push_set(obj->swp, 0, 1);179 180 spin_lock(&dpio_list_lock);181 list_add_tail(&obj->node, &dpio_list);182 if (desc->cpu >= 0 && !dpio_by_cpu[desc->cpu])183 dpio_by_cpu[desc->cpu] = obj;184 spin_unlock(&dpio_list_lock);185 186 obj->dev = dev;187 188 memset(&obj->rx_dim, 0, sizeof(obj->rx_dim));189 INIT_WORK(&obj->rx_dim.work, dpaa2_io_dim_work);190 obj->event_ctr = 0;191 obj->bytes = 0;192 obj->frames = 0;193 194 return obj;195}196 197/**198 * dpaa2_io_down() - release the dpaa2_io object.199 * @d: the dpaa2_io object to be released.200 *201 * The "struct dpaa2_io" type can represent an individual DPIO object (as202 * described by "struct dpaa2_io_desc") or an instance of a "DPIO service",203 * which can be used to group/encapsulate multiple DPIO objects. In all cases,204 * each handle obtained should be released using this function.205 */206void dpaa2_io_down(struct dpaa2_io *d)207{208 spin_lock(&dpio_list_lock);209 dpio_by_cpu[d->dpio_desc.cpu] = NULL;210 list_del(&d->node);211 spin_unlock(&dpio_list_lock);212 213 kfree(d);214}215 216#define DPAA_POLL_MAX 32217 218/**219 * dpaa2_io_irq() - ISR for DPIO interrupts220 *221 * @obj: the given DPIO object.222 *223 * Return IRQ_HANDLED for success or IRQ_NONE if there224 * were no pending interrupts.225 */226irqreturn_t dpaa2_io_irq(struct dpaa2_io *obj)227{228 const struct dpaa2_dq *dq;229 int max = 0;230 struct qbman_swp *swp;231 u32 status;232 233 obj->event_ctr++;234 235 swp = obj->swp;236 status = qbman_swp_interrupt_read_status(swp);237 if (!status)238 return IRQ_NONE;239 240 dq = qbman_swp_dqrr_next(swp);241 while (dq) {242 if (qbman_result_is_SCN(dq)) {243 struct dpaa2_io_notification_ctx *ctx;244 u64 q64;245 246 q64 = qbman_result_SCN_ctx(dq);247 ctx = (void *)(uintptr_t)q64;248 ctx->cb(ctx);249 } else {250 pr_crit("fsl-mc-dpio: Unrecognised/ignored DQRR entry\n");251 }252 qbman_swp_dqrr_consume(swp, dq);253 ++max;254 if (max > DPAA_POLL_MAX)255 goto done;256 dq = qbman_swp_dqrr_next(swp);257 }258done:259 qbman_swp_interrupt_clear_status(swp, status);260 qbman_swp_interrupt_set_inhibit(swp, 0);261 return IRQ_HANDLED;262}263 264/**265 * dpaa2_io_get_cpu() - get the cpu associated with a given DPIO object266 *267 * @d: the given DPIO object.268 *269 * Return the cpu associated with the DPIO object270 */271int dpaa2_io_get_cpu(struct dpaa2_io *d)272{273 return d->dpio_desc.cpu;274}275EXPORT_SYMBOL(dpaa2_io_get_cpu);276 277/**278 * dpaa2_io_service_register() - Prepare for servicing of FQDAN or CDAN279 * notifications on the given DPIO service.280 * @d: the given DPIO service.281 * @ctx: the notification context.282 * @dev: the device that requests the register283 *284 * The caller should make the MC command to attach a DPAA2 object to285 * a DPIO after this function completes successfully. In that way:286 * (a) The DPIO service is "ready" to handle a notification arrival287 * (which might happen before the "attach" command to MC has288 * returned control of execution back to the caller)289 * (b) The DPIO service can provide back to the caller the 'dpio_id' and290 * 'qman64' parameters that it should pass along in the MC command291 * in order for the object to be configured to produce the right292 * notification fields to the DPIO service.293 *294 * Return 0 for success, or -ENODEV for failure.295 */296int dpaa2_io_service_register(struct dpaa2_io *d,297 struct dpaa2_io_notification_ctx *ctx,298 struct device *dev)299{300 struct device_link *link;301 unsigned long irqflags;302 303 d = service_select_by_cpu(d, ctx->desired_cpu);304 if (!d)305 return -ENODEV;306 307 link = device_link_add(dev, d->dev, DL_FLAG_AUTOREMOVE_CONSUMER);308 if (!link)309 return -EINVAL;310 311 ctx->dpio_id = d->dpio_desc.dpio_id;312 ctx->qman64 = (u64)(uintptr_t)ctx;313 ctx->dpio_private = d;314 spin_lock_irqsave(&d->lock_notifications, irqflags);315 list_add(&ctx->node, &d->notifications);316 spin_unlock_irqrestore(&d->lock_notifications, irqflags);317 318 /* Enable the generation of CDAN notifications */319 if (ctx->is_cdan)320 return qbman_swp_CDAN_set_context_enable(d->swp,321 (u16)ctx->id,322 ctx->qman64);323 return 0;324}325EXPORT_SYMBOL_GPL(dpaa2_io_service_register);326 327/**328 * dpaa2_io_service_deregister - The opposite of 'register'.329 * @service: the given DPIO service.330 * @ctx: the notification context.331 * @dev: the device that requests to be deregistered332 *333 * This function should be called only after sending the MC command to334 * to detach the notification-producing device from the DPIO.335 */336void dpaa2_io_service_deregister(struct dpaa2_io *service,337 struct dpaa2_io_notification_ctx *ctx,338 struct device *dev)339{340 struct dpaa2_io *d = ctx->dpio_private;341 unsigned long irqflags;342 343 if (ctx->is_cdan)344 qbman_swp_CDAN_disable(d->swp, (u16)ctx->id);345 346 spin_lock_irqsave(&d->lock_notifications, irqflags);347 list_del(&ctx->node);348 spin_unlock_irqrestore(&d->lock_notifications, irqflags);349 350}351EXPORT_SYMBOL_GPL(dpaa2_io_service_deregister);352 353/**354 * dpaa2_io_service_rearm() - Rearm the notification for the given DPIO service.355 * @d: the given DPIO service.356 * @ctx: the notification context.357 *358 * Once a FQDAN/CDAN has been produced, the corresponding FQ/channel is359 * considered "disarmed". Ie. the user can issue pull dequeue operations on that360 * traffic source for as long as it likes. Eventually it may wish to "rearm"361 * that source to allow it to produce another FQDAN/CDAN, that's what this362 * function achieves.363 *364 * Return 0 for success.365 */366int dpaa2_io_service_rearm(struct dpaa2_io *d,367 struct dpaa2_io_notification_ctx *ctx)368{369 unsigned long irqflags;370 int err;371 372 d = service_select_by_cpu(d, ctx->desired_cpu);373 if (!unlikely(d))374 return -ENODEV;375 376 spin_lock_irqsave(&d->lock_mgmt_cmd, irqflags);377 if (ctx->is_cdan)378 err = qbman_swp_CDAN_enable(d->swp, (u16)ctx->id);379 else380 err = qbman_swp_fq_schedule(d->swp, ctx->id);381 spin_unlock_irqrestore(&d->lock_mgmt_cmd, irqflags);382 383 return err;384}385EXPORT_SYMBOL_GPL(dpaa2_io_service_rearm);386 387/**388 * dpaa2_io_service_pull_fq() - pull dequeue functions from a fq.389 * @d: the given DPIO service.390 * @fqid: the given frame queue id.391 * @s: the dpaa2_io_store object for the result.392 *393 * Return 0 for success, or error code for failure.394 */395int dpaa2_io_service_pull_fq(struct dpaa2_io *d, u32 fqid,396 struct dpaa2_io_store *s)397{398 struct qbman_pull_desc pd;399 int err;400 401 qbman_pull_desc_clear(&pd);402 qbman_pull_desc_set_storage(&pd, s->vaddr, s->paddr, 1);403 qbman_pull_desc_set_numframes(&pd, (u8)s->max);404 qbman_pull_desc_set_fq(&pd, fqid);405 406 d = service_select(d);407 if (!d)408 return -ENODEV;409 s->swp = d->swp;410 err = qbman_swp_pull(d->swp, &pd);411 if (err)412 s->swp = NULL;413 414 return err;415}416EXPORT_SYMBOL(dpaa2_io_service_pull_fq);417 418/**419 * dpaa2_io_service_pull_channel() - pull dequeue functions from a channel.420 * @d: the given DPIO service.421 * @channelid: the given channel id.422 * @s: the dpaa2_io_store object for the result.423 *424 * Return 0 for success, or error code for failure.425 */426int dpaa2_io_service_pull_channel(struct dpaa2_io *d, u32 channelid,427 struct dpaa2_io_store *s)428{429 struct qbman_pull_desc pd;430 int err;431 432 qbman_pull_desc_clear(&pd);433 qbman_pull_desc_set_storage(&pd, s->vaddr, s->paddr, 1);434 qbman_pull_desc_set_numframes(&pd, (u8)s->max);435 qbman_pull_desc_set_channel(&pd, channelid, qbman_pull_type_prio);436 437 d = service_select(d);438 if (!d)439 return -ENODEV;440 441 s->swp = d->swp;442 err = qbman_swp_pull(d->swp, &pd);443 if (err)444 s->swp = NULL;445 446 return err;447}448EXPORT_SYMBOL_GPL(dpaa2_io_service_pull_channel);449 450/**451 * dpaa2_io_service_enqueue_fq() - Enqueue a frame to a frame queue.452 * @d: the given DPIO service.453 * @fqid: the given frame queue id.454 * @fd: the frame descriptor which is enqueued.455 *456 * Return 0 for successful enqueue, -EBUSY if the enqueue ring is not ready,457 * or -ENODEV if there is no dpio service.458 */459int dpaa2_io_service_enqueue_fq(struct dpaa2_io *d,460 u32 fqid,461 const struct dpaa2_fd *fd)462{463 struct qbman_eq_desc ed;464 465 d = service_select(d);466 if (!d)467 return -ENODEV;468 469 qbman_eq_desc_clear(&ed);470 qbman_eq_desc_set_no_orp(&ed, 0);471 qbman_eq_desc_set_fq(&ed, fqid);472 473 return qbman_swp_enqueue(d->swp, &ed, fd);474}475EXPORT_SYMBOL(dpaa2_io_service_enqueue_fq);476 477/**478 * dpaa2_io_service_enqueue_multiple_fq() - Enqueue multiple frames479 * to a frame queue using one fqid.480 * @d: the given DPIO service.481 * @fqid: the given frame queue id.482 * @fd: the frame descriptor which is enqueued.483 * @nb: number of frames to be enqueud484 *485 * Return 0 for successful enqueue, -EBUSY if the enqueue ring is not ready,486 * or -ENODEV if there is no dpio service.487 */488int dpaa2_io_service_enqueue_multiple_fq(struct dpaa2_io *d,489 u32 fqid,490 const struct dpaa2_fd *fd,491 int nb)492{493 struct qbman_eq_desc ed;494 495 d = service_select(d);496 if (!d)497 return -ENODEV;498 499 qbman_eq_desc_clear(&ed);500 qbman_eq_desc_set_no_orp(&ed, 0);501 qbman_eq_desc_set_fq(&ed, fqid);502 503 return qbman_swp_enqueue_multiple(d->swp, &ed, fd, NULL, nb);504}505EXPORT_SYMBOL(dpaa2_io_service_enqueue_multiple_fq);506 507/**508 * dpaa2_io_service_enqueue_multiple_desc_fq() - Enqueue multiple frames509 * to different frame queue using a list of fqids.510 * @d: the given DPIO service.511 * @fqid: the given list of frame queue ids.512 * @fd: the frame descriptor which is enqueued.513 * @nb: number of frames to be enqueud514 *515 * Return 0 for successful enqueue, -EBUSY if the enqueue ring is not ready,516 * or -ENODEV if there is no dpio service.517 */518int dpaa2_io_service_enqueue_multiple_desc_fq(struct dpaa2_io *d,519 u32 *fqid,520 const struct dpaa2_fd *fd,521 int nb)522{523 struct qbman_eq_desc *ed;524 int i, ret;525 526 ed = kcalloc(32, sizeof(struct qbman_eq_desc), GFP_KERNEL);527 if (!ed)528 return -ENOMEM;529 530 d = service_select(d);531 if (!d) {532 ret = -ENODEV;533 goto out;534 }535 536 for (i = 0; i < nb; i++) {537 qbman_eq_desc_clear(&ed[i]);538 qbman_eq_desc_set_no_orp(&ed[i], 0);539 qbman_eq_desc_set_fq(&ed[i], fqid[i]);540 }541 542 ret = qbman_swp_enqueue_multiple_desc(d->swp, &ed[0], fd, nb);543out:544 kfree(ed);545 return ret;546}547EXPORT_SYMBOL(dpaa2_io_service_enqueue_multiple_desc_fq);548 549/**550 * dpaa2_io_service_enqueue_qd() - Enqueue a frame to a QD.551 * @d: the given DPIO service.552 * @qdid: the given queuing destination id.553 * @prio: the given queuing priority.554 * @qdbin: the given queuing destination bin.555 * @fd: the frame descriptor which is enqueued.556 *557 * Return 0 for successful enqueue, or -EBUSY if the enqueue ring is not ready,558 * or -ENODEV if there is no dpio service.559 */560int dpaa2_io_service_enqueue_qd(struct dpaa2_io *d,561 u32 qdid, u8 prio, u16 qdbin,562 const struct dpaa2_fd *fd)563{564 struct qbman_eq_desc ed;565 566 d = service_select(d);567 if (!d)568 return -ENODEV;569 570 qbman_eq_desc_clear(&ed);571 qbman_eq_desc_set_no_orp(&ed, 0);572 qbman_eq_desc_set_qd(&ed, qdid, qdbin, prio);573 574 return qbman_swp_enqueue(d->swp, &ed, fd);575}576EXPORT_SYMBOL_GPL(dpaa2_io_service_enqueue_qd);577 578/**579 * dpaa2_io_service_release() - Release buffers to a buffer pool.580 * @d: the given DPIO object.581 * @bpid: the buffer pool id.582 * @buffers: the buffers to be released.583 * @num_buffers: the number of the buffers to be released.584 *585 * Return 0 for success, and negative error code for failure.586 */587int dpaa2_io_service_release(struct dpaa2_io *d,588 u16 bpid,589 const u64 *buffers,590 unsigned int num_buffers)591{592 struct qbman_release_desc rd;593 594 d = service_select(d);595 if (!d)596 return -ENODEV;597 598 qbman_release_desc_clear(&rd);599 qbman_release_desc_set_bpid(&rd, bpid);600 601 return qbman_swp_release(d->swp, &rd, buffers, num_buffers);602}603EXPORT_SYMBOL_GPL(dpaa2_io_service_release);604 605/**606 * dpaa2_io_service_acquire() - Acquire buffers from a buffer pool.607 * @d: the given DPIO object.608 * @bpid: the buffer pool id.609 * @buffers: the buffer addresses for acquired buffers.610 * @num_buffers: the expected number of the buffers to acquire.611 *612 * Return a negative error code if the command failed, otherwise it returns613 * the number of buffers acquired, which may be less than the number requested.614 * Eg. if the buffer pool is empty, this will return zero.615 */616int dpaa2_io_service_acquire(struct dpaa2_io *d,617 u16 bpid,618 u64 *buffers,619 unsigned int num_buffers)620{621 unsigned long irqflags;622 int err;623 624 d = service_select(d);625 if (!d)626 return -ENODEV;627 628 spin_lock_irqsave(&d->lock_mgmt_cmd, irqflags);629 err = qbman_swp_acquire(d->swp, bpid, buffers, num_buffers);630 spin_unlock_irqrestore(&d->lock_mgmt_cmd, irqflags);631 632 return err;633}634EXPORT_SYMBOL_GPL(dpaa2_io_service_acquire);635 636/*637 * 'Stores' are reusable memory blocks for holding dequeue results, and to638 * assist with parsing those results.639 */640 641/**642 * dpaa2_io_store_create() - Create the dma memory storage for dequeue result.643 * @max_frames: the maximum number of dequeued result for frames, must be <= 32.644 * @dev: the device to allow mapping/unmapping the DMAable region.645 *646 * The size of the storage is "max_frames*sizeof(struct dpaa2_dq)".647 * The 'dpaa2_io_store' returned is a DPIO service managed object.648 *649 * Return pointer to dpaa2_io_store struct for successfully created storage650 * memory, or NULL on error.651 */652struct dpaa2_io_store *dpaa2_io_store_create(unsigned int max_frames,653 struct device *dev)654{655 struct dpaa2_io_store *ret;656 size_t size;657 658 if (!max_frames || (max_frames > 32))659 return NULL;660 661 ret = kmalloc(sizeof(*ret), GFP_KERNEL);662 if (!ret)663 return NULL;664 665 ret->max = max_frames;666 size = max_frames * sizeof(struct dpaa2_dq) + 64;667 ret->alloced_addr = kzalloc(size, GFP_KERNEL);668 if (!ret->alloced_addr) {669 kfree(ret);670 return NULL;671 }672 673 ret->vaddr = PTR_ALIGN(ret->alloced_addr, 64);674 ret->paddr = dma_map_single(dev, ret->vaddr,675 sizeof(struct dpaa2_dq) * max_frames,676 DMA_FROM_DEVICE);677 if (dma_mapping_error(dev, ret->paddr)) {678 kfree(ret->alloced_addr);679 kfree(ret);680 return NULL;681 }682 683 ret->idx = 0;684 ret->dev = dev;685 686 return ret;687}688EXPORT_SYMBOL_GPL(dpaa2_io_store_create);689 690/**691 * dpaa2_io_store_destroy() - Frees the dma memory storage for dequeue692 * result.693 * @s: the storage memory to be destroyed.694 */695void dpaa2_io_store_destroy(struct dpaa2_io_store *s)696{697 dma_unmap_single(s->dev, s->paddr, sizeof(struct dpaa2_dq) * s->max,698 DMA_FROM_DEVICE);699 kfree(s->alloced_addr);700 kfree(s);701}702EXPORT_SYMBOL_GPL(dpaa2_io_store_destroy);703 704/**705 * dpaa2_io_store_next() - Determine when the next dequeue result is available.706 * @s: the dpaa2_io_store object.707 * @is_last: indicate whether this is the last frame in the pull command.708 *709 * When an object driver performs dequeues to a dpaa2_io_store, this function710 * can be used to determine when the next frame result is available. Once711 * this function returns non-NULL, a subsequent call to it will try to find712 * the next dequeue result.713 *714 * Note that if a pull-dequeue has a NULL result because the target FQ/channel715 * was empty, then this function will also return NULL (rather than expecting716 * the caller to always check for this. As such, "is_last" can be used to717 * differentiate between "end-of-empty-dequeue" and "still-waiting".718 *719 * Return dequeue result for a valid dequeue result, or NULL for empty dequeue.720 */721struct dpaa2_dq *dpaa2_io_store_next(struct dpaa2_io_store *s, int *is_last)722{723 int match;724 struct dpaa2_dq *ret = &s->vaddr[s->idx];725 726 match = qbman_result_has_new_result(s->swp, ret);727 if (!match) {728 *is_last = 0;729 return NULL;730 }731 732 s->idx++;733 734 if (dpaa2_dq_is_pull_complete(ret)) {735 *is_last = 1;736 s->idx = 0;737 /*738 * If we get an empty dequeue result to terminate a zero-results739 * vdqcr, return NULL to the caller rather than expecting him to740 * check non-NULL results every time.741 */742 if (!(dpaa2_dq_flags(ret) & DPAA2_DQ_STAT_VALIDFRAME))743 ret = NULL;744 } else {745 prefetch(&s->vaddr[s->idx]);746 *is_last = 0;747 }748 749 return ret;750}751EXPORT_SYMBOL_GPL(dpaa2_io_store_next);752 753/**754 * dpaa2_io_query_fq_count() - Get the frame and byte count for a given fq.755 * @d: the given DPIO object.756 * @fqid: the id of frame queue to be queried.757 * @fcnt: the queried frame count.758 * @bcnt: the queried byte count.759 *760 * Knowing the FQ count at run-time can be useful in debugging situations.761 * The instantaneous frame- and byte-count are hereby returned.762 *763 * Return 0 for a successful query, and negative error code if query fails.764 */765int dpaa2_io_query_fq_count(struct dpaa2_io *d, u32 fqid,766 u32 *fcnt, u32 *bcnt)767{768 struct qbman_fq_query_np_rslt state;769 struct qbman_swp *swp;770 unsigned long irqflags;771 int ret;772 773 d = service_select(d);774 if (!d)775 return -ENODEV;776 777 swp = d->swp;778 spin_lock_irqsave(&d->lock_mgmt_cmd, irqflags);779 ret = qbman_fq_query_state(swp, fqid, &state);780 spin_unlock_irqrestore(&d->lock_mgmt_cmd, irqflags);781 if (ret)782 return ret;783 *fcnt = qbman_fq_state_frame_count(&state);784 *bcnt = qbman_fq_state_byte_count(&state);785 786 return 0;787}788EXPORT_SYMBOL_GPL(dpaa2_io_query_fq_count);789 790/**791 * dpaa2_io_query_bp_count() - Query the number of buffers currently in a792 * buffer pool.793 * @d: the given DPIO object.794 * @bpid: the index of buffer pool to be queried.795 * @num: the queried number of buffers in the buffer pool.796 *797 * Return 0 for a successful query, and negative error code if query fails.798 */799int dpaa2_io_query_bp_count(struct dpaa2_io *d, u16 bpid, u32 *num)800{801 struct qbman_bp_query_rslt state;802 struct qbman_swp *swp;803 unsigned long irqflags;804 int ret;805 806 d = service_select(d);807 if (!d)808 return -ENODEV;809 810 swp = d->swp;811 spin_lock_irqsave(&d->lock_mgmt_cmd, irqflags);812 ret = qbman_bp_query(swp, bpid, &state);813 spin_unlock_irqrestore(&d->lock_mgmt_cmd, irqflags);814 if (ret)815 return ret;816 *num = qbman_bp_info_num_free_bufs(&state);817 return 0;818}819EXPORT_SYMBOL_GPL(dpaa2_io_query_bp_count);820 821/**822 * dpaa2_io_set_irq_coalescing() - Set new IRQ coalescing values823 * @d: the given DPIO object824 * @irq_holdoff: interrupt holdoff (timeout) period in us825 *826 * Return 0 for success, or negative error code on error.827 */828int dpaa2_io_set_irq_coalescing(struct dpaa2_io *d, u32 irq_holdoff)829{830 struct qbman_swp *swp = d->swp;831 832 return qbman_swp_set_irq_coalescing(swp, swp->dqrr.dqrr_size - 1,833 irq_holdoff);834}835EXPORT_SYMBOL(dpaa2_io_set_irq_coalescing);836 837/**838 * dpaa2_io_get_irq_coalescing() - Get the current IRQ coalescing parameters839 * @d: the given DPIO object840 * @irq_holdoff: interrupt holdoff (timeout) period in us841 */842void dpaa2_io_get_irq_coalescing(struct dpaa2_io *d, u32 *irq_holdoff)843{844 struct qbman_swp *swp = d->swp;845 846 qbman_swp_get_irq_coalescing(swp, NULL, irq_holdoff);847}848EXPORT_SYMBOL(dpaa2_io_get_irq_coalescing);849 850/**851 * dpaa2_io_set_adaptive_coalescing() - Enable/disable adaptive coalescing852 * @d: the given DPIO object853 * @use_adaptive_rx_coalesce: adaptive coalescing state854 */855void dpaa2_io_set_adaptive_coalescing(struct dpaa2_io *d,856 int use_adaptive_rx_coalesce)857{858 d->swp->use_adaptive_rx_coalesce = use_adaptive_rx_coalesce;859}860EXPORT_SYMBOL(dpaa2_io_set_adaptive_coalescing);861 862/**863 * dpaa2_io_get_adaptive_coalescing() - Query adaptive coalescing state864 * @d: the given DPIO object865 *866 * Return 1 when adaptive coalescing is enabled on the DPIO object and 0867 * otherwise.868 */869int dpaa2_io_get_adaptive_coalescing(struct dpaa2_io *d)870{871 return d->swp->use_adaptive_rx_coalesce;872}873EXPORT_SYMBOL(dpaa2_io_get_adaptive_coalescing);874 875/**876 * dpaa2_io_update_net_dim() - Update Net DIM877 * @d: the given DPIO object878 * @frames: how many frames have been dequeued by the user since the last call879 * @bytes: how many bytes have been dequeued by the user since the last call880 */881void dpaa2_io_update_net_dim(struct dpaa2_io *d, __u64 frames, __u64 bytes)882{883 struct dim_sample dim_sample = {};884 885 if (!d->swp->use_adaptive_rx_coalesce)886 return;887 888 spin_lock(&d->dim_lock);889 890 d->bytes += bytes;891 d->frames += frames;892 893 dim_update_sample(d->event_ctr, d->frames, d->bytes, &dim_sample);894 net_dim(&d->rx_dim, dim_sample);895 896 spin_unlock(&d->dim_lock);897}898EXPORT_SYMBOL(dpaa2_io_update_net_dim);899