820 lines · c
1/* Copyright 2008 - 2016 Freescale Semiconductor, Inc.2 *3 * Redistribution and use in source and binary forms, with or without4 * modification, are permitted provided that the following conditions are met:5 * * Redistributions of source code must retain the above copyright6 * notice, this list of conditions and the following disclaimer.7 * * Redistributions in binary form must reproduce the above copyright8 * notice, this list of conditions and the following disclaimer in the9 * documentation and/or other materials provided with the distribution.10 * * Neither the name of Freescale Semiconductor nor the11 * names of its contributors may be used to endorse or promote products12 * derived from this software without specific prior written permission.13 *14 * ALTERNATIVELY, this software may be distributed under the terms of the15 * GNU General Public License ("GPL") as published by the Free Software16 * Foundation, either version 2 of that License or (at your option) any17 * later version.18 *19 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY20 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE22 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND26 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS28 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.29 */30 31#include "bman_priv.h"32 33#define IRQNAME "BMan portal %d"34#define MAX_IRQNAME 16 /* big enough for "BMan portal %d" */35 36/* Portal register assists */37 38#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)39/* Cache-inhibited register offsets */40#define BM_REG_RCR_PI_CINH 0x300041#define BM_REG_RCR_CI_CINH 0x310042#define BM_REG_RCR_ITR 0x320043#define BM_REG_CFG 0x330044#define BM_REG_SCN(n) (0x3400 + ((n) << 6))45#define BM_REG_ISR 0x3e0046#define BM_REG_IER 0x3e4047#define BM_REG_ISDR 0x3e8048#define BM_REG_IIR 0x3ec049 50/* Cache-enabled register offsets */51#define BM_CL_CR 0x000052#define BM_CL_RR0 0x010053#define BM_CL_RR1 0x014054#define BM_CL_RCR 0x100055#define BM_CL_RCR_PI_CENA 0x300056#define BM_CL_RCR_CI_CENA 0x310057 58#else59/* Cache-inhibited register offsets */60#define BM_REG_RCR_PI_CINH 0x000061#define BM_REG_RCR_CI_CINH 0x000462#define BM_REG_RCR_ITR 0x000863#define BM_REG_CFG 0x010064#define BM_REG_SCN(n) (0x0200 + ((n) << 2))65#define BM_REG_ISR 0x0e0066#define BM_REG_IER 0x0e0467#define BM_REG_ISDR 0x0e0868#define BM_REG_IIR 0x0e0c69 70/* Cache-enabled register offsets */71#define BM_CL_CR 0x000072#define BM_CL_RR0 0x010073#define BM_CL_RR1 0x014074#define BM_CL_RCR 0x100075#define BM_CL_RCR_PI_CENA 0x300076#define BM_CL_RCR_CI_CENA 0x310077#endif78 79/*80 * Portal modes.81 * Enum types;82 * pmode == production mode83 * cmode == consumption mode,84 * Enum values use 3 letter codes. First letter matches the portal mode,85 * remaining two letters indicate;86 * ci == cache-inhibited portal register87 * ce == cache-enabled portal register88 * vb == in-band valid-bit (cache-enabled)89 */90enum bm_rcr_pmode { /* matches BCSP_CFG::RPM */91 bm_rcr_pci = 0, /* PI index, cache-inhibited */92 bm_rcr_pce = 1, /* PI index, cache-enabled */93 bm_rcr_pvb = 2 /* valid-bit */94};95enum bm_rcr_cmode { /* s/w-only */96 bm_rcr_cci, /* CI index, cache-inhibited */97 bm_rcr_cce /* CI index, cache-enabled */98};99 100 101/* --- Portal structures --- */102 103#define BM_RCR_SIZE 8104 105/* Release Command */106struct bm_rcr_entry {107 union {108 struct {109 u8 _ncw_verb; /* writes to this are non-coherent */110 u8 bpid; /* used with BM_RCR_VERB_CMD_BPID_SINGLE */111 u8 __reserved1[62];112 };113 struct bm_buffer bufs[8];114 };115};116#define BM_RCR_VERB_VBIT 0x80117#define BM_RCR_VERB_CMD_MASK 0x70 /* one of two values; */118#define BM_RCR_VERB_CMD_BPID_SINGLE 0x20119#define BM_RCR_VERB_CMD_BPID_MULTI 0x30120#define BM_RCR_VERB_BUFCOUNT_MASK 0x0f /* values 1..8 */121 122struct bm_rcr {123 struct bm_rcr_entry *ring, *cursor;124 u8 ci, available, ithresh, vbit;125#ifdef CONFIG_FSL_DPAA_CHECKING126 u32 busy;127 enum bm_rcr_pmode pmode;128 enum bm_rcr_cmode cmode;129#endif130};131 132/* MC (Management Command) command */133struct bm_mc_command {134 u8 _ncw_verb; /* writes to this are non-coherent */135 u8 bpid; /* used by acquire command */136 u8 __reserved[62];137};138#define BM_MCC_VERB_VBIT 0x80139#define BM_MCC_VERB_CMD_MASK 0x70 /* where the verb contains; */140#define BM_MCC_VERB_CMD_ACQUIRE 0x10141#define BM_MCC_VERB_CMD_QUERY 0x40142#define BM_MCC_VERB_ACQUIRE_BUFCOUNT 0x0f /* values 1..8 go here */143 144/* MC result, Acquire and Query Response */145union bm_mc_result {146 struct {147 u8 verb;148 u8 bpid;149 u8 __reserved[62];150 };151 struct bm_buffer bufs[8];152};153#define BM_MCR_VERB_VBIT 0x80154#define BM_MCR_VERB_CMD_MASK BM_MCC_VERB_CMD_MASK155#define BM_MCR_VERB_CMD_ACQUIRE BM_MCC_VERB_CMD_ACQUIRE156#define BM_MCR_VERB_CMD_QUERY BM_MCC_VERB_CMD_QUERY157#define BM_MCR_VERB_CMD_ERR_INVALID 0x60158#define BM_MCR_VERB_CMD_ERR_ECC 0x70159#define BM_MCR_VERB_ACQUIRE_BUFCOUNT BM_MCC_VERB_ACQUIRE_BUFCOUNT /* 0..8 */160#define BM_MCR_TIMEOUT 10000 /* us */161 162struct bm_mc {163 struct bm_mc_command *cr;164 union bm_mc_result *rr;165 u8 rridx, vbit;166#ifdef CONFIG_FSL_DPAA_CHECKING167 enum {168 /* Can only be _mc_start()ed */169 mc_idle,170 /* Can only be _mc_commit()ed or _mc_abort()ed */171 mc_user,172 /* Can only be _mc_retry()ed */173 mc_hw174 } state;175#endif176};177 178struct bm_addr {179 void *ce; /* cache-enabled */180 __be32 *ce_be; /* Same as above but for direct access */181 void __iomem *ci; /* cache-inhibited */182};183 184struct bm_portal {185 struct bm_addr addr;186 struct bm_rcr rcr;187 struct bm_mc mc;188} ____cacheline_aligned;189 190/* Cache-inhibited register access. */191static inline u32 bm_in(struct bm_portal *p, u32 offset)192{193 return ioread32be(p->addr.ci + offset);194}195 196static inline void bm_out(struct bm_portal *p, u32 offset, u32 val)197{198 iowrite32be(val, p->addr.ci + offset);199}200 201/* Cache Enabled Portal Access */202static inline void bm_cl_invalidate(struct bm_portal *p, u32 offset)203{204 dpaa_invalidate(p->addr.ce + offset);205}206 207static inline void bm_cl_touch_ro(struct bm_portal *p, u32 offset)208{209 dpaa_touch_ro(p->addr.ce + offset);210}211 212static inline u32 bm_ce_in(struct bm_portal *p, u32 offset)213{214 return be32_to_cpu(*(p->addr.ce_be + (offset/4)));215}216 217struct bman_portal {218 struct bm_portal p;219 /* interrupt sources processed by portal_isr(), configurable */220 unsigned long irq_sources;221 /* probing time config params for cpu-affine portals */222 const struct bm_portal_config *config;223 char irqname[MAX_IRQNAME];224};225 226static cpumask_t affine_mask;227static DEFINE_SPINLOCK(affine_mask_lock);228static DEFINE_PER_CPU(struct bman_portal, bman_affine_portal);229 230static inline struct bman_portal *get_affine_portal(void)231{232 return &get_cpu_var(bman_affine_portal);233}234 235static inline void put_affine_portal(void)236{237 put_cpu_var(bman_affine_portal);238}239 240/*241 * This object type refers to a pool, it isn't *the* pool. There may be242 * more than one such object per BMan buffer pool, eg. if different users of the243 * pool are operating via different portals.244 */245struct bman_pool {246 /* index of the buffer pool to encapsulate (0-63) */247 u32 bpid;248 /* Used for hash-table admin when using depletion notifications. */249 struct bman_portal *portal;250 struct bman_pool *next;251};252 253static u32 poll_portal_slow(struct bman_portal *p, u32 is);254 255static irqreturn_t portal_isr(int irq, void *ptr)256{257 struct bman_portal *p = ptr;258 struct bm_portal *portal = &p->p;259 u32 clear = p->irq_sources;260 u32 is = bm_in(portal, BM_REG_ISR) & p->irq_sources;261 262 if (unlikely(!is))263 return IRQ_NONE;264 265 clear |= poll_portal_slow(p, is);266 bm_out(portal, BM_REG_ISR, clear);267 return IRQ_HANDLED;268}269 270/* --- RCR API --- */271 272#define RCR_SHIFT ilog2(sizeof(struct bm_rcr_entry))273#define RCR_CARRY (uintptr_t)(BM_RCR_SIZE << RCR_SHIFT)274 275/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */276static struct bm_rcr_entry *rcr_carryclear(struct bm_rcr_entry *p)277{278 uintptr_t addr = (uintptr_t)p;279 280 addr &= ~RCR_CARRY;281 282 return (struct bm_rcr_entry *)addr;283}284 285#ifdef CONFIG_FSL_DPAA_CHECKING286/* Bit-wise logic to convert a ring pointer to a ring index */287static int rcr_ptr2idx(struct bm_rcr_entry *e)288{289 return ((uintptr_t)e >> RCR_SHIFT) & (BM_RCR_SIZE - 1);290}291#endif292 293/* Increment the 'cursor' ring pointer, taking 'vbit' into account */294static inline void rcr_inc(struct bm_rcr *rcr)295{296 /* increment to the next RCR pointer and handle overflow and 'vbit' */297 struct bm_rcr_entry *partial = rcr->cursor + 1;298 299 rcr->cursor = rcr_carryclear(partial);300 if (partial != rcr->cursor)301 rcr->vbit ^= BM_RCR_VERB_VBIT;302}303 304static int bm_rcr_get_avail(struct bm_portal *portal)305{306 struct bm_rcr *rcr = &portal->rcr;307 308 return rcr->available;309}310 311static int bm_rcr_get_fill(struct bm_portal *portal)312{313 struct bm_rcr *rcr = &portal->rcr;314 315 return BM_RCR_SIZE - 1 - rcr->available;316}317 318static void bm_rcr_set_ithresh(struct bm_portal *portal, u8 ithresh)319{320 struct bm_rcr *rcr = &portal->rcr;321 322 rcr->ithresh = ithresh;323 bm_out(portal, BM_REG_RCR_ITR, ithresh);324}325 326static void bm_rcr_cce_prefetch(struct bm_portal *portal)327{328 __maybe_unused struct bm_rcr *rcr = &portal->rcr;329 330 DPAA_ASSERT(rcr->cmode == bm_rcr_cce);331 bm_cl_touch_ro(portal, BM_CL_RCR_CI_CENA);332}333 334static u8 bm_rcr_cce_update(struct bm_portal *portal)335{336 struct bm_rcr *rcr = &portal->rcr;337 u8 diff, old_ci = rcr->ci;338 339 DPAA_ASSERT(rcr->cmode == bm_rcr_cce);340 rcr->ci = bm_ce_in(portal, BM_CL_RCR_CI_CENA) & (BM_RCR_SIZE - 1);341 bm_cl_invalidate(portal, BM_CL_RCR_CI_CENA);342 diff = dpaa_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);343 rcr->available += diff;344 return diff;345}346 347static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)348{349 struct bm_rcr *rcr = &portal->rcr;350 351 DPAA_ASSERT(!rcr->busy);352 if (!rcr->available)353 return NULL;354#ifdef CONFIG_FSL_DPAA_CHECKING355 rcr->busy = 1;356#endif357 dpaa_zero(rcr->cursor);358 return rcr->cursor;359}360 361static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb)362{363 struct bm_rcr *rcr = &portal->rcr;364 struct bm_rcr_entry *rcursor;365 366 DPAA_ASSERT(rcr->busy);367 DPAA_ASSERT(rcr->pmode == bm_rcr_pvb);368 DPAA_ASSERT(rcr->available >= 1);369 dma_wmb();370 rcursor = rcr->cursor;371 rcursor->_ncw_verb = myverb | rcr->vbit;372 dpaa_flush(rcursor);373 rcr_inc(rcr);374 rcr->available--;375#ifdef CONFIG_FSL_DPAA_CHECKING376 rcr->busy = 0;377#endif378}379 380static int bm_rcr_init(struct bm_portal *portal, enum bm_rcr_pmode pmode,381 enum bm_rcr_cmode cmode)382{383 struct bm_rcr *rcr = &portal->rcr;384 u32 cfg;385 u8 pi;386 387 rcr->ring = portal->addr.ce + BM_CL_RCR;388 rcr->ci = bm_in(portal, BM_REG_RCR_CI_CINH) & (BM_RCR_SIZE - 1);389 pi = bm_in(portal, BM_REG_RCR_PI_CINH) & (BM_RCR_SIZE - 1);390 rcr->cursor = rcr->ring + pi;391 rcr->vbit = (bm_in(portal, BM_REG_RCR_PI_CINH) & BM_RCR_SIZE) ?392 BM_RCR_VERB_VBIT : 0;393 rcr->available = BM_RCR_SIZE - 1394 - dpaa_cyc_diff(BM_RCR_SIZE, rcr->ci, pi);395 rcr->ithresh = bm_in(portal, BM_REG_RCR_ITR);396#ifdef CONFIG_FSL_DPAA_CHECKING397 rcr->busy = 0;398 rcr->pmode = pmode;399 rcr->cmode = cmode;400#endif401 cfg = (bm_in(portal, BM_REG_CFG) & 0xffffffe0)402 | (pmode & 0x3); /* BCSP_CFG::RPM */403 bm_out(portal, BM_REG_CFG, cfg);404 return 0;405}406 407static void bm_rcr_finish(struct bm_portal *portal)408{409#ifdef CONFIG_FSL_DPAA_CHECKING410 struct bm_rcr *rcr = &portal->rcr;411 int i;412 413 DPAA_ASSERT(!rcr->busy);414 415 i = bm_in(portal, BM_REG_RCR_PI_CINH) & (BM_RCR_SIZE - 1);416 if (i != rcr_ptr2idx(rcr->cursor))417 pr_crit("losing uncommitted RCR entries\n");418 419 i = bm_in(portal, BM_REG_RCR_CI_CINH) & (BM_RCR_SIZE - 1);420 if (i != rcr->ci)421 pr_crit("missing existing RCR completions\n");422 if (rcr->ci != rcr_ptr2idx(rcr->cursor))423 pr_crit("RCR destroyed unquiesced\n");424#endif425}426 427/* --- Management command API --- */428static int bm_mc_init(struct bm_portal *portal)429{430 struct bm_mc *mc = &portal->mc;431 432 mc->cr = portal->addr.ce + BM_CL_CR;433 mc->rr = portal->addr.ce + BM_CL_RR0;434 mc->rridx = (mc->cr->_ncw_verb & BM_MCC_VERB_VBIT) ?435 0 : 1;436 mc->vbit = mc->rridx ? BM_MCC_VERB_VBIT : 0;437#ifdef CONFIG_FSL_DPAA_CHECKING438 mc->state = mc_idle;439#endif440 return 0;441}442 443static void bm_mc_finish(struct bm_portal *portal)444{445#ifdef CONFIG_FSL_DPAA_CHECKING446 struct bm_mc *mc = &portal->mc;447 448 DPAA_ASSERT(mc->state == mc_idle);449 if (mc->state != mc_idle)450 pr_crit("Losing incomplete MC command\n");451#endif452}453 454static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal)455{456 struct bm_mc *mc = &portal->mc;457 458 DPAA_ASSERT(mc->state == mc_idle);459#ifdef CONFIG_FSL_DPAA_CHECKING460 mc->state = mc_user;461#endif462 dpaa_zero(mc->cr);463 return mc->cr;464}465 466static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb)467{468 struct bm_mc *mc = &portal->mc;469 union bm_mc_result *rr = mc->rr + mc->rridx;470 471 DPAA_ASSERT(mc->state == mc_user);472 dma_wmb();473 mc->cr->_ncw_verb = myverb | mc->vbit;474 dpaa_flush(mc->cr);475 dpaa_invalidate_touch_ro(rr);476#ifdef CONFIG_FSL_DPAA_CHECKING477 mc->state = mc_hw;478#endif479}480 481static inline union bm_mc_result *bm_mc_result(struct bm_portal *portal)482{483 struct bm_mc *mc = &portal->mc;484 union bm_mc_result *rr = mc->rr + mc->rridx;485 486 DPAA_ASSERT(mc->state == mc_hw);487 /*488 * The inactive response register's verb byte always returns zero until489 * its command is submitted and completed. This includes the valid-bit,490 * in case you were wondering...491 */492 if (!rr->verb) {493 dpaa_invalidate_touch_ro(rr);494 return NULL;495 }496 mc->rridx ^= 1;497 mc->vbit ^= BM_MCC_VERB_VBIT;498#ifdef CONFIG_FSL_DPAA_CHECKING499 mc->state = mc_idle;500#endif501 return rr;502}503 504static inline int bm_mc_result_timeout(struct bm_portal *portal,505 union bm_mc_result **mcr)506{507 int timeout = BM_MCR_TIMEOUT;508 509 do {510 *mcr = bm_mc_result(portal);511 if (*mcr)512 break;513 udelay(1);514 } while (--timeout);515 516 return timeout;517}518 519/* Disable all BSCN interrupts for the portal */520static void bm_isr_bscn_disable(struct bm_portal *portal)521{522 bm_out(portal, BM_REG_SCN(0), 0);523 bm_out(portal, BM_REG_SCN(1), 0);524}525 526static int bman_create_portal(struct bman_portal *portal,527 const struct bm_portal_config *c)528{529 struct bm_portal *p;530 int ret;531 532 p = &portal->p;533 /*534 * prep the low-level portal struct with the mapped addresses from the535 * config, everything that follows depends on it and "config" is more536 * for (de)reference...537 */538 p->addr.ce = c->addr_virt_ce;539 p->addr.ce_be = c->addr_virt_ce;540 p->addr.ci = c->addr_virt_ci;541 if (bm_rcr_init(p, bm_rcr_pvb, bm_rcr_cce)) {542 dev_err(c->dev, "RCR initialisation failed\n");543 goto fail_rcr;544 }545 if (bm_mc_init(p)) {546 dev_err(c->dev, "MC initialisation failed\n");547 goto fail_mc;548 }549 /*550 * Default to all BPIDs disabled, we enable as required at551 * run-time.552 */553 bm_isr_bscn_disable(p);554 555 /* Write-to-clear any stale interrupt status bits */556 bm_out(p, BM_REG_ISDR, 0xffffffff);557 portal->irq_sources = 0;558 bm_out(p, BM_REG_IER, 0);559 bm_out(p, BM_REG_ISR, 0xffffffff);560 snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);561 if (request_irq(c->irq, portal_isr, 0, portal->irqname, portal)) {562 dev_err(c->dev, "request_irq() failed\n");563 goto fail_irq;564 }565 566 if (dpaa_set_portal_irq_affinity(c->dev, c->irq, c->cpu))567 goto fail_affinity;568 569 /* Need RCR to be empty before continuing */570 ret = bm_rcr_get_fill(p);571 if (ret) {572 dev_err(c->dev, "RCR unclean\n");573 goto fail_rcr_empty;574 }575 /* Success */576 portal->config = c;577 578 bm_out(p, BM_REG_ISDR, 0);579 bm_out(p, BM_REG_IIR, 0);580 581 return 0;582 583fail_rcr_empty:584fail_affinity:585 free_irq(c->irq, portal);586fail_irq:587 bm_mc_finish(p);588fail_mc:589 bm_rcr_finish(p);590fail_rcr:591 return -EIO;592}593 594struct bman_portal *bman_create_affine_portal(const struct bm_portal_config *c)595{596 struct bman_portal *portal;597 int err;598 599 portal = &per_cpu(bman_affine_portal, c->cpu);600 err = bman_create_portal(portal, c);601 if (err)602 return NULL;603 604 spin_lock(&affine_mask_lock);605 cpumask_set_cpu(c->cpu, &affine_mask);606 spin_unlock(&affine_mask_lock);607 608 return portal;609}610 611static u32 poll_portal_slow(struct bman_portal *p, u32 is)612{613 u32 ret = is;614 615 if (is & BM_PIRQ_RCRI) {616 bm_rcr_cce_update(&p->p);617 bm_rcr_set_ithresh(&p->p, 0);618 bm_out(&p->p, BM_REG_ISR, BM_PIRQ_RCRI);619 is &= ~BM_PIRQ_RCRI;620 }621 622 /* There should be no status register bits left undefined */623 DPAA_ASSERT(!is);624 return ret;625}626 627int bman_p_irqsource_add(struct bman_portal *p, u32 bits)628{629 unsigned long irqflags;630 631 local_irq_save(irqflags);632 p->irq_sources |= bits & BM_PIRQ_VISIBLE;633 bm_out(&p->p, BM_REG_IER, p->irq_sources);634 local_irq_restore(irqflags);635 return 0;636}637 638int bm_shutdown_pool(u32 bpid)639{640 int err = 0;641 struct bm_mc_command *bm_cmd;642 union bm_mc_result *bm_res;643 644 645 struct bman_portal *p = get_affine_portal();646 while (1) {647 /* Acquire buffers until empty */648 bm_cmd = bm_mc_start(&p->p);649 bm_cmd->bpid = bpid;650 bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE | 1);651 if (!bm_mc_result_timeout(&p->p, &bm_res)) {652 pr_crit("BMan Acquire Command timedout\n");653 err = -ETIMEDOUT;654 goto done;655 }656 if (!(bm_res->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT)) {657 /* Pool is empty */658 goto done;659 }660 }661done:662 put_affine_portal();663 return err;664}665 666struct gen_pool *bm_bpalloc;667 668static int bm_alloc_bpid_range(u32 *result, u32 count)669{670 unsigned long addr;671 672 addr = gen_pool_alloc(bm_bpalloc, count);673 if (!addr)674 return -ENOMEM;675 676 *result = addr & ~DPAA_GENALLOC_OFF;677 678 return 0;679}680 681static int bm_release_bpid(u32 bpid)682{683 int ret;684 685 ret = bm_shutdown_pool(bpid);686 if (ret) {687 pr_debug("BPID %d leaked\n", bpid);688 return ret;689 }690 691 gen_pool_free(bm_bpalloc, bpid | DPAA_GENALLOC_OFF, 1);692 return 0;693}694 695struct bman_pool *bman_new_pool(void)696{697 struct bman_pool *pool = NULL;698 u32 bpid;699 700 if (bm_alloc_bpid_range(&bpid, 1))701 return NULL;702 703 pool = kmalloc(sizeof(*pool), GFP_KERNEL);704 if (!pool)705 goto err;706 707 pool->bpid = bpid;708 709 return pool;710err:711 bm_release_bpid(bpid);712 return NULL;713}714EXPORT_SYMBOL(bman_new_pool);715 716void bman_free_pool(struct bman_pool *pool)717{718 bm_release_bpid(pool->bpid);719 720 kfree(pool);721}722EXPORT_SYMBOL(bman_free_pool);723 724int bman_get_bpid(const struct bman_pool *pool)725{726 return pool->bpid;727}728EXPORT_SYMBOL(bman_get_bpid);729 730static void update_rcr_ci(struct bman_portal *p, int avail)731{732 if (avail)733 bm_rcr_cce_prefetch(&p->p);734 else735 bm_rcr_cce_update(&p->p);736}737 738int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num)739{740 struct bman_portal *p;741 struct bm_rcr_entry *r;742 unsigned long irqflags;743 int avail, timeout = 1000; /* 1ms */744 int i = num - 1;745 746 DPAA_ASSERT(num > 0 && num <= 8);747 748 do {749 p = get_affine_portal();750 local_irq_save(irqflags);751 avail = bm_rcr_get_avail(&p->p);752 if (avail < 2)753 update_rcr_ci(p, avail);754 r = bm_rcr_start(&p->p);755 local_irq_restore(irqflags);756 put_affine_portal();757 if (likely(r))758 break;759 760 udelay(1);761 } while (--timeout);762 763 if (unlikely(!timeout))764 return -ETIMEDOUT;765 766 p = get_affine_portal();767 local_irq_save(irqflags);768 /*769 * we can copy all but the first entry, as this can trigger badness770 * with the valid-bit771 */772 bm_buffer_set64(r->bufs, bm_buffer_get64(bufs));773 bm_buffer_set_bpid(r->bufs, pool->bpid);774 if (i)775 memcpy(&r->bufs[1], &bufs[1], i * sizeof(bufs[0]));776 777 bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE |778 (num & BM_RCR_VERB_BUFCOUNT_MASK));779 780 local_irq_restore(irqflags);781 put_affine_portal();782 return 0;783}784EXPORT_SYMBOL(bman_release);785 786int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num)787{788 struct bman_portal *p = get_affine_portal();789 struct bm_mc_command *mcc;790 union bm_mc_result *mcr;791 int ret;792 793 DPAA_ASSERT(num > 0 && num <= 8);794 795 mcc = bm_mc_start(&p->p);796 mcc->bpid = pool->bpid;797 bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE |798 (num & BM_MCC_VERB_ACQUIRE_BUFCOUNT));799 if (!bm_mc_result_timeout(&p->p, &mcr)) {800 put_affine_portal();801 pr_crit("BMan Acquire Timeout\n");802 return -ETIMEDOUT;803 }804 ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT;805 if (bufs)806 memcpy(&bufs[0], &mcr->bufs[0], num * sizeof(bufs[0]));807 808 put_affine_portal();809 if (ret != num)810 ret = -ENOMEM;811 return ret;812}813EXPORT_SYMBOL(bman_acquire);814 815const struct bm_portal_config *816bman_get_bm_portal_config(const struct bman_portal *portal)817{818 return portal->config;819}820