517 lines · c
1/*2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.3 *4 * This software is available to you under a choice of one of two5 * licenses. You may choose to be licensed under the terms of the GNU6 * General Public License (GPL) Version 2, available from the file7 * COPYING in the main directory of this source tree, or the8 * OpenIB.org BSD license below:9 *10 * Redistribution and use in source and binary forms, with or11 * without modification, are permitted provided that the following12 * conditions are met:13 *14 * - Redistributions of source code must retain the above15 * copyright notice, this list of conditions and the following16 * disclaimer.17 *18 * - Redistributions in binary form must reproduce the above19 * copyright notice, this list of conditions and the following20 * disclaimer in the documentation and/or other materials21 * provided with the distribution.22 *23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE30 * SOFTWARE.31 */32/* Crude resource management */33#include <linux/spinlock.h>34#include <linux/genalloc.h>35#include <linux/ratelimit.h>36#include "iw_cxgb4.h"37 38static int c4iw_init_qid_table(struct c4iw_rdev *rdev)39{40 u32 i;41 42 if (c4iw_id_table_alloc(&rdev->resource.qid_table,43 rdev->lldi.vr->qp.start,44 rdev->lldi.vr->qp.size,45 rdev->lldi.vr->qp.size, 0))46 return -ENOMEM;47 48 for (i = rdev->lldi.vr->qp.start;49 i < rdev->lldi.vr->qp.start + rdev->lldi.vr->qp.size; i++)50 if (!(i & rdev->qpmask))51 c4iw_id_free(&rdev->resource.qid_table, i);52 return 0;53}54 55/* nr_* must be power of 2 */56int c4iw_init_resource(struct c4iw_rdev *rdev, u32 nr_tpt,57 u32 nr_pdid, u32 nr_srqt)58{59 int err = 0;60 err = c4iw_id_table_alloc(&rdev->resource.tpt_table, 0, nr_tpt, 1,61 C4IW_ID_TABLE_F_RANDOM);62 if (err)63 goto tpt_err;64 err = c4iw_init_qid_table(rdev);65 if (err)66 goto qid_err;67 err = c4iw_id_table_alloc(&rdev->resource.pdid_table, 0,68 nr_pdid, 1, 0);69 if (err)70 goto pdid_err;71 if (!nr_srqt)72 err = c4iw_id_table_alloc(&rdev->resource.srq_table, 0,73 1, 1, 0);74 else75 err = c4iw_id_table_alloc(&rdev->resource.srq_table, 0,76 nr_srqt, 0, 0);77 if (err)78 goto srq_err;79 return 0;80 srq_err:81 c4iw_id_table_free(&rdev->resource.pdid_table);82 pdid_err:83 c4iw_id_table_free(&rdev->resource.qid_table);84 qid_err:85 c4iw_id_table_free(&rdev->resource.tpt_table);86 tpt_err:87 return -ENOMEM;88}89 90/*91 * returns 0 if no resource available92 */93u32 c4iw_get_resource(struct c4iw_id_table *id_table)94{95 u32 entry;96 entry = c4iw_id_alloc(id_table);97 if (entry == (u32)(-1))98 return 0;99 return entry;100}101 102void c4iw_put_resource(struct c4iw_id_table *id_table, u32 entry)103{104 pr_debug("entry 0x%x\n", entry);105 c4iw_id_free(id_table, entry);106}107 108u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)109{110 struct c4iw_qid_list *entry;111 u32 qid;112 int i;113 114 mutex_lock(&uctx->lock);115 if (!list_empty(&uctx->cqids)) {116 entry = list_entry(uctx->cqids.next, struct c4iw_qid_list,117 entry);118 list_del(&entry->entry);119 qid = entry->qid;120 kfree(entry);121 } else {122 qid = c4iw_get_resource(&rdev->resource.qid_table);123 if (!qid)124 goto out;125 mutex_lock(&rdev->stats.lock);126 rdev->stats.qid.cur += rdev->qpmask + 1;127 mutex_unlock(&rdev->stats.lock);128 for (i = qid+1; i & rdev->qpmask; i++) {129 entry = kmalloc(sizeof(*entry), GFP_KERNEL);130 if (!entry)131 goto out;132 entry->qid = i;133 list_add_tail(&entry->entry, &uctx->cqids);134 }135 136 /*137 * now put the same ids on the qp list since they all138 * map to the same db/gts page.139 */140 entry = kmalloc(sizeof(*entry), GFP_KERNEL);141 if (!entry)142 goto out;143 entry->qid = qid;144 list_add_tail(&entry->entry, &uctx->qpids);145 for (i = qid+1; i & rdev->qpmask; i++) {146 entry = kmalloc(sizeof(*entry), GFP_KERNEL);147 if (!entry)148 goto out;149 entry->qid = i;150 list_add_tail(&entry->entry, &uctx->qpids);151 }152 }153out:154 mutex_unlock(&uctx->lock);155 pr_debug("qid 0x%x\n", qid);156 mutex_lock(&rdev->stats.lock);157 if (rdev->stats.qid.cur > rdev->stats.qid.max)158 rdev->stats.qid.max = rdev->stats.qid.cur;159 mutex_unlock(&rdev->stats.lock);160 return qid;161}162 163void c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid,164 struct c4iw_dev_ucontext *uctx)165{166 struct c4iw_qid_list *entry;167 168 entry = kmalloc(sizeof(*entry), GFP_KERNEL);169 if (!entry)170 return;171 pr_debug("qid 0x%x\n", qid);172 entry->qid = qid;173 mutex_lock(&uctx->lock);174 list_add_tail(&entry->entry, &uctx->cqids);175 mutex_unlock(&uctx->lock);176}177 178u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)179{180 struct c4iw_qid_list *entry;181 u32 qid;182 int i;183 184 mutex_lock(&uctx->lock);185 if (!list_empty(&uctx->qpids)) {186 entry = list_entry(uctx->qpids.next, struct c4iw_qid_list,187 entry);188 list_del(&entry->entry);189 qid = entry->qid;190 kfree(entry);191 } else {192 qid = c4iw_get_resource(&rdev->resource.qid_table);193 if (!qid) {194 mutex_lock(&rdev->stats.lock);195 rdev->stats.qid.fail++;196 mutex_unlock(&rdev->stats.lock);197 goto out;198 }199 mutex_lock(&rdev->stats.lock);200 rdev->stats.qid.cur += rdev->qpmask + 1;201 mutex_unlock(&rdev->stats.lock);202 for (i = qid+1; i & rdev->qpmask; i++) {203 entry = kmalloc(sizeof(*entry), GFP_KERNEL);204 if (!entry)205 goto out;206 entry->qid = i;207 list_add_tail(&entry->entry, &uctx->qpids);208 }209 210 /*211 * now put the same ids on the cq list since they all212 * map to the same db/gts page.213 */214 entry = kmalloc(sizeof(*entry), GFP_KERNEL);215 if (!entry)216 goto out;217 entry->qid = qid;218 list_add_tail(&entry->entry, &uctx->cqids);219 for (i = qid + 1; i & rdev->qpmask; i++) {220 entry = kmalloc(sizeof(*entry), GFP_KERNEL);221 if (!entry)222 goto out;223 entry->qid = i;224 list_add_tail(&entry->entry, &uctx->cqids);225 }226 }227out:228 mutex_unlock(&uctx->lock);229 pr_debug("qid 0x%x\n", qid);230 mutex_lock(&rdev->stats.lock);231 if (rdev->stats.qid.cur > rdev->stats.qid.max)232 rdev->stats.qid.max = rdev->stats.qid.cur;233 mutex_unlock(&rdev->stats.lock);234 return qid;235}236 237void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid,238 struct c4iw_dev_ucontext *uctx)239{240 struct c4iw_qid_list *entry;241 242 entry = kmalloc(sizeof(*entry), GFP_KERNEL);243 if (!entry)244 return;245 pr_debug("qid 0x%x\n", qid);246 entry->qid = qid;247 mutex_lock(&uctx->lock);248 list_add_tail(&entry->entry, &uctx->qpids);249 mutex_unlock(&uctx->lock);250}251 252void c4iw_destroy_resource(struct c4iw_resource *rscp)253{254 c4iw_id_table_free(&rscp->tpt_table);255 c4iw_id_table_free(&rscp->qid_table);256 c4iw_id_table_free(&rscp->pdid_table);257}258 259/*260 * PBL Memory Manager. Uses Linux generic allocator.261 */262 263#define MIN_PBL_SHIFT 8 /* 256B == min PBL size (32 entries) */264 265u32 c4iw_pblpool_alloc(struct c4iw_rdev *rdev, int size)266{267 unsigned long addr = gen_pool_alloc(rdev->pbl_pool, size);268 pr_debug("addr 0x%x size %d\n", (u32)addr, size);269 mutex_lock(&rdev->stats.lock);270 if (addr) {271 rdev->stats.pbl.cur += roundup(size, 1 << MIN_PBL_SHIFT);272 if (rdev->stats.pbl.cur > rdev->stats.pbl.max)273 rdev->stats.pbl.max = rdev->stats.pbl.cur;274 kref_get(&rdev->pbl_kref);275 } else276 rdev->stats.pbl.fail++;277 mutex_unlock(&rdev->stats.lock);278 return (u32)addr;279}280 281static void destroy_pblpool(struct kref *kref)282{283 struct c4iw_rdev *rdev;284 285 rdev = container_of(kref, struct c4iw_rdev, pbl_kref);286 gen_pool_destroy(rdev->pbl_pool);287 complete(&rdev->pbl_compl);288}289 290void c4iw_pblpool_free(struct c4iw_rdev *rdev, u32 addr, int size)291{292 pr_debug("addr 0x%x size %d\n", addr, size);293 mutex_lock(&rdev->stats.lock);294 rdev->stats.pbl.cur -= roundup(size, 1 << MIN_PBL_SHIFT);295 mutex_unlock(&rdev->stats.lock);296 gen_pool_free(rdev->pbl_pool, (unsigned long)addr, size);297 kref_put(&rdev->pbl_kref, destroy_pblpool);298}299 300int c4iw_pblpool_create(struct c4iw_rdev *rdev)301{302 unsigned pbl_start, pbl_chunk, pbl_top;303 304 rdev->pbl_pool = gen_pool_create(MIN_PBL_SHIFT, -1);305 if (!rdev->pbl_pool)306 return -ENOMEM;307 308 pbl_start = rdev->lldi.vr->pbl.start;309 pbl_chunk = rdev->lldi.vr->pbl.size;310 pbl_top = pbl_start + pbl_chunk;311 312 while (pbl_start < pbl_top) {313 pbl_chunk = min(pbl_top - pbl_start + 1, pbl_chunk);314 if (gen_pool_add(rdev->pbl_pool, pbl_start, pbl_chunk, -1)) {315 pr_debug("failed to add PBL chunk (%x/%x)\n",316 pbl_start, pbl_chunk);317 if (pbl_chunk <= 1024 << MIN_PBL_SHIFT) {318 pr_warn("Failed to add all PBL chunks (%x/%x)\n",319 pbl_start, pbl_top - pbl_start);320 return 0;321 }322 pbl_chunk >>= 1;323 } else {324 pr_debug("added PBL chunk (%x/%x)\n",325 pbl_start, pbl_chunk);326 pbl_start += pbl_chunk;327 }328 }329 330 return 0;331}332 333void c4iw_pblpool_destroy(struct c4iw_rdev *rdev)334{335 kref_put(&rdev->pbl_kref, destroy_pblpool);336}337 338/*339 * RQT Memory Manager. Uses Linux generic allocator.340 */341 342#define MIN_RQT_SHIFT 10 /* 1KB == min RQT size (16 entries) */343 344u32 c4iw_rqtpool_alloc(struct c4iw_rdev *rdev, int size)345{346 unsigned long addr = gen_pool_alloc(rdev->rqt_pool, size << 6);347 pr_debug("addr 0x%x size %d\n", (u32)addr, size << 6);348 if (!addr)349 pr_warn_ratelimited("%s: Out of RQT memory\n",350 pci_name(rdev->lldi.pdev));351 mutex_lock(&rdev->stats.lock);352 if (addr) {353 rdev->stats.rqt.cur += roundup(size << 6, 1 << MIN_RQT_SHIFT);354 if (rdev->stats.rqt.cur > rdev->stats.rqt.max)355 rdev->stats.rqt.max = rdev->stats.rqt.cur;356 kref_get(&rdev->rqt_kref);357 } else358 rdev->stats.rqt.fail++;359 mutex_unlock(&rdev->stats.lock);360 return (u32)addr;361}362 363static void destroy_rqtpool(struct kref *kref)364{365 struct c4iw_rdev *rdev;366 367 rdev = container_of(kref, struct c4iw_rdev, rqt_kref);368 gen_pool_destroy(rdev->rqt_pool);369 complete(&rdev->rqt_compl);370}371 372void c4iw_rqtpool_free(struct c4iw_rdev *rdev, u32 addr, int size)373{374 pr_debug("addr 0x%x size %d\n", addr, size << 6);375 mutex_lock(&rdev->stats.lock);376 rdev->stats.rqt.cur -= roundup(size << 6, 1 << MIN_RQT_SHIFT);377 mutex_unlock(&rdev->stats.lock);378 gen_pool_free(rdev->rqt_pool, (unsigned long)addr, size << 6);379 kref_put(&rdev->rqt_kref, destroy_rqtpool);380}381 382int c4iw_rqtpool_create(struct c4iw_rdev *rdev)383{384 unsigned rqt_start, rqt_chunk, rqt_top;385 int skip = 0;386 387 rdev->rqt_pool = gen_pool_create(MIN_RQT_SHIFT, -1);388 if (!rdev->rqt_pool)389 return -ENOMEM;390 391 /*392 * If SRQs are supported, then never use the first RQE from393 * the RQT region. This is because HW uses RQT index 0 as NULL.394 */395 if (rdev->lldi.vr->srq.size)396 skip = T4_RQT_ENTRY_SIZE;397 398 rqt_start = rdev->lldi.vr->rq.start + skip;399 rqt_chunk = rdev->lldi.vr->rq.size - skip;400 rqt_top = rqt_start + rqt_chunk;401 402 while (rqt_start < rqt_top) {403 rqt_chunk = min(rqt_top - rqt_start + 1, rqt_chunk);404 if (gen_pool_add(rdev->rqt_pool, rqt_start, rqt_chunk, -1)) {405 pr_debug("failed to add RQT chunk (%x/%x)\n",406 rqt_start, rqt_chunk);407 if (rqt_chunk <= 1024 << MIN_RQT_SHIFT) {408 pr_warn("Failed to add all RQT chunks (%x/%x)\n",409 rqt_start, rqt_top - rqt_start);410 return 0;411 }412 rqt_chunk >>= 1;413 } else {414 pr_debug("added RQT chunk (%x/%x)\n",415 rqt_start, rqt_chunk);416 rqt_start += rqt_chunk;417 }418 }419 return 0;420}421 422void c4iw_rqtpool_destroy(struct c4iw_rdev *rdev)423{424 kref_put(&rdev->rqt_kref, destroy_rqtpool);425}426 427int c4iw_alloc_srq_idx(struct c4iw_rdev *rdev)428{429 int idx;430 431 idx = c4iw_id_alloc(&rdev->resource.srq_table);432 mutex_lock(&rdev->stats.lock);433 if (idx == -1) {434 rdev->stats.srqt.fail++;435 mutex_unlock(&rdev->stats.lock);436 return -ENOMEM;437 }438 rdev->stats.srqt.cur++;439 if (rdev->stats.srqt.cur > rdev->stats.srqt.max)440 rdev->stats.srqt.max = rdev->stats.srqt.cur;441 mutex_unlock(&rdev->stats.lock);442 return idx;443}444 445void c4iw_free_srq_idx(struct c4iw_rdev *rdev, int idx)446{447 c4iw_id_free(&rdev->resource.srq_table, idx);448 mutex_lock(&rdev->stats.lock);449 rdev->stats.srqt.cur--;450 mutex_unlock(&rdev->stats.lock);451}452 453/*454 * On-Chip QP Memory.455 */456#define MIN_OCQP_SHIFT 12 /* 4KB == min ocqp size */457 458u32 c4iw_ocqp_pool_alloc(struct c4iw_rdev *rdev, int size)459{460 unsigned long addr = gen_pool_alloc(rdev->ocqp_pool, size);461 pr_debug("addr 0x%x size %d\n", (u32)addr, size);462 if (addr) {463 mutex_lock(&rdev->stats.lock);464 rdev->stats.ocqp.cur += roundup(size, 1 << MIN_OCQP_SHIFT);465 if (rdev->stats.ocqp.cur > rdev->stats.ocqp.max)466 rdev->stats.ocqp.max = rdev->stats.ocqp.cur;467 mutex_unlock(&rdev->stats.lock);468 }469 return (u32)addr;470}471 472void c4iw_ocqp_pool_free(struct c4iw_rdev *rdev, u32 addr, int size)473{474 pr_debug("addr 0x%x size %d\n", addr, size);475 mutex_lock(&rdev->stats.lock);476 rdev->stats.ocqp.cur -= roundup(size, 1 << MIN_OCQP_SHIFT);477 mutex_unlock(&rdev->stats.lock);478 gen_pool_free(rdev->ocqp_pool, (unsigned long)addr, size);479}480 481int c4iw_ocqp_pool_create(struct c4iw_rdev *rdev)482{483 unsigned start, chunk, top;484 485 rdev->ocqp_pool = gen_pool_create(MIN_OCQP_SHIFT, -1);486 if (!rdev->ocqp_pool)487 return -ENOMEM;488 489 start = rdev->lldi.vr->ocq.start;490 chunk = rdev->lldi.vr->ocq.size;491 top = start + chunk;492 493 while (start < top) {494 chunk = min(top - start + 1, chunk);495 if (gen_pool_add(rdev->ocqp_pool, start, chunk, -1)) {496 pr_debug("failed to add OCQP chunk (%x/%x)\n",497 start, chunk);498 if (chunk <= 1024 << MIN_OCQP_SHIFT) {499 pr_warn("Failed to add all OCQP chunks (%x/%x)\n",500 start, top - start);501 return 0;502 }503 chunk >>= 1;504 } else {505 pr_debug("added OCQP chunk (%x/%x)\n",506 start, chunk);507 start += chunk;508 }509 }510 return 0;511}512 513void c4iw_ocqp_pool_destroy(struct c4iw_rdev *rdev)514{515 gen_pool_destroy(rdev->ocqp_pool);516}517