764 lines · c
1/*2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.3 * Copyright (c) 2005 Cisco Systems. All rights reserved.4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.5 *6 * This software is available to you under a choice of one of two7 * licenses. You may choose to be licensed under the terms of the GNU8 * General Public License (GPL) Version 2, available from the file9 * COPYING in the main directory of this source tree, or the10 * OpenIB.org BSD license below:11 *12 * Redistribution and use in source and binary forms, with or13 * without modification, are permitted provided that the following14 * conditions are met:15 *16 * - Redistributions of source code must retain the above17 * copyright notice, this list of conditions and the following18 * disclaimer.19 *20 * - Redistributions in binary form must reproduce the above21 * copyright notice, this list of conditions and the following22 * disclaimer in the documentation and/or other materials23 * provided with the distribution.24 *25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE32 * SOFTWARE.33 */34 35#include <linux/mm.h>36#include <linux/scatterlist.h>37#include <linux/sched.h>38#include <linux/slab.h>39 40#include <asm/page.h>41 42#include "mthca_memfree.h"43#include "mthca_dev.h"44#include "mthca_cmd.h"45 46/*47 * We allocate in as big chunks as we can, up to a maximum of 256 KB48 * per chunk.49 */50enum {51 MTHCA_ICM_ALLOC_SIZE = 1 << 18,52 MTHCA_TABLE_CHUNK_SIZE = 1 << 1853};54 55struct mthca_user_db_table {56 struct mutex mutex;57 struct {58 u64 uvirt;59 struct scatterlist mem;60 int refcount;61 } page[];62};63 64static void mthca_free_icm_pages(struct mthca_dev *dev, struct mthca_icm_chunk *chunk)65{66 int i;67 68 if (chunk->nsg > 0)69 dma_unmap_sg(&dev->pdev->dev, chunk->mem, chunk->npages,70 DMA_BIDIRECTIONAL);71 72 for (i = 0; i < chunk->npages; ++i)73 __free_pages(sg_page(&chunk->mem[i]),74 get_order(chunk->mem[i].length));75}76 77static void mthca_free_icm_coherent(struct mthca_dev *dev, struct mthca_icm_chunk *chunk)78{79 int i;80 81 for (i = 0; i < chunk->npages; ++i) {82 dma_free_coherent(&dev->pdev->dev, chunk->mem[i].length,83 lowmem_page_address(sg_page(&chunk->mem[i])),84 sg_dma_address(&chunk->mem[i]));85 }86}87 88void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm, int coherent)89{90 struct mthca_icm_chunk *chunk, *tmp;91 92 if (!icm)93 return;94 95 list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) {96 if (coherent)97 mthca_free_icm_coherent(dev, chunk);98 else99 mthca_free_icm_pages(dev, chunk);100 101 kfree(chunk);102 }103 104 kfree(icm);105}106 107static int mthca_alloc_icm_pages(struct scatterlist *mem, int order, gfp_t gfp_mask)108{109 struct page *page;110 111 /*112 * Use __GFP_ZERO because buggy firmware assumes ICM pages are113 * cleared, and subtle failures are seen if they aren't.114 */115 page = alloc_pages(gfp_mask | __GFP_ZERO, order);116 if (!page)117 return -ENOMEM;118 119 sg_set_page(mem, page, PAGE_SIZE << order, 0);120 return 0;121}122 123static int mthca_alloc_icm_coherent(struct device *dev, struct scatterlist *mem,124 int order, gfp_t gfp_mask)125{126 void *buf = dma_alloc_coherent(dev, PAGE_SIZE << order, &sg_dma_address(mem),127 gfp_mask);128 if (!buf)129 return -ENOMEM;130 131 sg_set_buf(mem, buf, PAGE_SIZE << order);132 BUG_ON(mem->offset);133 sg_dma_len(mem) = PAGE_SIZE << order;134 return 0;135}136 137struct mthca_icm *mthca_alloc_icm(struct mthca_dev *dev, int npages,138 gfp_t gfp_mask, int coherent)139{140 struct mthca_icm *icm;141 struct mthca_icm_chunk *chunk = NULL;142 int cur_order;143 int ret;144 145 /* We use sg_set_buf for coherent allocs, which assumes low memory */146 BUG_ON(coherent && (gfp_mask & __GFP_HIGHMEM));147 148 icm = kmalloc(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));149 if (!icm)150 return icm;151 152 icm->refcount = 0;153 INIT_LIST_HEAD(&icm->chunk_list);154 155 cur_order = get_order(MTHCA_ICM_ALLOC_SIZE);156 157 while (npages > 0) {158 if (!chunk) {159 chunk = kmalloc(sizeof *chunk,160 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));161 if (!chunk)162 goto fail;163 164 sg_init_table(chunk->mem, MTHCA_ICM_CHUNK_LEN);165 chunk->npages = 0;166 chunk->nsg = 0;167 list_add_tail(&chunk->list, &icm->chunk_list);168 }169 170 while (1 << cur_order > npages)171 --cur_order;172 173 if (coherent)174 ret = mthca_alloc_icm_coherent(&dev->pdev->dev,175 &chunk->mem[chunk->npages],176 cur_order, gfp_mask);177 else178 ret = mthca_alloc_icm_pages(&chunk->mem[chunk->npages],179 cur_order, gfp_mask);180 181 if (!ret) {182 ++chunk->npages;183 184 if (coherent)185 ++chunk->nsg;186 else if (chunk->npages == MTHCA_ICM_CHUNK_LEN) {187 chunk->nsg =188 dma_map_sg(&dev->pdev->dev, chunk->mem,189 chunk->npages,190 DMA_BIDIRECTIONAL);191 192 if (chunk->nsg <= 0)193 goto fail;194 }195 196 if (chunk->npages == MTHCA_ICM_CHUNK_LEN)197 chunk = NULL;198 199 npages -= 1 << cur_order;200 } else {201 --cur_order;202 if (cur_order < 0)203 goto fail;204 }205 }206 207 if (!coherent && chunk) {208 chunk->nsg = dma_map_sg(&dev->pdev->dev, chunk->mem,209 chunk->npages, DMA_BIDIRECTIONAL);210 211 if (chunk->nsg <= 0)212 goto fail;213 }214 215 return icm;216 217fail:218 mthca_free_icm(dev, icm, coherent);219 return NULL;220}221 222int mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int obj)223{224 int i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE;225 int ret = 0;226 227 mutex_lock(&table->mutex);228 229 if (table->icm[i]) {230 ++table->icm[i]->refcount;231 goto out;232 }233 234 table->icm[i] = mthca_alloc_icm(dev, MTHCA_TABLE_CHUNK_SIZE >> PAGE_SHIFT,235 (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) |236 __GFP_NOWARN, table->coherent);237 if (!table->icm[i]) {238 ret = -ENOMEM;239 goto out;240 }241 242 if (mthca_MAP_ICM(dev, table->icm[i],243 table->virt + i * MTHCA_TABLE_CHUNK_SIZE)) {244 mthca_free_icm(dev, table->icm[i], table->coherent);245 table->icm[i] = NULL;246 ret = -ENOMEM;247 goto out;248 }249 250 ++table->icm[i]->refcount;251 252out:253 mutex_unlock(&table->mutex);254 return ret;255}256 257void mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int obj)258{259 int i;260 261 if (!mthca_is_memfree(dev))262 return;263 264 i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE;265 266 mutex_lock(&table->mutex);267 268 if (--table->icm[i]->refcount == 0) {269 mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE,270 MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE);271 mthca_free_icm(dev, table->icm[i], table->coherent);272 table->icm[i] = NULL;273 }274 275 mutex_unlock(&table->mutex);276}277 278void *mthca_table_find(struct mthca_icm_table *table, int obj, dma_addr_t *dma_handle)279{280 int idx, offset, dma_offset, i;281 struct mthca_icm_chunk *chunk;282 struct mthca_icm *icm;283 struct page *page = NULL;284 285 if (!table->lowmem)286 return NULL;287 288 mutex_lock(&table->mutex);289 290 idx = (obj & (table->num_obj - 1)) * table->obj_size;291 icm = table->icm[idx / MTHCA_TABLE_CHUNK_SIZE];292 dma_offset = offset = idx % MTHCA_TABLE_CHUNK_SIZE;293 294 if (!icm)295 goto out;296 297 list_for_each_entry(chunk, &icm->chunk_list, list) {298 for (i = 0; i < chunk->npages; ++i) {299 if (dma_handle && dma_offset >= 0) {300 if (sg_dma_len(&chunk->mem[i]) > dma_offset)301 *dma_handle = sg_dma_address(&chunk->mem[i]) +302 dma_offset;303 dma_offset -= sg_dma_len(&chunk->mem[i]);304 }305 /* DMA mapping can merge pages but not split them,306 * so if we found the page, dma_handle has already307 * been assigned to. */308 if (chunk->mem[i].length > offset) {309 page = sg_page(&chunk->mem[i]);310 goto out;311 }312 offset -= chunk->mem[i].length;313 }314 }315 316out:317 mutex_unlock(&table->mutex);318 return page ? lowmem_page_address(page) + offset : NULL;319}320 321int mthca_table_get_range(struct mthca_dev *dev, struct mthca_icm_table *table,322 int start, int end)323{324 int inc = MTHCA_TABLE_CHUNK_SIZE / table->obj_size;325 int i, err;326 327 for (i = start; i <= end; i += inc) {328 err = mthca_table_get(dev, table, i);329 if (err)330 goto fail;331 }332 333 return 0;334 335fail:336 while (i > start) {337 i -= inc;338 mthca_table_put(dev, table, i);339 }340 341 return err;342}343 344void mthca_table_put_range(struct mthca_dev *dev, struct mthca_icm_table *table,345 int start, int end)346{347 int i;348 349 if (!mthca_is_memfree(dev))350 return;351 352 for (i = start; i <= end; i += MTHCA_TABLE_CHUNK_SIZE / table->obj_size)353 mthca_table_put(dev, table, i);354}355 356struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev,357 u64 virt, int obj_size,358 int nobj, int reserved,359 int use_lowmem, int use_coherent)360{361 struct mthca_icm_table *table;362 int obj_per_chunk;363 int num_icm;364 unsigned chunk_size;365 int i;366 367 obj_per_chunk = MTHCA_TABLE_CHUNK_SIZE / obj_size;368 num_icm = DIV_ROUND_UP(nobj, obj_per_chunk);369 370 table = kmalloc(struct_size(table, icm, num_icm), GFP_KERNEL);371 if (!table)372 return NULL;373 374 table->virt = virt;375 table->num_icm = num_icm;376 table->num_obj = nobj;377 table->obj_size = obj_size;378 table->lowmem = use_lowmem;379 table->coherent = use_coherent;380 mutex_init(&table->mutex);381 382 for (i = 0; i < num_icm; ++i)383 table->icm[i] = NULL;384 385 for (i = 0; i * MTHCA_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) {386 chunk_size = MTHCA_TABLE_CHUNK_SIZE;387 if ((i + 1) * MTHCA_TABLE_CHUNK_SIZE > nobj * obj_size)388 chunk_size = nobj * obj_size - i * MTHCA_TABLE_CHUNK_SIZE;389 390 table->icm[i] = mthca_alloc_icm(dev, chunk_size >> PAGE_SHIFT,391 (use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) |392 __GFP_NOWARN, use_coherent);393 if (!table->icm[i])394 goto err;395 if (mthca_MAP_ICM(dev, table->icm[i],396 virt + i * MTHCA_TABLE_CHUNK_SIZE)) {397 mthca_free_icm(dev, table->icm[i], table->coherent);398 table->icm[i] = NULL;399 goto err;400 }401 402 /*403 * Add a reference to this ICM chunk so that it never404 * gets freed (since it contains reserved firmware objects).405 */406 ++table->icm[i]->refcount;407 }408 409 return table;410 411err:412 for (i = 0; i < num_icm; ++i)413 if (table->icm[i]) {414 mthca_UNMAP_ICM(dev, virt + i * MTHCA_TABLE_CHUNK_SIZE,415 MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE);416 mthca_free_icm(dev, table->icm[i], table->coherent);417 }418 419 kfree(table);420 421 return NULL;422}423 424void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table)425{426 int i;427 428 for (i = 0; i < table->num_icm; ++i)429 if (table->icm[i]) {430 mthca_UNMAP_ICM(dev,431 table->virt + i * MTHCA_TABLE_CHUNK_SIZE,432 MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE);433 mthca_free_icm(dev, table->icm[i], table->coherent);434 }435 436 kfree(table);437}438 439static u64 mthca_uarc_virt(struct mthca_dev *dev, struct mthca_uar *uar, int page)440{441 return dev->uar_table.uarc_base +442 uar->index * dev->uar_table.uarc_size +443 page * MTHCA_ICM_PAGE_SIZE;444}445 446int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar,447 struct mthca_user_db_table *db_tab, int index, u64 uaddr)448{449 struct page *pages[1];450 int ret = 0;451 int i;452 453 if (!mthca_is_memfree(dev))454 return 0;455 456 if (index < 0 || index > dev->uar_table.uarc_size / 8)457 return -EINVAL;458 459 mutex_lock(&db_tab->mutex);460 461 i = index / MTHCA_DB_REC_PER_PAGE;462 463 if ((db_tab->page[i].refcount >= MTHCA_DB_REC_PER_PAGE) ||464 (db_tab->page[i].uvirt && db_tab->page[i].uvirt != uaddr) ||465 (uaddr & 4095)) {466 ret = -EINVAL;467 goto out;468 }469 470 if (db_tab->page[i].refcount) {471 ++db_tab->page[i].refcount;472 goto out;473 }474 475 ret = pin_user_pages_fast(uaddr & PAGE_MASK, 1,476 FOLL_WRITE | FOLL_LONGTERM, pages);477 if (ret < 0)478 goto out;479 480 sg_set_page(&db_tab->page[i].mem, pages[0], MTHCA_ICM_PAGE_SIZE,481 uaddr & ~PAGE_MASK);482 483 ret = dma_map_sg(&dev->pdev->dev, &db_tab->page[i].mem, 1,484 DMA_TO_DEVICE);485 if (ret < 0) {486 unpin_user_page(pages[0]);487 goto out;488 }489 490 ret = mthca_MAP_ICM_page(dev, sg_dma_address(&db_tab->page[i].mem),491 mthca_uarc_virt(dev, uar, i));492 if (ret) {493 dma_unmap_sg(&dev->pdev->dev, &db_tab->page[i].mem, 1,494 DMA_TO_DEVICE);495 unpin_user_page(sg_page(&db_tab->page[i].mem));496 goto out;497 }498 499 db_tab->page[i].uvirt = uaddr;500 db_tab->page[i].refcount = 1;501 502out:503 mutex_unlock(&db_tab->mutex);504 return ret;505}506 507void mthca_unmap_user_db(struct mthca_dev *dev, struct mthca_uar *uar,508 struct mthca_user_db_table *db_tab, int index)509{510 if (!mthca_is_memfree(dev))511 return;512 513 /*514 * To make our bookkeeping simpler, we don't unmap DB515 * pages until we clean up the whole db table.516 */517 518 mutex_lock(&db_tab->mutex);519 520 --db_tab->page[index / MTHCA_DB_REC_PER_PAGE].refcount;521 522 mutex_unlock(&db_tab->mutex);523}524 525struct mthca_user_db_table *mthca_init_user_db_tab(struct mthca_dev *dev)526{527 struct mthca_user_db_table *db_tab;528 int npages;529 int i;530 531 if (!mthca_is_memfree(dev))532 return NULL;533 534 npages = dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE;535 db_tab = kmalloc(struct_size(db_tab, page, npages), GFP_KERNEL);536 if (!db_tab)537 return ERR_PTR(-ENOMEM);538 539 mutex_init(&db_tab->mutex);540 for (i = 0; i < npages; ++i) {541 db_tab->page[i].refcount = 0;542 db_tab->page[i].uvirt = 0;543 sg_init_table(&db_tab->page[i].mem, 1);544 }545 546 return db_tab;547}548 549void mthca_cleanup_user_db_tab(struct mthca_dev *dev, struct mthca_uar *uar,550 struct mthca_user_db_table *db_tab)551{552 int i;553 554 if (!mthca_is_memfree(dev))555 return;556 557 for (i = 0; i < dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE; ++i) {558 if (db_tab->page[i].uvirt) {559 mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, uar, i), 1);560 dma_unmap_sg(&dev->pdev->dev, &db_tab->page[i].mem, 1,561 DMA_TO_DEVICE);562 unpin_user_page(sg_page(&db_tab->page[i].mem));563 }564 }565 566 kfree(db_tab);567}568 569int mthca_alloc_db(struct mthca_dev *dev, enum mthca_db_type type,570 u32 qn, __be32 **db)571{572 int group;573 int start, end, dir;574 int i, j;575 struct mthca_db_page *page;576 int ret = 0;577 578 mutex_lock(&dev->db_tab->mutex);579 580 switch (type) {581 case MTHCA_DB_TYPE_CQ_ARM:582 case MTHCA_DB_TYPE_SQ:583 group = 0;584 start = 0;585 end = dev->db_tab->max_group1;586 dir = 1;587 break;588 589 case MTHCA_DB_TYPE_CQ_SET_CI:590 case MTHCA_DB_TYPE_RQ:591 case MTHCA_DB_TYPE_SRQ:592 group = 1;593 start = dev->db_tab->npages - 1;594 end = dev->db_tab->min_group2;595 dir = -1;596 break;597 598 default:599 ret = -EINVAL;600 goto out;601 }602 603 for (i = start; i != end; i += dir)604 if (dev->db_tab->page[i].db_rec &&605 !bitmap_full(dev->db_tab->page[i].used,606 MTHCA_DB_REC_PER_PAGE)) {607 page = dev->db_tab->page + i;608 goto found;609 }610 611 for (i = start; i != end; i += dir)612 if (!dev->db_tab->page[i].db_rec) {613 page = dev->db_tab->page + i;614 goto alloc;615 }616 617 if (dev->db_tab->max_group1 >= dev->db_tab->min_group2 - 1) {618 ret = -ENOMEM;619 goto out;620 }621 622 if (group == 0)623 ++dev->db_tab->max_group1;624 else625 --dev->db_tab->min_group2;626 627 page = dev->db_tab->page + end;628 629alloc:630 page->db_rec = dma_alloc_coherent(&dev->pdev->dev,631 MTHCA_ICM_PAGE_SIZE, &page->mapping,632 GFP_KERNEL);633 if (!page->db_rec) {634 ret = -ENOMEM;635 goto out;636 }637 638 ret = mthca_MAP_ICM_page(dev, page->mapping,639 mthca_uarc_virt(dev, &dev->driver_uar, i));640 if (ret) {641 dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE,642 page->db_rec, page->mapping);643 goto out;644 }645 646 bitmap_zero(page->used, MTHCA_DB_REC_PER_PAGE);647 648found:649 j = find_first_zero_bit(page->used, MTHCA_DB_REC_PER_PAGE);650 set_bit(j, page->used);651 652 if (group == 1)653 j = MTHCA_DB_REC_PER_PAGE - 1 - j;654 655 ret = i * MTHCA_DB_REC_PER_PAGE + j;656 657 page->db_rec[j] = cpu_to_be64((qn << 8) | (type << 5));658 659 *db = (__be32 *) &page->db_rec[j];660 661out:662 mutex_unlock(&dev->db_tab->mutex);663 664 return ret;665}666 667void mthca_free_db(struct mthca_dev *dev, int type, int db_index)668{669 int i, j;670 struct mthca_db_page *page;671 672 i = db_index / MTHCA_DB_REC_PER_PAGE;673 j = db_index % MTHCA_DB_REC_PER_PAGE;674 675 page = dev->db_tab->page + i;676 677 mutex_lock(&dev->db_tab->mutex);678 679 page->db_rec[j] = 0;680 if (i >= dev->db_tab->min_group2)681 j = MTHCA_DB_REC_PER_PAGE - 1 - j;682 clear_bit(j, page->used);683 684 if (bitmap_empty(page->used, MTHCA_DB_REC_PER_PAGE) &&685 i >= dev->db_tab->max_group1 - 1) {686 mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1);687 688 dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE,689 page->db_rec, page->mapping);690 page->db_rec = NULL;691 692 if (i == dev->db_tab->max_group1) {693 --dev->db_tab->max_group1;694 /* XXX may be able to unmap more pages now */695 }696 if (i == dev->db_tab->min_group2)697 ++dev->db_tab->min_group2;698 }699 700 mutex_unlock(&dev->db_tab->mutex);701}702 703int mthca_init_db_tab(struct mthca_dev *dev)704{705 int i;706 707 if (!mthca_is_memfree(dev))708 return 0;709 710 dev->db_tab = kmalloc(sizeof *dev->db_tab, GFP_KERNEL);711 if (!dev->db_tab)712 return -ENOMEM;713 714 mutex_init(&dev->db_tab->mutex);715 716 dev->db_tab->npages = dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE;717 dev->db_tab->max_group1 = 0;718 dev->db_tab->min_group2 = dev->db_tab->npages - 1;719 720 dev->db_tab->page = kmalloc_array(dev->db_tab->npages,721 sizeof(*dev->db_tab->page),722 GFP_KERNEL);723 if (!dev->db_tab->page) {724 kfree(dev->db_tab);725 return -ENOMEM;726 }727 728 for (i = 0; i < dev->db_tab->npages; ++i)729 dev->db_tab->page[i].db_rec = NULL;730 731 return 0;732}733 734void mthca_cleanup_db_tab(struct mthca_dev *dev)735{736 int i;737 738 if (!mthca_is_memfree(dev))739 return;740 741 /*742 * Because we don't always free our UARC pages when they743 * become empty to make mthca_free_db() simpler we need to744 * make a sweep through the doorbell pages and free any745 * leftover pages now.746 */747 for (i = 0; i < dev->db_tab->npages; ++i) {748 if (!dev->db_tab->page[i].db_rec)749 continue;750 751 if (!bitmap_empty(dev->db_tab->page[i].used, MTHCA_DB_REC_PER_PAGE))752 mthca_warn(dev, "Kernel UARC page %d not empty\n", i);753 754 mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1);755 756 dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE,757 dev->db_tab->page[i].db_rec,758 dev->db_tab->page[i].mapping);759 }760 761 kfree(dev->db_tab->page);762 kfree(dev->db_tab);763}764