494 lines · c
1/*2 * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved.3 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.4 *5 * This software is available to you under a choice of one of two6 * licenses. You may choose to be licensed under the terms of the GNU7 * General Public License (GPL) Version 2, available from the file8 * COPYING in the main directory of this source tree, or the9 * OpenIB.org BSD license below:10 *11 * Redistribution and use in source and binary forms, with or12 * without modification, are permitted provided that the following13 * conditions are met:14 *15 * - Redistributions of source code must retain the above16 * copyright notice, this list of conditions and the following17 * disclaimer.18 *19 * - Redistributions in binary form must reproduce the above20 * copyright notice, this list of conditions and the following21 * disclaimer in the documentation and/or other materials22 * provided with the distribution.23 *24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE31 * SOFTWARE.32 */33 34#include <linux/errno.h>35#include <linux/mm.h>36#include <linux/scatterlist.h>37#include <linux/slab.h>38 39#include <linux/mlx4/cmd.h>40 41#include "mlx4.h"42#include "icm.h"43#include "fw.h"44 45/*46 * We allocate in as big chunks as we can, up to a maximum of 256 KB47 * per chunk. Note that the chunks are not necessarily in contiguous48 * physical memory.49 */50enum {51 MLX4_ICM_ALLOC_SIZE = 1 << 18,52 MLX4_TABLE_CHUNK_SIZE = 1 << 18,53};54 55static void mlx4_free_icm_pages(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk)56{57 int i;58 59 if (chunk->nsg > 0)60 dma_unmap_sg(&dev->persist->pdev->dev, chunk->sg, chunk->npages,61 DMA_BIDIRECTIONAL);62 63 for (i = 0; i < chunk->npages; ++i)64 __free_pages(sg_page(&chunk->sg[i]),65 get_order(chunk->sg[i].length));66}67 68static void mlx4_free_icm_coherent(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk)69{70 int i;71 72 for (i = 0; i < chunk->npages; ++i)73 dma_free_coherent(&dev->persist->pdev->dev,74 chunk->buf[i].size,75 chunk->buf[i].addr,76 chunk->buf[i].dma_addr);77}78 79void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm, int coherent)80{81 struct mlx4_icm_chunk *chunk, *tmp;82 83 if (!icm)84 return;85 86 list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) {87 if (coherent)88 mlx4_free_icm_coherent(dev, chunk);89 else90 mlx4_free_icm_pages(dev, chunk);91 92 kfree(chunk);93 }94 95 kfree(icm);96}97 98static int mlx4_alloc_icm_pages(struct scatterlist *mem, int order,99 gfp_t gfp_mask, int node)100{101 struct page *page;102 103 page = alloc_pages_node(node, gfp_mask, order);104 if (!page) {105 page = alloc_pages(gfp_mask, order);106 if (!page)107 return -ENOMEM;108 }109 110 sg_set_page(mem, page, PAGE_SIZE << order, 0);111 return 0;112}113 114static int mlx4_alloc_icm_coherent(struct device *dev, struct mlx4_icm_buf *buf,115 int order, gfp_t gfp_mask)116{117 buf->addr = dma_alloc_coherent(dev, PAGE_SIZE << order,118 &buf->dma_addr, gfp_mask);119 if (!buf->addr)120 return -ENOMEM;121 122 if (offset_in_page(buf->addr)) {123 dma_free_coherent(dev, PAGE_SIZE << order, buf->addr,124 buf->dma_addr);125 return -ENOMEM;126 }127 128 buf->size = PAGE_SIZE << order;129 return 0;130}131 132struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages,133 gfp_t gfp_mask, int coherent)134{135 struct mlx4_icm *icm;136 struct mlx4_icm_chunk *chunk = NULL;137 int cur_order;138 gfp_t mask;139 int ret;140 141 /* We use sg_set_buf for coherent allocs, which assumes low memory */142 BUG_ON(coherent && (gfp_mask & __GFP_HIGHMEM));143 144 icm = kmalloc_node(sizeof(*icm),145 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN),146 dev->numa_node);147 if (!icm) {148 icm = kmalloc(sizeof(*icm),149 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));150 if (!icm)151 return NULL;152 }153 154 icm->refcount = 0;155 INIT_LIST_HEAD(&icm->chunk_list);156 157 cur_order = get_order(MLX4_ICM_ALLOC_SIZE);158 159 while (npages > 0) {160 if (!chunk) {161 chunk = kzalloc_node(sizeof(*chunk),162 gfp_mask & ~(__GFP_HIGHMEM |163 __GFP_NOWARN),164 dev->numa_node);165 if (!chunk) {166 chunk = kzalloc(sizeof(*chunk),167 gfp_mask & ~(__GFP_HIGHMEM |168 __GFP_NOWARN));169 if (!chunk)170 goto fail;171 }172 chunk->coherent = coherent;173 174 if (!coherent)175 sg_init_table(chunk->sg, MLX4_ICM_CHUNK_LEN);176 list_add_tail(&chunk->list, &icm->chunk_list);177 }178 179 while (1 << cur_order > npages)180 --cur_order;181 182 mask = gfp_mask;183 if (cur_order)184 mask &= ~__GFP_DIRECT_RECLAIM;185 186 if (coherent)187 ret = mlx4_alloc_icm_coherent(&dev->persist->pdev->dev,188 &chunk->buf[chunk->npages],189 cur_order, mask);190 else191 ret = mlx4_alloc_icm_pages(&chunk->sg[chunk->npages],192 cur_order, mask,193 dev->numa_node);194 195 if (ret) {196 if (--cur_order < 0)197 goto fail;198 else199 continue;200 }201 202 ++chunk->npages;203 204 if (coherent)205 ++chunk->nsg;206 else if (chunk->npages == MLX4_ICM_CHUNK_LEN) {207 chunk->nsg = dma_map_sg(&dev->persist->pdev->dev,208 chunk->sg, chunk->npages,209 DMA_BIDIRECTIONAL);210 211 if (!chunk->nsg)212 goto fail;213 }214 215 if (chunk->npages == MLX4_ICM_CHUNK_LEN)216 chunk = NULL;217 218 npages -= 1 << cur_order;219 }220 221 if (!coherent && chunk) {222 chunk->nsg = dma_map_sg(&dev->persist->pdev->dev, chunk->sg,223 chunk->npages, DMA_BIDIRECTIONAL);224 225 if (!chunk->nsg)226 goto fail;227 }228 229 return icm;230 231fail:232 mlx4_free_icm(dev, icm, coherent);233 return NULL;234}235 236static int mlx4_MAP_ICM(struct mlx4_dev *dev, struct mlx4_icm *icm, u64 virt)237{238 return mlx4_map_cmd(dev, MLX4_CMD_MAP_ICM, icm, virt);239}240 241static int mlx4_UNMAP_ICM(struct mlx4_dev *dev, u64 virt, u32 page_count)242{243 return mlx4_cmd(dev, virt, page_count, 0, MLX4_CMD_UNMAP_ICM,244 MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE);245}246 247int mlx4_MAP_ICM_AUX(struct mlx4_dev *dev, struct mlx4_icm *icm)248{249 return mlx4_map_cmd(dev, MLX4_CMD_MAP_ICM_AUX, icm, -1);250}251 252int mlx4_UNMAP_ICM_AUX(struct mlx4_dev *dev)253{254 return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_UNMAP_ICM_AUX,255 MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE);256}257 258int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj)259{260 u32 i = (obj & (table->num_obj - 1)) /261 (MLX4_TABLE_CHUNK_SIZE / table->obj_size);262 int ret = 0;263 264 mutex_lock(&table->mutex);265 266 if (table->icm[i]) {267 ++table->icm[i]->refcount;268 goto out;269 }270 271 table->icm[i] = mlx4_alloc_icm(dev, MLX4_TABLE_CHUNK_SIZE >> PAGE_SHIFT,272 (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) |273 __GFP_NOWARN, table->coherent);274 if (!table->icm[i]) {275 ret = -ENOMEM;276 goto out;277 }278 279 if (mlx4_MAP_ICM(dev, table->icm[i], table->virt +280 (u64) i * MLX4_TABLE_CHUNK_SIZE)) {281 mlx4_free_icm(dev, table->icm[i], table->coherent);282 table->icm[i] = NULL;283 ret = -ENOMEM;284 goto out;285 }286 287 ++table->icm[i]->refcount;288 289out:290 mutex_unlock(&table->mutex);291 return ret;292}293 294void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj)295{296 u32 i;297 u64 offset;298 299 i = (obj & (table->num_obj - 1)) / (MLX4_TABLE_CHUNK_SIZE / table->obj_size);300 301 mutex_lock(&table->mutex);302 303 if (--table->icm[i]->refcount == 0) {304 offset = (u64) i * MLX4_TABLE_CHUNK_SIZE;305 mlx4_UNMAP_ICM(dev, table->virt + offset,306 MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE);307 mlx4_free_icm(dev, table->icm[i], table->coherent);308 table->icm[i] = NULL;309 }310 311 mutex_unlock(&table->mutex);312}313 314void *mlx4_table_find(struct mlx4_icm_table *table, u32 obj,315 dma_addr_t *dma_handle)316{317 int offset, dma_offset, i;318 u64 idx;319 struct mlx4_icm_chunk *chunk;320 struct mlx4_icm *icm;321 void *addr = NULL;322 323 if (!table->lowmem)324 return NULL;325 326 mutex_lock(&table->mutex);327 328 idx = (u64) (obj & (table->num_obj - 1)) * table->obj_size;329 icm = table->icm[idx / MLX4_TABLE_CHUNK_SIZE];330 dma_offset = offset = idx % MLX4_TABLE_CHUNK_SIZE;331 332 if (!icm)333 goto out;334 335 list_for_each_entry(chunk, &icm->chunk_list, list) {336 for (i = 0; i < chunk->npages; ++i) {337 dma_addr_t dma_addr;338 size_t len;339 340 if (table->coherent) {341 len = chunk->buf[i].size;342 dma_addr = chunk->buf[i].dma_addr;343 addr = chunk->buf[i].addr;344 } else {345 struct page *page;346 347 len = sg_dma_len(&chunk->sg[i]);348 dma_addr = sg_dma_address(&chunk->sg[i]);349 350 /* XXX: we should never do this for highmem351 * allocation. This function either needs352 * to be split, or the kernel virtual address353 * return needs to be made optional.354 */355 page = sg_page(&chunk->sg[i]);356 addr = lowmem_page_address(page);357 }358 359 if (dma_handle && dma_offset >= 0) {360 if (len > dma_offset)361 *dma_handle = dma_addr + dma_offset;362 dma_offset -= len;363 }364 365 /*366 * DMA mapping can merge pages but not split them,367 * so if we found the page, dma_handle has already368 * been assigned to.369 */370 if (len > offset)371 goto out;372 offset -= len;373 }374 }375 376 addr = NULL;377out:378 mutex_unlock(&table->mutex);379 return addr ? addr + offset : NULL;380}381 382int mlx4_table_get_range(struct mlx4_dev *dev, struct mlx4_icm_table *table,383 u32 start, u32 end)384{385 int inc = MLX4_TABLE_CHUNK_SIZE / table->obj_size;386 int err;387 u32 i;388 389 for (i = start; i <= end; i += inc) {390 err = mlx4_table_get(dev, table, i);391 if (err)392 goto fail;393 }394 395 return 0;396 397fail:398 while (i > start) {399 i -= inc;400 mlx4_table_put(dev, table, i);401 }402 403 return err;404}405 406void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table,407 u32 start, u32 end)408{409 u32 i;410 411 for (i = start; i <= end; i += MLX4_TABLE_CHUNK_SIZE / table->obj_size)412 mlx4_table_put(dev, table, i);413}414 415int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table,416 u64 virt, int obj_size, u32 nobj, int reserved,417 int use_lowmem, int use_coherent)418{419 int obj_per_chunk;420 int num_icm;421 unsigned chunk_size;422 int i;423 u64 size;424 425 obj_per_chunk = MLX4_TABLE_CHUNK_SIZE / obj_size;426 if (WARN_ON(!obj_per_chunk))427 return -EINVAL;428 num_icm = DIV_ROUND_UP(nobj, obj_per_chunk);429 430 table->icm = kvcalloc(num_icm, sizeof(*table->icm), GFP_KERNEL);431 if (!table->icm)432 return -ENOMEM;433 table->virt = virt;434 table->num_icm = num_icm;435 table->num_obj = nobj;436 table->obj_size = obj_size;437 table->lowmem = use_lowmem;438 table->coherent = use_coherent;439 mutex_init(&table->mutex);440 441 size = (u64) nobj * obj_size;442 for (i = 0; i * MLX4_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) {443 chunk_size = MLX4_TABLE_CHUNK_SIZE;444 if ((i + 1) * MLX4_TABLE_CHUNK_SIZE > size)445 chunk_size = PAGE_ALIGN(size -446 i * MLX4_TABLE_CHUNK_SIZE);447 448 table->icm[i] = mlx4_alloc_icm(dev, chunk_size >> PAGE_SHIFT,449 (use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) |450 __GFP_NOWARN, use_coherent);451 if (!table->icm[i])452 goto err;453 if (mlx4_MAP_ICM(dev, table->icm[i], virt + i * MLX4_TABLE_CHUNK_SIZE)) {454 mlx4_free_icm(dev, table->icm[i], use_coherent);455 table->icm[i] = NULL;456 goto err;457 }458 459 /*460 * Add a reference to this ICM chunk so that it never461 * gets freed (since it contains reserved firmware objects).462 */463 ++table->icm[i]->refcount;464 }465 466 return 0;467 468err:469 for (i = 0; i < num_icm; ++i)470 if (table->icm[i]) {471 mlx4_UNMAP_ICM(dev, virt + i * MLX4_TABLE_CHUNK_SIZE,472 MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE);473 mlx4_free_icm(dev, table->icm[i], use_coherent);474 }475 476 kvfree(table->icm);477 478 return -ENOMEM;479}480 481void mlx4_cleanup_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table)482{483 int i;484 485 for (i = 0; i < table->num_icm; ++i)486 if (table->icm[i]) {487 mlx4_UNMAP_ICM(dev, table->virt + i * MLX4_TABLE_CHUNK_SIZE,488 MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE);489 mlx4_free_icm(dev, table->icm[i], table->coherent);490 }491 492 kvfree(table->icm);493}494