181 lines · c
1/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */2/*3 * Copyright (c) 2017 Hisilicon Limited.4 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.5 */6 7#include <rdma/ib_umem.h>8#include "hns_roce_device.h"9 10int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt,11 struct hns_roce_db *db)12{13 unsigned long page_addr = virt & PAGE_MASK;14 struct hns_roce_user_db_page *page;15 unsigned int offset;16 int ret = 0;17 18 mutex_lock(&context->page_mutex);19 20 list_for_each_entry(page, &context->page_list, list)21 if (page->user_virt == page_addr)22 goto found;23 24 page = kmalloc(sizeof(*page), GFP_KERNEL);25 if (!page) {26 ret = -ENOMEM;27 goto out;28 }29 30 refcount_set(&page->refcount, 1);31 page->user_virt = page_addr;32 page->umem = ib_umem_get(context->ibucontext.device, page_addr,33 PAGE_SIZE, 0);34 if (IS_ERR(page->umem)) {35 ret = PTR_ERR(page->umem);36 kfree(page);37 goto out;38 }39 40 list_add(&page->list, &context->page_list);41 42found:43 offset = virt - page_addr;44 db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + offset;45 db->virt_addr = sg_virt(page->umem->sgt_append.sgt.sgl) + offset;46 db->u.user_page = page;47 refcount_inc(&page->refcount);48 49out:50 mutex_unlock(&context->page_mutex);51 52 return ret;53}54 55void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,56 struct hns_roce_db *db)57{58 mutex_lock(&context->page_mutex);59 60 refcount_dec(&db->u.user_page->refcount);61 if (refcount_dec_if_one(&db->u.user_page->refcount)) {62 list_del(&db->u.user_page->list);63 ib_umem_release(db->u.user_page->umem);64 kfree(db->u.user_page);65 }66 67 mutex_unlock(&context->page_mutex);68}69 70static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir(71 struct device *dma_device)72{73 struct hns_roce_db_pgdir *pgdir;74 75 pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL);76 if (!pgdir)77 return NULL;78 79 bitmap_fill(pgdir->order1,80 HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT);81 pgdir->bits[0] = pgdir->order0;82 pgdir->bits[1] = pgdir->order1;83 pgdir->page = dma_alloc_coherent(dma_device, PAGE_SIZE,84 &pgdir->db_dma, GFP_KERNEL);85 if (!pgdir->page) {86 kfree(pgdir);87 return NULL;88 }89 90 return pgdir;91}92 93static int hns_roce_alloc_db_from_pgdir(struct hns_roce_db_pgdir *pgdir,94 struct hns_roce_db *db, int order)95{96 unsigned long o;97 unsigned long i;98 99 for (o = order; o <= 1; ++o) {100 i = find_first_bit(pgdir->bits[o], HNS_ROCE_DB_PER_PAGE >> o);101 if (i < HNS_ROCE_DB_PER_PAGE >> o)102 goto found;103 }104 105 return -ENOMEM;106 107found:108 clear_bit(i, pgdir->bits[o]);109 110 i <<= o;111 112 if (o > order)113 set_bit(i ^ 1, pgdir->bits[order]);114 115 db->u.pgdir = pgdir;116 db->index = i;117 db->db_record = pgdir->page + db->index;118 db->dma = pgdir->db_dma + db->index * HNS_ROCE_DB_UNIT_SIZE;119 db->order = order;120 121 return 0;122}123 124int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db,125 int order)126{127 struct hns_roce_db_pgdir *pgdir;128 int ret = 0;129 130 mutex_lock(&hr_dev->pgdir_mutex);131 132 list_for_each_entry(pgdir, &hr_dev->pgdir_list, list)133 if (!hns_roce_alloc_db_from_pgdir(pgdir, db, order))134 goto out;135 136 pgdir = hns_roce_alloc_db_pgdir(hr_dev->dev);137 if (!pgdir) {138 ret = -ENOMEM;139 goto out;140 }141 142 list_add(&pgdir->list, &hr_dev->pgdir_list);143 144 /* This should never fail -- we just allocated an empty page: */145 WARN_ON(hns_roce_alloc_db_from_pgdir(pgdir, db, order));146 147out:148 mutex_unlock(&hr_dev->pgdir_mutex);149 150 return ret;151}152 153void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db)154{155 unsigned long o;156 unsigned long i;157 158 mutex_lock(&hr_dev->pgdir_mutex);159 160 o = db->order;161 i = db->index;162 163 if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {164 clear_bit(i ^ 1, db->u.pgdir->order0);165 ++o;166 }167 168 i >>= o;169 set_bit(i, db->u.pgdir->bits[o]);170 171 if (bitmap_full(db->u.pgdir->order1,172 HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT)) {173 dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page,174 db->u.pgdir->db_dma);175 list_del(&db->u.pgdir->list);176 kfree(db->u.pgdir);177 }178 179 mutex_unlock(&hr_dev->pgdir_mutex);180}181