551 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB2/*3 * Copyright (c) 2018 Hisilicon Limited.4 */5 6#include <linux/pci.h>7#include <rdma/ib_umem.h>8#include <rdma/uverbs_ioctl.h>9#include "hns_roce_device.h"10#include "hns_roce_cmd.h"11#include "hns_roce_hem.h"12 13void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type)14{15 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;16 struct hns_roce_srq *srq;17 18 xa_lock(&srq_table->xa);19 srq = xa_load(&srq_table->xa, srqn & (hr_dev->caps.num_srqs - 1));20 if (srq)21 refcount_inc(&srq->refcount);22 xa_unlock(&srq_table->xa);23 24 if (!srq) {25 dev_warn(hr_dev->dev, "Async event for bogus SRQ %08x\n", srqn);26 return;27 }28 29 srq->event(srq, event_type);30 31 if (refcount_dec_and_test(&srq->refcount))32 complete(&srq->free);33}34 35static void hns_roce_ib_srq_event(struct hns_roce_srq *srq,36 enum hns_roce_event event_type)37{38 struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device);39 struct ib_srq *ibsrq = &srq->ibsrq;40 struct ib_event event;41 42 if (ibsrq->event_handler) {43 event.device = ibsrq->device;44 event.element.srq = ibsrq;45 switch (event_type) {46 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:47 event.event = IB_EVENT_SRQ_LIMIT_REACHED;48 break;49 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:50 event.event = IB_EVENT_SRQ_ERR;51 break;52 default:53 dev_err(hr_dev->dev,54 "hns_roce:Unexpected event type 0x%x on SRQ %06lx\n",55 event_type, srq->srqn);56 return;57 }58 59 ibsrq->event_handler(&event, ibsrq->srq_context);60 }61}62 63static int alloc_srqn(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)64{65 struct hns_roce_ida *srq_ida = &hr_dev->srq_table.srq_ida;66 int id;67 68 id = ida_alloc_range(&srq_ida->ida, srq_ida->min, srq_ida->max,69 GFP_KERNEL);70 if (id < 0) {71 ibdev_err(&hr_dev->ib_dev, "failed to alloc srq(%d).\n", id);72 return -ENOMEM;73 }74 75 srq->srqn = id;76 77 return 0;78}79 80static void free_srqn(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)81{82 ida_free(&hr_dev->srq_table.srq_ida.ida, (int)srq->srqn);83}84 85static int hns_roce_create_srqc(struct hns_roce_dev *hr_dev,86 struct hns_roce_srq *srq)87{88 struct ib_device *ibdev = &hr_dev->ib_dev;89 struct hns_roce_cmd_mailbox *mailbox;90 int ret;91 92 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);93 if (IS_ERR(mailbox)) {94 ibdev_err(ibdev, "failed to alloc mailbox for SRQC.\n");95 return PTR_ERR(mailbox);96 }97 98 ret = hr_dev->hw->write_srqc(srq, mailbox->buf);99 if (ret) {100 ibdev_err(ibdev, "failed to write SRQC.\n");101 goto err_mbox;102 }103 104 ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_SRQ,105 srq->srqn);106 if (ret)107 ibdev_err(ibdev, "failed to config SRQC, ret = %d.\n", ret);108 109err_mbox:110 hns_roce_free_cmd_mailbox(hr_dev, mailbox);111 return ret;112}113 114static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)115{116 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;117 struct ib_device *ibdev = &hr_dev->ib_dev;118 int ret;119 120 ret = hns_roce_table_get(hr_dev, &srq_table->table, srq->srqn);121 if (ret) {122 ibdev_err(ibdev, "failed to get SRQC table, ret = %d.\n", ret);123 return ret;124 }125 126 ret = xa_err(xa_store_irq(&srq_table->xa, srq->srqn, srq, GFP_KERNEL));127 if (ret) {128 ibdev_err(ibdev, "failed to store SRQC, ret = %d.\n", ret);129 goto err_put;130 }131 132 ret = hns_roce_create_srqc(hr_dev, srq);133 if (ret)134 goto err_xa;135 136 return 0;137 138err_xa:139 xa_erase_irq(&srq_table->xa, srq->srqn);140err_put:141 hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn);142 143 return ret;144}145 146static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)147{148 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;149 int ret;150 151 ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_SRQ,152 srq->srqn);153 if (ret)154 dev_err(hr_dev->dev, "DESTROY_SRQ failed (%d) for SRQN %06lx\n",155 ret, srq->srqn);156 157 xa_erase_irq(&srq_table->xa, srq->srqn);158 159 if (refcount_dec_and_test(&srq->refcount))160 complete(&srq->free);161 wait_for_completion(&srq->free);162 163 hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn);164}165 166static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,167 struct ib_udata *udata, unsigned long addr)168{169 struct hns_roce_idx_que *idx_que = &srq->idx_que;170 struct ib_device *ibdev = &hr_dev->ib_dev;171 struct hns_roce_buf_attr buf_attr = {};172 int ret;173 174 srq->idx_que.entry_shift = ilog2(HNS_ROCE_IDX_QUE_ENTRY_SZ);175 176 buf_attr.page_shift = hr_dev->caps.idx_buf_pg_sz + PAGE_SHIFT;177 buf_attr.region[0].size = to_hr_hem_entries_size(srq->wqe_cnt,178 srq->idx_que.entry_shift);179 buf_attr.region[0].hopnum = hr_dev->caps.idx_hop_num;180 buf_attr.region_count = 1;181 182 ret = hns_roce_mtr_create(hr_dev, &idx_que->mtr, &buf_attr,183 hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT,184 udata, addr);185 if (ret) {186 ibdev_err(ibdev,187 "failed to alloc SRQ idx mtr, ret = %d.\n", ret);188 return ret;189 }190 191 if (!udata) {192 idx_que->bitmap = bitmap_zalloc(srq->wqe_cnt, GFP_KERNEL);193 if (!idx_que->bitmap) {194 ibdev_err(ibdev, "failed to alloc SRQ idx bitmap.\n");195 ret = -ENOMEM;196 goto err_idx_mtr;197 }198 }199 200 idx_que->head = 0;201 idx_que->tail = 0;202 203 return 0;204err_idx_mtr:205 hns_roce_mtr_destroy(hr_dev, &idx_que->mtr);206 207 return ret;208}209 210static void free_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)211{212 struct hns_roce_idx_que *idx_que = &srq->idx_que;213 214 bitmap_free(idx_que->bitmap);215 idx_que->bitmap = NULL;216 hns_roce_mtr_destroy(hr_dev, &idx_que->mtr);217}218 219static int alloc_srq_wqe_buf(struct hns_roce_dev *hr_dev,220 struct hns_roce_srq *srq,221 struct ib_udata *udata, unsigned long addr)222{223 struct ib_device *ibdev = &hr_dev->ib_dev;224 struct hns_roce_buf_attr buf_attr = {};225 int ret;226 227 srq->wqe_shift = ilog2(roundup_pow_of_two(max(HNS_ROCE_SGE_SIZE,228 HNS_ROCE_SGE_SIZE *229 srq->max_gs)));230 231 buf_attr.page_shift = hr_dev->caps.srqwqe_buf_pg_sz + PAGE_SHIFT;232 buf_attr.region[0].size = to_hr_hem_entries_size(srq->wqe_cnt,233 srq->wqe_shift);234 buf_attr.region[0].hopnum = hr_dev->caps.srqwqe_hop_num;235 buf_attr.region_count = 1;236 237 ret = hns_roce_mtr_create(hr_dev, &srq->buf_mtr, &buf_attr,238 hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT,239 udata, addr);240 if (ret)241 ibdev_err(ibdev,242 "failed to alloc SRQ buf mtr, ret = %d.\n", ret);243 244 return ret;245}246 247static void free_srq_wqe_buf(struct hns_roce_dev *hr_dev,248 struct hns_roce_srq *srq)249{250 hns_roce_mtr_destroy(hr_dev, &srq->buf_mtr);251}252 253static int alloc_srq_wrid(struct hns_roce_srq *srq)254{255 srq->wrid = kvmalloc_array(srq->wqe_cnt, sizeof(u64), GFP_KERNEL);256 if (!srq->wrid)257 return -ENOMEM;258 259 return 0;260}261 262static void free_srq_wrid(struct hns_roce_srq *srq)263{264 kvfree(srq->wrid);265 srq->wrid = NULL;266}267 268static u32 proc_srq_sge(struct hns_roce_dev *dev, struct hns_roce_srq *hr_srq,269 bool user)270{271 u32 max_sge = dev->caps.max_srq_sges;272 273 if (dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)274 return max_sge;275 276 /* Reserve SGEs only for HIP08 in kernel; The userspace driver will277 * calculate number of max_sge with reserved SGEs when allocating wqe278 * buf, so there is no need to do this again in kernel. But the number279 * may exceed the capacity of SGEs recorded in the firmware, so the280 * kernel driver should just adapt the value accordingly.281 */282 if (user)283 max_sge = roundup_pow_of_two(max_sge + 1);284 else285 hr_srq->rsv_sge = 1;286 287 return max_sge;288}289 290static int set_srq_basic_param(struct hns_roce_srq *srq,291 struct ib_srq_init_attr *init_attr,292 struct ib_udata *udata)293{294 struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device);295 struct ib_srq_attr *attr = &init_attr->attr;296 u32 max_sge;297 298 max_sge = proc_srq_sge(hr_dev, srq, !!udata);299 if (attr->max_wr > hr_dev->caps.max_srq_wrs ||300 attr->max_sge > max_sge || !attr->max_sge) {301 ibdev_err(&hr_dev->ib_dev,302 "invalid SRQ attr, depth = %u, sge = %u.\n",303 attr->max_wr, attr->max_sge);304 return -EINVAL;305 }306 307 attr->max_wr = max_t(u32, attr->max_wr, HNS_ROCE_MIN_SRQ_WQE_NUM);308 srq->wqe_cnt = roundup_pow_of_two(attr->max_wr);309 srq->max_gs = roundup_pow_of_two(attr->max_sge + srq->rsv_sge);310 311 attr->max_wr = srq->wqe_cnt;312 attr->max_sge = srq->max_gs - srq->rsv_sge;313 attr->srq_limit = 0;314 315 return 0;316}317 318static void set_srq_ext_param(struct hns_roce_srq *srq,319 struct ib_srq_init_attr *init_attr)320{321 srq->cqn = ib_srq_has_cq(init_attr->srq_type) ?322 to_hr_cq(init_attr->ext.cq)->cqn : 0;323 324 srq->xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ?325 to_hr_xrcd(init_attr->ext.xrc.xrcd)->xrcdn : 0;326}327 328static int set_srq_param(struct hns_roce_srq *srq,329 struct ib_srq_init_attr *init_attr,330 struct ib_udata *udata)331{332 int ret;333 334 ret = set_srq_basic_param(srq, init_attr, udata);335 if (ret)336 return ret;337 338 set_srq_ext_param(srq, init_attr);339 340 return 0;341}342 343static int alloc_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,344 struct ib_udata *udata)345{346 struct hns_roce_ib_create_srq ucmd = {};347 int ret;348 349 if (udata) {350 ret = ib_copy_from_udata(&ucmd, udata,351 min(udata->inlen, sizeof(ucmd)));352 if (ret) {353 ibdev_err(&hr_dev->ib_dev,354 "failed to copy SRQ udata, ret = %d.\n",355 ret);356 return ret;357 }358 }359 360 ret = alloc_srq_idx(hr_dev, srq, udata, ucmd.que_addr);361 if (ret)362 return ret;363 364 ret = alloc_srq_wqe_buf(hr_dev, srq, udata, ucmd.buf_addr);365 if (ret)366 goto err_idx;367 368 if (!udata) {369 ret = alloc_srq_wrid(srq);370 if (ret)371 goto err_wqe_buf;372 }373 374 return 0;375 376err_wqe_buf:377 free_srq_wqe_buf(hr_dev, srq);378err_idx:379 free_srq_idx(hr_dev, srq);380 381 return ret;382}383 384static void free_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)385{386 free_srq_wrid(srq);387 free_srq_wqe_buf(hr_dev, srq);388 free_srq_idx(hr_dev, srq);389}390 391static int get_srq_ucmd(struct hns_roce_srq *srq, struct ib_udata *udata,392 struct hns_roce_ib_create_srq *ucmd)393{394 struct ib_device *ibdev = srq->ibsrq.device;395 int ret;396 397 ret = ib_copy_from_udata(ucmd, udata, min(udata->inlen, sizeof(*ucmd)));398 if (ret) {399 ibdev_err(ibdev, "failed to copy SRQ udata, ret = %d.\n", ret);400 return ret;401 }402 403 return 0;404}405 406static void free_srq_db(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,407 struct ib_udata *udata)408{409 struct hns_roce_ucontext *uctx;410 411 if (!(srq->cap_flags & HNS_ROCE_SRQ_CAP_RECORD_DB))412 return;413 414 srq->cap_flags &= ~HNS_ROCE_SRQ_CAP_RECORD_DB;415 if (udata) {416 uctx = rdma_udata_to_drv_context(udata,417 struct hns_roce_ucontext,418 ibucontext);419 hns_roce_db_unmap_user(uctx, &srq->rdb);420 } else {421 hns_roce_free_db(hr_dev, &srq->rdb);422 }423}424 425static int alloc_srq_db(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,426 struct ib_udata *udata,427 struct hns_roce_ib_create_srq_resp *resp)428{429 struct hns_roce_ib_create_srq ucmd = {};430 struct hns_roce_ucontext *uctx;431 int ret;432 433 if (udata) {434 ret = get_srq_ucmd(srq, udata, &ucmd);435 if (ret)436 return ret;437 438 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ_RECORD_DB) &&439 (ucmd.req_cap_flags & HNS_ROCE_SRQ_CAP_RECORD_DB)) {440 uctx = rdma_udata_to_drv_context(udata,441 struct hns_roce_ucontext, ibucontext);442 ret = hns_roce_db_map_user(uctx, ucmd.db_addr,443 &srq->rdb);444 if (ret)445 return ret;446 447 srq->cap_flags |= HNS_ROCE_RSP_SRQ_CAP_RECORD_DB;448 }449 } else {450 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ_RECORD_DB) {451 ret = hns_roce_alloc_db(hr_dev, &srq->rdb, 1);452 if (ret)453 return ret;454 455 *srq->rdb.db_record = 0;456 srq->cap_flags |= HNS_ROCE_RSP_SRQ_CAP_RECORD_DB;457 }458 srq->db_reg = hr_dev->reg_base + SRQ_DB_REG;459 }460 461 return 0;462}463 464int hns_roce_create_srq(struct ib_srq *ib_srq,465 struct ib_srq_init_attr *init_attr,466 struct ib_udata *udata)467{468 struct hns_roce_dev *hr_dev = to_hr_dev(ib_srq->device);469 struct hns_roce_ib_create_srq_resp resp = {};470 struct hns_roce_srq *srq = to_hr_srq(ib_srq);471 int ret;472 473 mutex_init(&srq->mutex);474 spin_lock_init(&srq->lock);475 476 ret = set_srq_param(srq, init_attr, udata);477 if (ret)478 goto err_out;479 480 ret = alloc_srq_buf(hr_dev, srq, udata);481 if (ret)482 goto err_out;483 484 ret = alloc_srq_db(hr_dev, srq, udata, &resp);485 if (ret)486 goto err_srq_buf;487 488 ret = alloc_srqn(hr_dev, srq);489 if (ret)490 goto err_srq_db;491 492 ret = alloc_srqc(hr_dev, srq);493 if (ret)494 goto err_srqn;495 496 if (udata) {497 resp.cap_flags = srq->cap_flags;498 resp.srqn = srq->srqn;499 if (ib_copy_to_udata(udata, &resp,500 min(udata->outlen, sizeof(resp)))) {501 ret = -EFAULT;502 goto err_srqc;503 }504 }505 506 srq->event = hns_roce_ib_srq_event;507 refcount_set(&srq->refcount, 1);508 init_completion(&srq->free);509 510 return 0;511 512err_srqc:513 free_srqc(hr_dev, srq);514err_srqn:515 free_srqn(hr_dev, srq);516err_srq_db:517 free_srq_db(hr_dev, srq, udata);518err_srq_buf:519 free_srq_buf(hr_dev, srq);520err_out:521 mutex_destroy(&srq->mutex);522 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_SRQ_CREATE_ERR_CNT]);523 524 return ret;525}526 527int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)528{529 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);530 struct hns_roce_srq *srq = to_hr_srq(ibsrq);531 532 free_srqc(hr_dev, srq);533 free_srqn(hr_dev, srq);534 free_srq_db(hr_dev, srq, udata);535 free_srq_buf(hr_dev, srq);536 mutex_destroy(&srq->mutex);537 return 0;538}539 540void hns_roce_init_srq_table(struct hns_roce_dev *hr_dev)541{542 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;543 struct hns_roce_ida *srq_ida = &srq_table->srq_ida;544 545 xa_init(&srq_table->xa);546 547 ida_init(&srq_ida->ida);548 srq_ida->max = hr_dev->caps.num_srqs - 1;549 srq_ida->min = hr_dev->caps.reserved_srqs;550}551