1842 lines · c
1// SPDX-License-Identifier: GPL-2.02 3/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */4/* Kai Shen <kaishen@linux.alibaba.com> */5/* Copyright (c) 2020-2022, Alibaba Group. */6 7/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */8/* Copyright (c) 2008-2019, IBM Corporation */9 10/* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */11 12#include <linux/vmalloc.h>13#include <net/addrconf.h>14#include <rdma/erdma-abi.h>15#include <rdma/ib_umem.h>16#include <rdma/uverbs_ioctl.h>17 18#include "erdma.h"19#include "erdma_cm.h"20#include "erdma_verbs.h"21 22static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg,23 u64 *addr0, u64 *addr1)24{25 struct erdma_mtt *mtt = mem->mtt;26 27 if (mem->mtt_nents > ERDMA_MAX_INLINE_MTT_ENTRIES) {28 *addr0 = mtt->buf_dma;29 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,30 ERDMA_MR_MTT_1LEVEL);31 } else {32 *addr0 = mtt->buf[0];33 memcpy(addr1, mtt->buf + 1, MTT_SIZE(mem->mtt_nents - 1));34 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,35 ERDMA_MR_MTT_0LEVEL);36 }37}38 39static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)40{41 struct erdma_dev *dev = to_edev(qp->ibqp.device);42 struct erdma_pd *pd = to_epd(qp->ibqp.pd);43 struct erdma_cmdq_create_qp_req req;44 struct erdma_uqp *user_qp;45 u64 resp0, resp1;46 int err;47 48 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,49 CMDQ_OPCODE_CREATE_QP);50 51 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK,52 ilog2(qp->attrs.sq_size)) |53 FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp));54 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK,55 ilog2(qp->attrs.rq_size)) |56 FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn);57 58 if (rdma_is_kernel_res(&qp->ibqp.res)) {59 u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;60 61 req.sq_cqn_mtt_cfg =62 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,63 pgsz_range) |64 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);65 req.rq_cqn_mtt_cfg =66 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,67 pgsz_range) |68 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);69 70 req.sq_mtt_cfg =71 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) |72 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) |73 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,74 ERDMA_MR_MTT_0LEVEL);75 req.rq_mtt_cfg = req.sq_mtt_cfg;76 77 req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr;78 req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr;79 req.sq_dbrec_dma = qp->kern_qp.sq_dbrec_dma;80 req.rq_dbrec_dma = qp->kern_qp.rq_dbrec_dma;81 } else {82 user_qp = &qp->user_qp;83 req.sq_cqn_mtt_cfg = FIELD_PREP(84 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,85 ilog2(user_qp->sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);86 req.sq_cqn_mtt_cfg |=87 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);88 89 req.rq_cqn_mtt_cfg = FIELD_PREP(90 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,91 ilog2(user_qp->rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);92 req.rq_cqn_mtt_cfg |=93 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);94 95 req.sq_mtt_cfg = user_qp->sq_mem.page_offset;96 req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,97 user_qp->sq_mem.mtt_nents);98 99 req.rq_mtt_cfg = user_qp->rq_mem.page_offset;100 req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,101 user_qp->rq_mem.mtt_nents);102 103 assemble_qbuf_mtt_for_cmd(&user_qp->sq_mem, &req.sq_mtt_cfg,104 &req.sq_buf_addr, req.sq_mtt_entry);105 assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg,106 &req.rq_buf_addr, req.rq_mtt_entry);107 108 req.sq_dbrec_dma = user_qp->sq_dbrec_dma;109 req.rq_dbrec_dma = user_qp->rq_dbrec_dma;110 111 if (uctx->ext_db.enable) {112 req.sq_cqn_mtt_cfg |=113 FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);114 req.db_cfg =115 FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,116 uctx->ext_db.sdb_off) |117 FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,118 uctx->ext_db.rdb_off);119 }120 }121 122 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0,123 &resp1);124 if (!err)125 qp->attrs.cookie =126 FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0);127 128 return err;129}130 131static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr)132{133 struct erdma_pd *pd = to_epd(mr->ibmr.pd);134 u32 mtt_level = ERDMA_MR_MTT_0LEVEL;135 struct erdma_cmdq_reg_mr_req req;136 137 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR);138 139 if (mr->type == ERDMA_MR_TYPE_FRMR ||140 mr->mem.page_cnt > ERDMA_MAX_INLINE_MTT_ENTRIES) {141 if (mr->mem.mtt->continuous) {142 req.phy_addr[0] = mr->mem.mtt->buf_dma;143 mtt_level = ERDMA_MR_MTT_1LEVEL;144 } else {145 req.phy_addr[0] = sg_dma_address(mr->mem.mtt->sglist);146 mtt_level = mr->mem.mtt->level;147 }148 } else if (mr->type != ERDMA_MR_TYPE_DMA) {149 memcpy(req.phy_addr, mr->mem.mtt->buf,150 MTT_SIZE(mr->mem.page_cnt));151 }152 153 req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr->valid) |154 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF) |155 FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8);156 req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) |157 FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) |158 FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access);159 req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK,160 ilog2(mr->mem.page_size)) |161 FIELD_PREP(ERDMA_CMD_REGMR_MTT_LEVEL_MASK, mtt_level) |162 FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr->mem.page_cnt);163 164 if (mr->type == ERDMA_MR_TYPE_DMA)165 goto post_cmd;166 167 if (mr->type == ERDMA_MR_TYPE_NORMAL) {168 req.start_va = mr->mem.va;169 req.size = mr->mem.len;170 }171 172 if (!mr->mem.mtt->continuous && mr->mem.mtt->level > 1) {173 req.cfg0 |= FIELD_PREP(ERDMA_CMD_MR_VERSION_MASK, 1);174 req.cfg2 |= FIELD_PREP(ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK,175 PAGE_SHIFT - ERDMA_HW_PAGE_SHIFT);176 req.size_h = upper_32_bits(mr->mem.len);177 req.mtt_cnt_h = mr->mem.page_cnt >> 20;178 }179 180post_cmd:181 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);182}183 184static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)185{186 struct erdma_dev *dev = to_edev(cq->ibcq.device);187 struct erdma_cmdq_create_cq_req req;188 struct erdma_mem *mem;189 u32 page_size;190 191 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,192 CMDQ_OPCODE_CREATE_CQ);193 194 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq->cqn) |195 FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq->depth));196 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn);197 198 if (rdma_is_kernel_res(&cq->ibcq.res)) {199 page_size = SZ_32M;200 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,201 ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);202 req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);203 req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);204 205 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) |206 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,207 ERDMA_MR_MTT_0LEVEL);208 209 req.first_page_offset = 0;210 req.cq_dbrec_dma = cq->kern_cq.dbrec_dma;211 } else {212 mem = &cq->user_cq.qbuf_mem;213 req.cfg0 |=214 FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,215 ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT);216 if (mem->mtt_nents == 1) {217 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]);218 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]);219 req.cfg1 |=220 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,221 ERDMA_MR_MTT_0LEVEL);222 } else {223 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma);224 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma);225 req.cfg1 |=226 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,227 ERDMA_MR_MTT_1LEVEL);228 }229 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK,230 mem->mtt_nents);231 232 req.first_page_offset = mem->page_offset;233 req.cq_dbrec_dma = cq->user_cq.dbrec_dma;234 235 if (uctx->ext_db.enable) {236 req.cfg1 |= FIELD_PREP(237 ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);238 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,239 uctx->ext_db.cdb_off);240 }241 }242 243 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);244}245 246static int erdma_alloc_idx(struct erdma_resource_cb *res_cb)247{248 int idx;249 unsigned long flags;250 251 spin_lock_irqsave(&res_cb->lock, flags);252 idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap,253 res_cb->next_alloc_idx);254 if (idx == res_cb->max_cap) {255 idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap);256 if (idx == res_cb->max_cap) {257 res_cb->next_alloc_idx = 1;258 spin_unlock_irqrestore(&res_cb->lock, flags);259 return -ENOSPC;260 }261 }262 263 set_bit(idx, res_cb->bitmap);264 res_cb->next_alloc_idx = idx + 1;265 spin_unlock_irqrestore(&res_cb->lock, flags);266 267 return idx;268}269 270static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx)271{272 unsigned long flags;273 u32 used;274 275 spin_lock_irqsave(&res_cb->lock, flags);276 used = __test_and_clear_bit(idx, res_cb->bitmap);277 spin_unlock_irqrestore(&res_cb->lock, flags);278 WARN_ON(!used);279}280 281static struct rdma_user_mmap_entry *282erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address,283 u32 size, u8 mmap_flag, u64 *mmap_offset)284{285 struct erdma_user_mmap_entry *entry =286 kzalloc(sizeof(*entry), GFP_KERNEL);287 int ret;288 289 if (!entry)290 return NULL;291 292 entry->address = (u64)address;293 entry->mmap_flag = mmap_flag;294 295 size = PAGE_ALIGN(size);296 297 ret = rdma_user_mmap_entry_insert(&uctx->ibucontext, &entry->rdma_entry,298 size);299 if (ret) {300 kfree(entry);301 return NULL;302 }303 304 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry);305 306 return &entry->rdma_entry;307}308 309int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,310 struct ib_udata *unused)311{312 struct erdma_dev *dev = to_edev(ibdev);313 314 memset(attr, 0, sizeof(*attr));315 316 attr->max_mr_size = dev->attrs.max_mr_size;317 attr->vendor_id = PCI_VENDOR_ID_ALIBABA;318 attr->vendor_part_id = dev->pdev->device;319 attr->hw_ver = dev->pdev->revision;320 attr->max_qp = dev->attrs.max_qp - 1;321 attr->max_qp_wr = min(dev->attrs.max_send_wr, dev->attrs.max_recv_wr);322 attr->max_qp_rd_atom = dev->attrs.max_ord;323 attr->max_qp_init_rd_atom = dev->attrs.max_ird;324 attr->max_res_rd_atom = dev->attrs.max_qp * dev->attrs.max_ird;325 attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;326 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;327 ibdev->local_dma_lkey = dev->attrs.local_dma_key;328 attr->max_send_sge = dev->attrs.max_send_sge;329 attr->max_recv_sge = dev->attrs.max_recv_sge;330 attr->max_sge_rd = dev->attrs.max_sge_rd;331 attr->max_cq = dev->attrs.max_cq - 1;332 attr->max_cqe = dev->attrs.max_cqe;333 attr->max_mr = dev->attrs.max_mr;334 attr->max_pd = dev->attrs.max_pd;335 attr->max_mw = dev->attrs.max_mw;336 attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA;337 attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT;338 339 if (dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_ATOMIC)340 attr->atomic_cap = IB_ATOMIC_GLOB;341 342 attr->fw_ver = dev->attrs.fw_version;343 344 if (dev->netdev)345 addrconf_addr_eui48((u8 *)&attr->sys_image_guid,346 dev->netdev->dev_addr);347 348 return 0;349}350 351int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx,352 union ib_gid *gid)353{354 struct erdma_dev *dev = to_edev(ibdev);355 356 memset(gid, 0, sizeof(*gid));357 ether_addr_copy(gid->raw, dev->attrs.peer_addr);358 359 return 0;360}361 362int erdma_query_port(struct ib_device *ibdev, u32 port,363 struct ib_port_attr *attr)364{365 struct erdma_dev *dev = to_edev(ibdev);366 struct net_device *ndev = dev->netdev;367 368 memset(attr, 0, sizeof(*attr));369 370 attr->gid_tbl_len = 1;371 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;372 attr->max_msg_sz = -1;373 374 if (!ndev)375 goto out;376 377 ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width);378 attr->max_mtu = ib_mtu_int_to_enum(ndev->mtu);379 attr->active_mtu = ib_mtu_int_to_enum(ndev->mtu);380 if (netif_running(ndev) && netif_carrier_ok(ndev))381 dev->state = IB_PORT_ACTIVE;382 else383 dev->state = IB_PORT_DOWN;384 attr->state = dev->state;385 386out:387 if (dev->state == IB_PORT_ACTIVE)388 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;389 else390 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;391 392 return 0;393}394 395int erdma_get_port_immutable(struct ib_device *ibdev, u32 port,396 struct ib_port_immutable *port_immutable)397{398 port_immutable->gid_tbl_len = 1;399 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;400 401 return 0;402}403 404int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)405{406 struct erdma_pd *pd = to_epd(ibpd);407 struct erdma_dev *dev = to_edev(ibpd->device);408 int pdn;409 410 pdn = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_PD]);411 if (pdn < 0)412 return pdn;413 414 pd->pdn = pdn;415 416 return 0;417}418 419int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)420{421 struct erdma_pd *pd = to_epd(ibpd);422 struct erdma_dev *dev = to_edev(ibpd->device);423 424 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_PD], pd->pdn);425 426 return 0;427}428 429static void erdma_flush_worker(struct work_struct *work)430{431 struct delayed_work *dwork = to_delayed_work(work);432 struct erdma_qp *qp =433 container_of(dwork, struct erdma_qp, reflush_dwork);434 struct erdma_cmdq_reflush_req req;435 436 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,437 CMDQ_OPCODE_REFLUSH);438 req.qpn = QP_ID(qp);439 req.sq_pi = qp->kern_qp.sq_pi;440 req.rq_pi = qp->kern_qp.rq_pi;441 erdma_post_cmd_wait(&qp->dev->cmdq, &req, sizeof(req), NULL, NULL);442}443 444static int erdma_qp_validate_cap(struct erdma_dev *dev,445 struct ib_qp_init_attr *attrs)446{447 if ((attrs->cap.max_send_wr > dev->attrs.max_send_wr) ||448 (attrs->cap.max_recv_wr > dev->attrs.max_recv_wr) ||449 (attrs->cap.max_send_sge > dev->attrs.max_send_sge) ||450 (attrs->cap.max_recv_sge > dev->attrs.max_recv_sge) ||451 (attrs->cap.max_inline_data > ERDMA_MAX_INLINE) ||452 !attrs->cap.max_send_wr || !attrs->cap.max_recv_wr) {453 return -EINVAL;454 }455 456 return 0;457}458 459static int erdma_qp_validate_attr(struct erdma_dev *dev,460 struct ib_qp_init_attr *attrs)461{462 if (attrs->qp_type != IB_QPT_RC)463 return -EOPNOTSUPP;464 465 if (attrs->srq)466 return -EOPNOTSUPP;467 468 if (!attrs->send_cq || !attrs->recv_cq)469 return -EOPNOTSUPP;470 471 return 0;472}473 474static void free_kernel_qp(struct erdma_qp *qp)475{476 struct erdma_dev *dev = qp->dev;477 478 vfree(qp->kern_qp.swr_tbl);479 vfree(qp->kern_qp.rwr_tbl);480 481 if (qp->kern_qp.sq_buf)482 dma_free_coherent(&dev->pdev->dev,483 qp->attrs.sq_size << SQEBB_SHIFT,484 qp->kern_qp.sq_buf,485 qp->kern_qp.sq_buf_dma_addr);486 487 if (qp->kern_qp.sq_dbrec)488 dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,489 qp->kern_qp.sq_dbrec_dma);490 491 if (qp->kern_qp.rq_buf)492 dma_free_coherent(&dev->pdev->dev,493 qp->attrs.rq_size << RQE_SHIFT,494 qp->kern_qp.rq_buf,495 qp->kern_qp.rq_buf_dma_addr);496 497 if (qp->kern_qp.rq_dbrec)498 dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,499 qp->kern_qp.rq_dbrec_dma);500}501 502static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,503 struct ib_qp_init_attr *attrs)504{505 struct erdma_kqp *kqp = &qp->kern_qp;506 int size;507 508 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)509 kqp->sig_all = 1;510 511 kqp->sq_pi = 0;512 kqp->sq_ci = 0;513 kqp->rq_pi = 0;514 kqp->rq_ci = 0;515 kqp->hw_sq_db =516 dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);517 kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;518 519 kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));520 kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));521 if (!kqp->swr_tbl || !kqp->rwr_tbl)522 goto err_out;523 524 size = qp->attrs.sq_size << SQEBB_SHIFT;525 kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size,526 &kqp->sq_buf_dma_addr, GFP_KERNEL);527 if (!kqp->sq_buf)528 goto err_out;529 530 kqp->sq_dbrec =531 dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);532 if (!kqp->sq_dbrec)533 goto err_out;534 535 size = qp->attrs.rq_size << RQE_SHIFT;536 kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size,537 &kqp->rq_buf_dma_addr, GFP_KERNEL);538 if (!kqp->rq_buf)539 goto err_out;540 541 kqp->rq_dbrec =542 dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);543 if (!kqp->rq_dbrec)544 goto err_out;545 546 return 0;547 548err_out:549 free_kernel_qp(qp);550 return -ENOMEM;551}552 553static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem)554{555 struct erdma_mtt *mtt = mem->mtt;556 struct ib_block_iter biter;557 u32 idx = 0;558 559 while (mtt->low_level)560 mtt = mtt->low_level;561 562 rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size)563 mtt->buf[idx++] = rdma_block_iter_dma_address(&biter);564}565 566static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev,567 size_t size)568{569 struct erdma_mtt *mtt;570 571 mtt = kzalloc(sizeof(*mtt), GFP_KERNEL);572 if (!mtt)573 return ERR_PTR(-ENOMEM);574 575 mtt->size = size;576 mtt->buf = kzalloc(mtt->size, GFP_KERNEL);577 if (!mtt->buf)578 goto err_free_mtt;579 580 mtt->continuous = true;581 mtt->buf_dma = dma_map_single(&dev->pdev->dev, mtt->buf, mtt->size,582 DMA_TO_DEVICE);583 if (dma_mapping_error(&dev->pdev->dev, mtt->buf_dma))584 goto err_free_mtt_buf;585 586 return mtt;587 588err_free_mtt_buf:589 kfree(mtt->buf);590 591err_free_mtt:592 kfree(mtt);593 594 return ERR_PTR(-ENOMEM);595}596 597static void erdma_destroy_mtt_buf_sg(struct erdma_dev *dev,598 struct erdma_mtt *mtt)599{600 dma_unmap_sg(&dev->pdev->dev, mtt->sglist, mtt->nsg, DMA_TO_DEVICE);601 vfree(mtt->sglist);602}603 604static void erdma_destroy_scatter_mtt(struct erdma_dev *dev,605 struct erdma_mtt *mtt)606{607 erdma_destroy_mtt_buf_sg(dev, mtt);608 vfree(mtt->buf);609 kfree(mtt);610}611 612static void erdma_init_middle_mtt(struct erdma_mtt *mtt,613 struct erdma_mtt *low_mtt)614{615 struct scatterlist *sg;616 u32 idx = 0, i;617 618 for_each_sg(low_mtt->sglist, sg, low_mtt->nsg, i)619 mtt->buf[idx++] = sg_dma_address(sg);620}621 622static int erdma_create_mtt_buf_sg(struct erdma_dev *dev, struct erdma_mtt *mtt)623{624 struct scatterlist *sglist;625 void *buf = mtt->buf;626 u32 npages, i, nsg;627 struct page *pg;628 629 /* Failed if buf is not page aligned */630 if ((uintptr_t)buf & ~PAGE_MASK)631 return -EINVAL;632 633 npages = DIV_ROUND_UP(mtt->size, PAGE_SIZE);634 sglist = vzalloc(npages * sizeof(*sglist));635 if (!sglist)636 return -ENOMEM;637 638 sg_init_table(sglist, npages);639 for (i = 0; i < npages; i++) {640 pg = vmalloc_to_page(buf);641 if (!pg)642 goto err;643 sg_set_page(&sglist[i], pg, PAGE_SIZE, 0);644 buf += PAGE_SIZE;645 }646 647 nsg = dma_map_sg(&dev->pdev->dev, sglist, npages, DMA_TO_DEVICE);648 if (!nsg)649 goto err;650 651 mtt->sglist = sglist;652 mtt->nsg = nsg;653 654 return 0;655err:656 vfree(sglist);657 658 return -ENOMEM;659}660 661static struct erdma_mtt *erdma_create_scatter_mtt(struct erdma_dev *dev,662 size_t size)663{664 struct erdma_mtt *mtt;665 int ret = -ENOMEM;666 667 mtt = kzalloc(sizeof(*mtt), GFP_KERNEL);668 if (!mtt)669 return ERR_PTR(-ENOMEM);670 671 mtt->size = ALIGN(size, PAGE_SIZE);672 mtt->buf = vzalloc(mtt->size);673 mtt->continuous = false;674 if (!mtt->buf)675 goto err_free_mtt;676 677 ret = erdma_create_mtt_buf_sg(dev, mtt);678 if (ret)679 goto err_free_mtt_buf;680 681 ibdev_dbg(&dev->ibdev, "create scatter mtt, size:%lu, nsg:%u\n",682 mtt->size, mtt->nsg);683 684 return mtt;685 686err_free_mtt_buf:687 vfree(mtt->buf);688 689err_free_mtt:690 kfree(mtt);691 692 return ERR_PTR(ret);693}694 695static struct erdma_mtt *erdma_create_mtt(struct erdma_dev *dev, size_t size,696 bool force_continuous)697{698 struct erdma_mtt *mtt, *tmp_mtt;699 int ret, level = 0;700 701 ibdev_dbg(&dev->ibdev, "create_mtt, size:%lu, force cont:%d\n", size,702 force_continuous);703 704 if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_MTT_VA))705 force_continuous = true;706 707 if (force_continuous)708 return erdma_create_cont_mtt(dev, size);709 710 mtt = erdma_create_scatter_mtt(dev, size);711 if (IS_ERR(mtt))712 return mtt;713 level = 1;714 715 /* convergence the mtt table. */716 while (mtt->nsg != 1 && level <= 3) {717 tmp_mtt = erdma_create_scatter_mtt(dev, MTT_SIZE(mtt->nsg));718 if (IS_ERR(tmp_mtt)) {719 ret = PTR_ERR(tmp_mtt);720 goto err_free_mtt;721 }722 erdma_init_middle_mtt(tmp_mtt, mtt);723 tmp_mtt->low_level = mtt;724 mtt = tmp_mtt;725 level++;726 }727 728 if (level > 3) {729 ret = -ENOMEM;730 goto err_free_mtt;731 }732 733 mtt->level = level;734 ibdev_dbg(&dev->ibdev, "top mtt: level:%d, dma_addr 0x%llx\n",735 mtt->level, mtt->sglist[0].dma_address);736 737 return mtt;738err_free_mtt:739 while (mtt) {740 tmp_mtt = mtt->low_level;741 erdma_destroy_scatter_mtt(dev, mtt);742 mtt = tmp_mtt;743 }744 745 return ERR_PTR(ret);746}747 748static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)749{750 struct erdma_mtt *tmp_mtt;751 752 if (mtt->continuous) {753 dma_unmap_single(&dev->pdev->dev, mtt->buf_dma, mtt->size,754 DMA_TO_DEVICE);755 kfree(mtt->buf);756 kfree(mtt);757 } else {758 while (mtt) {759 tmp_mtt = mtt->low_level;760 erdma_destroy_scatter_mtt(dev, mtt);761 mtt = tmp_mtt;762 }763 }764}765 766static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,767 u64 start, u64 len, int access, u64 virt,768 unsigned long req_page_size, bool force_continuous)769{770 int ret = 0;771 772 mem->umem = ib_umem_get(&dev->ibdev, start, len, access);773 if (IS_ERR(mem->umem)) {774 ret = PTR_ERR(mem->umem);775 mem->umem = NULL;776 return ret;777 }778 779 mem->va = virt;780 mem->len = len;781 mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt);782 mem->page_offset = start & (mem->page_size - 1);783 mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);784 mem->page_cnt = mem->mtt_nents;785 mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),786 force_continuous);787 if (IS_ERR(mem->mtt)) {788 ret = PTR_ERR(mem->mtt);789 goto error_ret;790 }791 792 erdma_fill_bottom_mtt(dev, mem);793 794 return 0;795 796error_ret:797 if (mem->umem) {798 ib_umem_release(mem->umem);799 mem->umem = NULL;800 }801 802 return ret;803}804 805static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)806{807 if (mem->mtt)808 erdma_destroy_mtt(dev, mem->mtt);809 810 if (mem->umem) {811 ib_umem_release(mem->umem);812 mem->umem = NULL;813 }814}815 816static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,817 u64 dbrecords_va,818 struct erdma_user_dbrecords_page **dbr_page,819 dma_addr_t *dma_addr)820{821 struct erdma_user_dbrecords_page *page = NULL;822 int rv = 0;823 824 mutex_lock(&ctx->dbrecords_page_mutex);825 826 list_for_each_entry(page, &ctx->dbrecords_page_list, list)827 if (page->va == (dbrecords_va & PAGE_MASK))828 goto found;829 830 page = kmalloc(sizeof(*page), GFP_KERNEL);831 if (!page) {832 rv = -ENOMEM;833 goto out;834 }835 836 page->va = (dbrecords_va & PAGE_MASK);837 page->refcnt = 0;838 839 page->umem = ib_umem_get(ctx->ibucontext.device,840 dbrecords_va & PAGE_MASK, PAGE_SIZE, 0);841 if (IS_ERR(page->umem)) {842 rv = PTR_ERR(page->umem);843 kfree(page);844 goto out;845 }846 847 list_add(&page->list, &ctx->dbrecords_page_list);848 849found:850 *dma_addr = sg_dma_address(page->umem->sgt_append.sgt.sgl) +851 (dbrecords_va & ~PAGE_MASK);852 *dbr_page = page;853 page->refcnt++;854 855out:856 mutex_unlock(&ctx->dbrecords_page_mutex);857 return rv;858}859 860static void861erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,862 struct erdma_user_dbrecords_page **dbr_page)863{864 if (!ctx || !(*dbr_page))865 return;866 867 mutex_lock(&ctx->dbrecords_page_mutex);868 if (--(*dbr_page)->refcnt == 0) {869 list_del(&(*dbr_page)->list);870 ib_umem_release((*dbr_page)->umem);871 kfree(*dbr_page);872 }873 874 *dbr_page = NULL;875 mutex_unlock(&ctx->dbrecords_page_mutex);876}877 878static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,879 u64 va, u32 len, u64 dbrec_va)880{881 dma_addr_t dbrec_dma;882 u32 rq_offset;883 int ret;884 885 if (len < (ALIGN(qp->attrs.sq_size * SQEBB_SIZE, ERDMA_HW_PAGE_SIZE) +886 qp->attrs.rq_size * RQE_SIZE))887 return -EINVAL;888 889 ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va,890 qp->attrs.sq_size << SQEBB_SHIFT, 0, va,891 (SZ_1M - SZ_4K), true);892 if (ret)893 return ret;894 895 rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);896 qp->user_qp.rq_offset = rq_offset;897 898 ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset,899 qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,900 (SZ_1M - SZ_4K), true);901 if (ret)902 goto put_sq_mtt;903 904 ret = erdma_map_user_dbrecords(uctx, dbrec_va,905 &qp->user_qp.user_dbr_page,906 &dbrec_dma);907 if (ret)908 goto put_rq_mtt;909 910 qp->user_qp.sq_dbrec_dma = dbrec_dma;911 qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE;912 913 return 0;914 915put_rq_mtt:916 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);917 918put_sq_mtt:919 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);920 921 return ret;922}923 924static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)925{926 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);927 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);928 erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page);929}930 931int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,932 struct ib_udata *udata)933{934 struct erdma_qp *qp = to_eqp(ibqp);935 struct erdma_dev *dev = to_edev(ibqp->device);936 struct erdma_ucontext *uctx = rdma_udata_to_drv_context(937 udata, struct erdma_ucontext, ibucontext);938 struct erdma_ureq_create_qp ureq;939 struct erdma_uresp_create_qp uresp;940 int ret;941 942 ret = erdma_qp_validate_cap(dev, attrs);943 if (ret)944 goto err_out;945 946 ret = erdma_qp_validate_attr(dev, attrs);947 if (ret)948 goto err_out;949 950 qp->scq = to_ecq(attrs->send_cq);951 qp->rcq = to_ecq(attrs->recv_cq);952 qp->dev = dev;953 qp->attrs.cc = dev->attrs.cc;954 955 init_rwsem(&qp->state_lock);956 kref_init(&qp->ref);957 init_completion(&qp->safe_free);958 959 ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp,960 XA_LIMIT(1, dev->attrs.max_qp - 1),961 &dev->next_alloc_qpn, GFP_KERNEL);962 if (ret < 0) {963 ret = -ENOMEM;964 goto err_out;965 }966 967 qp->attrs.sq_size = roundup_pow_of_two(attrs->cap.max_send_wr *968 ERDMA_MAX_WQEBB_PER_SQE);969 qp->attrs.rq_size = roundup_pow_of_two(attrs->cap.max_recv_wr);970 971 if (uctx) {972 ret = ib_copy_from_udata(&ureq, udata,973 min(sizeof(ureq), udata->inlen));974 if (ret)975 goto err_out_xa;976 977 ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len,978 ureq.db_record_va);979 if (ret)980 goto err_out_xa;981 982 memset(&uresp, 0, sizeof(uresp));983 984 uresp.num_sqe = qp->attrs.sq_size;985 uresp.num_rqe = qp->attrs.rq_size;986 uresp.qp_id = QP_ID(qp);987 uresp.rq_offset = qp->user_qp.rq_offset;988 989 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));990 if (ret)991 goto err_out_cmd;992 } else {993 init_kernel_qp(dev, qp, attrs);994 }995 996 qp->attrs.max_send_sge = attrs->cap.max_send_sge;997 qp->attrs.max_recv_sge = attrs->cap.max_recv_sge;998 qp->attrs.state = ERDMA_QP_STATE_IDLE;999 INIT_DELAYED_WORK(&qp->reflush_dwork, erdma_flush_worker);1000 1001 ret = create_qp_cmd(uctx, qp);1002 if (ret)1003 goto err_out_cmd;1004 1005 spin_lock_init(&qp->lock);1006 1007 return 0;1008 1009err_out_cmd:1010 if (uctx)1011 free_user_qp(qp, uctx);1012 else1013 free_kernel_qp(qp);1014err_out_xa:1015 xa_erase(&dev->qp_xa, QP_ID(qp));1016err_out:1017 return ret;1018}1019 1020static int erdma_create_stag(struct erdma_dev *dev, u32 *stag)1021{1022 int stag_idx;1023 1024 stag_idx = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX]);1025 if (stag_idx < 0)1026 return stag_idx;1027 1028 /* For now, we always let key field be zero. */1029 *stag = (stag_idx << 8);1030 1031 return 0;1032}1033 1034struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc)1035{1036 struct erdma_dev *dev = to_edev(ibpd->device);1037 struct erdma_mr *mr;1038 u32 stag;1039 int ret;1040 1041 mr = kzalloc(sizeof(*mr), GFP_KERNEL);1042 if (!mr)1043 return ERR_PTR(-ENOMEM);1044 1045 ret = erdma_create_stag(dev, &stag);1046 if (ret)1047 goto out_free;1048 1049 mr->type = ERDMA_MR_TYPE_DMA;1050 1051 mr->ibmr.lkey = stag;1052 mr->ibmr.rkey = stag;1053 mr->ibmr.pd = ibpd;1054 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc);1055 ret = regmr_cmd(dev, mr);1056 if (ret)1057 goto out_remove_stag;1058 1059 return &mr->ibmr;1060 1061out_remove_stag:1062 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],1063 mr->ibmr.lkey >> 8);1064 1065out_free:1066 kfree(mr);1067 1068 return ERR_PTR(ret);1069}1070 1071struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,1072 u32 max_num_sg)1073{1074 struct erdma_mr *mr;1075 struct erdma_dev *dev = to_edev(ibpd->device);1076 int ret;1077 u32 stag;1078 1079 if (mr_type != IB_MR_TYPE_MEM_REG)1080 return ERR_PTR(-EOPNOTSUPP);1081 1082 if (max_num_sg > ERDMA_MR_MAX_MTT_CNT)1083 return ERR_PTR(-EINVAL);1084 1085 mr = kzalloc(sizeof(*mr), GFP_KERNEL);1086 if (!mr)1087 return ERR_PTR(-ENOMEM);1088 1089 ret = erdma_create_stag(dev, &stag);1090 if (ret)1091 goto out_free;1092 1093 mr->type = ERDMA_MR_TYPE_FRMR;1094 1095 mr->ibmr.lkey = stag;1096 mr->ibmr.rkey = stag;1097 mr->ibmr.pd = ibpd;1098 /* update it in FRMR. */1099 mr->access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR |1100 ERDMA_MR_ACC_RW;1101 1102 mr->mem.page_size = PAGE_SIZE; /* update it later. */1103 mr->mem.page_cnt = max_num_sg;1104 mr->mem.mtt = erdma_create_mtt(dev, MTT_SIZE(max_num_sg), true);1105 if (IS_ERR(mr->mem.mtt)) {1106 ret = PTR_ERR(mr->mem.mtt);1107 goto out_remove_stag;1108 }1109 1110 ret = regmr_cmd(dev, mr);1111 if (ret)1112 goto out_destroy_mtt;1113 1114 return &mr->ibmr;1115 1116out_destroy_mtt:1117 erdma_destroy_mtt(dev, mr->mem.mtt);1118 1119out_remove_stag:1120 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],1121 mr->ibmr.lkey >> 8);1122 1123out_free:1124 kfree(mr);1125 1126 return ERR_PTR(ret);1127}1128 1129static int erdma_set_page(struct ib_mr *ibmr, u64 addr)1130{1131 struct erdma_mr *mr = to_emr(ibmr);1132 1133 if (mr->mem.mtt_nents >= mr->mem.page_cnt)1134 return -1;1135 1136 mr->mem.mtt->buf[mr->mem.mtt_nents] = addr;1137 mr->mem.mtt_nents++;1138 1139 return 0;1140}1141 1142int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,1143 unsigned int *sg_offset)1144{1145 struct erdma_mr *mr = to_emr(ibmr);1146 int num;1147 1148 mr->mem.mtt_nents = 0;1149 1150 num = ib_sg_to_pages(&mr->ibmr, sg, sg_nents, sg_offset,1151 erdma_set_page);1152 1153 return num;1154}1155 1156struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,1157 u64 virt, int access, struct ib_udata *udata)1158{1159 struct erdma_mr *mr = NULL;1160 struct erdma_dev *dev = to_edev(ibpd->device);1161 u32 stag;1162 int ret;1163 1164 if (!len || len > dev->attrs.max_mr_size)1165 return ERR_PTR(-EINVAL);1166 1167 mr = kzalloc(sizeof(*mr), GFP_KERNEL);1168 if (!mr)1169 return ERR_PTR(-ENOMEM);1170 1171 ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt,1172 SZ_2G - SZ_4K, false);1173 if (ret)1174 goto err_out_free;1175 1176 ret = erdma_create_stag(dev, &stag);1177 if (ret)1178 goto err_out_put_mtt;1179 1180 mr->ibmr.lkey = mr->ibmr.rkey = stag;1181 mr->ibmr.pd = ibpd;1182 mr->mem.va = virt;1183 mr->mem.len = len;1184 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);1185 mr->valid = 1;1186 mr->type = ERDMA_MR_TYPE_NORMAL;1187 1188 ret = regmr_cmd(dev, mr);1189 if (ret)1190 goto err_out_mr;1191 1192 return &mr->ibmr;1193 1194err_out_mr:1195 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],1196 mr->ibmr.lkey >> 8);1197 1198err_out_put_mtt:1199 put_mtt_entries(dev, &mr->mem);1200 1201err_out_free:1202 kfree(mr);1203 1204 return ERR_PTR(ret);1205}1206 1207int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)1208{1209 struct erdma_mr *mr;1210 struct erdma_dev *dev = to_edev(ibmr->device);1211 struct erdma_cmdq_dereg_mr_req req;1212 int ret;1213 1214 mr = to_emr(ibmr);1215 1216 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,1217 CMDQ_OPCODE_DEREG_MR);1218 1219 req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) |1220 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF);1221 1222 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);1223 if (ret)1224 return ret;1225 1226 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);1227 1228 put_mtt_entries(dev, &mr->mem);1229 1230 kfree(mr);1231 return 0;1232}1233 1234int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)1235{1236 struct erdma_cq *cq = to_ecq(ibcq);1237 struct erdma_dev *dev = to_edev(ibcq->device);1238 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(1239 udata, struct erdma_ucontext, ibucontext);1240 int err;1241 struct erdma_cmdq_destroy_cq_req req;1242 1243 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,1244 CMDQ_OPCODE_DESTROY_CQ);1245 req.cqn = cq->cqn;1246 1247 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);1248 if (err)1249 return err;1250 1251 if (rdma_is_kernel_res(&cq->ibcq.res)) {1252 dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,1253 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);1254 dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,1255 cq->kern_cq.dbrec_dma);1256 } else {1257 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);1258 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);1259 }1260 1261 xa_erase(&dev->cq_xa, cq->cqn);1262 1263 return 0;1264}1265 1266int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)1267{1268 struct erdma_qp *qp = to_eqp(ibqp);1269 struct erdma_dev *dev = to_edev(ibqp->device);1270 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(1271 udata, struct erdma_ucontext, ibucontext);1272 struct erdma_qp_attrs qp_attrs;1273 int err;1274 struct erdma_cmdq_destroy_qp_req req;1275 1276 down_write(&qp->state_lock);1277 qp_attrs.state = ERDMA_QP_STATE_ERROR;1278 erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE);1279 up_write(&qp->state_lock);1280 1281 cancel_delayed_work_sync(&qp->reflush_dwork);1282 1283 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,1284 CMDQ_OPCODE_DESTROY_QP);1285 req.qpn = QP_ID(qp);1286 1287 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);1288 if (err)1289 return err;1290 1291 erdma_qp_put(qp);1292 wait_for_completion(&qp->safe_free);1293 1294 if (rdma_is_kernel_res(&qp->ibqp.res)) {1295 free_kernel_qp(qp);1296 } else {1297 put_mtt_entries(dev, &qp->user_qp.sq_mem);1298 put_mtt_entries(dev, &qp->user_qp.rq_mem);1299 erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page);1300 }1301 1302 if (qp->cep)1303 erdma_cep_put(qp->cep);1304 xa_erase(&dev->qp_xa, QP_ID(qp));1305 1306 return 0;1307}1308 1309void erdma_qp_get_ref(struct ib_qp *ibqp)1310{1311 erdma_qp_get(to_eqp(ibqp));1312}1313 1314void erdma_qp_put_ref(struct ib_qp *ibqp)1315{1316 erdma_qp_put(to_eqp(ibqp));1317}1318 1319int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)1320{1321 struct rdma_user_mmap_entry *rdma_entry;1322 struct erdma_user_mmap_entry *entry;1323 pgprot_t prot;1324 int err;1325 1326 rdma_entry = rdma_user_mmap_entry_get(ctx, vma);1327 if (!rdma_entry)1328 return -EINVAL;1329 1330 entry = to_emmap(rdma_entry);1331 1332 switch (entry->mmap_flag) {1333 case ERDMA_MMAP_IO_NC:1334 /* map doorbell. */1335 prot = pgprot_device(vma->vm_page_prot);1336 break;1337 default:1338 err = -EINVAL;1339 goto put_entry;1340 }1341 1342 err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE,1343 prot, rdma_entry);1344 1345put_entry:1346 rdma_user_mmap_entry_put(rdma_entry);1347 return err;1348}1349 1350void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)1351{1352 struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry);1353 1354 kfree(entry);1355}1356 1357static int alloc_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx,1358 bool ext_db_en)1359{1360 struct erdma_cmdq_ext_db_req req = {};1361 u64 val0, val1;1362 int ret;1363 1364 /*1365 * CAP_SYS_RAWIO is required if hardware does not support extend1366 * doorbell mechanism.1367 */1368 if (!ext_db_en && !capable(CAP_SYS_RAWIO))1369 return -EPERM;1370 1371 if (!ext_db_en) {1372 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET;1373 ctx->rdb = dev->func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET;1374 ctx->cdb = dev->func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET;1375 return 0;1376 }1377 1378 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,1379 CMDQ_OPCODE_ALLOC_DB);1380 1381 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |1382 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |1383 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);1384 1385 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &val0, &val1);1386 if (ret)1387 return ret;1388 1389 ctx->ext_db.enable = true;1390 ctx->ext_db.sdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_SDB);1391 ctx->ext_db.rdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_RDB);1392 ctx->ext_db.cdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_CDB);1393 1394 ctx->sdb = dev->func_bar_addr + (ctx->ext_db.sdb_off << PAGE_SHIFT);1395 ctx->cdb = dev->func_bar_addr + (ctx->ext_db.rdb_off << PAGE_SHIFT);1396 ctx->rdb = dev->func_bar_addr + (ctx->ext_db.cdb_off << PAGE_SHIFT);1397 1398 return 0;1399}1400 1401static void free_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx)1402{1403 struct erdma_cmdq_ext_db_req req = {};1404 int ret;1405 1406 if (!ctx->ext_db.enable)1407 return;1408 1409 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,1410 CMDQ_OPCODE_FREE_DB);1411 1412 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |1413 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |1414 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);1415 1416 req.sdb_off = ctx->ext_db.sdb_off;1417 req.rdb_off = ctx->ext_db.rdb_off;1418 req.cdb_off = ctx->ext_db.cdb_off;1419 1420 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);1421 if (ret)1422 ibdev_err_ratelimited(&dev->ibdev,1423 "free db resources failed %d", ret);1424}1425 1426static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx)1427{1428 rdma_user_mmap_entry_remove(uctx->sq_db_mmap_entry);1429 rdma_user_mmap_entry_remove(uctx->rq_db_mmap_entry);1430 rdma_user_mmap_entry_remove(uctx->cq_db_mmap_entry);1431}1432 1433int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)1434{1435 struct erdma_ucontext *ctx = to_ectx(ibctx);1436 struct erdma_dev *dev = to_edev(ibctx->device);1437 int ret;1438 struct erdma_uresp_alloc_ctx uresp = {};1439 1440 if (atomic_inc_return(&dev->num_ctx) > ERDMA_MAX_CONTEXT) {1441 ret = -ENOMEM;1442 goto err_out;1443 }1444 1445 if (udata->outlen < sizeof(uresp)) {1446 ret = -EINVAL;1447 goto err_out;1448 }1449 1450 INIT_LIST_HEAD(&ctx->dbrecords_page_list);1451 mutex_init(&ctx->dbrecords_page_mutex);1452 1453 ret = alloc_db_resources(dev, ctx,1454 !!(dev->attrs.cap_flags &1455 ERDMA_DEV_CAP_FLAGS_EXTEND_DB));1456 if (ret)1457 goto err_out;1458 1459 ctx->sq_db_mmap_entry = erdma_user_mmap_entry_insert(1460 ctx, (void *)ctx->sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.sdb);1461 if (!ctx->sq_db_mmap_entry) {1462 ret = -ENOMEM;1463 goto err_free_ext_db;1464 }1465 1466 ctx->rq_db_mmap_entry = erdma_user_mmap_entry_insert(1467 ctx, (void *)ctx->rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.rdb);1468 if (!ctx->rq_db_mmap_entry) {1469 ret = -EINVAL;1470 goto err_put_mmap_entries;1471 }1472 1473 ctx->cq_db_mmap_entry = erdma_user_mmap_entry_insert(1474 ctx, (void *)ctx->cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.cdb);1475 if (!ctx->cq_db_mmap_entry) {1476 ret = -EINVAL;1477 goto err_put_mmap_entries;1478 }1479 1480 uresp.dev_id = dev->pdev->device;1481 1482 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));1483 if (ret)1484 goto err_put_mmap_entries;1485 1486 return 0;1487 1488err_put_mmap_entries:1489 erdma_uctx_user_mmap_entries_remove(ctx);1490 1491err_free_ext_db:1492 free_db_resources(dev, ctx);1493 1494err_out:1495 atomic_dec(&dev->num_ctx);1496 return ret;1497}1498 1499void erdma_dealloc_ucontext(struct ib_ucontext *ibctx)1500{1501 struct erdma_dev *dev = to_edev(ibctx->device);1502 struct erdma_ucontext *ctx = to_ectx(ibctx);1503 1504 erdma_uctx_user_mmap_entries_remove(ctx);1505 free_db_resources(dev, ctx);1506 atomic_dec(&dev->num_ctx);1507}1508 1509static int ib_qp_state_to_erdma_qp_state[IB_QPS_ERR + 1] = {1510 [IB_QPS_RESET] = ERDMA_QP_STATE_IDLE,1511 [IB_QPS_INIT] = ERDMA_QP_STATE_IDLE,1512 [IB_QPS_RTR] = ERDMA_QP_STATE_RTR,1513 [IB_QPS_RTS] = ERDMA_QP_STATE_RTS,1514 [IB_QPS_SQD] = ERDMA_QP_STATE_CLOSING,1515 [IB_QPS_SQE] = ERDMA_QP_STATE_TERMINATE,1516 [IB_QPS_ERR] = ERDMA_QP_STATE_ERROR1517};1518 1519int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,1520 struct ib_udata *udata)1521{1522 struct erdma_qp_attrs new_attrs;1523 enum erdma_qp_attr_mask erdma_attr_mask = 0;1524 struct erdma_qp *qp = to_eqp(ibqp);1525 int ret = 0;1526 1527 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)1528 return -EOPNOTSUPP;1529 1530 memset(&new_attrs, 0, sizeof(new_attrs));1531 1532 if (attr_mask & IB_QP_STATE) {1533 new_attrs.state = ib_qp_state_to_erdma_qp_state[attr->qp_state];1534 1535 erdma_attr_mask |= ERDMA_QP_ATTR_STATE;1536 }1537 1538 down_write(&qp->state_lock);1539 1540 ret = erdma_modify_qp_internal(qp, &new_attrs, erdma_attr_mask);1541 1542 up_write(&qp->state_lock);1543 1544 return ret;1545}1546 1547static enum ib_qp_state query_qp_state(struct erdma_qp *qp)1548{1549 switch (qp->attrs.state) {1550 case ERDMA_QP_STATE_IDLE:1551 return IB_QPS_INIT;1552 case ERDMA_QP_STATE_RTR:1553 return IB_QPS_RTR;1554 case ERDMA_QP_STATE_RTS:1555 return IB_QPS_RTS;1556 case ERDMA_QP_STATE_CLOSING:1557 return IB_QPS_ERR;1558 case ERDMA_QP_STATE_TERMINATE:1559 return IB_QPS_ERR;1560 case ERDMA_QP_STATE_ERROR:1561 return IB_QPS_ERR;1562 default:1563 return IB_QPS_ERR;1564 }1565}1566 1567int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,1568 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)1569{1570 struct erdma_dev *dev;1571 struct erdma_qp *qp;1572 1573 if (ibqp && qp_attr && qp_init_attr) {1574 qp = to_eqp(ibqp);1575 dev = to_edev(ibqp->device);1576 } else {1577 return -EINVAL;1578 }1579 1580 qp_attr->cap.max_inline_data = ERDMA_MAX_INLINE;1581 qp_init_attr->cap.max_inline_data = ERDMA_MAX_INLINE;1582 1583 qp_attr->cap.max_send_wr = qp->attrs.sq_size;1584 qp_attr->cap.max_recv_wr = qp->attrs.rq_size;1585 qp_attr->cap.max_send_sge = qp->attrs.max_send_sge;1586 qp_attr->cap.max_recv_sge = qp->attrs.max_recv_sge;1587 1588 qp_attr->path_mtu = ib_mtu_int_to_enum(dev->netdev->mtu);1589 qp_attr->max_rd_atomic = qp->attrs.irq_size;1590 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;1591 1592 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |1593 IB_ACCESS_REMOTE_WRITE |1594 IB_ACCESS_REMOTE_READ;1595 1596 qp_init_attr->cap = qp_attr->cap;1597 1598 qp_attr->qp_state = query_qp_state(qp);1599 qp_attr->cur_qp_state = query_qp_state(qp);1600 1601 return 0;1602}1603 1604static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,1605 struct erdma_ureq_create_cq *ureq)1606{1607 int ret;1608 struct erdma_dev *dev = to_edev(cq->ibcq.device);1609 1610 ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va,1611 ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,1612 true);1613 if (ret)1614 return ret;1615 1616 ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va,1617 &cq->user_cq.user_dbr_page,1618 &cq->user_cq.dbrec_dma);1619 if (ret)1620 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);1621 1622 return ret;1623}1624 1625static int erdma_init_kernel_cq(struct erdma_cq *cq)1626{1627 struct erdma_dev *dev = to_edev(cq->ibcq.device);1628 1629 cq->kern_cq.qbuf =1630 dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,1631 &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL);1632 if (!cq->kern_cq.qbuf)1633 return -ENOMEM;1634 1635 cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL,1636 &cq->kern_cq.dbrec_dma);1637 if (!cq->kern_cq.dbrec)1638 goto err_out;1639 1640 spin_lock_init(&cq->kern_cq.lock);1641 /* use default cqdb addr */1642 cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET;1643 1644 return 0;1645 1646err_out:1647 dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,1648 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);1649 1650 return -ENOMEM;1651}1652 1653int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,1654 struct uverbs_attr_bundle *attrs)1655{1656 struct ib_udata *udata = &attrs->driver_udata;1657 struct erdma_cq *cq = to_ecq(ibcq);1658 struct erdma_dev *dev = to_edev(ibcq->device);1659 unsigned int depth = attr->cqe;1660 int ret;1661 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(1662 udata, struct erdma_ucontext, ibucontext);1663 1664 if (depth > dev->attrs.max_cqe)1665 return -EINVAL;1666 1667 depth = roundup_pow_of_two(depth);1668 cq->ibcq.cqe = depth;1669 cq->depth = depth;1670 cq->assoc_eqn = attr->comp_vector + 1;1671 1672 ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq,1673 XA_LIMIT(1, dev->attrs.max_cq - 1),1674 &dev->next_alloc_cqn, GFP_KERNEL);1675 if (ret < 0)1676 return ret;1677 1678 if (!rdma_is_kernel_res(&ibcq->res)) {1679 struct erdma_ureq_create_cq ureq;1680 struct erdma_uresp_create_cq uresp;1681 1682 ret = ib_copy_from_udata(&ureq, udata,1683 min(udata->inlen, sizeof(ureq)));1684 if (ret)1685 goto err_out_xa;1686 1687 ret = erdma_init_user_cq(ctx, cq, &ureq);1688 if (ret)1689 goto err_out_xa;1690 1691 uresp.cq_id = cq->cqn;1692 uresp.num_cqe = depth;1693 1694 ret = ib_copy_to_udata(udata, &uresp,1695 min(sizeof(uresp), udata->outlen));1696 if (ret)1697 goto err_free_res;1698 } else {1699 ret = erdma_init_kernel_cq(cq);1700 if (ret)1701 goto err_out_xa;1702 }1703 1704 ret = create_cq_cmd(ctx, cq);1705 if (ret)1706 goto err_free_res;1707 1708 return 0;1709 1710err_free_res:1711 if (!rdma_is_kernel_res(&ibcq->res)) {1712 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);1713 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);1714 } else {1715 dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT,1716 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);1717 dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,1718 cq->kern_cq.dbrec_dma);1719 }1720 1721err_out_xa:1722 xa_erase(&dev->cq_xa, cq->cqn);1723 1724 return ret;1725}1726 1727void erdma_disassociate_ucontext(struct ib_ucontext *ibcontext)1728{1729}1730 1731void erdma_set_mtu(struct erdma_dev *dev, u32 mtu)1732{1733 struct erdma_cmdq_config_mtu_req req;1734 1735 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,1736 CMDQ_OPCODE_CONF_MTU);1737 req.mtu = mtu;1738 1739 erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);1740}1741 1742void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason)1743{1744 struct ib_event event;1745 1746 event.device = &dev->ibdev;1747 event.element.port_num = 1;1748 event.event = reason;1749 1750 ib_dispatch_event(&event);1751}1752 1753enum counters {1754 ERDMA_STATS_TX_REQS_CNT,1755 ERDMA_STATS_TX_PACKETS_CNT,1756 ERDMA_STATS_TX_BYTES_CNT,1757 ERDMA_STATS_TX_DISABLE_DROP_CNT,1758 ERDMA_STATS_TX_BPS_METER_DROP_CNT,1759 ERDMA_STATS_TX_PPS_METER_DROP_CNT,1760 1761 ERDMA_STATS_RX_PACKETS_CNT,1762 ERDMA_STATS_RX_BYTES_CNT,1763 ERDMA_STATS_RX_DISABLE_DROP_CNT,1764 ERDMA_STATS_RX_BPS_METER_DROP_CNT,1765 ERDMA_STATS_RX_PPS_METER_DROP_CNT,1766 1767 ERDMA_STATS_MAX1768};1769 1770static const struct rdma_stat_desc erdma_descs[] = {1771 [ERDMA_STATS_TX_REQS_CNT].name = "tx_reqs_cnt",1772 [ERDMA_STATS_TX_PACKETS_CNT].name = "tx_packets_cnt",1773 [ERDMA_STATS_TX_BYTES_CNT].name = "tx_bytes_cnt",1774 [ERDMA_STATS_TX_DISABLE_DROP_CNT].name = "tx_disable_drop_cnt",1775 [ERDMA_STATS_TX_BPS_METER_DROP_CNT].name = "tx_bps_limit_drop_cnt",1776 [ERDMA_STATS_TX_PPS_METER_DROP_CNT].name = "tx_pps_limit_drop_cnt",1777 [ERDMA_STATS_RX_PACKETS_CNT].name = "rx_packets_cnt",1778 [ERDMA_STATS_RX_BYTES_CNT].name = "rx_bytes_cnt",1779 [ERDMA_STATS_RX_DISABLE_DROP_CNT].name = "rx_disable_drop_cnt",1780 [ERDMA_STATS_RX_BPS_METER_DROP_CNT].name = "rx_bps_limit_drop_cnt",1781 [ERDMA_STATS_RX_PPS_METER_DROP_CNT].name = "rx_pps_limit_drop_cnt",1782};1783 1784struct rdma_hw_stats *erdma_alloc_hw_port_stats(struct ib_device *device,1785 u32 port_num)1786{1787 return rdma_alloc_hw_stats_struct(erdma_descs, ERDMA_STATS_MAX,1788 RDMA_HW_STATS_DEFAULT_LIFESPAN);1789}1790 1791static int erdma_query_hw_stats(struct erdma_dev *dev,1792 struct rdma_hw_stats *stats)1793{1794 struct erdma_cmdq_query_stats_resp *resp;1795 struct erdma_cmdq_query_req req;1796 dma_addr_t dma_addr;1797 int err;1798 1799 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,1800 CMDQ_OPCODE_GET_STATS);1801 1802 resp = dma_pool_zalloc(dev->resp_pool, GFP_KERNEL, &dma_addr);1803 if (!resp)1804 return -ENOMEM;1805 1806 req.target_addr = dma_addr;1807 req.target_length = ERDMA_HW_RESP_SIZE;1808 1809 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);1810 if (err)1811 goto out;1812 1813 if (resp->hdr.magic != ERDMA_HW_RESP_MAGIC) {1814 err = -EINVAL;1815 goto out;1816 }1817 1818 memcpy(&stats->value[0], &resp->tx_req_cnt,1819 sizeof(u64) * stats->num_counters);1820 1821out:1822 dma_pool_free(dev->resp_pool, resp, dma_addr);1823 1824 return err;1825}1826 1827int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,1828 u32 port, int index)1829{1830 struct erdma_dev *dev = to_edev(ibdev);1831 int ret;1832 1833 if (port == 0)1834 return 0;1835 1836 ret = erdma_query_hw_stats(dev, stats);1837 if (ret)1838 return ret;1839 1840 return stats->num_counters;1841}1842